diff --git a/AGENTS.md b/AGENTS.md index 892f301b..e5fa9d8a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -323,6 +323,25 @@ If it touches user input, files, dynamic imports, or logs, what trust boundary w Prefer small, load-bearing PRs. Do not mix baseline fixes, feature work, and large reorganization unless the coupling is unavoidable. +## Kernel Substrate / No-New-Legacy Rule + +After PR #829, the preferred math comprehension construction path is: + +```text +raw problem text → KernelFacts → ProblemFrame → contract-backed derivation organs +``` + +- Use `generate/problem_frame_builder.py::build_problem_frame` for substrate-backed + fact extraction (scalars, units, hazards, process-frame candidates). +- New derivation capabilities must consume ProblemFrame facts where the substrate can + represent the needed meaning. +- New raw-prose/local-regex parsing inside a derivation organ requires an explicit + `LEGACY_EXCEPTION` comment and migration rationale. +- Guard test: `tests/test_kernel_no_new_legacy_derivation_surfaces.py`. +- Audit map: `docs/analysis/kernel-substrate-deprecation-audit-2026-06-18.md`. + +Do not add another isolated benchmark organ with a local prose parser. + ## Architecture in One Sentence Raw input becomes a closed versor field once; thought evolves through exact diff --git a/CLAUDE.md b/CLAUDE.md index 97529fe3..bf2ac299 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -123,6 +123,20 @@ runtime path. Vault recall is exact and deterministic. - `calibration/*` — bounded replay-based calibration. - `docs/runtime_contracts.md` — response, telemetry, memory, identity, and testing contracts. +### Kernel substrate / ProblemFrame (operational path after PR #829) + +New derivation capabilities must consume `KernelFacts` / `ProblemFrame` facts where the +substrate can represent the needed meaning (`generate/problem_frame_builder.py`). + +```text +raw problem text → KernelFacts → ProblemFrame → contract-backed derivation organs +``` + +New raw-prose/local-regex parsing inside a derivation organ requires an explicit +`LEGACY_EXCEPTION` note and a migration rationale. Guard: +`tests/test_kernel_no_new_legacy_derivation_surfaces.py`. Migration map: +`docs/analysis/kernel-substrate-deprecation-audit-2026-06-18.md`. + ### GSM8K math comprehension substrate (sealed; serving `7/43/0`, wrong=0 — moves only via ratified PRs) - `core/reliability_gate/` — calibrated-learning ledger + gate (ADR-0175): `ClassTally` counts, `conservative_floor` (one-sided Wilson, N_MIN=10), θ ceilings. diff --git a/GROK.md b/GROK.md index ee79a255..46fa263c 100644 --- a/GROK.md +++ b/GROK.md @@ -284,6 +284,24 @@ This is not optional. It is the only continuity mechanism across your stateless --- +## Kernel Substrate / ProblemFrame Doctrine + +New derivation capabilities must consume `KernelFacts` / `ProblemFrame` facts where the +substrate can represent the needed meaning (`generate/problem_frame_builder.py`). + +```text +raw problem text → KernelFacts → ProblemFrame → contract-backed derivation organs +``` + +New raw-prose/local-regex parsing inside a derivation organ requires an explicit +`LEGACY_EXCEPTION` note and a migration rationale. Guard: +`tests/test_kernel_no_new_legacy_derivation_surfaces.py`. + +Do not add isolated benchmark organs with local prose parsers. Do not treat #829 +substrate modules as optional helpers. + +--- + ## Architecture Summary Raw input becomes a closed versor field once; thought evolves through exact versor transitions and CGA recall; cognition is structured as intent, proposition graph, articulation target, deterministic realization, reviewed memory, eval/calibration replay, and traceable evidence. diff --git a/docs/analysis/kernel-operationalization-implementation-2026-06-18.md b/docs/analysis/kernel-operationalization-implementation-2026-06-18.md new file mode 100644 index 00000000..99a146b8 --- /dev/null +++ b/docs/analysis/kernel-operationalization-implementation-2026-06-18.md @@ -0,0 +1,186 @@ +# Kernel Operationalization Implementation (2026-06-18) + +Records the operationalization pass after PR #829 (Kernel Substrate Tranche 1). + +--- + +## 1. Purpose + +Make the #829 kernel substrate the **preferred operational path**: + +```text +raw problem text → KernelFacts → ProblemFrame → contract-backed derivation organs +``` + +Begin retiring legacy `raw text → local regex organ → one-off answer` habits without +breaking serving (`train_sample: 30 / 20 / 0`, `wrong_ids: []`). + +--- + +## 2. Relationship to #829 + +PR #829 (`58a94c8e`) added: + +- `generate/kernel_facts.py` +- `language_packs/scalar_equivalence.py` +- `language_packs/unit_dimensions.py` +- `language_packs/ambiguity_hazards.py` +- `generate/process_frames.py` +- `generate/problem_frame.py` (IR skeleton) +- `scripts/gsm8k_substrate_morphology.py` (v1 labels) + +This PR **operationalizes** those modules via `build_problem_frame`, adds deprecation +audit + no-new-legacy guardrails, and upgrades morphology to planner v2. No serving +organ rewrite in this PR. + +--- + +## 3. Legacy audit summary + +See `docs/analysis/kernel-substrate-deprecation-audit-2026-06-18.md`. + +Key findings: + +- **Shared debt:** `generate/derivation/extract.py` fans out to most organs. +- **Serving path (keep):** `math_candidate_parser.py`, `math_candidate_graph.py`. +- **First migration targets:** `percent_partition`, `nested_fraction_remainder_total`, `fraction_decrease`, `temporal_tariff`. +- **Recommended first organ:** `percent_partition`. + +--- + +## 4. No-new-legacy rule + +Added to agent/architecture guidance: + +> New derivation capabilities must consume KernelFacts / ProblemFrame facts where the +> substrate can represent the needed meaning. New raw-prose/local-regex parsing inside a +> derivation organ requires an explicit `LEGACY_EXCEPTION` note and a migration rationale. + +**Locations:** + +- `AGENTS.md` +- `GROK.md` +- `CLAUDE.md` +- `docs/architecture/kernel-knowledge-layer-v1.md` +- `docs/runtime_contracts.md` + +**Guard test:** `tests/test_kernel_no_new_legacy_derivation_surfaces.py` + +--- + +## 5. ProblemFrame builder behavior + +**Module:** `generate/problem_frame_builder.py` + +**Entry point:** `build_problem_frame(problem_text: str) -> ProblemFrame` + +Pipeline: + +1. `extract_scalar_candidates` → scalar facts with exact spans (ordinal contexts like `third place` refused as fractions) +2. Unit token scan + `classify_dimension` → `GroundedUnit` with `problem_text` spans +3. `ambiguity_hazards` registry scan (+ `%` → percent hazards) +4. `process_frames` trigger scan → candidate frames (not conclusions) +5. `CandidateRelation` per matched process frame +6. Optional `QuestionTarget` from `how many` / `how much` / `?` + +**Non-goals enforced:** no answer derivation, no case-id behavior, no ADR-0128 broadening +for unsupported surfaces (`.5`, `1 / 2`). + +**Tests:** `tests/test_problem_frame_builder.py` (8 scenarios from brief) + +--- + +## 6. Morphology planner v2 behavior + +**Module:** `scripts/gsm8k_substrate_morphology.py` + +**New API:** `plan_substrate_case(...)`, `recommend_migration_target(...)` + +**CLI:** `--planner` emits v2 records; optional `--verdicts` attaches serving verdicts. + +Output fields: + +| Field | Source | +|---|---| +| `case_id` | caller / cases JSONL | +| `current_verdict` | optional report JSON | +| `recognized_scalars` | `build_problem_frame` | +| `recognized_units` | `build_problem_frame` | +| `recognized_process_frames` | `build_problem_frame` | +| `recognized_hazards` | `build_problem_frame` | +| `missing_substrate_labels` | `classify_missing_substrate` (v1) | +| `legacy_parser_dependency` | heuristic organ/module map | +| `recommended_migration_target` | organ or `substrate:*` extension | + +No pack mutation, no sealed artifact analysis, no answer mining, no report rebaseline. + +--- + +## 7. First organ migration recommendation + +**Target:** `generate/derivation/percent_partition.py` + +See audit doc § "Recommended first migration: percent_partition". + +--- + +## 8. Serving integration status + +| Surface | Status | +|---|---| +| `build_problem_frame` | **Available** — inspection/diagnostics/tests only | +| Morphology planner v2 | **Available** — CLI diagnostics | +| Serving answer admission from ProblemFrame | **Not wired** — intentional | +| `report.json` | **Unchanged** | +| Sealed artifacts | **Unchanged** | +| Train/holdout scores | **Unchanged** (validated below) | + +--- + +## 9. Validation + +```bash +git diff --check origin/main...HEAD +pytest tests/test_kernel_facts.py -q +pytest tests/test_problem_frame_skeleton.py -q +pytest tests/test_problem_frame_builder.py -q +pytest tests/test_gsm8k_morphology_missing_kernel_labels.py -q +pytest tests/test_kernel_no_new_legacy_derivation_surfaces.py -q +pytest tests/test_adr_0128_numeric_formats.py -q +pytest tests/test_math_candidate_graph_xhigh_sprint13_lift.py -q +pytest tests/test_math_candidate_graph_sprint12_singleton_contract_lift.py -q +pytest tests/test_math_candidate_graph_sprint11_cluster_contract_lift.py -q +``` + +Capability stability (expected `train_sample: 30 20 0`, `wrong_ids: []`): + +```bash +uv run python - <<'PY' +from evals.gsm8k_math.train_sample.v1.runner import _CASES_PATH, _load_cases, build_report +r = build_report(_load_cases(_CASES_PATH)) +c = r["counts"] +print("train_sample:", c["correct"], c["refused"], c["wrong"]) +print("wrong_ids:", sorted(x["case_id"] for x in r["per_case"] if x["verdict"] == "wrong")) +PY +``` + +Holdout safety (expected `wrong_ids: []`): + +```bash +uv run python - <<'PY' +from evals.gsm8k_math.holdout_dev.v1.runner import build_report +r = build_report() +print("wrong_ids:", [x["case_id"] for x in r["per_case"] if x["verdict"] == "wrong"]) +PY +``` + +--- + +## 10. Next PR recommendation + +**`feat(kernel): migrate percent_partition to ProblemFrame`** + +- Consume `build_problem_frame` facts inside `percent_partition` +- Keep verifier gate and `wrong == 0` +- No `report.json` rebaseline unless ratified +- Remove redundant local percent regex only after parity tests pass \ No newline at end of file diff --git a/docs/analysis/kernel-substrate-deprecation-audit-2026-06-18.md b/docs/analysis/kernel-substrate-deprecation-audit-2026-06-18.md new file mode 100644 index 00000000..b7847eeb --- /dev/null +++ b/docs/analysis/kernel-substrate-deprecation-audit-2026-06-18.md @@ -0,0 +1,151 @@ +# Kernel Substrate Legacy Deprecation Audit (2026-06-18) + +Migration map for retiring raw-prose / local-regex derivation habits after PR #829 +(Kernel Substrate Tranche 1). This is not a blame list — it classifies what still +serves, what should migrate, and what can be deleted after migration. + +## Audit method + +Searches run on `origin/main` at merge `58a94c8e` plus operationalization branch +additions: + +```bash +git grep -n "re.compile" generate/derivation generate/math_candidate_graph.py \ + generate/math_candidate_parser.py generate/math_completeness.py generate/math_roundtrip.py +git grep -n "half\|quarter\|third\|percent\|per\|each\|remaining\|altogether" generate/ tests/ +git grep -n "case_id" generate/ evals/ tests/ +``` + +## Classification legend + +| Class | Meaning | +|---|---| +| `current_runtime_dependency` | Still on the serving path; do not break during migration | +| `migrate_to_problemframe` | Should consume `ProblemFrame` / `KernelFacts` instead of scraping prose | +| `wrap_with_substrate_adapter` | Can be fed by substrate facades without full organ rewrite yet | +| `delete_after_migration` | One-off helper that should disappear once ProblemFrame path owns the surface | +| `allowed_non_derivation_regex` | Harmless test/doc/format regex outside derivation admission | + +--- + +## Shared legacy substrate (highest leverage) + +| File | Class | Notes | +|---|---|---| +| `generate/derivation/extract.py` | `migrate_to_problemframe` | Central quantity/word-qty regex (`_HALF_OF_RE`, `_FRACTION_OF_RE`, `_WORD_QTY_RE`). Every organ importing `extract_quantities` inherits this debt. | +| `generate/derivation/clauses.py` | `wrap_with_substrate_adapter` | Sentence split regex only; keep until clause segmentation moves behind a substrate contract. | +| `generate/derivation/comparatives.py` | `migrate_to_problemframe` | Local comparative scalar extraction (`half`, `times`, `more than`). | +| `generate/math_candidate_parser.py` | `current_runtime_dependency` | 47 regex sites; recognizer/candidate-graph serving path. Migrate incrementally via ProblemFrame inspection, not big-bang delete. | +| `generate/math_candidate_graph.py` | `current_runtime_dependency` | Organ dispatch hub; must keep until each organ migrates. | +| `generate/math_roundtrip.py` | `current_runtime_dependency` | Tokenization helper used by organs and verifier. | +| `generate/math_completeness.py` | `wrap_with_substrate_adapter` | Completeness checks; not a prose parser but coupled to legacy graph shape. | + +--- + +## First-wave organ migration candidates + +| Organ | Class | Local parsing debt | ProblemFrame feed | +|---|---|---|---| +| `generate/derivation/percent_partition.py` | `migrate_to_problemframe` | `_PERCENT_OF_GROUP_RE`, `_FRACTION_RE`, half-split phrase scan | `extract_scalar_candidates`, partition/consumption process frames, percent hazards | +| `generate/derivation/nested_fraction_remainder_total.py` | `migrate_to_problemframe` | `_OUTER_HALF_RE`, `_INNER_QUARTER_RE`, morning/afternoon regex | partition frame + grounded scalars + remainder hazards | +| `generate/derivation/fraction_decrease.py` | `migrate_to_problemframe` | `_DECREASE_TO_FRACTION_RE`, extra-fraction scan | scalar candidates + consumption/partition frames | +| `generate/derivation/temporal_tariff.py` | `migrate_to_problemframe` | hourly rate / threshold shift regex | labor_rate frame + unit dimensions (time/money) | + +All four have train-sample coverage, perform local scalar/phrase parsing today, can be +fed by `build_problem_frame`, and can preserve `wrong == 0` when migrated under +contract-backed admission. + +### Recommended first migration: `percent_partition` + +**Before** + +```text +raw-prose local parsing inside organ + → _PERCENT_OF_GROUP_RE / half-split phrase heuristics + → extract_quantities + comparative_step + → answer derivation +``` + +**After (next PR)** + +```text +build_problem_frame(problem_text) + → grounded scalars (50%, half) + → partition + consumption process-frame candidates + → percent / remainder hazards preserved + → explicit percent_partition contract consumes ProblemFrame facts + → answer derivation (unchanged verifier gate) +``` + +**Why first:** smallest surface area among the four candidates, clear scalar/frame +coverage in substrate, existing train coverage, and morphology planner v2 already +routes half+percent split problems to this organ. + +--- + +## Other derivation organs (deferred) + +| File | Class | Notes | +|---|---|---| +| `generate/derivation/affine_fraction_delta.py` | `migrate_to_problemframe` | Fraction decrease/increase affine cues | +| `generate/derivation/piecewise_daily_hours_total.py` | `migrate_to_problemframe` | Calendar + halfway-month local tables | +| `generate/derivation/loose_crayon_box_capacity.py` | `migrate_to_problemframe` | Container/box regex family | +| `generate/derivation/r1_reconstruction.py` | `current_runtime_dependency` | Broad fact-reconstruction regex suite | +| `generate/derivation/sequential_comparative_scale.py` | `migrate_to_problemframe` | Reader/scale regex | +| `generate/derivation/round_trip_trip_duration.py` | `migrate_to_problemframe` | Travel fraction cues | +| `generate/derivation/bounded_rate_projection.py` | `migrate_to_problemframe` | Percent projection regex | +| `generate/derivation/calendar_grounding.py` | `wrap_with_substrate_adapter` | Month-name regex pending `kernel_calendar` pack | +| `generate/derivation/state/bind.py` | `delete_after_migration` | Word token regex subsumed by substrate spans | +| `generate/derivation/state/source.py` | `delete_after_migration` | Conjunction split regex | + +--- + +## New substrate path (target design) + +| File | Class | Notes | +|---|---|---| +| `generate/problem_frame_builder.py` | **target path** | `build_problem_frame` — substrate-backed, no solving | +| `language_packs/scalar_equivalence.py` | **target path** | ADR-0128 facade; `extract_scalar_candidates` | +| `language_packs/unit_dimensions.py` | **target path** | ADR-0127 facade | +| `language_packs/ambiguity_hazards.py` | **target path** | Hazard registry | +| `generate/process_frames.py` | **target path** | Declarative process-frame candidates | +| `generate/kernel_facts.py` | **target path** | Typed fact/provenance model | +| `generate/problem_frame.py` | **target path** | ProblemFrame IR skeleton | +| `scripts/gsm8k_substrate_morphology.py` | **target path** | Planner v2 diagnostics | + +--- + +## Phrase-surface grep summary (`generate/`) + +Surfaces like `half`, `quarter`, `percent`, `remaining` appear in: + +- **Substrate (keep):** `scalar_equivalence.py`, `ambiguity_hazards.py`, `process_frames.py`, `problem_frame_builder.py` +- **Legacy organs (migrate):** `derivation/extract.py`, `percent_partition.py`, `nested_fraction_remainder_total.py`, `fraction_decrease.py`, `comparatives.py`, and sprint lift organs +- **Serving graph (keep temporarily):** `math_candidate_graph.py`, `math_candidate_parser.py` +- **Non-derivation (allowed):** `process_frames.py` trigger tables, tests, docs + +--- + +## `case_id` usage + +| Area | Class | Notes | +|---|---|---| +| `evals/gsm8k_math/**` | `allowed_non_derivation_regex` | Case identifiers for measurement harnesses only | +| `tests/test_math_candidate_graph_*` | `allowed_non_derivation_regex` | Fixture pinning by case_id | +| `generate/derivation/**` | **none found** | Organs do not hardcode case_id (good) | +| `generate/problem_frame_builder.py` | **none** | Builder explicitly case-agnostic | + +--- + +## Guardrails added in this PR + +- **No-new-legacy rule** in `AGENTS.md`, `GROK.md`, `CLAUDE.md`, `docs/architecture/kernel-knowledge-layer-v1.md`, `docs/runtime_contracts.md` +- **`tests/test_kernel_no_new_legacy_derivation_surfaces.py`** — allowlisted legacy regex files; new regex in derivation paths requires `LEGACY_EXCEPTION` + +--- + +## Next PR recommendation + +Migrate **`percent_partition`** to consume `ProblemFrame` facts while keeping the +existing verifier gate and `wrong == 0` invariant. Do not change `report.json` or +sealed artifacts in the migration PR unless a ratified score change is intended. \ No newline at end of file diff --git a/docs/architecture/kernel-knowledge-layer-v1.md b/docs/architecture/kernel-knowledge-layer-v1.md index 5378539e..aa92c75b 100644 --- a/docs/architecture/kernel-knowledge-layer-v1.md +++ b/docs/architecture/kernel-knowledge-layer-v1.md @@ -75,6 +75,17 @@ To prevent architectural drift and maintain a clean separation of concerns, the 2. **Dimension Tables:** Standard conversion coefficients (e.g., $60 \text{ minutes} = 1 \text{ hour}$) must reside in the units pack conversions, not in organ-level multiplication constants. 3. **Calendar Metrics:** Day-counts and calendar logic must be resolved strictly through the calendar pack. +### No-new-legacy rule (enforced after operationalization PR) + +New derivation capabilities must consume `KernelFacts` / `ProblemFrame` facts where the +substrate can represent the needed meaning. Construction entry point: +`generate/problem_frame_builder.py::build_problem_frame`. + +New raw-prose/local-regex parsing inside a derivation organ requires an explicit +`LEGACY_EXCEPTION` comment and migration rationale. CI guard: +`tests/test_kernel_no_new_legacy_derivation_surfaces.py`. Legacy inventory: +`docs/analysis/kernel-substrate-deprecation-audit-2026-06-18.md`. + --- ## 8. Kernel Pack Model diff --git a/docs/runtime_contracts.md b/docs/runtime_contracts.md index af779202..61c4bdb4 100644 --- a/docs/runtime_contracts.md +++ b/docs/runtime_contracts.md @@ -812,6 +812,28 @@ from ADR-0119.5 and ADR-0114a Obligation #8: The `subtle_in_grammar` family (4 cases, all `correct`) proves the gate is not trivially satisfied by refusing everything. +## Kernel substrate / ProblemFrame construction contract + +Diagnostics and future organ migrations may construct a `ProblemFrame` from raw +problem text via `generate/problem_frame_builder.py::build_problem_frame`. + +Required properties: + +- Deterministic ordering of scalars, units, hazards, and process-frame candidates. +- Exact `problem_text` source spans where grounding is available. +- Hazards preserved from `language_packs.ambiguity_hazards` and scalar candidates. +- Process frames attached as **candidates**, not conclusions. +- No answer derivation, no case-id behavior, no serving admission from ProblemFrame alone. + +No-new-legacy rule: new derivation capabilities must consume ProblemFrame facts where +the substrate can represent the needed meaning. New raw-prose/local-regex parsing +inside a derivation organ requires an explicit `LEGACY_EXCEPTION` note and migration +rationale (`tests/test_kernel_no_new_legacy_derivation_surfaces.py`). + +Serving policy (current): ProblemFrame construction is inspection/diagnostic only. +Changing `train_sample` / holdout scores via ProblemFrame requires a ratified organ +migration PR with verifier parity evidence. + ### Evidence location - Lane runner: `evals/gsm8k_math/runner.py` diff --git a/generate/problem_frame_builder.py b/generate/problem_frame_builder.py new file mode 100644 index 00000000..817f38b1 --- /dev/null +++ b/generate/problem_frame_builder.py @@ -0,0 +1,346 @@ +"""ProblemFrame builder — substrate-backed construction from raw problem text. + +Operationalizes the #829 kernel substrate path: + + raw text → scalar/unit/hazard/process-frame facts → ProblemFrame + +Non-goals: + - answer derivation + - case-id behavior + - serving admission + - guessing unsupported or ambiguous surfaces +""" +from __future__ import annotations + +import re +from fractions import Fraction + +from generate.kernel_facts import ( + CandidateRelation, + GroundedScalar, + GroundedUnit, + KernelHazard, + KernelProvenance, + RelationRole, + SourceSpan, +) +from generate.problem_frame import ProblemFrame, ProblemFrameBuilder, QuestionTarget +from generate.process_frames import ProcessFrame, all_frames, lookup_frame +from language_packs.ambiguity_hazards import ( + AmbiguityHazard, + all_registered_surfaces, + lookup_hazards, +) +from language_packs.scalar_equivalence import ( + ScalarCandidate, + extract_scalar_candidates, + list_unsupported_surfaces, +) +from language_packs.unit_dimensions import classify_dimension + +_UNIT_TOKEN_RE: re.Pattern[str] = re.compile(r"\b\d+(?:\.\d+)?\s+([a-zA-Z]+)\b") + +_UNIT_STOPWORDS: frozenset[str] = frozenset({ + "more", "less", "times", "percent", "percentage", "of", "and", "or", + "the", "a", "an", "in", "to", "for", "with", "at", "by", "from", + "each", "per", "way", "ways", +}) + +_ORDINAL_SUFFIX_RE: re.Pattern[str] = re.compile( + r"\b(half|third|quarter)\s+(place|position|grade|rank)\b", + re.IGNORECASE, +) + + +def _surface_in_text(surface: str, text_lower: str) -> bool: + token = surface.lower() + padded = f" {text_lower} " + return ( + f" {token} " in padded + or text_lower.startswith(f"{token} ") + or text_lower.endswith(f" {token}") + or text_lower == token + ) + + +def _hazard_to_kernel(hazard: AmbiguityHazard) -> KernelHazard: + return KernelHazard( + hazard_id=hazard.hazard_id, + category=hazard.category, + surface=hazard.surface, + description=hazard.description, + context_required=hazard.context_required, + ) + + +def _extract_unit_candidates(text: str) -> tuple[GroundedUnit, ...]: + units: list[GroundedUnit] = [] + seen: set[tuple[str, int, int]] = set() + + for match in _UNIT_TOKEN_RE.finditer(text): + token = match.group(1) + token_lower = token.lower() + if token_lower in _UNIT_STOPWORDS: + continue + dim_fact = classify_dimension(token_lower) + if dim_fact is None: + continue + start = match.start(1) + end = match.end(1) + key = (token_lower, start, end) + if key in seen: + continue + seen.add(key) + span = SourceSpan(text[start:end], start, end) + provenance = KernelProvenance(kind="problem_text", source_spans=(span,)) + units.append( + GroundedUnit( + fact_id=f"unit-{len(units):04d}", + surface=token_lower, + dimension=dim_fact.dimension, + singular=dim_fact.singular, + provenance=provenance, + ) + ) + + return tuple(sorted(units, key=lambda u: (u.provenance.source_spans[0].start, u.surface))) + + +def _extract_hazards(text: str) -> tuple[KernelHazard, ...]: + text_lower = text.lower() + hazards: list[KernelHazard] = [] + seen: set[str] = set() + + for surface in all_registered_surfaces(): + if not _surface_in_text(surface, text_lower): + continue + for hazard in lookup_hazards(surface): + if hazard.hazard_id in seen: + continue + seen.add(hazard.hazard_id) + hazards.append(_hazard_to_kernel(hazard)) + + if "%" in text: + for hazard in lookup_hazards("percent"): + if hazard.hazard_id in seen: + continue + seen.add(hazard.hazard_id) + hazards.append(_hazard_to_kernel(hazard)) + + return tuple(sorted(hazards, key=lambda h: h.hazard_id)) + + +def _is_ordinal_scalar_span(text: str, start: int, end: int) -> bool: + """Refuse fraction readings for ordinals like ``third place``.""" + window_start = max(0, start - 20) + window_end = min(len(text), end + 20) + window = text[window_start:window_end] + for match in _ORDINAL_SUFFIX_RE.finditer(window): + abs_start = window_start + match.start() + abs_end = window_start + match.end() + if start >= abs_start and end <= abs_end: + return True + return False + + +def _filter_scalar_candidates( + text: str, + candidates: tuple[ScalarCandidate, ...], +) -> tuple[ScalarCandidate, ...]: + kept: list[ScalarCandidate] = [] + for candidate in candidates: + if candidate.source_span is None: + kept.append(candidate) + continue + start, end = candidate.source_span + if _is_ordinal_scalar_span(text, start, end): + continue + kept.append(candidate) + return tuple(kept) + + +def _trigger_span(text: str, trigger: str) -> SourceSpan | None: + text_lower = text.lower() + trigger_lower = trigger.lower() + idx = text_lower.find(trigger_lower) + if idx < 0: + return None + return SourceSpan(text[idx:idx + len(trigger_lower)], idx, idx + len(trigger_lower)) + + +def _extract_process_frame_candidates(text: str) -> tuple[ProcessFrame, ...]: + text_lower = text.lower() + matched: dict[str, ProcessFrame] = {} + + for frame in all_frames(): + for trigger in frame.trigger_surfaces: + if _surface_in_text(trigger, text_lower): + matched[frame.name] = frame + break + + return tuple(matched[name] for name in sorted(matched)) + + +def _frame_roles(frame: ProcessFrame) -> tuple[RelationRole, ...]: + roles: list[RelationRole] = [] + for role in frame.required_roles: + roles.append(RelationRole(role.name, True, role.description)) + for role in frame.optional_roles: + roles.append(RelationRole(role.name, False, role.description)) + return tuple(roles) + + +def _extract_candidate_relations( + text: str, + frames: tuple[ProcessFrame, ...], +) -> tuple[CandidateRelation, ...]: + relations: list[CandidateRelation] = [] + + for frame in frames: + span: SourceSpan | None = None + for trigger in frame.trigger_surfaces: + span = _trigger_span(text, trigger) + if span is not None: + break + provenance = ( + KernelProvenance(kind="problem_text", source_spans=(span,)) + if span is not None + else None + ) + frame_hazards = tuple( + KernelHazard( + hazard_id=f"frame-{frame.name}-{category}", + category=category, + surface=frame.name, + description=f"Process frame {frame.name} hazard {category}", + ) + for category in frame.hazards + ) + relations.append( + CandidateRelation( + relation_id=f"rel-{frame.name}", + relation_type=frame.candidate_relation, + roles=_frame_roles(frame), + provenance=provenance, + hazards=frame_hazards, + ) + ) + + return tuple(relations) + + +def _scalar_to_grounded( + candidate: ScalarCandidate, + text: str, + index: int, +) -> GroundedScalar | None: + if candidate.source_span is None or candidate.source_surface is None: + return None + + start, end = candidate.source_span + span = SourceSpan(candidate.source_surface, start, end) + provenance = KernelProvenance(kind="problem_text", source_spans=(span,)) + hazards = tuple( + KernelHazard( + hazard_id=hid, + category=hid, + surface=candidate.surface, + description=f"Scalar hazard {hid}", + ) + for hid in candidate.hazards + ) + return GroundedScalar( + fact_id=f"scalar-{index:04d}", + surface=candidate.surface, + value=candidate.canonical, + provenance=provenance, + hazards=hazards, + ) + + +def _detect_question_target(text: str) -> QuestionTarget | None: + text_lower = text.lower() + if "how many" in text_lower: + return QuestionTarget("how many", "count") + if "how much" in text_lower: + return QuestionTarget("how much", "quantity") + if "?" in text: + return QuestionTarget("?", "unknown") + return None + + +def _has_unsupported_scalar_surface(text: str) -> bool: + for surface in list_unsupported_surfaces(): + if surface in text: + return True + return False + + +def build_problem_frame(problem_text: str) -> ProblemFrame: + """Build a substrate-backed ProblemFrame from raw problem text. + + Deterministic ordering; preserves hazards and provenance; does not derive + answers or bind case-specific behavior. + """ + builder = ProblemFrameBuilder() + + scalars = _filter_scalar_candidates(problem_text, extract_scalar_candidates(problem_text)) + for scalar in scalars: + builder.add_scalar(scalar) + + for index, scalar in enumerate(scalars): + grounded = _scalar_to_grounded(scalar, problem_text, index) + if grounded is not None: + builder.add_quantity(grounded) + + for unit in _extract_unit_candidates(problem_text): + builder.add_unit(unit) + + for hazard in _extract_hazards(problem_text): + builder.add_hazard(hazard) + + frames = _extract_process_frame_candidates(problem_text) + for frame in frames: + builder.add_process_frame(frame) + + for relation in _extract_candidate_relations(problem_text, frames): + builder.add_relation(relation) + + question_target = _detect_question_target(problem_text) + if question_target is not None: + builder.set_question_target(question_target) + + # Unsupported scalar tokenisations remain absent from scalars; callers can + # consult list_unsupported_surfaces() — we do not broaden ADR-0128 here. + _ = _has_unsupported_scalar_surface(problem_text) + + return builder.build() + + +def recognized_scalar_surfaces(frame: ProblemFrame) -> tuple[str, ...]: + """Return sorted scalar surfaces recognized in a ProblemFrame.""" + surfaces = {s.surface for s in frame.scalars} + surfaces.update(q.surface for q in frame.quantities) + return tuple(sorted(surfaces)) + + +def recognized_unit_surfaces(frame: ProblemFrame) -> tuple[str, ...]: + """Return sorted unit surfaces recognized in a ProblemFrame.""" + return tuple(sorted({u.surface for u in frame.units})) + + +def recognized_process_frame_names(frame: ProblemFrame) -> tuple[str, ...]: + """Return sorted process-frame names attached as candidates.""" + return tuple(sorted({f.name for f in frame.process_frames})) + + +def recognized_hazard_ids(frame: ProblemFrame) -> tuple[str, ...]: + """Return sorted hazard IDs preserved on the frame.""" + return tuple(sorted({h.hazard_id for h in frame.hazards})) + + +def scalar_canonical_values(frame: ProblemFrame) -> tuple[Fraction, ...]: + """Return canonical scalar values in deterministic order.""" + values = [s.canonical for s in frame.scalars] + values.extend(q.value for q in frame.quantities) + return tuple(sorted(values, key=lambda v: (v.denominator, v.numerator))) \ No newline at end of file diff --git a/scripts/gsm8k_substrate_morphology.py b/scripts/gsm8k_substrate_morphology.py index 300753f2..8a1099c0 100644 --- a/scripts/gsm8k_substrate_morphology.py +++ b/scripts/gsm8k_substrate_morphology.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 -"""Classify GSM8K problems by missing substrate category. +"""Classify GSM8K problems by missing substrate category and plan migrations. Tranche 1 — broad base-layer foundations. +Planner v2 — operationalization pass: recognize substrate facts and recommend +legacy-parser migration targets without answer mining or pack mutation. Labels are semantically honest: ``missing_*`` categories fire only when a needed substrate lookup actually fails, not merely because a trigger @@ -13,8 +15,15 @@ import argparse import json import re from pathlib import Path -from typing import Sequence +from typing import Any, Sequence +from generate.problem_frame_builder import ( + build_problem_frame, + recognized_hazard_ids, + recognized_process_frame_names, + recognized_scalar_surfaces, + recognized_unit_surfaces, +) from language_packs.scalar_equivalence import list_unsupported_surfaces from language_packs.unit_dimensions import classify_dimension from language_packs.loader import lookup_container @@ -157,11 +166,141 @@ def classify_missing_substrate(problem_text: str) -> tuple[str, ...]: return tuple(sorted(labels)) +_FIRST_MIGRATION_ORGANS: tuple[str, ...] = ( + "percent_partition", + "nested_fraction_remainder_total", + "fraction_decrease", + "temporal_tariff", +) + +_ORGAN_MODULE_PATHS: dict[str, str] = { + "percent_partition": "generate/derivation/percent_partition.py", + "nested_fraction_remainder_total": "generate/derivation/nested_fraction_remainder_total.py", + "fraction_decrease": "generate/derivation/fraction_decrease.py", + "temporal_tariff": "generate/derivation/temporal_tariff.py", + "extract_shared": "generate/derivation/extract.py", + "math_candidate_parser": "generate/math_candidate_parser.py", +} + + +def _legacy_parser_dependency( + problem_text: str, + process_frames: tuple[str, ...], + missing_labels: tuple[str, ...], +) -> tuple[str, ...]: + """Map problem surfaces to currently-serving legacy parser modules.""" + deps: set[str] = set() + lowered = problem_text.lower() + + if "%" in problem_text or "percent" in lowered: + deps.add(_ORGAN_MODULE_PATHS["percent_partition"]) + if "other half" in lowered: + deps.add(_ORGAN_MODULE_PATHS["percent_partition"]) + if "remaining" in lowered and ("half" in lowered or "quarter" in lowered): + deps.add(_ORGAN_MODULE_PATHS["nested_fraction_remainder_total"]) + if any(word in lowered for word in ("decrease", "decreased", "decreases")): + deps.add(_ORGAN_MODULE_PATHS["fraction_decrease"]) + if any( + token in lowered + for token in ("hour", "hours", "per hour", "overtime", "threshold") + ): + deps.add(_ORGAN_MODULE_PATHS["temporal_tariff"]) + if "labor_rate" in process_frames: + deps.add(_ORGAN_MODULE_PATHS["temporal_tariff"]) + + if re.search(r"\d", problem_text): + deps.add(_ORGAN_MODULE_PATHS["extract_shared"]) + if "missing_scalar_equivalence" in missing_labels: + deps.add(_ORGAN_MODULE_PATHS["math_candidate_parser"]) + + return tuple(sorted(deps)) + + +def recommend_migration_target( + problem_text: str, + process_frames: tuple[str, ...], + missing_labels: tuple[str, ...], +) -> str: + """Recommend the next organ or substrate extension for this problem.""" + lowered = problem_text.lower() + + if "%" in problem_text and ("half" in lowered or "partition" in process_frames): + return "percent_partition" + if "other half" in lowered and "%" in problem_text: + return "percent_partition" + + if "missing_scalar_equivalence" in missing_labels: + return "substrate:scalar_equivalence" + if "missing_unit_dimension" in missing_labels: + return "substrate:unit_dimensions" + if "blocked_provenance_gap" in missing_labels: + return "substrate:kernel_calendar" + if "remaining" in lowered and ("half" in lowered or "quarter" in lowered): + return "nested_fraction_remainder_total" + if any(word in lowered for word in ("decrease", "decreased")): + return "fraction_decrease" + if "labor_rate" in process_frames or any( + token in lowered for token in ("per hour", "hourly", "overtime") + ): + return "temporal_tariff" + if "blocked_ambiguity_hazard" in missing_labels: + return "substrate:ambiguity_hazards" + + if process_frames: + return process_frames[0] + + return "substrate:problem_frame_builder" + + +def plan_substrate_case( + *, + case_id: str, + problem_text: str, + current_verdict: str | None = None, +) -> dict[str, Any]: + """Planner v2 record for one problem — diagnostics only, no solving.""" + frame = build_problem_frame(problem_text) + missing_labels = classify_missing_substrate(problem_text) + process_frames = recognized_process_frame_names(frame) + + return { + "case_id": case_id, + "current_verdict": current_verdict, + "recognized_scalars": recognized_scalar_surfaces(frame), + "recognized_units": recognized_unit_surfaces(frame), + "recognized_process_frames": process_frames, + "recognized_hazards": recognized_hazard_ids(frame), + "missing_substrate_labels": missing_labels, + "legacy_parser_dependency": _legacy_parser_dependency( + problem_text, + process_frames, + missing_labels, + ), + "recommended_migration_target": recommend_migration_target( + problem_text, + process_frames, + missing_labels, + ), + } + + def main() -> None: - parser = argparse.ArgumentParser(description="Classify GSM8K problems by missing substrate.") + parser = argparse.ArgumentParser( + description="Classify GSM8K problems by missing substrate and plan migrations.", + ) parser.add_argument("--cases", type=str, help="Path to JSONL cases file") parser.add_argument("--out", type=str, help="Path to write classified output JSONL") parser.add_argument("--limit", type=int, help="Limit number of cases to process") + parser.add_argument( + "--planner", + action="store_true", + help="Emit morphology planner v2 records (recognized substrate + migration targets)", + ) + parser.add_argument( + "--verdicts", + type=str, + help="Optional JSON report with per_case verdicts keyed by case_id", + ) args = parser.parse_args() @@ -174,7 +313,18 @@ def main() -> None: print(f"Cases file not found at {args.cases}") return - out_lines = [] + verdicts: dict[str, str] = {} + if args.verdicts: + report_path = Path(args.verdicts) + if report_path.exists(): + report = json.loads(report_path.read_text(encoding="utf-8")) + for row in report.get("per_case", []): + cid = row.get("case_id") + verdict = row.get("verdict") + if cid and verdict: + verdicts[cid] = verdict + + out_lines: list[dict[str, Any]] = [] count = 0 with cases_path.open("r", encoding="utf-8") as f: for line in f: @@ -185,13 +335,21 @@ def main() -> None: if not problem_text: continue - labels = classify_missing_substrate(problem_text) case_id = case.get("case_id") or f"case_{count}" - out_lines.append({ - "case_id": case_id, - "problem_text": problem_text, - "missing_substrate_labels": labels - }) + if args.planner: + record = plan_substrate_case( + case_id=case_id, + problem_text=problem_text, + current_verdict=verdicts.get(case_id), + ) + else: + labels = classify_missing_substrate(problem_text) + record = { + "case_id": case_id, + "problem_text": problem_text, + "missing_substrate_labels": labels, + } + out_lines.append(record) count += 1 if args.limit and count >= args.limit: diff --git a/tests/test_gsm8k_morphology_missing_kernel_labels.py b/tests/test_gsm8k_morphology_missing_kernel_labels.py index ea6c3dca..684ea623 100644 --- a/tests/test_gsm8k_morphology_missing_kernel_labels.py +++ b/tests/test_gsm8k_morphology_missing_kernel_labels.py @@ -1,7 +1,11 @@ """Tests for scripts/gsm8k_substrate_morphology.py.""" from __future__ import annotations -from scripts.gsm8k_substrate_morphology import classify_missing_substrate +from scripts.gsm8k_substrate_morphology import ( + classify_missing_substrate, + plan_substrate_case, + recommend_migration_target, +) def test_classify_missing_substrate_labels() -> None: @@ -69,4 +73,40 @@ def test_deterministic_and_sorted() -> None: assert "missing_unit_dimension" in labels1 assert "missing_part_whole_frame" not in labels1 assert "missing_container_frame" not in labels1 - assert "blocked_provenance_gap" in labels1 \ No newline at end of file + assert "blocked_provenance_gap" in labels1 + + +def test_planner_v2_recognizes_substrate_without_solving() -> None: + record = plan_substrate_case( + case_id="test-0001", + problem_text="Mia spent 50% of her money.", + current_verdict="refused", + ) + + assert record["case_id"] == "test-0001" + assert record["current_verdict"] == "refused" + assert record["recognized_scalars"] + assert "consumption" in record["recognized_process_frames"] + assert record["recognized_hazards"] + assert isinstance(record["legacy_parser_dependency"], tuple) + assert record["recommended_migration_target"] in { + "percent_partition", + "substrate:scalar_equivalence", + "substrate:ambiguity_hazards", + "consumption", + "substrate:problem_frame_builder", + } + + +def test_planner_v2_recommends_percent_partition_for_half_percent_split() -> None: + text = ( + "There are 100 students. Half are girls and the other half are boys. " + "30% of the girls own pets and 20% of the boys own pets. " + "How many students own pets?" + ) + target = recommend_migration_target( + text, + ("partition", "consumption"), + classify_missing_substrate(text), + ) + assert target == "percent_partition" \ No newline at end of file diff --git a/tests/test_kernel_no_new_legacy_derivation_surfaces.py b/tests/test_kernel_no_new_legacy_derivation_surfaces.py new file mode 100644 index 00000000..04cc18fc --- /dev/null +++ b/tests/test_kernel_no_new_legacy_derivation_surfaces.py @@ -0,0 +1,106 @@ +"""Guard against new legacy raw-text parsing in derivation paths.""" +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[1] + +# Current serving legacy surfaces — explicit allowlist while migrations proceed. +ALLOWLISTED_LEGACY_DERIVATION_FILES: frozenset[str] = frozenset({ + "generate/derivation/affine_comparative_inversion_total.py", + "generate/derivation/affine_fraction_delta.py", + "generate/derivation/bounded_rate_projection.py", + "generate/derivation/calendar_grounding.py", + "generate/derivation/clauses.py", + "generate/derivation/closed_reference_affine_aggregate.py", + "generate/derivation/comparatives.py", + "generate/derivation/compose.py", + "generate/derivation/duration_segment_total.py", + "generate/derivation/extract.py", + "generate/derivation/fraction_decrease.py", + "generate/derivation/giveaway_target_residual.py", + "generate/derivation/goal_residual.py", + "generate/derivation/loose_crayon_box_capacity.py", + "generate/derivation/multistep.py", + "generate/derivation/nested_fraction_remainder_total.py", + "generate/derivation/percent_partition.py", + "generate/derivation/piecewise_daily_hours_total.py", + "generate/derivation/product_bridge.py", + "generate/derivation/question_bound_product.py", + "generate/derivation/r1_reconstruction.py", + "generate/derivation/round_trip_trip_duration.py", + "generate/derivation/search.py", + "generate/derivation/sequential_comparative_scale.py", + "generate/derivation/state/bind.py", + "generate/derivation/state/source.py", + "generate/derivation/survey_rate_earnings.py", + "generate/derivation/target.py", + "generate/derivation/temporal_tariff.py", + "generate/math_candidate_parser.py", + "generate/math_candidate_graph.py", + "generate/math_completeness.py", + "generate/math_roundtrip.py", +}) + +_REGEX_USAGE = re.compile( + r"\bre\.(?:compile|search|match|findall|finditer|sub|split)\b" +) + +_SCAN_ROOTS = ( + _REPO_ROOT / "generate" / "derivation", + _REPO_ROOT / "generate" / "math_candidate_parser.py", + _REPO_ROOT / "generate" / "math_candidate_graph.py", + _REPO_ROOT / "generate" / "math_completeness.py", + _REPO_ROOT / "generate" / "math_roundtrip.py", +) + + +def _iter_scanned_files() -> list[Path]: + files: list[Path] = [] + for root in _SCAN_ROOTS: + if root.is_file(): + files.append(root) + continue + for path in sorted(root.rglob("*.py")): + if path.name == "__init__.py": + continue + files.append(path) + return files + + +def test_no_new_legacy_derivation_regex_without_exception() -> None: + violations: list[str] = [] + + for path in _iter_scanned_files(): + rel = path.relative_to(_REPO_ROOT).as_posix() + text = path.read_text(encoding="utf-8") + if not _REGEX_USAGE.search(text): + continue + if rel in ALLOWLISTED_LEGACY_DERIVATION_FILES: + continue + if "LEGACY_EXCEPTION" in text: + continue + violations.append(rel) + + assert not violations, ( + "New legacy raw-text regex surfaces detected. Either migrate to " + "ProblemFrame/substrate extraction or add an explicit LEGACY_EXCEPTION " + f"with migration rationale:\n" + "\n".join(violations) + ) + + +def test_allowlist_covers_current_legacy_derivation_regex_files() -> None: + """Ensure the allowlist stays aligned with scanned legacy regex files.""" + regex_files = { + path.relative_to(_REPO_ROOT).as_posix() + for path in _iter_scanned_files() + if _REGEX_USAGE.search(path.read_text(encoding="utf-8")) + } + missing = sorted(regex_files - ALLOWLISTED_LEGACY_DERIVATION_FILES) + assert not missing, ( + "Allowlist missing current legacy regex files — update " + "ALLOWLISTED_LEGACY_DERIVATION_FILES:\n" + "\n".join(missing) + ) \ No newline at end of file diff --git a/tests/test_problem_frame_builder.py b/tests/test_problem_frame_builder.py new file mode 100644 index 00000000..a23f29a0 --- /dev/null +++ b/tests/test_problem_frame_builder.py @@ -0,0 +1,105 @@ +"""Tests for generate/problem_frame_builder.py.""" +from __future__ import annotations + +from fractions import Fraction + +from generate.problem_frame_builder import build_problem_frame +from language_packs.scalar_equivalence import extract_scalar_candidates, list_unsupported_surfaces + + +def _frame_names(text: str) -> tuple[str, ...]: + return tuple(f.name for f in build_problem_frame(text).process_frames) + + +def _hazard_categories(text: str) -> tuple[str, ...]: + return tuple(sorted({h.category for h in build_problem_frame(text).hazards})) + + +def test_percent_text_produces_scalar_facts_and_hazards_without_solving() -> None: + text = "Mia spent 50% of her money." + frame = build_problem_frame(text) + + assert frame.scalars + assert any(s.canonical == Fraction(1, 2) for s in frame.scalars) + assert "percent_change_vs_percent_of" in _hazard_categories(text) + assert frame.question_target is None + + +def test_transfer_text_produces_transfer_process_frame_without_solving() -> None: + text = "Tom gave Ana 3 marbles." + frame = build_problem_frame(text) + + assert "transfer" in _frame_names(text) + assert any(r.relation_type == "transfer" for r in frame.candidate_relations) + assert not frame.question_target + + +def test_container_text_produces_container_process_frame_without_solving() -> None: + text = "There are 4 full boxes and 3 loose crayons." + frame = build_problem_frame(text) + + assert "container_packing" in _frame_names(text) + assert frame.units or frame.scalars + + +def test_travel_text_produces_travel_process_frame_without_solving() -> None: + text = "A car drove 30 miles each way." + frame = build_problem_frame(text) + + assert "travel" in _frame_names(text) + + +def test_ambiguous_quarter_surfaces_carry_hazards() -> None: + text = "A quarter of the class left." + frame = build_problem_frame(text) + + categories = _hazard_categories(text) + assert "unbound_base_quantity" in categories + assert any( + h.surface == "quarter" or h.category.startswith("quarter_") + for h in frame.hazards + ) + + +def test_third_ordinal_carries_hazard() -> None: + text = "Sam finished in third place." + frame = build_problem_frame(text) + + assert "third_ordinal" in _hazard_categories(text) + assert not any(s.canonical == Fraction(1, 3) for s in frame.scalars) + + +def test_unsupported_scalar_surfaces_do_not_broaden_adr_0128() -> None: + text = "The remaining amount is .5 of the total." + frame = build_problem_frame(text) + + assert frame.scalars == () + assert ".5" in list_unsupported_surfaces() + assert extract_scalar_candidates(text) == () + + +def test_exact_spans_slice_original_text() -> None: + text = "Mia spent 50% of her money." + frame = build_problem_frame(text) + + for scalar in frame.scalars: + assert scalar.source_span is not None + assert scalar.source_surface is not None + start, end = scalar.source_span + assert text[start:end] == scalar.source_surface + + for quantity in frame.quantities: + span = quantity.provenance.source_spans[0] + assert text[span.start:span.end] == span.text + + +def test_deterministic_ordering_across_repeated_runs() -> None: + text = "Tom gave Ana 3 marbles and spent 50% of her money." + frame_a = build_problem_frame(text) + frame_b = build_problem_frame(text) + + assert frame_a.scalars == frame_b.scalars + assert frame_a.units == frame_b.units + assert frame_a.process_frames == frame_b.process_frames + assert frame_a.hazards == frame_b.hazards + assert frame_a.candidate_relations == frame_b.candidate_relations \ No newline at end of file