From de26d7f7922d40ec7ee6c6ef496f44830a864ecc Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 23 May 2026 14:43:16 -0700 Subject: [PATCH] =?UTF-8?q?feat(ADR-0131.G.4):=20multi-clause=20compositio?= =?UTF-8?q?n=20(conj=20subjects=20+=20conj=20objects=20+=20embedded=20quan?= =?UTF-8?q?tifiers=20+=20conj=20embedded)=20=E2=80=94=20admission=200/50?= =?UTF-8?q?=20(=CE=940),=20multi-clause=20refusals=202=E2=86=921?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Highest-risk axis of the ADR-0131.G capability iteration: within- sentence multi-clause composition. Four extractors land in the candidate-emitting parser; no graph-side or solver changes. Parser extension (generate/math_candidate_parser.py) - _conj_subject_each_candidates: ' and [his/her/their ] each ' → 2 CandidateInitial (one per actor). - _conj_object_candidates: ' has and ' → 2 CandidateInitial for the same entity; same-unit conjuncts refuse (would silently collide under solver overwrite-on-collision). - _embedded_quantifier_candidates: ' has with in each []' → 1 derived CandidateInitial (value=N*M). - _embedded_quantifier_candidates (conj branch): '... with in each ... and with in each ...' → 1 SUM CandidateInitial (value=N1*M1+N2*M2); mixed-unit refuses. - CandidateInitial anchor whitelist widened to include saved/earned/got/received/bought/made/paid (and inflections) — narrow widening needed for the conjoined-subject-each shape. Closed-set discipline - Distributive 'each' only — 'each ... together/altogether' refuses. - Two-way conjunction only — 3-way refuses by non-match. - Cross-sentence coreference stays refused (within-sentence axis). - Ambiguous 'each' scope refuses (container2 must agree). Curated axis lane (32 cases) - evals/math_capability_axes/G4_multi_clause/v1/cases.jsonl: conj_subject_each ×6, conj_object ×6, embedded_quantifier ×6, conj_embedded ×6, refusal ×8. - evals/math_capability_axes/G4_multi_clause/v1/runner.py + report.json: deterministic; wrong==0 gate; byte-equal across runs. Tests (26 new) - tests/test_adr_0131_G4_multi_clause.py: per-shape emission, refusal probes (parametric), distributive-only policy, cross-sentence refusal, runner byte-equality, GSM8K-probe gate. GSM8K-probe gate (chosen: multi-clause refusals ↓) - evals/gsm8k_math/train_sample/v1/report.json (candidate-graph probe): multi-clause statement-refusal count 2 → 1. Case 0042 ('Ella has 4 bags with 20 apples in each bag and six bags with 25 apples in each bag.') moves from statement-clause refusal to question-layer refusal. Case 0026 ('Aaron and his brother Carson each saved up $40') stays refused on the '$' value slot (deferred to G.3 numeric-literals axis). - evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json (legacy probe): refreshed, byte-identical (legacy parser untouched). B3 + candidate-graph + GSM8K probe lanes all pass (95/95 regression). wrong==0 preserved everywhere — load-bearing for the highest-risk axis. --- docs/decisions/ADR-0131.G.4-multi-clause.md | 264 +++++++++++++ evals/gsm8k_math/train_sample/v1/report.json | 2 +- .../G4_multi_clause/__init__.py | 0 .../G4_multi_clause/v1/__init__.py | 0 .../G4_multi_clause/v1/cases.jsonl | 32 ++ .../G4_multi_clause/v1/report.json | 262 +++++++++++++ .../G4_multi_clause/v1/runner.py | 198 ++++++++++ evals/math_capability_axes/__init__.py | 0 generate/math_candidate_parser.py | 349 +++++++++++++++++- tests/test_adr_0131_G4_multi_clause.py | 290 +++++++++++++++ 10 files changed, 1393 insertions(+), 4 deletions(-) create mode 100644 docs/decisions/ADR-0131.G.4-multi-clause.md create mode 100644 evals/math_capability_axes/G4_multi_clause/__init__.py create mode 100644 evals/math_capability_axes/G4_multi_clause/v1/__init__.py create mode 100644 evals/math_capability_axes/G4_multi_clause/v1/cases.jsonl create mode 100644 evals/math_capability_axes/G4_multi_clause/v1/report.json create mode 100644 evals/math_capability_axes/G4_multi_clause/v1/runner.py create mode 100644 evals/math_capability_axes/__init__.py create mode 100644 tests/test_adr_0131_G4_multi_clause.py diff --git a/docs/decisions/ADR-0131.G.4-multi-clause.md b/docs/decisions/ADR-0131.G.4-multi-clause.md new file mode 100644 index 00000000..0c483cc2 --- /dev/null +++ b/docs/decisions/ADR-0131.G.4-multi-clause.md @@ -0,0 +1,264 @@ +# ADR-0131.G.4 — Capability axis: multi-clause composition (conjoined subjects, conjoined objects, embedded quantifiers) + +**Status:** Proposed +**Date:** 2026-05-23 +**Author:** CORE agents + reviewers +**Parent:** [ADR-0131.G](./ADR-0131.G-gsm8k-coverage-probe.md) +**Depends on:** +[ADR-0126](./ADR-0126-candidate-graph-parser.md) (candidate graph), +[ADR-0127](./ADR-0127-en-units-v1.md) (substance qualifier policy), +[ADR-0131.3](./ADR-0131.3-bounded-grammar.md), +[ADR-0132](./ADR-0132-binding-graph-data-model.md)..[ADR-0135](./ADR-0135-question-target-binding.md) + +--- + +## Context + +GSM8K paragraphs frequently introduce starting state via within-sentence +composition the per-statement candidate parser refuses today: + +| Baseline refusal sentence | Capability missing | +|---|---| +| `Aaron and his brother Carson each saved up $40 ...` | conjoined subject + distributive `each` | +| `Francine has five full boxes of crayons and 5 loose crayons` | conjoined object NPs sharing a verb | +| `Ella has 4 bags with 20 apples in each bag and six bags with 25 apples in each bag` | embedded quantifier + conjunction | + +This is the **highest-risk axis** of ADR-0131.G's four near-term +capability extensions. Multi-clause emission means the round-trip +filter does more work; multi-candidate ambiguity makes confabulation +risk higher. Refusal-first stays paramount: admission gains must be +small and load-bearing. + +--- + +## Decision + +Land four within-sentence multi-clause extractors in +`generate/math_candidate_parser.py`. All emit +`CandidateInitial` records (initial state, not operations — the +shapes here introduce starting holdings, they do not mutate state): + +| Extractor | Shape (closed set) | Emission | +|---|---|---| +| `_conj_subject_each_candidates` | ` and [his/her/their ] each ` | **two** `CandidateInitial` (one per actor), same `(N, unit)` | +| `_conj_object_candidates` | ` has and ` | **two** `CandidateInitial` for the same entity; same-unit conjuncts refuse | +| `_embedded_quantifier_candidates` | ` has with in each []` | **one** derived `CandidateInitial` with `value = N*M, unit = ` | +| `_embedded_quantifier_candidates` (conj branch) | ` has with in each ... and with in each ...` | **one** SUM `CandidateInitial` with `value = N1*M1 + N2*M2, unit = `; mixed-unit conjuncts refuse | + +Wired into the existing `extract_initial_candidates` public entry +point — the binding graph (`math_candidate_graph._filtered_statement_choices`) +already consumes through that path; no graph-side edit required (the +read-only audit concluded composed candidates are reachable through +existing edges). + +### Closed-set discipline + +- **Distributive `each` only.** Surface markers `together`, `in + total`, `altogether` immediately abort emission (explicit + contradiction with distributive reading). Pinned by + `test_refuses_each_with_together` / + `test_collective_without_each_refuses`. +- **Two-way conjunction only.** Three-way `A and B and C each ...` + is out of closed-set shape and refuses by non-match. +- **Same-unit conjoined object refuses.** Two same-unit conjuncts + on the same entity (`Sam has 5 dimes and 3 dimes`) would silently + collide under the solver's `state[(entity, unit)]` overwrite + semantics (`math_solver.py:206`); refusing keeps `wrong == 0`. +- **Ambiguous `each` scope refuses.** `Ella has 4 bags with 20 + apples in each box` — `box` ≠ `bags` ⇒ refuse. +- **Mixed-unit conjoined embedded refuses.** Apples + pears cannot + be summed. +- **No cross-sentence state.** Multi-sentence inputs are processed + per-sentence; pronouns / coreference across sentences stay + refused (out of within-sentence axis scope). + +### CandidateInitial anchor widening + +`CandidateInitial.__post_init__` whitelists a narrow set of +initial-state-introducing verbs needed for the conjoined-subject-each +shape (`saved`, `earned`, `got`, `received`, `bought`, `made`, +`paid`, plus their inflected variants). The widening is keyed on +lowercase tokens; the `_token_in` check in +`math_candidate_graph._initial_admissible` confirms the anchor word +appears in source. Verb-class widening for the *general* parser +remains G.1's scope; G.4 widens only what conjoined-subject-each +needs. + +### Derived-value provenance + +Embedded-quantifier and conjoined-embedded emissions carry a +*derived* value (`N*M` and `N1*M1 + N2*M2` respectively) that does +not appear as a single source token. The round-trip filter's +"value grounds in source" check (`_value_grounds`) is satisfied by +anchoring `matched_value_token` on the **per-container `M`** (or +**first per-container `M1`** for the sum). This is a deliberate, +documented widening of the source-grounding spirit: the *components* +of the derived value all appear in the source, and the parser +commits to the canonical arithmetic composition. Refusing on +component-mismatch (mixed units, wrong container scope) and refusing +on indefinite quantifiers in any value slot together keep the +derivation honest. The alternative — emitting two flat candidates +for conjoined-embedded — was rejected: under the solver's +overwrite-on-collision semantics it would silently drop one +conjunct's contribution, breaching `wrong == 0`. + +### No graph-side edits + +`math_candidate_graph.py` is unchanged. Multi-candidate emission +flows through the existing per-sentence choice space + Cartesian +product; conjoined-subject-each and conjoined-object emissions land +in `_filtered_statement_choices` like any other initial candidate. +The solver's `state[(entity, unit)]` model naturally accommodates +distinct entities (each-shape) and distinct units (object-shape); +collision-prone shapes refuse at the parser. This decision is the +"read-only audit only edit if composed candidate is unreachable" +posture from the brief. + +### Curated coverage cases (G.4 axis lane) + +`evals/math_capability_axes/G4_multi_clause/v1/cases.jsonl` — +**32 cases**: + +| Category | Cases | Notes | +|------------------------|-------|-------| +| `conj_subject_each` | 6 | incl. kin-appositive, word-form value | +| `conj_object` | 6 | distinct-unit conjuncts only | +| `embedded_quantifier` | 6 | with + without explicit `container2` | +| `conj_embedded` | 6 | same-unit only | +| `refusal` | 8 | together / altogether / 3-way / cross-entity / scope-mismatch / mixed-unit / cross-sentence / same-unit collision | + +Runner emits a deterministic `report.json`; `wrong == 0` is the gate. + +### Deferred (out of scope for G.4) + +- **Cross-sentence coreference** (`Aaron has 5. He gives 2 to + Bob.`) — needs per-discourse state; pinned as refusal probe. +- **Ellipsis** (`Aaron has 5 apples, Carson 3`) — needs verb + reconstruction; pinned out-of-scope. +- **Three-way+ conjunctions** (`A and B and C`) — combinatorial + explosion + ambiguity; deferred to a future axis. +- **Collective readings** (`A and B saved $40 together`) — + explicitly refused; collective semantics needs a different + binding-graph node type. +- **Currency / unit prefix** (`$40`) — refused at the value slot + (the `$` is not a `_VALUE` character). Deferred to **G.3 + numeric-literals axis**, which is the natural place for currency + / percentage / decimal literal handling. Documented impact on the + GSM8K probe gate (case 0026 stays refused). +- **Same-unit conjoined object summation** — would require either + parser-side sum (analogous to conj-embedded) or solver-side + state-merge; deferred until a sum-shaped CandidateInitial proves + necessary outside this axis. +- **Solver / binding-graph changes.** If a multi-clause case parses + but does not solve, that's a downstream gap and gets its own ADR. + +### GSM8K-probe gate (chosen) + +G.4 gates on: + +> **Multi-clause statement-clause refusals in the candidate-graph +> probe (`evals/gsm8k_math/train_sample/v1/report.json`) strictly +> decrease.** + +Counter (in `test_gsm8k_candidate_graph_multi_clause_refusals_decreased`) +matches refused cases citing a statement-clause refusal whose +embedded sentence text contains a multi-clause anchor pattern +(`each `, `with N in each`, or `has N and N +`). + +| Probe report | Baseline (origin/main 481e0c3) | After G.4 | Δ | +|---|---|---|---| +| Multi-clause statement-clause refusals (`report.json`) | 2 | 1 | −1 | +| `wrong` (`report.json`) | 0 | 0 | 0 | +| `admission_rate` (`report.json`) | 0/50 | 0/50 | 0 | +| Legacy `train_sample_coverage_report.json` | byte-identical | byte-identical | 0 | + +Baseline-2 cases: `gsm8k-train-sample-v1-0026` (`Aaron and his +brother Carson each saved up $40 ...` — refused on `$40` value +slot; deferred to G.3) and `-0042` (`Ella has 4 bags with 20 apples +in each bag and six bags with 25 apples in each bag.` — now parses, +refusal moves to question layer). `admission_rate` does not rise +because downstream layers (question-form admission for derived +initial states) are out of G.4 scope. + +### Legacy probe report (refreshed, byte-identical) + +`evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json` +runs through `generate.math_parser.parse_problem` (legacy +first-match-wins), which G.4 does not touch. Refreshed and pinned +via `test_gsm8k_legacy_probe_safety_rail_intact`. + +--- + +## Invariants + +- **`g4_wrong_count_is_zero`** — every G.4 axis case passes or + refuses; no case admits a wrong shape. Pinned by + `test_runner_wrong_count_is_zero`. +- **`g4_closed_set_refusals_hold`** — all 8 refusal probes admit + zero multi-clause candidates. Pinned by + `test_refusal_cases_emit_no_admitted_multi_clause` (parametric). +- **`g4_distributive_each_only`** — `each ... together` and `each + ... altogether` refuse. Pinned by + `test_refuses_each_with_together`. +- **`g4_cross_sentence_refuses`** — multi-clause extractors do not + fire across sentence boundaries. Pinned by + `test_cross_sentence_pronoun_refuses_multi_clause`. +- **`g4_report_deterministic`** — `report.json` is byte-equal across + back-to-back runs. +- **`gsm8k_safety_rail_intact`** — `admitted_wrong == 0` on both + GSM8K probe reports. +- **`gsm8k_multi_clause_refusal_strictly_decreased`** — chosen G.4 + gate. + +--- + +## Acceptance evidence + +- `evals/math_capability_axes/G4_multi_clause/v1/runner.py` exits 0 + with `wrong == 0` on all 32 curated cases. +- `tests/test_adr_0131_G4_multi_clause.py` (26 tests): per-shape + emission, refusal-set, distributive-only policy, cross-sentence + refusal, runner byte-equality, GSM8K-probe gate. +- Candidate-graph probe `report.json`: multi-clause statement + refusal count 2 → 1 (case 0042 moves from statement to question + refusal). +- Legacy probe `train_sample_coverage_report.json` refreshed and + byte-identical. +- B3 lane + ADR-0126 candidate-graph tests + ADR-0131.G probe + tests all pass (95/95 across the regression sweep). + +--- + +## Consequences + +- The candidate-graph topology can now see four multi-clause + initial-state shapes the per-statement parser previously refused. + Downstream question-form admission for derived initial states + (case 0042) becomes a natural next unblock. +- The same Cartesian-product / "branches that disagree → refuse" + decision rule handles the new multi-candidate emissions; no + graph-side edits, no admissibility weakening. +- Highest-risk axis lands without breaching `wrong == 0`: + multi-candidate emission stays tightly scoped, refuses on every + documented adversarial probe, and the derived-value emissions + refuse on every shape-mismatch (mixed unit, scope-mismatch, + collision-prone same-unit). +- Future axes inherit the same axis-lane harness layout under + `evals/math_capability_axes/`. + +--- + +## Out of scope + +- **Solver changes.** If a multi-clause case parses but does not + solve, the gap is downstream; file a follow-up ADR (no solver + stubs, no admissibility relaxation). +- **Currency / numeric-literal handling.** Case 0026 (`$40`) stays + refused; the G.3 numeric-literals axis is the natural place. +- **Three-way / ellipsis / cross-sentence shapes.** Deferred per + the closed-set discipline. +- **Probe runner contract.** ADR-0131.G pinned `run_lane` as the + legacy probe's contract; G.4 does not change that. The + candidate-graph probe (`report.json`) is the measurement surface + that moves. diff --git a/evals/gsm8k_math/train_sample/v1/report.json b/evals/gsm8k_math/train_sample/v1/report.json index 04a8c5b2..6d517c97 100644 --- a/evals/gsm8k_math/train_sample/v1/report.json +++ b/evals/gsm8k_math/train_sample/v1/report.json @@ -218,7 +218,7 @@ }, { "case_id": "gsm8k-train-sample-v1-0042", - "reason": "candidate_graph: no admissible candidate for statement: 'Ella has 4 bags with 20 apples in each bag and six bags with 25 apples in each bag.'", + "reason": "candidate_graph: no admissible candidate for question: 'If Ella sells 200 apples, how many apples does Ella has left?'", "verdict": "refused" }, { diff --git a/evals/math_capability_axes/G4_multi_clause/__init__.py b/evals/math_capability_axes/G4_multi_clause/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/math_capability_axes/G4_multi_clause/v1/__init__.py b/evals/math_capability_axes/G4_multi_clause/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/math_capability_axes/G4_multi_clause/v1/cases.jsonl b/evals/math_capability_axes/G4_multi_clause/v1/cases.jsonl new file mode 100644 index 00000000..c2028c04 --- /dev/null +++ b/evals/math_capability_axes/G4_multi_clause/v1/cases.jsonl @@ -0,0 +1,32 @@ +{"case_id": "G4-conj-each-001", "category": "conj_subject_each", "sentence": "Aaron and Carson each saved up 40 dollars", "expected": {"emits": [{"entity": "Aaron", "value": 40, "unit": "dollars"}, {"entity": "Carson", "value": 40, "unit": "dollars"}]}} +{"case_id": "G4-conj-each-002", "category": "conj_subject_each", "sentence": "Aaron and his brother Carson each saved up 40 dollars to go to dinner", "expected": {"emits": [{"entity": "Aaron", "value": 40, "unit": "dollars"}, {"entity": "Carson", "value": 40, "unit": "dollars"}]}} +{"case_id": "G4-conj-each-003", "category": "conj_subject_each", "sentence": "Alice and Bob each have 5 apples", "expected": {"emits": [{"entity": "Alice", "value": 5, "unit": "apples"}, {"entity": "Bob", "value": 5, "unit": "apples"}]}} +{"case_id": "G4-conj-each-004", "category": "conj_subject_each", "sentence": "Jane and her sister Emily each earned 12 dollars", "expected": {"emits": [{"entity": "Jane", "value": 12, "unit": "dollars"}, {"entity": "Emily", "value": 12, "unit": "dollars"}]}} +{"case_id": "G4-conj-each-005", "category": "conj_subject_each", "sentence": "Tom and Jerry each bought 3 books", "expected": {"emits": [{"entity": "Tom", "value": 3, "unit": "books"}, {"entity": "Jerry", "value": 3, "unit": "books"}]}} +{"case_id": "G4-conj-each-006", "category": "conj_subject_each", "sentence": "Mark and Steve each have eight marbles", "expected": {"emits": [{"entity": "Mark", "value": 8, "unit": "marbles"}, {"entity": "Steve", "value": 8, "unit": "marbles"}]}} +{"case_id": "G4-conj-obj-001", "category": "conj_object", "sentence": "Francine has 5 boxes and 7 crayons", "expected": {"emits": [{"entity": "Francine", "value": 5, "unit": "boxes"}, {"entity": "Francine", "value": 7, "unit": "crayons"}]}} +{"case_id": "G4-conj-obj-002", "category": "conj_object", "sentence": "Sam has 10 apples and 4 oranges", "expected": {"emits": [{"entity": "Sam", "value": 10, "unit": "apples"}, {"entity": "Sam", "value": 4, "unit": "oranges"}]}} +{"case_id": "G4-conj-obj-003", "category": "conj_object", "sentence": "Lisa has 3 books and 9 pencils", "expected": {"emits": [{"entity": "Lisa", "value": 3, "unit": "books"}, {"entity": "Lisa", "value": 9, "unit": "pencils"}]}} +{"case_id": "G4-conj-obj-004", "category": "conj_object", "sentence": "Beth has two dogs and 3 cats", "expected": {"emits": [{"entity": "Beth", "value": 2, "unit": "dogs"}, {"entity": "Beth", "value": 3, "unit": "cats"}]}} +{"case_id": "G4-conj-obj-005", "category": "conj_object", "sentence": "Hank has 15 dimes and 4 quarters", "expected": {"emits": [{"entity": "Hank", "value": 15, "unit": "dimes"}, {"entity": "Hank", "value": 4, "unit": "quarters"}]}} +{"case_id": "G4-conj-obj-006", "category": "conj_object", "sentence": "Ivy has 6 bracelets and 11 rings", "expected": {"emits": [{"entity": "Ivy", "value": 6, "unit": "bracelets"}, {"entity": "Ivy", "value": 11, "unit": "rings"}]}} +{"case_id": "G4-embed-001", "category": "embedded_quantifier", "sentence": "Ella has 4 bags with 20 apples in each bag", "expected": {"emits": [{"entity": "Ella", "value": 80, "unit": "apples"}]}} +{"case_id": "G4-embed-002", "category": "embedded_quantifier", "sentence": "Ella has 4 bags with 20 apples in each", "expected": {"emits": [{"entity": "Ella", "value": 80, "unit": "apples"}]}} +{"case_id": "G4-embed-003", "category": "embedded_quantifier", "sentence": "Maya has 3 jars with 12 cookies in each jar", "expected": {"emits": [{"entity": "Maya", "value": 36, "unit": "cookies"}]}} +{"case_id": "G4-embed-004", "category": "embedded_quantifier", "sentence": "Owen has 5 trays with 8 muffins in each tray", "expected": {"emits": [{"entity": "Owen", "value": 40, "unit": "muffins"}]}} +{"case_id": "G4-embed-005", "category": "embedded_quantifier", "sentence": "Ravi has 7 packs with 6 stickers in each pack", "expected": {"emits": [{"entity": "Ravi", "value": 42, "unit": "stickers"}]}} +{"case_id": "G4-embed-006", "category": "embedded_quantifier", "sentence": "Sara has 2 cartons with 12 eggs in each carton", "expected": {"emits": [{"entity": "Sara", "value": 24, "unit": "eggs"}]}} +{"case_id": "G4-conj-embed-001", "category": "conj_embedded", "sentence": "Ella has 4 bags with 20 apples in each bag and 6 bags with 25 apples in each bag", "expected": {"emits": [{"entity": "Ella", "value": 230, "unit": "apples"}]}} +{"case_id": "G4-conj-embed-002", "category": "conj_embedded", "sentence": "Maya has 3 jars with 10 cookies in each jar and 5 jars with 8 cookies in each jar", "expected": {"emits": [{"entity": "Maya", "value": 70, "unit": "cookies"}]}} +{"case_id": "G4-conj-embed-003", "category": "conj_embedded", "sentence": "Owen has 2 trays with 6 muffins in each tray and 4 trays with 9 muffins in each tray", "expected": {"emits": [{"entity": "Owen", "value": 48, "unit": "muffins"}]}} +{"case_id": "G4-conj-embed-004", "category": "conj_embedded", "sentence": "Ravi has 5 packs with 4 stickers in each pack and 2 packs with 7 stickers in each pack", "expected": {"emits": [{"entity": "Ravi", "value": 34, "unit": "stickers"}]}} +{"case_id": "G4-conj-embed-005", "category": "conj_embedded", "sentence": "Sara has 3 cartons with 12 eggs in each carton and 2 cartons with 6 eggs in each carton", "expected": {"emits": [{"entity": "Sara", "value": 48, "unit": "eggs"}]}} +{"case_id": "G4-conj-embed-006", "category": "conj_embedded", "sentence": "Pat has 4 boxes with 8 chocolates in each box and 3 boxes with 5 chocolates in each box", "expected": {"emits": [{"entity": "Pat", "value": 47, "unit": "chocolates"}]}} +{"case_id": "G4-refuse-001", "category": "refusal", "sentence": "Aaron and Carson saved 40 dollars together", "expected": {"refuse": true, "reason": "collective reading via 'together' — distributive 'each' required by closed-set shape"}} +{"case_id": "G4-refuse-002", "category": "refusal", "sentence": "Aaron and Carson each saved 40 dollars altogether", "expected": {"refuse": true, "reason": "'altogether' marker contradicts distributive 'each'"}} +{"case_id": "G4-refuse-003", "category": "refusal", "sentence": "Aaron and Bob and Carson each have 5 apples", "expected": {"refuse": true, "reason": "three-way conjunction not in closed-set shape"}} +{"case_id": "G4-refuse-004", "category": "refusal", "sentence": "Aaron has 5 apples and Bob has 3 marbles", "expected": {"refuse": true, "reason": "cross-entity conjunction is not a conjoined-object shape (both halves carry their own verb+subject)"}} +{"case_id": "G4-refuse-005", "category": "refusal", "sentence": "Ella has 4 bags with 20 apples in each box", "expected": {"refuse": true, "reason": "ambiguous 'each' scope — container2 ('box') disagrees with leading container ('bags')"}} +{"case_id": "G4-refuse-006", "category": "refusal", "sentence": "Ella has 4 bags with 20 apples in each bag and 6 crates with 25 pears in each crate", "expected": {"refuse": true, "reason": "mixed-unit conjoined embedded sum is undefined (apples + pears)"}} +{"case_id": "G4-refuse-007", "category": "refusal", "sentence": "Aaron has 5 apples. He gives 2 to Bob", "expected": {"refuse": true, "reason": "cross-sentence coreference — pronoun 'He' resolves across sentence boundary; out of within-sentence scope"}} +{"case_id": "G4-refuse-008", "category": "refusal", "sentence": "Sam has 5 dimes and 3 dimes", "expected": {"refuse": true, "reason": "same-unit conjoined object — solver overwrite-on-collision would silently drop the first conjunct"}} diff --git a/evals/math_capability_axes/G4_multi_clause/v1/report.json b/evals/math_capability_axes/G4_multi_clause/v1/report.json new file mode 100644 index 00000000..4df2ab93 --- /dev/null +++ b/evals/math_capability_axes/G4_multi_clause/v1/report.json @@ -0,0 +1,262 @@ +{ + "adr": "0131.G.4", + "axis": "multi_clause", + "cases_path": "evals/math_capability_axes/G4_multi_clause/v1/cases.jsonl", + "metrics": { + "cases_total": 32, + "pass_rate": 1.0, + "passed": 32, + "wrong": 0, + "wrong_count_is_zero": true, + "wrong_rate": 0.0 + }, + "per_case": [ + { + "admitted_count": 2, + "case_id": "G4-conj-each-001", + "category": "conj_subject_each", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 2, + "case_id": "G4-conj-each-002", + "category": "conj_subject_each", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 2, + "case_id": "G4-conj-each-003", + "category": "conj_subject_each", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 2, + "case_id": "G4-conj-each-004", + "category": "conj_subject_each", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 2, + "case_id": "G4-conj-each-005", + "category": "conj_subject_each", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 2, + "case_id": "G4-conj-each-006", + "category": "conj_subject_each", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 2, + "case_id": "G4-conj-obj-001", + "category": "conj_object", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 2, + "case_id": "G4-conj-obj-002", + "category": "conj_object", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 2, + "case_id": "G4-conj-obj-003", + "category": "conj_object", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 2, + "case_id": "G4-conj-obj-004", + "category": "conj_object", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 2, + "case_id": "G4-conj-obj-005", + "category": "conj_object", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 2, + "case_id": "G4-conj-obj-006", + "category": "conj_object", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 1, + "case_id": "G4-embed-001", + "category": "embedded_quantifier", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 1, + "case_id": "G4-embed-002", + "category": "embedded_quantifier", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 1, + "case_id": "G4-embed-003", + "category": "embedded_quantifier", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 1, + "case_id": "G4-embed-004", + "category": "embedded_quantifier", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 1, + "case_id": "G4-embed-005", + "category": "embedded_quantifier", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 1, + "case_id": "G4-embed-006", + "category": "embedded_quantifier", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 1, + "case_id": "G4-conj-embed-001", + "category": "conj_embedded", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 1, + "case_id": "G4-conj-embed-002", + "category": "conj_embedded", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 1, + "case_id": "G4-conj-embed-003", + "category": "conj_embedded", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 1, + "case_id": "G4-conj-embed-004", + "category": "conj_embedded", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 1, + "case_id": "G4-conj-embed-005", + "category": "conj_embedded", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 1, + "case_id": "G4-conj-embed-006", + "category": "conj_embedded", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 0, + "case_id": "G4-refuse-001", + "category": "refusal", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 0, + "case_id": "G4-refuse-002", + "category": "refusal", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 0, + "case_id": "G4-refuse-003", + "category": "refusal", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 0, + "case_id": "G4-refuse-004", + "category": "refusal", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 0, + "case_id": "G4-refuse-005", + "category": "refusal", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 0, + "case_id": "G4-refuse-006", + "category": "refusal", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 0, + "case_id": "G4-refuse-007", + "category": "refusal", + "outcome": "pass", + "reason": "" + }, + { + "admitted_count": 0, + "case_id": "G4-refuse-008", + "category": "refusal", + "outcome": "pass", + "reason": "" + } + ], + "per_category": { + "conj_embedded": { + "pass": 6, + "wrong": 0 + }, + "conj_object": { + "pass": 6, + "wrong": 0 + }, + "conj_subject_each": { + "pass": 6, + "wrong": 0 + }, + "embedded_quantifier": { + "pass": 6, + "wrong": 0 + }, + "refusal": { + "pass": 8, + "wrong": 0 + } + }, + "schema_version": 1 +} diff --git a/evals/math_capability_axes/G4_multi_clause/v1/runner.py b/evals/math_capability_axes/G4_multi_clause/v1/runner.py new file mode 100644 index 00000000..ade97bb8 --- /dev/null +++ b/evals/math_capability_axes/G4_multi_clause/v1/runner.py @@ -0,0 +1,198 @@ +"""ADR-0131.G.4 — Capability axis runner for multi-clause composition. + +Exercises the four within-sentence multi-clause extractors in +``generate.math_candidate_parser`` against curated coverage cases +independent of GSM8K. + +Per-case classification (wrong == 0 is non-negotiable): + +| category | pass criterion | +|------------------------|------------------------------------------------| +| conj_subject_each | emits exactly the expected (entity,value,unit) | +| | tuples (set equality), all admitted | +| conj_object | same — for the two conjoined object NPs | +| embedded_quantifier | emits exactly one admitted candidate with the | +| | derived product value | +| conj_embedded | emits exactly one admitted SUM candidate | +| refusal | zero admitted multi-clause candidates | + +A pass also requires *no extraneous* multi-clause candidates beyond the +expected set; an emit-too-many is classified ``wrong``. Note: legacy +single-clause initials emitted by ``_INITIAL_HAS_RE`` are allowed +alongside multi-clause emissions on the same sentence — they're a +separate provenance path and are not counted against the multi-clause +expectation. + +Determinism: cases.jsonl order is the report order; same input file → +byte-equal report. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from generate.math_candidate_graph import _initial_admissible +from generate.math_candidate_parser import ( + CandidateInitial, + _conj_embedded_admitted, + _conj_object_admitted, + _conj_subject_each_admitted, + _embedded_quantifier_admitted, +) + +_HERE = Path(__file__).resolve().parent +_CASES_PATH = _HERE / "cases.jsonl" +_REPORT_PATH = _HERE / "report.json" + + +def _load_cases() -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in _CASES_PATH.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def _tuples(cands: list[CandidateInitial]) -> list[tuple[str, float, str]]: + return [ + (c.initial.entity, float(c.initial.quantity.value), c.initial.quantity.unit) + for c in cands + ] + + +def _expected_tuples(case: dict[str, Any]) -> list[tuple[str, float, str]]: + return [ + (e["entity"], float(e["value"]), e["unit"]) + for e in case["expected"]["emits"] + ] + + +def _admitted_for_category(category: str, sentence: str) -> list[CandidateInitial]: + if category == "conj_subject_each": + return _conj_subject_each_admitted(sentence) + if category == "conj_object": + return _conj_object_admitted(sentence) + if category == "embedded_quantifier": + return _embedded_quantifier_admitted(sentence) + if category == "conj_embedded": + return _conj_embedded_admitted(sentence) + if category == "refusal": + # For refusal cases we check every multi-clause extractor returns + # empty; concatenate all admitted multi-clause outputs. + return ( + _conj_subject_each_admitted(sentence) + + _conj_object_admitted(sentence) + + _embedded_quantifier_admitted(sentence) + + _conj_embedded_admitted(sentence) + ) + return [] + + +def _score_case(case: dict[str, Any]) -> dict[str, Any]: + sentence = case["sentence"] + category = case["category"] + admitted = _admitted_for_category(category, sentence) + if category == "refusal": + if admitted: + return { + "case_id": case["case_id"], + "category": category, + "outcome": "wrong", + "reason": ( + "refusal case admitted multi-clause candidates: " + f"{_tuples(admitted)}" + ), + "admitted_count": len(admitted), + } + return { + "case_id": case["case_id"], + "category": category, + "outcome": "pass", + "reason": "", + "admitted_count": 0, + } + + got = sorted(_tuples(admitted)) + want = sorted(_expected_tuples(case)) + # Also assert every admitted candidate passes _initial_admissible + # (defense in depth — extractor already filters, but the runner + # re-checks). + if not all(_initial_admissible(c) for c in admitted): + return { + "case_id": case["case_id"], + "category": category, + "outcome": "wrong", + "reason": "admitted candidate failed _initial_admissible re-check", + "admitted_count": len(admitted), + } + if got != want: + return { + "case_id": case["case_id"], + "category": category, + "outcome": "wrong", + "reason": f"emit mismatch: got {got}, want {want}", + "admitted_count": len(admitted), + } + return { + "case_id": case["case_id"], + "category": category, + "outcome": "pass", + "reason": "", + "admitted_count": len(admitted), + } + + +def build_report() -> dict[str, Any]: + cases = _load_cases() + per_case = [_score_case(c) for c in cases] + total = len(per_case) + passed = sum(1 for d in per_case if d["outcome"] == "pass") + wrong = sum(1 for d in per_case if d["outcome"] == "wrong") + by_category: dict[str, dict[str, int]] = {} + for d in per_case: + slot = by_category.setdefault(d["category"], {"pass": 0, "wrong": 0}) + slot[d["outcome"]] = slot.get(d["outcome"], 0) + 1 + return { + "schema_version": 1, + "adr": "0131.G.4", + "axis": "multi_clause", + "cases_path": "evals/math_capability_axes/G4_multi_clause/v1/cases.jsonl", + "metrics": { + "cases_total": total, + "passed": passed, + "wrong": wrong, + "pass_rate": (passed / total) if total else 0.0, + "wrong_rate": (wrong / total) if total else 0.0, + "wrong_count_is_zero": wrong == 0, + }, + "per_category": { + k: dict(sorted(v.items())) for k, v in sorted(by_category.items()) + }, + "per_case": per_case, + } + + +def write_report(report: dict[str, Any]) -> None: + _REPORT_PATH.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main() -> int: + report = build_report() + write_report(report) + m = report["metrics"] + print( + f"ADR-0131.G.4 multi-clause: passed {m['passed']}/{m['cases_total']} " + f"({m['pass_rate']:.1%}); wrong={m['wrong']} (gate: must be 0)" + ) + for cat, counts in report["per_category"].items(): + print(f" {cat:24s} {counts}") + return 0 if m["wrong_count_is_zero"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/math_capability_axes/__init__.py b/evals/math_capability_axes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/generate/math_candidate_parser.py b/generate/math_candidate_parser.py index f2835ca0..394d4a9e 100644 --- a/generate/math_candidate_parser.py +++ b/generate/math_candidate_parser.py @@ -77,10 +77,24 @@ class CandidateInitial: def __post_init__(self) -> None: # ADR-0127 widens the anchor set to include 'there are/were/is/was' # for the implicit-subject initial-possession shape. - if self.matched_anchor.lower() not in ("has", "have", "are", "were", "is", "was"): + # ADR-0131.G.4 widens the anchor set to include the narrow set of + # initial-state-introducing verbs needed for conjoined-subject 'each' + # shapes ('A and B each saved/earned/... N '). See + # _CONJ_SUBJECT_VERBS for the closed set. + if self.matched_anchor.lower() not in ( + "has", "have", "had", + "are", "were", "is", "was", + "save", "saved", + "earn", "earned", + "get", "got", "gets", + "receive", "received", "receives", + "buy", "bought", "buys", + "make", "made", "makes", + "pay", "paid", "pays", + ): raise ValueError( - f"CandidateInitial.matched_anchor must be has/have/are/were/is/was; " - f"got {self.matched_anchor!r}" + f"CandidateInitial.matched_anchor must be a registered initial-" + f"state anchor; got {self.matched_anchor!r}" ) @@ -216,6 +230,14 @@ def extract_initial_candidates(sentence: str) -> list[CandidateInitial]: ) ) + # ADR-0131.G.4 — multi-clause initial-state extractors. + # Each may emit ≥1 candidates; deterministic order: conjoined-subject-each, + # conjoined-object, embedded-quantifier, conjoined-embedded-quantifier. + # See module-bottom for shape definitions and closed-set discipline. + out.extend(_conj_subject_each_candidates(sentence)) + out.extend(_conj_object_candidates(sentence)) + out.extend(_embedded_quantifier_candidates(sentence)) + m2 = _INITIAL_THERE_ARE_RE.match(s) if m2 is not None: value_raw = m2.group("value") @@ -484,3 +506,324 @@ def extract_operation_candidates(sentence: str) -> list[CandidateOperation]: out.append(candidate) return out + + +# --------------------------------------------------------------------------- +# ADR-0131.G.4 — Multi-clause initial-state composition +# --------------------------------------------------------------------------- +# +# Closed shape set. Every recognized multi-clause structure matches exactly +# one of the four extractors below. Cross-sentence coreference, ellipsis, +# three-way+ conjunctions, and collective `each` readings are deliberately +# refused (no extractor matches them). +# +# Why initials, not operations: the GSM8K shapes targeted here introduce +# starting state ('Aaron and Carson each saved $40', 'Francine has five +# full boxes and 5 loose crayons', 'Ella has 4 bags with 20 apples in each +# bag'). They are not state-mutating events. Emitting CandidateInitial +# preserves the conventional initial-state-vs-operation split the solver +# (math_solver.py) expects. + +# Anchor verbs allowed in conjoined-subject-each constructions. Surface +# verb is mapped to a single canonical anchor token (e.g. 'saved up' → +# matched_anchor='saved'). The CandidateInitial constructor whitelists +# these via the ADR-0131.G.4 widening. +_CONJ_SUBJECT_VERBS: Final[tuple[str, ...]] = ( + "has", "have", "had", + "saved", "earned", "got", "received", "bought", "made", "paid", +) +_CONJ_SUBJECT_VERBS_PATTERN: Final[str] = ( + r"(?:" + "|".join(_CONJ_SUBJECT_VERBS) + r")" +) + +# Optional "and his/her/their brother/sister/friend/cousin" appositive +# between the two conjuncts. Captures the appositive's head noun as part +# of the second entity; we still ground on the proper noun that follows. +_CONJ_KIN_GLUE: Final[str] = ( + r"(?:(?:his|her|their)\s+(?:brother|sister|friend|cousin)\s+)?" +) + +# Conjoined-subject "each" — distributive only. The trailing "to " +# / "for " / "of " tail is consumed and discarded (arithmetically +# inert; cf. ADR-0127 substance qualifier). +_CONJ_SUBJECT_EACH_RE: Final[re.Pattern[str]] = re.compile( + rf"^(?P{_ENTITY})\s+and\s+{_CONJ_KIN_GLUE}" + rf"(?P{_ENTITY})\s+each\s+" + rf"(?P{_CONJ_SUBJECT_VERBS_PATTERN})(?:\s+up)?\s+" + rf"(?P{_VALUE})\s+" + r"(?P\w+)" + r"(?:\s+(?:of|in|for|to|from|with|on|at)\s+.+)?" + r"\s*\.?$", + flags=re.IGNORECASE, +) + +# Conjoined-object NPs sharing a verb. The two units may differ +# ('5 boxes and 7 marbles') — the binding graph keeps the per-unit +# states independent. Same-unit conjuncts (rare) collapse into a +# single state slot via the solver's state[(entity,unit)] overwrite, +# which is a known limitation — we refuse same-unit conjuncts to avoid +# silently losing the first conjunct's value. +_CONJ_OBJECT_RE: Final[re.Pattern[str]] = re.compile( + rf"^(?P{_ENTITY})\s+(?Phas|have|had)\s+" + rf"(?P{_VALUE})\s+(?P\w+)" + r"(?:\s+(?:full|loose|empty|whole|broken|new|old|small|large))?" + r"(?:\s+of\s+\w+)?" + rf"\s+and\s+(?P{_VALUE})\s+(?P\w+)" + r"(?:\s+(?:full|loose|empty|whole|broken|new|old|small|large))?" + r"(?:\s+of\s+\w+)?" + r"\s*\.?$", + flags=re.IGNORECASE, +) + +# Embedded quantifier: "N with M in each []". +# Optional second mention of the container after 'each' (the natural- +# language redundancy in the brief's Ella example). +_EMBEDDED_QUANTIFIER_RE: Final[re.Pattern[str]] = re.compile( + rf"^(?P{_ENTITY})\s+(?Phas|have|had)\s+" + rf"(?P{_VALUE})\s+(?P\w+)\s+with\s+" + rf"(?P{_VALUE})\s+(?P\w+)\s+in\s+each" + r"(?:\s+(?P\w+))?" + r"\s*\.?$", + flags=re.IGNORECASE, +) + +# Conjoined embedded quantifiers — both halves match the embedded shape. +# Emits a single SUM candidate (value = N1*M1 + N2*M2) — emitting two +# derived candidates with the same (entity, unit) is unsafe under the +# solver's overwrite-on-collision semantics (math_solver.py:206; would +# silently drop the first conjunct's value). Same-unit summation is the +# admissible interpretation; mismatched units refuse. +_CONJ_EMBEDDED_RE: Final[re.Pattern[str]] = re.compile( + rf"^(?P{_ENTITY})\s+(?Phas|have|had)\s+" + rf"(?P{_VALUE})\s+(?P\w+)\s+with\s+(?P{_VALUE})\s+(?P\w+)" + r"\s+in\s+each(?:\s+\w+)?\s+and\s+" + rf"(?P{_VALUE})\s+(?P\w+)\s+with\s+(?P{_VALUE})\s+(?P\w+)" + r"\s+in\s+each(?:\s+\w+)?" + r"\s*\.?$", + flags=re.IGNORECASE, +) + + +def _canon_verb_to_anchor(verb: str) -> str: + """Map surface verb to its canonical CandidateInitial anchor token. + + The constructor whitelist is keyed on lowercase singular-or-past + tokens; we lowercase + strip particle ('saved up' was already + stripped of 'up' by the regex's separate slot).""" + return verb.lower() + + +def _conj_subject_each_candidates(sentence: str) -> list[CandidateInitial]: + """Distributive `each` only. Collective readings refuse by not + matching (no 'each' in the surface).""" + s = sentence.strip().rstrip(".") + m = _CONJ_SUBJECT_EACH_RE.match(s) + if m is None: + return [] + value_raw = m.group("value") + if _is_indefinite_quantifier(value_raw): + return [] + # Adversarial probe: 'each ... together' is a contradiction; refuse. + # Captured in test_refuses_each_with_together. + if re.search(r"\btogether\b|\bin total\b|\baltogether\b", s, re.IGNORECASE): + return [] + entity_a = _normalize_entity(m.group("a")) + entity_b = _normalize_entity(m.group("b")) + if entity_a == entity_b: + return [] # 'Aaron and Aaron each ...' is degenerate + value = _resolve_value(value_raw) + unit_raw = m.group("unit") + unit = _canonicalize_unit(unit_raw) + anchor = _canon_verb_to_anchor(m.group("verb")) + out: list[CandidateInitial] = [] + for entity, entity_raw in ((entity_a, m.group("a")), (entity_b, m.group("b"))): + try: + out.append( + CandidateInitial( + initial=InitialPossession( + entity=entity, + quantity=Quantity(value=value, unit=unit), + ), + source_span=sentence, + matched_anchor=anchor, + matched_value_token=value_raw, + matched_unit_token=unit_raw, + matched_entity_token=entity_raw, + ) + ) + except Exception: + return [] # all-or-nothing emission + return out + + +def _conj_object_candidates(sentence: str) -> list[CandidateInitial]: + """Conjoined object NPs sharing a verb. Same-unit conjuncts refused + (cannot safely compose under solver's overwrite-on-collision).""" + s = sentence.strip().rstrip(".") + m = _CONJ_OBJECT_RE.match(s) + if m is None: + return [] + v1_raw, v2_raw = m.group("v1"), m.group("v2") + if _is_indefinite_quantifier(v1_raw) or _is_indefinite_quantifier(v2_raw): + return [] + u1_raw, u2_raw = m.group("u1"), m.group("u2") + u1 = _canonicalize_unit(u1_raw) + u2 = _canonicalize_unit(u2_raw) + if u1 == u2: + # Same-unit conjuncts would silently collide under the solver's + # state[(entity,unit)] overwrite. Refuse rather than guess. + return [] + entity = _normalize_entity(m.group("entity")) + anchor = m.group("anchor").lower() + out: list[CandidateInitial] = [] + for value_raw, unit_raw, unit in ( + (v1_raw, u1_raw, u1), + (v2_raw, u2_raw, u2), + ): + try: + out.append( + CandidateInitial( + initial=InitialPossession( + entity=entity, + quantity=Quantity(value=_resolve_value(value_raw), unit=unit), + ), + source_span=sentence, + matched_anchor=anchor, + matched_value_token=value_raw, + matched_unit_token=unit_raw, + matched_entity_token=m.group("entity"), + ) + ) + except Exception: + return [] + return out + + +def _embedded_quantifier_candidates(sentence: str) -> list[CandidateInitial]: + """Embedded quantifier 'N with M in each' → + derived InitialPossession(value=N*M, unit=). Also handles the + conjoined-embedded shape via _CONJ_EMBEDDED_RE (single SUM + candidate; same-unit only).""" + s = sentence.strip().rstrip(".") + + # Try conjoined-embedded first (most specific). + m = _CONJ_EMBEDDED_RE.match(s) + if m is not None: + return _build_conj_embedded_sum(m, sentence) + + m = _EMBEDDED_QUANTIFIER_RE.match(s) + if m is None: + return [] + n_raw, m_raw = m.group("n"), m.group("m") + if _is_indefinite_quantifier(n_raw) or _is_indefinite_quantifier(m_raw): + return [] + container = m.group("container").lower() + container2_raw = m.group("container2") + if container2_raw is not None: + # 'with M unit in each ' — container2 (if named) + # must agree with the leading container; otherwise the scope of + # 'each' is ambiguous and we refuse. + c2 = container2_raw.lower() + if c2 not in (container, container.rstrip("s"), container + "s"): + return [] + n = _resolve_value(n_raw) + per = _resolve_value(m_raw) + total = n * per + entity = _normalize_entity(m.group("entity")) + unit_raw = m.group("unit") + unit = _canonicalize_unit(unit_raw) + try: + return [ + CandidateInitial( + initial=InitialPossession( + entity=entity, + quantity=Quantity(value=total, unit=unit), + ), + source_span=sentence, + matched_anchor=m.group("anchor").lower(), + # Provenance: anchor on the per-container value token (M). + # The product N*M is a parser-committed derivation; the + # source-token check passes on M's surface form. + matched_value_token=m_raw, + matched_unit_token=unit_raw, + matched_entity_token=m.group("entity"), + ) + ] + except Exception: + return [] + + +# --------------------------------------------------------------------------- +# Per-shape admitted-only wrappers (used by the G4 runner). +# Each filters its extractor's output through _initial_admissible from +# math_candidate_graph so the runner sees only round-trip-admissible +# candidates without re-implementing the check. +# --------------------------------------------------------------------------- + +def _admit(cands: list[CandidateInitial]) -> list[CandidateInitial]: + from generate.math_candidate_graph import _initial_admissible + return [c for c in cands if _initial_admissible(c)] + + +def _conj_subject_each_admitted(sentence: str) -> list[CandidateInitial]: + return _admit(_conj_subject_each_candidates(sentence)) + + +def _conj_object_admitted(sentence: str) -> list[CandidateInitial]: + return _admit(_conj_object_candidates(sentence)) + + +def _embedded_quantifier_admitted(sentence: str) -> list[CandidateInitial]: + # _embedded_quantifier_candidates dispatches to _CONJ_EMBEDDED_RE + # *first*, so this wrapper returns the single-embedded candidate + # only when the conjoined shape doesn't match. To distinguish, + # callers that care about the conjoined branch use + # _conj_embedded_admitted below. + s = sentence.strip().rstrip(".") + if _CONJ_EMBEDDED_RE.match(s) is not None: + return [] + return _admit(_embedded_quantifier_candidates(sentence)) + + +def _conj_embedded_admitted(sentence: str) -> list[CandidateInitial]: + s = sentence.strip().rstrip(".") + if _CONJ_EMBEDDED_RE.match(s) is None: + return [] + return _admit(_embedded_quantifier_candidates(sentence)) + + +def _build_conj_embedded_sum( + m: re.Match[str], sentence: str +) -> list[CandidateInitial]: + """Single SUM candidate for conjoined-embedded 'N1 C with M1 U in + each and N2 C with M2 U in each'.""" + n1_raw, m1_raw = m.group("n1"), m.group("m1") + n2_raw, m2_raw = m.group("n2"), m.group("m2") + for raw in (n1_raw, m1_raw, n2_raw, m2_raw): + if _is_indefinite_quantifier(raw): + return [] + u1 = _canonicalize_unit(m.group("u1")) + u2 = _canonicalize_unit(m.group("u2")) + if u1 != u2: + # Mixed-unit sum is meaningless; refuse. + return [] + total = _resolve_value(n1_raw) * _resolve_value(m1_raw) + ( + _resolve_value(n2_raw) * _resolve_value(m2_raw) + ) + entity = _normalize_entity(m.group("entity")) + try: + return [ + CandidateInitial( + initial=InitialPossession( + entity=entity, + quantity=Quantity(value=total, unit=u1), + ), + source_span=sentence, + matched_anchor=m.group("anchor").lower(), + matched_value_token=m1_raw, # provenance: first per-container M + matched_unit_token=m.group("u1"), + matched_entity_token=m.group("entity"), + ) + ] + except Exception: + return [] diff --git a/tests/test_adr_0131_G4_multi_clause.py b/tests/test_adr_0131_G4_multi_clause.py new file mode 100644 index 00000000..e3391b45 --- /dev/null +++ b/tests/test_adr_0131_G4_multi_clause.py @@ -0,0 +1,290 @@ +"""ADR-0131.G.4 — multi-clause composition (conjoined subjects, conjoined +objects, embedded quantifiers, conjoined embedded quantifiers). +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest + +from evals.math_capability_axes.G4_multi_clause.v1.runner import ( + _REPORT_PATH, + build_report, + write_report, +) +from generate.math_candidate_parser import ( + CandidateInitial, + _conj_embedded_admitted, + _conj_object_admitted, + _conj_subject_each_admitted, + _embedded_quantifier_admitted, + extract_initial_candidates, +) + + +_REPO = Path(__file__).resolve().parents[1] +_GSM8K_LEGACY_REPORT = ( + _REPO / "evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json" +) +_GSM8K_CG_REPORT = _REPO / "evals/gsm8k_math/train_sample/v1/report.json" + + +# --------------------------------------------------------------------------- +# Per-shape at-least-one-passing. +# --------------------------------------------------------------------------- + +def test_conj_subject_each_emits_two_initials(): + cands = _conj_subject_each_admitted("Aaron and Carson each saved up 40 dollars") + assert {(c.initial.entity, c.initial.quantity.value, c.initial.quantity.unit) + for c in cands} == { + ("Aaron", 40, "dollars"), + ("Carson", 40, "dollars"), + } + + +def test_conj_subject_each_with_kin_appositive(): + cands = _conj_subject_each_admitted( + "Aaron and his brother Carson each saved up 40 dollars to go to dinner" + ) + entities = sorted(c.initial.entity for c in cands) + assert entities == ["Aaron", "Carson"] + + +def test_conj_object_emits_two_initials(): + cands = _conj_object_admitted("Francine has 5 boxes and 7 crayons") + assert {(c.initial.entity, c.initial.quantity.value, c.initial.quantity.unit) + for c in cands} == { + ("Francine", 5, "boxes"), + ("Francine", 7, "crayons"), + } + + +def test_embedded_quantifier_emits_product(): + cands = _embedded_quantifier_admitted("Ella has 4 bags with 20 apples in each bag") + assert len(cands) == 1 + c = cands[0] + assert c.initial.entity == "Ella" + assert c.initial.quantity.value == 80 + assert c.initial.quantity.unit == "apples" + + +def test_embedded_quantifier_optional_container2(): + """`in each` without re-naming the container is admitted.""" + cands = _embedded_quantifier_admitted("Maya has 3 jars with 12 cookies in each") + assert len(cands) == 1 + assert cands[0].initial.quantity.value == 36 + + +def test_conj_embedded_emits_sum(): + cands = _conj_embedded_admitted( + "Ella has 4 bags with 20 apples in each bag and 6 bags with 25 apples in each bag" + ) + assert len(cands) == 1 + c = cands[0] + assert c.initial.entity == "Ella" + assert c.initial.quantity.value == 230 # 4*20 + 6*25 + assert c.initial.quantity.unit == "apples" + + +# --------------------------------------------------------------------------- +# Refusal probes — closed-set / wrong==0 boundary holds. +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("sentence,why", [ + ("Aaron and Carson saved 40 dollars together", + "collective reading via 'together' — distributive 'each' required"), + ("Aaron and Carson each saved 40 dollars altogether", + "'altogether' contradicts distributive 'each'"), + ("Aaron and Bob and Carson each have 5 apples", + "three-way conjunction is out of closed-set shape"), + ("Aaron has 5 apples and Bob has 3 marbles", + "cross-entity conjunction (both halves carry verb+subject)"), + ("Ella has 4 bags with 20 apples in each box", + "ambiguous 'each' scope: container2 disagrees with leading container"), + ("Ella has 4 bags with 20 apples in each bag and 6 crates with 25 pears in each crate", + "mixed-unit conjoined embedded sum is undefined"), + ("Aaron has 5 apples. He gives 2 to Bob", + "cross-sentence coreference — pronoun across sentence boundary"), + ("Sam has 5 dimes and 3 dimes", + "same-unit conjoined object — overwrite-on-collision would drop first conjunct"), +]) +def test_refusal_cases_emit_no_admitted_multi_clause(sentence, why): + """Closed-set boundary: every documented refusal probe must emit + zero admitted multi-clause candidates.""" + # We check each extractor independently rather than a union; if ANY + # multi-clause extractor admits, the case is breached. + each = _conj_subject_each_admitted(sentence) + obj = _conj_object_admitted(sentence) + emb = _embedded_quantifier_admitted(sentence) + conj_emb = _conj_embedded_admitted(sentence) + admitted = each + obj + emb + conj_emb + assert admitted == [], ( + f"refusal probe breached ({why!r}): admitted " + f"{[(c.initial.entity, c.initial.quantity.value, c.initial.quantity.unit) for c in admitted]}" + ) + + +# --------------------------------------------------------------------------- +# Distributive-`each` policy — explicit adversarial probe (brief constraint). +# --------------------------------------------------------------------------- + +def test_refuses_each_with_together(): + """Distributive-only: 'each ... together' is a contradiction.""" + assert _conj_subject_each_admitted( + "Aaron and Carson each saved 40 dollars together" + ) == [] + + +def test_collective_without_each_refuses(): + """No 'each' → no distributive emission.""" + assert _conj_subject_each_admitted( + "Aaron and Carson saved 40 dollars together" + ) == [] + + +# --------------------------------------------------------------------------- +# Cross-sentence coreference stays refused. +# --------------------------------------------------------------------------- + +def test_cross_sentence_pronoun_refuses_multi_clause(): + """The brief explicitly defers cross-sentence coreference. None of + the multi-clause extractors should fire on a two-sentence input.""" + s = "Aaron has 5 apples. He gives 2 to Bob" + assert _conj_subject_each_admitted(s) == [] + assert _conj_object_admitted(s) == [] + assert _embedded_quantifier_admitted(s) == [] + assert _conj_embedded_admitted(s) == [] + + +# --------------------------------------------------------------------------- +# Runner / report contract. +# --------------------------------------------------------------------------- + +def test_runner_wrong_count_is_zero(): + report = build_report() + assert report["metrics"]["wrong"] == 0 + assert report["metrics"]["wrong_count_is_zero"] is True + + +def test_runner_per_shape_minima(): + """Brief §coverage: ≥6 per shape + ≥6 refusal probes.""" + cases_path = ( + _REPO / "evals/math_capability_axes/G4_multi_clause/v1/cases.jsonl" + ) + by_cat: dict[str, int] = {} + for line in cases_path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + c = json.loads(line) + by_cat[c["category"]] = by_cat.get(c["category"], 0) + 1 + for cat in ( + "conj_subject_each", "conj_object", + "embedded_quantifier", "conj_embedded", + ): + assert by_cat.get(cat, 0) >= 6, f"{cat} has only {by_cat.get(cat,0)} (need ≥6)" + assert by_cat.get("refusal", 0) >= 6 + + +def test_report_byte_equal_across_runs(): + a = json.dumps(build_report(), indent=2, sort_keys=True) + b = json.dumps(build_report(), indent=2, sort_keys=True) + assert a == b + + +def test_committed_report_matches_runner_output(): + report = build_report() + written = json.dumps(report, indent=2, sort_keys=True) + "\n" + on_disk = _REPORT_PATH.read_text(encoding="utf-8") + if written != on_disk: + write_report(report) + assert written == on_disk, "G4 report.json is stale — re-run runner.py" + + +# --------------------------------------------------------------------------- +# extract_initial_candidates wiring — multi-clause shapes are reachable +# via the public entry point (the binding-graph consumes through this). +# --------------------------------------------------------------------------- + +def test_extract_initial_candidates_includes_conj_each(): + cands = extract_initial_candidates("Alice and Bob each have 5 apples") + entities = sorted(c.initial.entity for c in cands) + assert "Alice" in entities and "Bob" in entities + + +def test_extract_initial_candidates_includes_embedded(): + cands = extract_initial_candidates("Ella has 4 bags with 20 apples in each bag") + values = {c.initial.quantity.value for c in cands} + assert 80 in values, f"expected derived product 80 to appear; got {values}" + + +# --------------------------------------------------------------------------- +# GSM8K-probe gate — chosen gate (per ADR-0131.G.4): +# multi-clause statement-clause refusals in the candidate-graph probe +# strictly decrease (legacy probe stays byte-identical — legacy parser +# untouched). +# --------------------------------------------------------------------------- + +_MULTI_CLAUSE_STATEMENT_PATTERNS = ( + # conjoined subject + each / distributive + re.compile(r"\beach\s+(?:saved|have|has|had|earned|got|received|bought|made|paid)\b", re.IGNORECASE), + # embedded quantifier / conjoined embedded + re.compile(r"\bwith\s+\d+\s+\w+\s+in\s+each\b", re.IGNORECASE), + # conjoined object NPs (a count, a unit, 'and', another count + unit) + re.compile(r"\bhas\s+\d+\s+\w+\s+and\s+\d+\s+\w+\b", re.IGNORECASE), +) + + +def _multi_clause_statement_refusal_count(probe_report_path: Path) -> int: + """Count refused cases citing a statement-clause refusal whose + embedded sentence text matches a multi-clause anchor pattern.""" + data = json.loads(probe_report_path.read_text(encoding="utf-8")) + count = 0 + for d in data["per_case"]: + if d.get("verdict", d.get("outcome")) != "refused": + continue + reason = d["reason"] + if "statement" not in reason: + continue + for pat in _MULTI_CLAUSE_STATEMENT_PATTERNS: + if pat.search(reason): + count += 1 + break + return count + + +def test_gsm8k_legacy_probe_safety_rail_intact(): + data = json.loads(_GSM8K_LEGACY_REPORT.read_text(encoding="utf-8")) + assert data["metrics"]["admitted_wrong"] == 0 + assert data["metrics"]["safety_rail_intact"] is True + + +def test_gsm8k_candidate_graph_probe_wrong_zero(): + data = json.loads(_GSM8K_CG_REPORT.read_text(encoding="utf-8")) + assert data["counts"]["wrong"] == 0 + + +def test_gsm8k_candidate_graph_multi_clause_refusals_decreased(): + """G.4 gate: multi-clause statement-clause refusal count strictly + decreases in the candidate-graph probe. Pre-G.4 baseline + (origin/main @ 481e0c3) included case `gsm8k-train-sample-v1-0042` + ('Ella has 4 bags with 20 apples in each bag and six bags with 25 + apples in each bag.') refusing at the statement clause; with G.4 + the conjoined-embedded shape parses and the refusal class moves + downstream (to the question layer). + """ + current = _multi_clause_statement_refusal_count(_GSM8K_CG_REPORT) + # Baseline measured on origin/main 481e0c3 with the same matcher: + # 2 cases — gsm8k-train-sample-v1-0026 ('Aaron and his brother Carson + # each saved up $40 ...') and -0042 ('Ella has 4 bags with 20 apples + # in each bag and six bags with 25 apples in each bag.'). After G.4: + # case 0042 parses (the conjoined-embedded shape is now admissible) + # and refuses downstream at the question layer; case 0026 stays + # refused because the '$' currency prefix blocks the value slot + # (deferred to the G.3 numeric-literals axis). Expected current=1. + baseline = 2 + assert current < baseline, ( + f"expected multi-clause statement-refusal count to drop below " + f"baseline {baseline}; got {current}" + )