From 65d857e72ae512ac2dc5c68c8eba8e23bcecb099 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 29 May 2026 08:43:03 -0700 Subject: [PATCH] feat(adr-0179): integrate EX-1/EX-4/EX-5 extraction richness (sealed lane) (#455) Reconciles ChatGPT's four independently-branched extraction PRs (#451/#452/ #453/#454) into one coherent generate/derivation/extract.py. They each rewrote the same file + same new test off main, so they conflicted pairwise and needed integration, not a merge. Integrated (span-tracked, most-specific-pass-first so numbers are never double counted): - EX-1 word-numbers (#452): reuses WORD_NUMBERS; tens-one hyphen compounds; factor-bearing half/third/quarter excluded. - EX-4 list-unit inheritance (#451): bare numeric list with one trailing unit. - EX-5 sentence-final numbers (#454): bare final number with empty unit. Deferred: EX-3 multi-word units (#453). Its greedy lowercase span reads "6 apples and 4 apples" as unit "apples and", regressing GB-2's test_same_unit_list_sums, and still can't recover real multi-word units from 0024-class text ("jumping jacks on"). Needs a tighter rule; see docs/handoff/AUDIT-ADR-0179-EX-RECONCILE.md. Verification (sealed lane only; chat/ does not import this module): - Serving frozen: lane-SHA 8/8 match, generate_claims --check OK -> 3/47/0 byte-identical, wrong=0 held. - Sealed practice improved 4/2/44 -> 4/1/45: case 0025 flips wrong->refused. EX-1 reads "three", so completeness sees a quantity the 6x50 chain omits and refuses the spurious 300 (gold 1200) instead of committing it. - No new test failures (3 pre-existing on main). Also fixes stale test drift from EX-2 (#447): TestDecimalGroundingGapIsDeferred asserted decimals still refuse, but #447 made $0.75-class resolve to 864. Renamed to TestDecimalGroundingResolves and updated to assert the flip. Honest scope note: EX-4 does NOT unblock real case 0024 (its PR test used a fabricated bare-list paraphrase). TestRealCase0024StillBlocked pins the true boundary. --- docs/handoff/AUDIT-ADR-0179-EX-RECONCILE.md | 70 +++++++++ generate/derivation/extract.py | 166 ++++++++++++++++++-- tests/test_adr_0176_ms3_search.py | 15 +- tests/test_adr_0179_extract.py | 118 ++++++++++++++ 4 files changed, 345 insertions(+), 24 deletions(-) create mode 100644 docs/handoff/AUDIT-ADR-0179-EX-RECONCILE.md create mode 100644 tests/test_adr_0179_extract.py diff --git a/docs/handoff/AUDIT-ADR-0179-EX-RECONCILE.md b/docs/handoff/AUDIT-ADR-0179-EX-RECONCILE.md new file mode 100644 index 00000000..be56935a --- /dev/null +++ b/docs/handoff/AUDIT-ADR-0179-EX-RECONCILE.md @@ -0,0 +1,70 @@ +# ADR-0179 EX-1/3/4/5 reconciliation (Claude, at the library) + +Reconciles the four sealed-lane extraction PRs ChatGPT opened from the remote +brief (`docs/handoff/CHATGPT-REMOTE-BRIEF.md`) into one coherent +`generate/derivation/extract.py`. Each PR was branched independently off `main` +and rewrote the *same* file + the *same* new test, so they conflict pairwise and +could not be merged as-is — they needed integration, not a fast-forward. + +## Disposition + +| PR | Sub-phase | Verdict | Action | +|----|-----------|---------|--------| +| #452 | EX-1 word-numbers | **Integrated** | folded in (reuses `WORD_NUMBERS`, factor-words excluded) | +| #453 | EX-3 multi-word units | **Deferred — regressed GB-2** | not integrated; see below | +| #451 | EX-4 list-unit inheritance | **Integrated** | folded in (span-tracked; does not flip real 0024) | +| #454 | EX-5 sentence-final numbers | **Integrated** | folded in (empty unit, span-excluded) | +| #450 | Stream A lookback audit | **Sound** | merged (read-only, feeds GB-3) | + +The four EX source PRs are superseded by the integration commit and closed with a +pointer here; their authored content survives in the merged file/tests. + +## Why EX-3 (multi-word units) was deferred + +The brief required "keep units tight" and "don't regress GB-1/GB-2 tests." EX-3's +greedy lowercase unit span (`[a-z]+(?:\s+[a-z]+)*`) does both wrongs: + +- **Regresses GB-2.** `compose_sequential("She picked 6 apples and 4 apples.")` + expects `10.0`. Greedy units read the first unit as `"apples and"` (it swallows + the connective up to the next digit), so `_same_unit` sees two distinct units + and the composer refuses. The existing test `test_same_unit_list_sums` flips + from pass to fail. +- **Doesn't even recover real multi-word units.** Real gold case 0024 is + `"20 jumping jacks on Monday, 36 on Tuesday, …"`. Greedy lowercase reads + `"jumping jacks on"` (stops only at the capital `Monday`), so the intended + `"jumping jacks"` unit is never produced anyway. + +EX-3's own tests pass only because they place the unit at a clause end (`"12 +jumping jacks."`) where punctuation halts the greedy run. That is a contrived +shape, not the GSM8K shape. A correct multi-word-unit extractor needs a tighter, +non-connective-crossing rule; tracked as future work, not shipped here. + +## Honest note on EX-4 and case 0024 + +EX-4's PR test asserted it "unblocks 0024" using a *fabricated* input +(`"20, 36, 40 and 50 jumping-jacks"`). The real case interleaves numbers with +temporal phrases (`"36 on Tuesday, 40 on Wednesday"`), so the bare-list regex +never fires and 36/40/50 do not inherit the unit. EX-4 is still a real, safe +orthographic primitive (some GSM8K problems do state a unit once after a bare +list), but it does **not** flip 0024. `TestRealCase0024StillBlocked` pins this so +no future change silently re-claims the unblock without proving 438 end-to-end. + +## Verification (run at the library) + +- **Serving frozen:** lane-SHA gate 8/8 match; `scripts/generate_claims.py + --check` OK → serving `3/47/0` byte-identical. wrong=0 held. +- **No new test failures:** the 3 failures present (`0163` pronoun, telemetry + round-trip, and the now-fixed `ms3` decimal-deferred) all pre-existed on `main`. +- **Sealed practice improved:** `build_search_report` went **4/2/44 → 4/1/45** + (one wrong eliminated, no correct lost). Case **0025** flipped wrong→refused: + EX-1 now reads `"three"`, so the completeness check sees a quantity the 6×50 + chain doesn't consume and refuses the spurious `300` (gold is `1200`). Richer + reading → the gate refuses rather than commits a wrong answer. This is the + intended direction. + +## Drift fixed in passing (cleanup-as-you-find) + +`tests/test_adr_0176_ms3_search.py::TestDecimalGroundingGapIsDeferred` asserted +decimals were "currently refused." EX-2 (#447) landed decimal grounding and made +that case resolve to the correct `864`; the test was stale on `main`. Renamed to +`TestDecimalGroundingResolves` and updated to assert the flip. diff --git a/generate/derivation/extract.py b/generate/derivation/extract.py index b7ed6896..e2f59953 100644 --- a/generate/derivation/extract.py +++ b/generate/derivation/extract.py @@ -1,10 +1,34 @@ -"""ADR-0175 Phase 3b — lexeme-level quantity extraction. +"""ADR-0175 Phase 3b / ADR-0179 — lexeme-level quantity extraction. -Pulls ``(value, unit, source_token)`` triples from a problem using a single -orthographic pattern: a number immediately followed by a unit word. Per -ADR-0165 this is a *lexeme* pattern ("what this piece looks like: a number, a -unit word") — not a grammar template ("how words combine to mean X"). The -*combining* is the search's job (search.py) gated by self-verification. +Pulls ``(value, unit, source_token)`` triples from a problem using conservative +orthographic patterns. Per ADR-0165 these are *lexeme* patterns ("what this +piece looks like: a number, a unit word") — never grammar templates ("how words +combine to mean X"). The *combining* is the search's job (search.py / compose.py) +gated by self-verification, which is refuse-preferring; over-extraction here can +only cost *refusals*, never a wrong answer. + +ADR-0179 enrichments integrated here (sealed lane only — ``chat/`` does not import +this module, so none of this can move the serving ``3/47/0``): + +* **EX-1 — word-numbers.** ``"three apples"`` → ``3.0``, including tens-one + hyphen compounds (``"twenty-four"`` → ``24.0``). Reuses the canonical + ``WORD_NUMBERS`` table from :mod:`generate.math_roundtrip` (single number + vocabulary). Factor-bearing forms (``half``/``third``/``quarter``) are excluded + — they read as divisors, not counts. +* **EX-4 — list-unit inheritance.** In a bare numeric list with the unit stated + once at the end (``"20, 36, 40 and 50 push-ups"``) the trailing unit attaches + to every number in the list. Still orthographic: a run of number tokens joined + by comma/``and`` delimiters, then one unit token. Whether the resulting + quantities may compose is the gate's decision, not the extractor's. +* **EX-5 — sentence-final numbers.** A number with no following unit word (end of + sentence/text or before terminal punctuation) extracts with an empty unit so it + stays available to the completeness check without inventing a unit lexeme. + +EX-3 (multi-word units) is deliberately **not** integrated: the greedy lowercase +unit span regresses GB-2's same-unit detection (``"6 apples and 4 apples"`` → +unit ``"apples and"``) and does not cleanly recover real multi-word units from +0024-class text (``"20 jumping jacks on Monday"`` → ``"jumping jacks on"``). See +``docs/handoff/AUDIT-ADR-0179-EX-RECONCILE.md``. """ from __future__ import annotations @@ -13,27 +37,135 @@ import re from typing import Final from generate.derivation.model import Quantity +from generate.math_roundtrip import WORD_NUMBERS -# Number (int or decimal) immediately followed by a unit word. Lexeme-level. +# Number (int or decimal) immediately followed by a single unit word. Lexeme-level. _QTY_RE: Final[re.Pattern[str]] = re.compile( r"(? Quantity | None: + """Build a quantity from an already-matched numeric token.""" + try: + value = float(value_token) + except ValueError: # pragma: no cover - regex guarantees numeric + return None + return Quantity(value=value, unit=unit.lower(), source_token=value_token) + + +def _resolve_word_number(first: str, second: str | None) -> float | None: + """Resolve a conservative word-number token from ``WORD_NUMBERS``. + + A bare word resolves directly. A hyphen compound resolves only as a tens-one + form (``twenty-four``, ``ninety-nine``); anything else returns ``None`` so the + extractor stays conservative rather than guessing a composition rule. + """ + first_value = WORD_NUMBERS.get(first.lower()) + if first_value is None: + return None + if second is None: + return float(first_value) + second_value = WORD_NUMBERS.get(second.lower()) + if second_value is None: + return None + if first_value < 20 or first_value >= 100 or not 0 < second_value < 10: + return None + return float(first_value + second_value) + + +def _claimed(pos: int, spans: list[tuple[int, int]]) -> bool: + """Whether a numeric token at ``pos`` was already claimed by an earlier pass.""" + return any(start <= pos < end for start, end in spans) + def extract_quantities(problem_text: str) -> tuple[Quantity, ...]: """Extract ``(value, unit, source_token)`` quantities in left-to-right order. Deterministic. ``source_token`` is the surface number string (used by the - self-verification gate to prove the value is grounded in the text). Units - are lowercased; the value's surface token is preserved verbatim. + self-verification gate to prove the value is grounded in the text). Units are + lowercased; the value's surface token is preserved verbatim. + + Passes run most-specific first and claim the digit spans they consume so later + passes never double-count a number: + + 1. EX-4 same-unit list (claims every number in the list); + 2. digit + single unit word (skips numbers a list already claimed); + 3. EX-1 word-number + unit word (alphabetic, disjoint from digit spans); + 4. EX-5 sentence-final bare number (skips any already-claimed digit). """ - out: list[Quantity] = [] - for match in _QTY_RE.finditer(problem_text): - value_token = match.group(1) + found: list[tuple[int, Quantity]] = [] + claimed: list[tuple[int, int]] = [] + + # 1. EX-4 — list with one trailing unit; the unit inherits to every number. + for match in _LIST_WITH_TRAILING_UNIT_RE.finditer(problem_text): unit = match.group(2).lower() - try: - value = float(value_token) - except ValueError: # pragma: no cover - regex guarantees numeric + for num in _NUMBER_RE.finditer(match.group(1)): + pos = match.start(1) + num.start() + quantity = _quantity(num.group(0), unit) + if quantity is not None: + found.append((pos, quantity)) + claimed.append((pos, pos + len(num.group(0)))) + + # 2. digit + single unit word — the original base pattern. + for match in _QTY_RE.finditer(problem_text): + if _claimed(match.start(1), claimed): continue - out.append(Quantity(value=value, unit=unit, source_token=value_token)) - return tuple(out) + quantity = _quantity(match.group(1), match.group(2)) + if quantity is not None: + found.append((match.start(1), quantity)) + claimed.append(match.span(1)) + + # 3. EX-1 — word-numbers (and tens-one hyphen compounds) with a unit word. + for match in _WORD_QTY_RE.finditer(problem_text): + value = _resolve_word_number(match.group(1), match.group(2)) + if value is None: + continue + source_token = ( + match.group(1) + if match.group(2) is None + else match.group(0).rsplit(maxsplit=1)[0] + ) + found.append( + ( + match.start(1), + Quantity(value=value, unit=match.group(3).lower(), source_token=source_token), + ) + ) + + # 4. EX-5 — sentence-final bare numbers (empty unit). + for match in _FINAL_NUMBER_RE.finditer(problem_text): + if _claimed(match.start(1), claimed): + continue + quantity = _quantity(match.group(1), "") + if quantity is not None: + found.append((match.start(1), quantity)) + + found.sort(key=lambda item: item[0]) + return tuple(quantity for _, quantity in found) diff --git a/tests/test_adr_0176_ms3_search.py b/tests/test_adr_0176_ms3_search.py index 7a3ece13..5d9d12ca 100644 --- a/tests/test_adr_0176_ms3_search.py +++ b/tests/test_adr_0176_ms3_search.py @@ -66,14 +66,15 @@ class TestTargetThreading: assert search_chain(text, target).answer == 300.0 -class TestDecimalGroundingGapIsDeferred: - def test_decimal_operand_currently_refused(self) -> None: - # Documents the known gap: $0.75 tokenizes to 0/75 so "0.75" is not grounded - # by the shared round-trip primitive -> the (correct, 864) product refuses. - # When decimal/currency grounding lands, this should flip. Asserting the - # CURRENT behaviour so the fix is detectable. +class TestDecimalGroundingResolves: + def test_decimal_operand_grounds_and_resolves(self) -> None: + # ADR-0179 EX-2 (#447) landed bare-decimal grounding in the shared round-trip + # primitive, so "0.75" is now grounded and the (correct, 864) product + # self-verifies. This previously refused (the decimal grounding gap); the + # flip is the EX-2 acceptance signal. text = ( "There are 48 boxes with 24 erasers in each box. " "They sell the erasers for $0.75 each. How much money will they make?" ) - assert search_chain(text) is None # refused today (decimal grounding gap) + res = search_chain(text) + assert res is not None and res.answer == 864.0 # 48 * 24 * 0.75 diff --git a/tests/test_adr_0179_extract.py b/tests/test_adr_0179_extract.py new file mode 100644 index 00000000..689a3952 --- /dev/null +++ b/tests/test_adr_0179_extract.py @@ -0,0 +1,118 @@ +"""ADR-0179 — extraction richness tests (sealed lane: generate/derivation/extract). + +Integration of the EX-1 (word-numbers), EX-4 (list-unit inheritance), and EX-5 +(sentence-final numbers) sub-phases. EX-3 (multi-word units) is deferred — see +the module docstring and docs/handoff/AUDIT-ADR-0179-EX-RECONCILE.md — so there +is no multi-word-unit test class here by design. + +Every assertion is a plain input-string → extracted-quantity check. Over-extraction +in this lane costs refusals (the gate is refuse-preferring), never wrong answers. +""" + +from __future__ import annotations + +from generate.derivation import extract_quantities + + +def _triples(text: str) -> list[tuple[float, str, str]]: + return [(q.value, q.unit, q.source_token) for q in extract_quantities(text)] + + +class TestEX1WordNumbers: + def test_word_number_extracts_as_quantity(self) -> None: + assert _triples("She picked three apples.") == [(3.0, "apples", "three")] + + def test_digit_and_word_numbers_extract_left_to_right(self) -> None: + assert _triples("She picked 2 apples and three oranges.") == [ + (2.0, "apples", "2"), + (3.0, "oranges", "three"), + ] + + def test_hyphenated_tens_one_compound(self) -> None: + assert _triples("The team scored twenty-four points.") == [ + (24.0, "points", "twenty-four"), + ] + + def test_non_tens_one_compound_is_not_guessed(self) -> None: + # "one-third" is not a tens-one compound; the resolver declines the whole + # hyphen match rather than invent a composition rule (no fraction support, + # no fall-back to "one" alone). Conservative: nothing extracted. + assert _triples("He ate one-third pizza.") == [] + + def test_factor_words_are_not_extracted_as_counts(self) -> None: + # half/third/quarter read as divisors, not counts — excluded from EX-1. + assert _triples("She drank half cup.") == [] + + +class TestEX4ListUnitInheritance: + def test_trailing_unit_attaches_to_every_number_in_list(self) -> None: + qs = extract_quantities("20, 36, 40 and 50 push-ups") + assert [q.value for q in qs] == [20.0, 36.0, 40.0, 50.0] + assert [q.unit for q in qs] == ["push-ups"] * 4 + assert [q.source_token for q in qs] == ["20", "36", "40", "50"] + + def test_left_to_right_order_with_surrounding_quantities(self) -> None: + assert _triples("She had 2 bags, then did 20, 36 and 40 reps.") == [ + (2.0, "bags", "2"), + (20.0, "reps", "20"), + (36.0, "reps", "36"), + (40.0, "reps", "40"), + ] + + def test_list_numbers_not_double_counted(self) -> None: + # the trailing number's "50 push-ups" must not also surface via the + # single-unit pass. + qs = extract_quantities("20, 36, 40 and 50 push-ups") + assert len(qs) == 4 + + +class TestEX5SentenceFinalNumbers: + def test_sentence_final_number_extracts_with_empty_unit(self) -> None: + assert _triples("She had 5.") == [(5.0, "", "5")] + + def test_question_final_number_extracts_with_empty_unit(self) -> None: + assert _triples("The answer is 7?") == [(7.0, "", "7")] + + def test_unit_quantity_and_final_number_extract_left_to_right(self) -> None: + assert _triples("She picked 6 apples and had 4.") == [ + (6.0, "apples", "6"), + (4.0, "", "4"), + ] + + def test_number_with_unit_is_not_duplicated_as_final_number(self) -> None: + assert _triples("She picked 6 apples.") == [(6.0, "apples", "6")] + + +class TestNoRegression: + def test_simple_digit_units_still_work(self) -> None: + assert _triples("She picked 6 apples and 4 apples.") == [ + (6.0, "apples", "6"), + (4.0, "apples", "4"), + ] + + def test_decimal_value_preserved(self) -> None: + assert _triples("It costs 0.75 dollars.") == [(0.75, "dollars", "0.75")] + + +class TestRealCase0024StillBlocked: + """Honest pin: the EX-4 unit-list pattern does NOT recover real case 0024. + + The actual gold text interleaves numbers with temporal phrases + ("36 on Tuesday, 40 on Wednesday"), so the bare-list regex never fires and the + 36/40/50 do not inherit "jumping jacks". This test documents the current + boundary so no future change silently claims 0024 is "unblocked" without + proving the (20+36+40+50)*3 = 438 chain end-to-end. + """ + + def test_only_first_number_carries_the_unit(self) -> None: + qs = extract_quantities( + "Sidney does 20 jumping jacks on Monday, 36 on Tuesday, " + "40 on Wednesday, and 50 on Thursday." + ) + values = [q.value for q in qs] + assert values == [20.0, 36.0, 40.0, 50.0] + # 20 keeps a unit word; the bare 36/40/50 do not inherit "jumping" — they + # are sentence-internal, not a bare comma-list, so EX-4 cannot reach them. + units = [q.unit for q in qs] + assert units[0] == "jumping" + assert len(set(units)) > 1 # NOT a same-unit list -> compose refuses