From 2544fda6f6febceadaa6364b94510782a474c3c3 Mon Sep 17 00:00:00 2001 From: Shay Date: Sun, 19 Jul 2026 20:04:27 -0700 Subject: [PATCH] fix(trackb): pure-S1 extract refuse multi-clause overmatch (0361 wrong=0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inverted-seed P2 regex treated optional "times" as optional, so additive "8 more" on multi-entity zoo case 0361 extracted as pure S1 and the three-gate corridor emitted 1.125 vs gold 114.0 (wrong≠0). - Require explicit multiplicative surface (times as many / twice the …) - Fail-closed purity gate: multi-clause markers, multi-numeric, multi-mult - Regression tests for 0361 and bare "N more" - measure --mode full-holdout-wrong0 over all 500 holdout cases [Verification]: full-holdout emit_ok=9 wrong=0 refused=491; unit 37 passed --- .../trackb-inc2-generalization-2026-07-20.md | 17 +- generate/structure_mapping/text_extract.py | 321 +++++++++++------- scripts/measure_trackb_inc2.py | 48 +++ tests/test_trackb_inc2_structure_mapping.py | 37 ++ 4 files changed, 292 insertions(+), 131 deletions(-) diff --git a/docs/research/trackb-inc2-generalization-2026-07-20.md b/docs/research/trackb-inc2-generalization-2026-07-20.md index 855ade14..e24735ab 100644 --- a/docs/research/trackb-inc2-generalization-2026-07-20.md +++ b/docs/research/trackb-inc2-generalization-2026-07-20.md @@ -123,6 +123,20 @@ SUMMARY right-reason: correct=9 wrong=0 refused=0 SUMMARY surface-variant: ok=True ``` +### 3g. Full holdout wrong=0 gate (all 500 — not curated subset) + +Regression: a loose inverted-seed regex matched multi-entity zoo case **0361** +(`John has 8 more pandas…`) as pure S1 and the corridor emitted `1.125` vs +gold `114.0`. Fixed by pure-family surface gates (require explicit multiplicative +language; refuse bare `N more` / multi-clause / multi-numeric surfaces). + +```text +$ uv run python scripts/measure_trackb_inc2.py --mode full-holdout-wrong0 +=== full holdout_dev/v1 wrong=0 gate (all 500 cases) === +SUMMARY full-holdout: emit_ok=9 wrong=0 refused=491 n=500 +PASS: wrong=0 absolute on full holdout_dev/v1 +``` + --- ## 4. Discipline checks @@ -138,8 +152,9 @@ SUMMARY surface-variant: ok=True 1. **S2–S4 holdout ratio = 0** until the serving reader (or a reviewed SM extract family) produces pure transfer/rate/additive graphs on holdout. Do not manufacture synthetic holdout clones for ratio. 2. **S3/S4 solve emit** blocked by multi-register scope (`apply_rate`, `compare_additive` not Tier-2a). Gates not weakened — refuse. -3. **SM extract is narrow** (regex pure-S1 templates only). It is not a general parser; expanding it is a deliberate future increment. +3. **SM extract is narrow** (regex pure-S1 templates only, with fail-closed purity gates). It is not a general parser; expanding it is a deliberate future increment. Overmatch risk is gated by `mode full-holdout-wrong0` on all 500 cases. 4. **Organ retirement** deferred; coverage-gain path met bar 2 without serving risk. +5. **0361 pure-S1 overmatch** (fixed in follow-up commit): optional `times` in P2 regex treated additive `8 more` as multiplicative; purity gate + required `times as many` closed it. Full-holdout scan is now mandatory before claiming wrong=0. --- diff --git a/generate/structure_mapping/text_extract.py b/generate/structure_mapping/text_extract.py index 9dab374a..c0436e4e 100644 --- a/generate/structure_mapping/text_extract.py +++ b/generate/structure_mapping/text_extract.py @@ -6,6 +6,14 @@ the serving reader or organ dispatch. Deterministic regex extractors — no LLM, no gold labels. Refuse is the default; only narrow pure-family patterns emit a graph. + +Pure-family surface gates (fail-closed — wrong=0 absolute): +- Exactly one multiplicative compare phrase (``times as many/much``, + ``twice|double|triple|thrice as``, ``times the number/size of``). +- Never match bare additive language (``N more``, ``N fewer``) as a factor — + that was the 0361 overmatch that emitted a certified wrong answer. +- Refuse multi-clause / multi-relation / fraction / percent surfaces. +- At most two numeric seeds and two named roles for inverted/day templates. """ from __future__ import annotations @@ -39,6 +47,57 @@ _WORD_FACTORS: dict[str, float] = { "ten": 10.0, } +# Multiplicative compare surface (must be present for pure S1 extract). +# Intentionally does NOT match bare "N more" / "N fewer" (additive). +_MULT_COMPARE = re.compile( + r"(?:" + r"twice\s+as\s+(?:many|much)|" + r"twice\s+the\s+(?:total\s+)?(?:number|amount|size)|" + r"double\s+(?:what|the)|" + r"triple\s+(?:what|the)|" + r"thrice\s+as\s+(?:many|much)|" + r"(?:two|three|four|five|six|seven|eight|nine|ten|\d+)\s+times\s+" + r"(?:as\s+(?:many|much)|the\s+(?:total\s+)?(?:number|size|amount)|more\s+than)|" + r"times\s+(?:as\s+(?:many|much)|the\s+(?:total\s+)?(?:number|size|amount)\s+of)" + r")", + re.IGNORECASE, +) + +# Additive / multi-step markers that disqualify pure-S1 extract. +_IMPURE_MARKERS = re.compile( + r"(?:" + r"\bfewer\b|" + r"\bless than\b|" + r"\bmore than\b|" # additive "more than" without being part of "times more than" + r"\b\d+\s+more\b|" + r"\b\d+\s+fewer\b|" + r"\b\d+\s+less\b|" + r"\b1/\d+\b|" + r"\bhalf as\b|" + r"\ba third\b|" + r"\bpercent\b|" + r"%|" + r"\band half\b|" + r"\bbut\b|" + r"\bthen\b|" + r"\bafter\b|" + r"\bbought\b|" + r"\bsold\b|" + r"\bgave\b|" + r"\bgives\b|" + r"\blost\b|" + r"\beach of\b|" + r"\bevery\b" + r")", + re.IGNORECASE, +) + +# "times more than" is multiplicative; strip those before impure "more than" check. +_TIMES_MORE_THAN = re.compile( + r"(?:twice|thrice|\d+|two|three|four|five|six|seven|eight|nine|ten)\s+times\s+more\s+than", + re.IGNORECASE, +) + @dataclass(frozen=True, slots=True) class TextExtractResult: @@ -58,33 +117,94 @@ def _factor_from_token(tok: str) -> float | None: return v if v > 0 else None +def _purity_gate(text: str) -> str | None: + """Return a refuse reason if text is not a pure single-relation S1 surface.""" + # Count multiplicative compare phrases — pure S1 has exactly one. + mults = list(_MULT_COMPARE.finditer(text)) + if len(mults) == 0: + return "no_multiplicative_compare" + if len(mults) > 1: + return "multi_multiplicative_compare" + + # Impure markers after masking legitimate "times more than". + masked = _TIMES_MORE_THAN.sub("TIMES_MORE_THAN", text) + # Also mask the single mult compare span so "more" inside "times as many" + # is not confused — the mult phrase itself is clean. + for m in mults: + masked = masked[: m.start()] + ("X" * (m.end() - m.start())) + masked[m.end() :] + if _IMPURE_MARKERS.search(masked): + return "impure_multi_clause_markers" + + # Too many independent numeric seeds → multi-step (pure S1: 1 seed + factor + # word, or 1 seed with numeric factor already in mult phrase). + nums = re.findall(r"\b\d+(?:\.\d+)?\b", text) + if len(nums) > 2: + return "too_many_numeric_seeds" + + return None + + +def _s1_graph( + *, + a: str, + b: str, + k: float, + a_value: float, + unit: str, +) -> MathProblemGraph: + return MathProblemGraph( + entities=(a, b), + initial_state=( + InitialPossession(entity=a, quantity=Quantity(value=a_value, unit=unit)), + ), + operations=( + Operation( + actor=b, + kind="compare_multiplicative", + operand=Comparison( + reference_actor=a, + delta=None, + factor=k, + direction="times", + ), + ), + ), + unknown=Unknown(entity=None, unit=unit), + ) + + def extract_pure_s1(text: str) -> TextExtractResult: """Extract pure S1 graphs from narrow surface templates. - Patterns (all require a total/altogether/combined/two-days query): - - P1 — reference seeded: - `` has|scored|taught times (as many|the number of) ... as . - If has ... total|altogether`` - or `` has . has times as many ... total`` + Patterns (all require a total/altogether/combined/two-days query and + pass the pure-family surface gate): P2 — scaled entity seeded (inverted): - `` has|wrote times as many ... as . If ... ... altogether`` - or `` has times as many as . If has , how many ... combined`` + `` wrote|has times as many as . If ... ... altogether`` - P3 — day/entity pair with ordinal: - ``first day was twice ... second day. If ... second day, how many ... two days`` + P2b — warehouse / first-second form with ``twice|N times as many ... as`` + + P2c — deaf/blind population form + + P3 — day pair: first day = k × second day; second seeded; total two days """ if not isinstance(text, str) or not text.strip(): return TextExtractResult(None, "empty_text") t = " ".join(text.split()) + refuse = _purity_gate(t) + if refuse is not None: + return TextExtractResult(None, f"not_pure_s1:{refuse}") + # --- P3: people-counting / day1 = k × day2, day2 seeded (case 0148 family) + # Requires explicit "times" when kword is numeric; "twice/double/triple/thrice" + # are inherently multiplicative. m = re.search( r"(?:the\s+)?(?:number of\s+\w+\s+counted on\s+)?the\s+first\s+day\s+was\s+" - r"(?Ptwice|double|triple|thrice|\d+|two|three|four|five|six|seven|eight|nine|ten)" - r"(?:\s+times)?\s+(?:the\s+)?(?:total\s+)?(?:number\s+)?(?:counted\s+)?on\s+the\s+second\s+day" + r"(?Ptwice|double|triple|thrice|" + r"(?:two|three|four|five|six|seven|eight|nine|ten|\d+)\s+times)" + r"\s+(?:the\s+)?(?:total\s+)?(?:number\s+)?(?:counted\s+)?on\s+the\s+second\s+day" r".{0,80}?" r"(?P\d+(?:\.\d+)?)\s+\w+\s+(?:were\s+)?counted\s+on\s+the\s+second\s+day" r".{0,40}?" @@ -93,169 +213,110 @@ def extract_pure_s1(text: str) -> TextExtractResult: flags=re.IGNORECASE | re.DOTALL, ) if m: - k = _factor_from_token(m.group("kword")) + k_raw = m.group("kword") + # Strip trailing "times" for numeric/word factors + k_tok = re.sub(r"\s+times$", "", k_raw.strip(), flags=re.IGNORECASE) + k = _factor_from_token(k_tok) n = float(m.group("n")) if k is not None: - a, b, unit = "second day", "first day", "people" - graph = MathProblemGraph( - entities=(a, b), - initial_state=( - InitialPossession( - entity=a, quantity=Quantity(value=n, unit=unit) - ), - ), - operations=( - Operation( - actor=b, - kind="compare_multiplicative", - operand=Comparison( - reference_actor=a, - delta=None, - factor=k, - direction="times", - ), - ), - ), - unknown=Unknown(entity=None, unit=unit), + graph = _s1_graph( + a="second day", b="first day", k=k, a_value=n, unit="people" ) return TextExtractResult(graph, None, pattern_id="s1_p3_day_pair") # --- P2: inverted seed — B = k×A, B has n, total A+B - # "Zig wrote four times as many books as Flo. If Zig wrote 60 books, how many ... altogether" + # REQUIRE "times as many/much" or word-factor "twice as many" — never bare "N more". m = re.search( - r"(?P[A-Z][A-Za-z]+(?:\s+[A-Z][A-Za-z]+)*)\s+" - r"(?:wrote|has|have|scored|taught|caught|produced|bought)\s+" - r"(?Ptwice|double|triple|thrice|\d+|two|three|four|five|six|seven|eight|nine|ten)" - r"(?:\s+times)?\s+(?:as many|as much|more|the number of)\s+" - r"(?P\w+)?\s*(?:as|than)\s+" - r"(?P[A-Z][A-Za-z]+(?:\s+[A-Z][A-Za-z]+)*)" - r".{0,60}?" - r"(?:if\s+)?(?P=b)\s+(?:wrote|has|have|scored|taught|caught|produced|bought)\s+" + r"(?P[A-Z][a-z]+)\s+" + r"(?:wrote|has|have|scored|taught|caught|produced)\s+" + r"(?Ptwice|double|triple|thrice|" + r"(?:two|three|four|five|six|seven|eight|nine|ten|\d+)\s+times)\s+" + r"(?:as many|as much)\s+" + r"(?P\w+)\s+as\s+" + r"(?P[A-Z][a-z]+)" + r"\.\s+" + r"(?:If\s+)?(?P=b)\s+(?:wrote|has|have|scored|taught|caught|produced)\s+" r"(?P\d+(?:\.\d+)?)" r".{0,80}?" - r"(?:altogether|in total|combined|total)", + r"(?:altogether|in total|combined)", t, flags=re.IGNORECASE | re.DOTALL, ) if m: - k = _factor_from_token(m.group("kword")) + k_tok = re.sub( + r"\s+times$", "", m.group("kword").strip(), flags=re.IGNORECASE + ) + k = _factor_from_token(k_tok) n = float(m.group("n")) a = m.group("a").strip() b = m.group("b").strip() - unit = (m.group("unit") or "units").lower() - if k is not None and a.lower() != b.lower(): - a_value = n / k - graph = MathProblemGraph( - entities=(a, b), - initial_state=( - InitialPossession( - entity=a, quantity=Quantity(value=a_value, unit=unit) - ), - ), - operations=( - Operation( - actor=b, - kind="compare_multiplicative", - operand=Comparison( - reference_actor=a, - delta=None, - factor=k, - direction="times", - ), - ), - ), - unknown=Unknown(entity=None, unit=unit), - ) + unit = m.group("unit").lower() + # Entity names must be single proper nouns (reject "he does lions"). + if ( + k is not None + and a.lower() != b.lower() + and a[0].isupper() + and b[0].isupper() + and " " not in a + and " " not in b + ): + graph = _s1_graph(a=a, b=b, k=k, a_value=n / k, unit=unit) return TextExtractResult(graph, None, pattern_id="s1_p2_b_seeded") - # Warehouse / population form: - # "The first warehouse has twice as many boxes as the second warehouse. - # If the first warehouse has 400 boxes, how many boxes ... combined" + # Warehouse form: first/second + "twice|N times as many UNIT as" m = re.search( - r"(?:the\s+)?(?Pfirst\s+\w+|deaf\s+\w+\s+population|[A-Z][A-Za-z]+(?:\s+[A-Z][A-Za-z]+)*)\s+" - r"has\s+" - r"(?Ptwice|double|triple|thrice|\d+|two|three|four|five|six|seven|eight|nine|ten)" - r"(?:\s+times)?\s+(?:as many|as much)\s+(?P\w+)\s+as\s+" - r"(?:the\s+)?(?Psecond\s+\w+|blind\s+\w+\s+population|[A-Z][A-Za-z]+(?:\s+[A-Z][A-Za-z]+)*)" + r"(?:the\s+)?(?Pfirst\s+\w+)\s+has\s+" + r"(?Ptwice|double|triple|thrice|" + r"(?:two|three|four|five|six|seven|eight|nine|ten|\d+)\s+times)\s+" + r"as many\s+(?P\w+)\s+as\s+" + r"(?:the\s+)?(?Psecond\s+\w+)" r".{0,80}?" r"(?:if\s+)?(?:the\s+)?(?P=b)\s+has\s+(?P\d+(?:\.\d+)?)" r".{0,80}?" - r"(?:combined|altogether|in total|total|how many)", + r"(?:combined|altogether|in total|how many)", t, flags=re.IGNORECASE | re.DOTALL, ) if m: - k = _factor_from_token(m.group("kword")) + k_tok = re.sub( + r"\s+times$", "", m.group("kword").strip(), flags=re.IGNORECASE + ) + k = _factor_from_token(k_tok) n = float(m.group("n")) a = re.sub(r"\s+", " ", m.group("a")).strip() b = re.sub(r"\s+", " ", m.group("b")).strip() unit = m.group("unit").lower() if k is not None and a.lower() != b.lower(): - a_value = n / k - graph = MathProblemGraph( - entities=(a, b), - initial_state=( - InitialPossession( - entity=a, quantity=Quantity(value=a_value, unit=unit) - ), - ), - operations=( - Operation( - actor=b, - kind="compare_multiplicative", - operand=Comparison( - reference_actor=a, - delta=None, - factor=k, - direction="times", - ), - ), - ), - unknown=Unknown(entity=None, unit=unit), - ) + graph = _s1_graph(a=a, b=b, k=k, a_value=n / k, unit=unit) return TextExtractResult(graph, None, pattern_id="s1_p2_b_seeded_has") - # Special: "deaf student population three times the size of blind student - # population. If the number of deaf students is 180, how many students - # are there altogether" + # Deaf/blind population form (fixed surface; still purity-gated). m = re.search( r"(?Pdeaf\s+student\s+population)\s+" - r"(?Ptwice|double|triple|thrice|\d+|two|three|four|five|six|seven|eight|nine|ten)" - r"(?:\s+times)?\s+(?:the size of|as many as)\s+" + r"(?Ptwice|double|triple|thrice|" + r"(?:two|three|four|five|six|seven|eight|nine|ten|\d+)\s+times)\s+" + r"(?:the size of|as many as)\s+" r"(?Pblind\s+student\s+population)" r".{0,80}?" r"(?:number of\s+deaf\s+students\s+is|deaf\s+students\s+is)\s+(?P\d+(?:\.\d+)?)" r".{0,60}?" - r"(?:altogether|in total|total|how many students)", + r"(?:altogether|in total|how many students)", t, flags=re.IGNORECASE | re.DOTALL, ) if m: - k = _factor_from_token(m.group("kword")) + k_tok = re.sub( + r"\s+times$", "", m.group("kword").strip(), flags=re.IGNORECASE + ) + k = _factor_from_token(k_tok) n = float(m.group("n")) if k is not None: - a, b, unit = "blind students", "deaf students", "students" - a_value = n / k - graph = MathProblemGraph( - entities=(a, b), - initial_state=( - InitialPossession( - entity=a, quantity=Quantity(value=a_value, unit=unit) - ), - ), - operations=( - Operation( - actor=b, - kind="compare_multiplicative", - operand=Comparison( - reference_actor=a, - delta=None, - factor=k, - direction="times", - ), - ), - ), - unknown=Unknown(entity=None, unit=unit), + graph = _s1_graph( + a="blind students", + b="deaf students", + k=k, + a_value=n / k, + unit="students", ) return TextExtractResult(graph, None, pattern_id="s1_p2_deaf_blind") diff --git a/scripts/measure_trackb_inc2.py b/scripts/measure_trackb_inc2.py index ee386f1e..70ff6df2 100644 --- a/scripts/measure_trackb_inc2.py +++ b/scripts/measure_trackb_inc2.py @@ -410,6 +410,50 @@ def mode_right_reason() -> None: print(f"SUMMARY s2_public_sample: ok={s2_ok} scanned_transferish={s2_n}") +def mode_full_holdout_wrong0() -> None: + """Absolute wrong=0 gate over **all** holdout_dev/v1 cases (not curated subset). + + Any pipeline emit whose answer disagrees with gold is a hard failure of the + wrong=0 bar. Overmatching SM extract that certifies garbage is the hazard + this mode is designed to catch (regression: 0361). + """ + print("=== full holdout_dev/v1 wrong=0 gate (all 500 cases) ===") + cases = _load_cases() + emit_ok = 0 + wrong: list[tuple[str, float | None, float, str | None, str | None]] = [] + refused = 0 + for cid, row in cases.items(): + gold = float(row["expected_answer"]) + out, trace, _ = run_structure_mapping_pipeline(row["problem"]) + if not out.emitted or out.answer is None: + refused += 1 + continue + if abs(float(out.answer) - gold) <= 1e-6 * max(1.0, abs(gold)): + emit_ok += 1 + else: + wrong.append( + ( + cid, + out.answer, + gold, + trace.graph_source, + trace.extract_pattern, + ) + ) + print( + f"SUMMARY full-holdout: emit_ok={emit_ok} wrong={len(wrong)} " + f"refused={refused} n={len(cases)}" + ) + for cid, ans, gold, src, pat in wrong: + print( + f" WRONG {cid} emit={ans} gold={gold} src={src} extract={pat}" + ) + if wrong: + print("FAIL: wrong≠0 on holdout_dev/v1 — pure-family extract overmatch") + else: + print("PASS: wrong=0 absolute on full holdout_dev/v1") + + def mode_all() -> None: mode_ratio() print() @@ -420,6 +464,8 @@ def mode_all() -> None: mode_parser_frontier() print() mode_right_reason() + print() + mode_full_holdout_wrong0() def main(argv: list[str] | None = None) -> int: @@ -432,6 +478,7 @@ def main(argv: list[str] | None = None) -> int: "selector", "parser-frontier", "right-reason", + "full-holdout-wrong0", "all", ), default="all", @@ -443,6 +490,7 @@ def main(argv: list[str] | None = None) -> int: "selector": mode_selector, "parser-frontier": mode_parser_frontier, "right-reason": mode_right_reason, + "full-holdout-wrong0": mode_full_holdout_wrong0, "all": mode_all, }[args.mode]() return 0 diff --git a/tests/test_trackb_inc2_structure_mapping.py b/tests/test_trackb_inc2_structure_mapping.py index 63a78cba..49ad0cab 100644 --- a/tests/test_trackb_inc2_structure_mapping.py +++ b/tests/test_trackb_inc2_structure_mapping.py @@ -274,6 +274,43 @@ def test_surface_variant_inverted_s1(): assert trace.graph_source == "sm_extract" +def test_extract_refuses_multi_entity_zoo_0361(): + """Regression: 0361 overmatch must refuse, never emit wrong certified answer. + + Prior loose P2 regex matched 'John has 8 more pandas than he does lions' + as pure S1 and the corridor emitted 1.125 vs gold 114.0 (wrong≠0). + """ + text = ( + "John wants to start a zoo. He has 15 snakes. He has twice as many " + "monkeys as he does snakes. He has 5 fewer lions than he does monkeys. " + "John has 8 more pandas than he does lions. John has 1/3 as many dogs as " + "he does pandas. How many total animals does John have?" + ) + from generate.structure_mapping.text_extract import extract_pure_s1 + + ext = extract_pure_s1(text) + assert ext.graph is None, f"must refuse extract, got {ext}" + assert ext.reason is not None + assert "not_pure_s1" in ext.reason or "no_pure" in ext.reason + + out, trace, _ = run_structure_mapping_pipeline(text) + assert not out.emitted + assert out.answer is None + assert not trace.extract_used + + +def test_extract_refuses_bare_n_more_as_factor(): + """Bare additive 'N more' must never be parsed as multiplicative factor.""" + from generate.structure_mapping.text_extract import extract_pure_s1 + + text = ( + "John has 8 more pandas than he does lions. If John has 8 pandas, " + "how many pandas and lions do they have altogether?" + ) + ext = extract_pure_s1(text) + assert ext.graph is None + + def test_mapper_modules_do_not_import_scoring_labels(): root = Path("generate/structure_mapping") for path in root.glob("*.py"):