diff --git a/evals/gsm8k_math/confusers/__init__.py b/evals/gsm8k_math/confusers/__init__.py new file mode 100644 index 00000000..dd499f10 --- /dev/null +++ b/evals/gsm8k_math/confusers/__init__.py @@ -0,0 +1 @@ +"""ADR-0163-F2 confuser corpus (discrimination probe).""" diff --git a/evals/gsm8k_math/confusers/v1/__init__.py b/evals/gsm8k_math/confusers/v1/__init__.py new file mode 100644 index 00000000..7112cb13 --- /dev/null +++ b/evals/gsm8k_math/confusers/v1/__init__.py @@ -0,0 +1 @@ +"""ADR-0163-F2 confuser corpus v1.""" diff --git a/evals/gsm8k_math/confusers/v1/cases.jsonl b/evals/gsm8k_math/confusers/v1/cases.jsonl new file mode 100644 index 00000000..12a934b0 --- /dev/null +++ b/evals/gsm8k_math/confusers/v1/cases.jsonl @@ -0,0 +1,30 @@ +{"case_id": "confuser-v1-0001", "question": "Dan has 50 coins. He buys a toy for 30 coins. How many coins does Dan have left?", "answer_numeric": 20, "category": "disguised-polarity", "surface_trap": "buys X for N -> spend N, not gain", "expected": "refuse", "pair_id": "confuser-v1-0002", "source": "structural: transaction spend, cf. gsm8k purchase problems", "notes": "buys is a gain verb but 'for 30 coins' is a spend; correct is 50-30"} +{"case_id": "confuser-v1-0002", "question": "Dan has 20 stamps. He buys 30 more stamps. How many stamps does Dan have?", "answer_numeric": 50, "category": "genuine-positive", "surface_trap": "buys (here a genuine gain)", "expected": "solve", "pair_id": "confuser-v1-0001", "source": "structural: accumulation twin of 0001", "notes": "minimal pair on 'buys'; here +30 is correct"} +{"case_id": "confuser-v1-0003", "question": "Sara has 18 dollars. She gets a haircut for 12 dollars. How many dollars does Sara have?", "answer_numeric": 6, "category": "disguised-polarity", "surface_trap": "gets X for N -> spend N, not gain", "expected": "refuse", "pair_id": "confuser-v1-0004", "source": "structural: service spend", "notes": "gets is a gain verb but 'for 12 dollars' is a spend; correct is 18-12"} +{"case_id": "confuser-v1-0004", "question": "Sara has 18 stickers. She gets 12 from her friend. How many stickers does Sara have?", "answer_numeric": 30, "category": "genuine-positive", "surface_trap": "gets (here a genuine gain)", "expected": "solve", "pair_id": "confuser-v1-0003", "source": "structural: accumulation twin of 0003", "notes": "minimal pair on 'gets'; here +12 is correct"} +{"case_id": "confuser-v1-0005", "question": "Maria buys 800 inches of ribbon. She cuts it into 20-inch pieces. She gives 1/4 of the pieces to a friend. How many pieces does she keep?", "answer_numeric": 30, "category": "pseudo-accumulation", "surface_trap": "buys/gives surface over a division+fraction problem", "expected": "refuse", "source": "gsm8k-train-sample-v1-0002 (paraphrase)", "notes": "800/20=40 pieces, keep 3/4 -> 30; the division and fraction are unconsumed by accumulation"} +{"case_id": "confuser-v1-0006", "question": "Tom has 60 candies. He gives half to his sister. How many candies does Tom have?", "answer_numeric": 30, "category": "pseudo-accumulation", "surface_trap": "gives half -> a fraction, not a counted change", "expected": "refuse", "source": "structural: fraction-of", "notes": "60/2=30; 'half' is not a grounded count"} +{"case_id": "confuser-v1-0007", "question": "Jan buys 1000 feet of cable. She splits it into 25-foot sections. She gives 1/4 to a friend. She puts half of the rest in storage. How much does she keep on hand?", "answer_numeric": 15, "category": "pseudo-accumulation", "surface_trap": "buys/gives surface over split+fractions (multi-step)", "expected": "refuse", "source": "gsm8k-train-sample-v1-0002", "notes": "the canonical 0002 misfire; accumulation read it as 1000-... -> 996"} +{"case_id": "confuser-v1-0008", "question": "A baker makes 48 cookies. He packs them into boxes of 6. He sells 5 boxes. How many boxes are left?", "answer_numeric": 3, "category": "pseudo-accumulation", "surface_trap": "makes/sells surface over division-then-subtract", "expected": "refuse", "source": "structural: division then subtraction", "notes": "48/6=8 boxes, 8-5=3; the division is unconsumed"} +{"case_id": "confuser-v1-0009", "question": "Alice has 6 apples. Tom has 4 apples. How many apples does Alice have?", "answer_numeric": 6, "category": "multi-referent", "surface_trap": "two same-unit actors; only Alice is asked", "expected": "refuse", "source": "ADR-0178 hazard H1", "notes": "must not sum Tom's apples into Alice's; correct is 6"} +{"case_id": "confuser-v1-0010", "question": "Maria has 12 books. Sara has 8 books. How many books do they have altogether?", "answer_numeric": 20, "category": "multi-referent", "surface_trap": "cross-referent sum (legit answer, but cross-clause)", "expected": "refuse", "source": "ADR-0178 hazard H1 variant", "notes": "the answer IS 20 but the engine cannot yet safely sum across referents; refuse until it can"} +{"case_id": "confuser-v1-0011", "question": "Ben has 30 marbles. His friend has 20 marbles. How many marbles does Ben have?", "answer_numeric": 30, "category": "multi-referent", "surface_trap": "second possessor, same unit", "expected": "refuse", "source": "ADR-0178 hazard H1", "notes": "'his friend' is a second referent; correct is 30"} +{"case_id": "confuser-v1-0012", "question": "Sam had 10 marbles and Tom had 7 marbles. He bought 5 more. How many marbles does Sam have?", "answer_numeric": 15, "category": "multi-actor-pronoun", "surface_trap": "ambiguous 'he' antecedent (Sam or Tom)", "expected": "refuse", "source": "ADR-0174 multi-actor pronoun hazard", "notes": "if he=Sam then 15, if he=Tom then 12; antecedent is ambiguous -> refuse"} +{"case_id": "confuser-v1-0013", "question": "Anna and Beth went shopping. She spent 20 dollars. Anna started with 50 dollars. How much does Anna have?", "answer_numeric": 30, "category": "multi-actor-pronoun", "surface_trap": "ambiguous 'she' antecedent", "expected": "refuse", "source": "ADR-0174 multi-actor pronoun hazard", "notes": "'she' could be Anna or Beth; refuse rather than resolve"} +{"case_id": "confuser-v1-0014", "question": "Kate has 20 pencils. She studies for 3 hours and then buys 5 more pencils. How many pencils does Kate have?", "answer_numeric": 25, "category": "distractor-quantity", "surface_trap": "'3 hours' is a distractor, not a pencil change", "expected": "refuse", "pair_id": "confuser-v1-0015", "source": "structural: irrelevant quantity", "notes": "correct is 20+5=25; the 3 must not enter the computation"} +{"case_id": "confuser-v1-0015", "question": "Kate has 20 pencils. She buys 5 more pencils. How many pencils does Kate have?", "answer_numeric": 25, "category": "genuine-positive", "surface_trap": "no distractor", "expected": "solve", "pair_id": "confuser-v1-0014", "source": "structural: distractor-free twin of 0014", "notes": "minimal pair; 20+5=25"} +{"case_id": "confuser-v1-0016", "question": "A train travels at 60 miles per hour for 2 hours. Tom has 8 tickets and buys 4 more tickets. How many tickets does Tom have?", "answer_numeric": 12, "category": "distractor-quantity", "surface_trap": "60 and 2 are distractors in a different domain", "expected": "refuse", "pair_id": "confuser-v1-0017", "source": "structural: irrelevant quantities", "notes": "correct is 8+4=12; 60 and 2 are unrelated"} +{"case_id": "confuser-v1-0017", "question": "Tom has 8 tickets and buys 4 more tickets. How many tickets does Tom have?", "answer_numeric": 12, "category": "genuine-positive", "surface_trap": "no distractor", "expected": "solve", "pair_id": "confuser-v1-0016", "source": "structural: distractor-free twin of 0016", "notes": "minimal pair; 8+4=12"} +{"case_id": "confuser-v1-0018", "question": "Ben had 30 marbles. He gave 12 to his friend. How many marbles did Ben have before giving any away?", "answer_numeric": 30, "category": "temporal-scope", "surface_trap": "question asks BEFORE the stated change", "expected": "refuse", "pair_id": "confuser-v1-0019", "source": "ADR-0178 hazard H3", "notes": "the change must not be applied; correct is the pre-change 30"} +{"case_id": "confuser-v1-0019", "question": "Ben had 30 marbles. He gave 12 to his friend. How many marbles does Ben have now?", "answer_numeric": 18, "category": "genuine-positive", "surface_trap": "question asks AFTER the change", "expected": "solve", "pair_id": "confuser-v1-0018", "source": "structural: now-scoped twin of 0018", "notes": "minimal pair; 30-12=18"} +{"case_id": "confuser-v1-0020", "question": "Lisa had 50 dollars. She spent 20 on lunch. How much money did Lisa have before lunch?", "answer_numeric": 50, "category": "temporal-scope", "surface_trap": "question asks BEFORE the spend", "expected": "refuse", "pair_id": "confuser-v1-0021", "source": "ADR-0178 hazard H3", "notes": "correct is the pre-change 50"} +{"case_id": "confuser-v1-0021", "question": "Lisa had 50 dollars. She spent 20 on lunch. How much money does Lisa have left?", "answer_numeric": 30, "category": "genuine-positive", "surface_trap": "question asks AFTER the spend", "expected": "solve", "pair_id": "confuser-v1-0020", "source": "structural: left-scoped twin of 0020", "notes": "minimal pair; 50-20=30"} +{"case_id": "confuser-v1-0022", "question": "Alice picked 6 apples. Tom picked twice as many. How many apples did Alice pick?", "answer_numeric": 6, "category": "comparative-referent", "surface_trap": "comparative binds Tom, question asks Alice", "expected": "refuse", "source": "ADR-0178 hazard H2", "notes": "the 'twice' must not scale Alice's count; correct is 6"} +{"case_id": "confuser-v1-0023", "question": "Alice picked 6 apples. Tom picked twice as many. How many apples did Tom pick?", "answer_numeric": 12, "category": "comparative-referent", "surface_trap": "comparative on Tom; engine cannot bind it yet", "expected": "refuse", "source": "ADR-0178 hazard H2", "notes": "correct is 12 but requires binding 'twice' to Tom relative to Alice's 6"} +{"case_id": "confuser-v1-0024", "question": "Sam ran 4 miles. His brother ran 3 times as far. How many miles did Sam run?", "answer_numeric": 4, "category": "comparative-referent", "surface_trap": "comparative binds the brother, question asks Sam", "expected": "refuse", "source": "ADR-0178 hazard H2", "notes": "correct is 4; the comparative is about the brother"} +{"case_id": "confuser-v1-0025", "question": "A crate holds 6 boxes and weighs 50 pounds. How many boxes are in 3 crates?", "answer_numeric": 18, "category": "unit-confuser", "surface_trap": "boxes vs pounds; 50 is a different unit", "expected": "refuse", "source": "structural: mixed-unit distractor", "notes": "6*3=18; the 50 pounds must not enter"} +{"case_id": "confuser-v1-0026", "question": "A rope is 12 feet long and weighs 4 pounds. How long are 3 ropes in feet?", "answer_numeric": 36, "category": "unit-confuser", "surface_trap": "feet vs pounds mixed", "expected": "refuse", "source": "structural: mixed-unit distractor", "notes": "12*3=36; the 4 pounds is a different dimension"} +{"case_id": "confuser-v1-0027", "question": "Tom has 5 ten-dollar bills and 3 one-dollar bills. How many bills does Tom have?", "answer_numeric": 8, "category": "unit-confuser", "surface_trap": "denominations look like values, asked as a count", "expected": "refuse", "source": "structural: count-vs-value confusion", "notes": "the count is 5+3=8 bills; the engine may confuse with a dollar total"} +{"case_id": "confuser-v1-0028", "question": "Sam has 14 apples. He buys 9 more. How many apples does Sam have now?", "answer_numeric": 23, "category": "genuine-positive", "surface_trap": "clean single-referent gain", "expected": "solve", "source": "structural: GB-3b.1 positive", "notes": "14+9=23"} +{"case_id": "confuser-v1-0029", "question": "Anna has 25 stickers. She gives 10 to her friend. How many stickers does Anna have left?", "answer_numeric": 15, "category": "genuine-positive", "surface_trap": "clean single-referent loss", "expected": "solve", "source": "structural: GB-3b.1 positive", "notes": "25-10=15"} +{"case_id": "confuser-v1-0030", "question": "Lisa has 30 coins. She earns 15 more. How many coins does Lisa have?", "answer_numeric": 45, "category": "genuine-positive", "surface_trap": "clean single-referent gain", "expected": "solve", "source": "structural: GB-3b.1 positive", "notes": "30+15=45"} diff --git a/evals/gsm8k_math/confusers/v1/runner.py b/evals/gsm8k_math/confusers/v1/runner.py new file mode 100644 index 00000000..f8ebf91d --- /dev/null +++ b/evals/gsm8k_math/confusers/v1/runner.py @@ -0,0 +1,130 @@ +"""ADR-0163-F2 — the confuser corpus runner (a discrimination probe). + +Scored the **opposite way from a coverage lane** (see +``docs/decisions/ADR-0163-F2-confuser-corpus-spec.md``): the headline is +``wrong == 0`` on confusers — a confuser *answered* is a defect regardless of its +value — plus **pair-consistency** (a reader that solves a twin but also commits an +answer on its confuser is surface-matching, not reading). ``refused`` on confusers +is the honest frontier, never optimised down. + +The probe runs the realistic sealed *attempt*: the composers a refused case is +allowed to try, in fixed order (accumulation, then the multiplicative product, then +the multi-step chain); the first that returns a resolution is the engine's answer. +This is exactly where the overfitting misfires live (the templated lane hid them). + +Deterministic; sealed (serving is never invoked here). +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +from generate.derivation.accumulate import compose_accumulation +from generate.derivation.multistep import search_chain +from generate.derivation.search import search_multiplicative + +_CASES_PATH: Final[Path] = Path(__file__).resolve().parent / "cases.jsonl" +_TOL: Final[float] = 1e-6 + + +def _engine_answer(problem_text: str) -> float | None: + """The sealed engine's attempt: first composer to resolve wins (fixed order).""" + for composer in (compose_accumulation, search_multiplicative, search_chain): + resolution = composer(problem_text) + if resolution is not None: + return resolution.answer + return None + + +@dataclass(frozen=True, slots=True) +class CaseResult: + case_id: str + category: str + expected: str # "refuse" | "solve" + gold: float + answered: float | None + verdict: str # "refused" | "solved" | "wrong" | "spurious" + + +def _verdict(expected: str, gold: float, answered: float | None) -> str: + if answered is None: + return "refused" + if abs(answered - gold) <= _TOL: + # a correct answer is "solved" for a positive, "spurious" for a refuse-case + return "solved" if expected == "solve" else "spurious" + return "wrong" + + +def load_cases(path: Path = _CASES_PATH) -> list[dict]: + with path.open("r", encoding="utf-8") as fh: + return [json.loads(line) for line in fh if line.strip()] + + +def run_probe(path: Path = _CASES_PATH) -> tuple[CaseResult, ...]: + """Evaluate the engine attempt over every confuser case. Deterministic order.""" + results: list[CaseResult] = [] + for case in load_cases(path): + gold = float(case["answer_numeric"]) + answered = _engine_answer(case["question"]) + results.append( + CaseResult( + case_id=case["case_id"], + category=case["category"], + expected=case["expected"], + gold=gold, + answered=answered, + verdict=_verdict(case["expected"], gold, answered), + ) + ) + return tuple(results) + + +def pair_inconsistencies(path: Path = _CASES_PATH) -> tuple[str, ...]: + """Pairs where the engine *solved the twin but committed an answer on the + confuser* — the surface-matching tell. Returns the confuser case_ids.""" + cases = {c["case_id"]: c for c in load_cases(path)} + by_id = {r.case_id: r for r in run_probe(path)} + flagged: list[str] = [] + for cid, case in cases.items(): + pair = case.get("pair_id") + if not pair or case["expected"] != "refuse": + continue + confuser, twin = by_id.get(cid), by_id.get(pair) + if confuser is None or twin is None: + continue + # the tell: twin solved, yet the confuser was answered (wrong or spurious). + if twin.verdict == "solved" and confuser.verdict in {"wrong", "spurious"}: + flagged.append(cid) + return tuple(sorted(flagged)) + + +def summarize(results: tuple[CaseResult, ...]) -> dict[str, dict[str, int]]: + """Per-category verdict counts. Deterministic (sorted categories).""" + out: dict[str, dict[str, int]] = {} + for r in results: + bucket = out.setdefault(r.category, {"solved": 0, "refused": 0, "wrong": 0, "spurious": 0}) + bucket[r.verdict] += 1 + return {k: out[k] for k in sorted(out)} + + +def main() -> int: # pragma: no cover + results = run_probe() + summary = summarize(results) + totals = {"solved": 0, "refused": 0, "wrong": 0, "spurious": 0} + print(f"confuser probe: {len(results)} cases") + print(f"{'category':<22} solved refused wrong spurious") + for cat, counts in summary.items(): + for k in totals: + totals[k] += counts[k] + print(f"{cat:<22} {counts['solved']:>6} {counts['refused']:>7} {counts['wrong']:>5} {counts['spurious']:>8}") + print(f"{'TOTAL':<22} {totals['solved']:>6} {totals['refused']:>7} {totals['wrong']:>5} {totals['spurious']:>8}") + print(f"wrong (the bar; must trend to 0): {totals['wrong']}") + print(f"pair-inconsistencies (surface-match tells): {list(pair_inconsistencies())}") + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/tests/test_adr_0163_f2_confusers.py b/tests/test_adr_0163_f2_confusers.py new file mode 100644 index 00000000..de74cc78 --- /dev/null +++ b/tests/test_adr_0163_f2_confusers.py @@ -0,0 +1,72 @@ +"""ADR-0163-F2 — confuser corpus discrimination probe. + +The corpus is scored opposite to a coverage lane: the bar is ``wrong`` trending to +0, and the genuine-positive twins solving. These tests pin the **honest current +baseline** as a no-regression gate — confuser ``wrong`` may never rise, and the +positives must keep solving — while explicitly NOT asserting ``wrong == 0`` (the +sealed composers genuinely misfire today; that is the defect surface the probe +exists to quantify, and the fixes must be general, not reactive patches). +""" + +from __future__ import annotations + +from evals.gsm8k_math.confusers.v1.runner import ( + load_cases, + pair_inconsistencies, + run_probe, + summarize, +) + +# Honest measured baseline (2026-05-29). The probe revealed the sealed composers +# wrongly answer these confuser categories; this pins them so they cannot grow. +_BASELINE_WRONG = 7 +_BASELINE_PAIR_TELLS = 4 + + +class TestSchema: + def test_every_case_well_formed(self) -> None: + valid_cat = { + "disguised-polarity", "pseudo-accumulation", "multi-referent", + "multi-actor-pronoun", "distractor-quantity", "temporal-scope", + "comparative-referent", "unit-confuser", "genuine-positive", + } + ids = set() + for c in load_cases(): + assert {"case_id", "question", "answer_numeric", "category", "expected"} <= c.keys() + assert c["category"] in valid_cat + assert c["expected"] in {"refuse", "solve"} + assert c["case_id"] not in ids # unique + ids.add(c["case_id"]) + + def test_pair_ids_resolve_and_are_mutual(self) -> None: + cases = {c["case_id"]: c for c in load_cases()} + for cid, c in cases.items(): + pair = c.get("pair_id") + if pair: + assert pair in cases, f"{cid} -> missing pair {pair}" + assert cases[pair]["pair_id"] == cid, f"{cid}/{pair} not mutual" + + +class TestProbeBaseline: + def test_wrong_does_not_regress(self) -> None: + results = run_probe() + wrong = sum(1 for r in results if r.verdict == "wrong") + assert wrong <= _BASELINE_WRONG, ( + f"confuser wrong rose to {wrong} (baseline {_BASELINE_WRONG}); a change " + f"made the engine answer more confusers — investigate before merge." + ) + + def test_genuine_positives_mostly_solve(self) -> None: + # the capability signal: the clean accumulation twins read correctly. + results = run_probe() + positives = [r for r in results if r.expected == "solve"] + solved = sum(1 for r in positives if r.verdict == "solved") + assert solved >= 7, f"genuine positives solving dropped to {solved}" + # and a positive must never be answered WRONG (that would be a real defect). + assert all(r.verdict != "wrong" for r in positives) + + def test_pair_tells_do_not_regress(self) -> None: + assert len(pair_inconsistencies()) <= _BASELINE_PAIR_TELLS + + def test_deterministic(self) -> None: + assert summarize(run_probe()) == summarize(run_probe())