From 9d19b8176f66bcdc4fb8cd30a13f79dba62010c8 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 23 May 2026 06:18:43 -0700 Subject: [PATCH] =?UTF-8?q?feat(gsm8k):=20ADR-0126=20P6=20=E2=80=94=20trai?= =?UTF-8?q?n-sample=20runner=20+=20exit-criterion=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wraps existing math pipeline (parser -> solver -> verifier) against PR #159's 50-case train sample. Emits deterministic report.json with per-case verdicts. CLI exit code reflects exit criterion (correct >= 10 AND wrong == 0). Baseline against current parser: 0 correct / 0 wrong / 50 refused. This baseline is the inner-loop gradient signal for ADR-0126's candidate-graph parser (in flight on feat/adr-0126-candidate-graph). Registers tests/test_adr_0126_train_sample_runner.py under 'core test --suite math' so the wrong == 0 invariant becomes a hard CI gate per ADR-0114a Obligation #4 (refuse rather than confabulate). Depends on PR #159 (gemini/adr-0126-train-sample). Rebase onto main after #159 lands. --- core/cli.py | 6 + evals/gsm8k_math/train_sample/__init__.py | 0 evals/gsm8k_math/train_sample/v1/__init__.py | 0 evals/gsm8k_math/train_sample/v1/report.json | 268 +++++++++++++++++++ evals/gsm8k_math/train_sample/v1/runner.py | 114 ++++++++ tests/test_adr_0126_train_sample_runner.py | 73 +++++ 6 files changed, 461 insertions(+) create mode 100644 evals/gsm8k_math/train_sample/__init__.py create mode 100644 evals/gsm8k_math/train_sample/v1/__init__.py create mode 100644 evals/gsm8k_math/train_sample/v1/report.json create mode 100644 evals/gsm8k_math/train_sample/v1/runner.py create mode 100644 tests/test_adr_0126_train_sample_runner.py diff --git a/core/cli.py b/core/cli.py index e4c7c37c..a036678b 100644 --- a/core/cli.py +++ b/core/cli.py @@ -129,6 +129,12 @@ _TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_phase5_corpus.py", "tests/test_phase6_demo.py", ), + # ADR-0126 P6 — measurement harness for the GSM8K candidate-graph + # parser exit criterion. ``wrong == 0`` is a hard gate (Obligation + # #4: refuse rather than confabulate). + "math": ( + "tests/test_adr_0126_train_sample_runner.py", + ), "full": ("tests/",), } diff --git a/evals/gsm8k_math/train_sample/__init__.py b/evals/gsm8k_math/train_sample/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/gsm8k_math/train_sample/v1/__init__.py b/evals/gsm8k_math/train_sample/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/gsm8k_math/train_sample/v1/report.json b/evals/gsm8k_math/train_sample/v1/report.json new file mode 100644 index 00000000..4663e240 --- /dev/null +++ b/evals/gsm8k_math/train_sample/v1/report.json @@ -0,0 +1,268 @@ +{ + "adr": "0126", + "counts": { + "correct": 0, + "refused": 50, + "wrong": 0 + }, + "exit_criterion": { + "correct_min": 10, + "passed": false, + "wrong_max": 0 + }, + "per_case": [ + { + "case_id": "gsm8k-train-sample-v1-0001", + "reason": "parser: could not parse statement clause: 'Tina makes $18.00 an hour' (in sentence: 'Tina makes $18.00 an hour.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0002", + "reason": "parser: could not parse statement clause: 'Jan buys 1000 feet of cable' (in sentence: 'Jan buys 1000 feet of cable.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0003", + "reason": "parser: could not parse statement clause: 'The student council sells scented erasers in the morning before school starts to help raise money for school dances' (in sentence: 'The student council sells scented erasers in the morning before school starts to help raise money for school dances.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0004", + "reason": "parser: could not parse statement clause: 'There are some kids in camp' (in sentence: 'There are some kids in camp.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0005", + "reason": "parser: could not parse statement clause: \"In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature\" (in sentence: \"In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature.\")", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0006", + "reason": "parser: could not parse statement clause: 'Mandy started reading books with only 8 pages when she was 6 years old' (in sentence: 'Mandy started reading books with only 8 pages when she was 6 years old.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0007", + "reason": "parser: could not parse statement clause: 'Francine has five full boxes of crayons and 5 loose crayons' (in sentence: 'Francine has five full boxes of crayons and 5 loose crayons, and her friend has 27 loose crayons.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0008", + "reason": "parser: could not parse statement clause: 'Marnie makes bead bracelets' (in sentence: 'Marnie makes bead bracelets.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0009", + "reason": "parser: could not parse statement clause: 'Jen has 10 more ducks than four times the number of chickens' (in sentence: 'Jen has 10 more ducks than four times the number of chickens.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0010", + "reason": "parser: could not parse statement clause: 'Yun had 20 paperclips initially, but then lost 12' (in sentence: 'Yun had 20 paperclips initially, but then lost 12.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0011", + "reason": "parser: could not parse statement clause: 'Alexa has a lemonade stand where she sells lemonade for $2 for one cup' (in sentence: 'Alexa has a lemonade stand where she sells lemonade for $2 for one cup.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0012", + "reason": "parser: could not parse statement clause: 'Dennis collected 10 rocks' (in sentence: 'Dennis collected 10 rocks.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0013", + "reason": "parser: could not parse statement clause: 'Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel' (in sentence: 'Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0014", + "reason": "parser: could not parse statement clause: 'Bob can shuck 10 oysters in 5 minutes' (in sentence: 'Bob can shuck 10 oysters in 5 minutes.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0015", + "reason": "parser: could not parse statement clause: 'Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride' (in sentence: 'Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0016", + "reason": "parser: could not parse statement clause: \"On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs\" (in sentence: \"On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs.\")", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0017", + "reason": "parser: could not parse statement clause: 'Jason has a carriage house that he rents out' (in sentence: 'Jason has a carriage house that he rents out.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0018", + "reason": "parser: could not parse statement clause: 'Xavier plays football with his friends' (in sentence: 'Xavier plays football with his friends.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0019", + "reason": "parser: could not parse statement clause: 'John adopts a dog from a shelter' (in sentence: 'John adopts a dog from a shelter.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0020", + "reason": "parser: could not parse statement clause: 'Two puppies, two kittens' (in sentence: 'Two puppies, two kittens, and three parakeets were for sale at the pet shop.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0021", + "reason": "parser: could not parse statement clause: 'John is lifting weights' (in sentence: 'John is lifting weights.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0022", + "reason": "parser: could not parse statement clause: 'Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish' (in sentence: 'Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0023", + "reason": "parser: could not parse statement clause: 'Nicole collected 400 Pokemon cards' (in sentence: 'Nicole collected 400 Pokemon cards.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0024", + "reason": "parser: could not parse statement clause: 'Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday' (in sentence: 'Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday, and 50 on Thursday.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0025", + "reason": "parser: could not parse statement clause: 'Lilibeth and her friends go strawberry picking' (in sentence: 'Lilibeth and her friends go strawberry picking.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0026", + "reason": "parser: could not parse statement clause: 'Aaron and his brother Carson each saved up $40 to go to dinner' (in sentence: 'Aaron and his brother Carson each saved up $40 to go to dinner.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0027", + "reason": "parser: could not parse statement clause: 'Malcolm has 240 followers on Instagram and 500 followers on Facebook' (in sentence: 'Malcolm has 240 followers on Instagram and 500 followers on Facebook.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0028", + "reason": "parser: could not parse statement clause: 'Tom opens an amusement park' (in sentence: 'Tom opens an amusement park.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0029", + "reason": "parser: could not parse statement clause: 'Fabian bought a brand new computer mouse and keyboard to be able to work from home' (in sentence: 'Fabian bought a brand new computer mouse and keyboard to be able to work from home.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0030", + "reason": "parser: could not parse statement clause: 'Jake decides to go to the beach for a fun day' (in sentence: 'Jake decides to go to the beach for a fun day.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0031", + "reason": "parser: could not parse statement clause: 'Jeremie wants to go to an amusement park with 3 friends at the end of summer' (in sentence: 'Jeremie wants to go to an amusement park with 3 friends at the end of summer.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0032", + "reason": "parser: could not parse statement clause: 'John decides to take up illustration' (in sentence: 'John decides to take up illustration.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0033", + "reason": "parser: could not parse statement clause: 'Rachel is 12 years old' (in sentence: 'Rachel is 12 years old, and her grandfather is 7 times her age.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0034", + "reason": "parser: could not parse statement clause: 'Georgie is a varsity player on a football team' (in sentence: 'Georgie is a varsity player on a football team.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0035", + "reason": "parser: could not parse statement clause: 'She decided to split them among her friends' (in sentence: 'She decided to split them among her friends.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0036", + "reason": "parser: could not parse statement clause: 'Monica way studying for an exam' (in sentence: 'Monica way studying for an exam.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0037", + "reason": "parser: could not parse statement clause: 'Michael wants to lose 10 pounds by June' (in sentence: 'Michael wants to lose 10 pounds by June.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0038", + "reason": "parser: could not parse statement clause: 'In a building, there are a hundred ladies on the first-floor studying' (in sentence: 'In a building, there are a hundred ladies on the first-floor studying.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0039", + "reason": "parser: could not parse statement clause: 'At the family reunion, everyone ate too much food and gained weight' (in sentence: 'At the family reunion, everyone ate too much food and gained weight.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0040", + "reason": "parser: could not parse statement clause: 'Over several years, Daniel has adopted any stray animals he sees on the side of the road' (in sentence: 'Over several years, Daniel has adopted any stray animals he sees on the side of the road.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0041", + "reason": "parser: could not parse statement clause: 'Troy bakes 2 pans of brownies, cut into 16 pieces per pan' (in sentence: 'Troy bakes 2 pans of brownies, cut into 16 pieces per pan.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0042", + "reason": "parser: could not parse statement clause: 'Ella has 4 bags with 20 apples in each bag and six bags with 25 apples in each bag' (in sentence: 'Ella has 4 bags with 20 apples in each bag and six bags with 25 apples in each bag.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0043", + "reason": "parser: could not parse statement clause: 'Sandra wants to buy some sweets' (in sentence: 'Sandra wants to buy some sweets.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0044", + "reason": "parser: could not parse statement clause: 'John invests in a bank and gets 10% simple interest' (in sentence: 'John invests in a bank and gets 10% simple interest.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0045", + "reason": "parser: could not parse statement clause: 'Bart fills out surveys to earn money' (in sentence: 'Bart fills out surveys to earn money.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0046", + "reason": "parser: could not parse statement clause: 'A school has 100 students' (in sentence: 'A school has 100 students.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0047", + "reason": "parser: could not parse statement clause: 'John bakes 12 coconut macaroons, each weighing 5 ounces' (in sentence: 'John bakes 12 coconut macaroons, each weighing 5 ounces.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0048", + "reason": "parser: could not parse statement clause: 'Jed collects stamp cards' (in sentence: 'Jed collects stamp cards.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0049", + "reason": "parser: could not parse statement clause: 'Malcolm is trying to find the fastest walk to school and is currently comparing two routes' (in sentence: 'Malcolm is trying to find the fastest walk to school and is currently comparing two routes.')", + "verdict": "refused" + }, + { + "case_id": "gsm8k-train-sample-v1-0050", + "reason": "parser: could not parse statement clause: 'Mark does a gig every other day for 2 weeks' (in sentence: 'Mark does a gig every other day for 2 weeks.')", + "verdict": "refused" + } + ], + "sample_count": 50, + "sample_path": "evals/gsm8k_math/train_sample/v1/cases.jsonl", + "schema_version": 1 +} diff --git a/evals/gsm8k_math/train_sample/v1/runner.py b/evals/gsm8k_math/train_sample/v1/runner.py new file mode 100644 index 00000000..5ba7375d --- /dev/null +++ b/evals/gsm8k_math/train_sample/v1/runner.py @@ -0,0 +1,114 @@ +"""ADR-0126 P6 — train-sample exit-criterion runner. + +Thin measurement harness around the canonical :mod:`evals.gsm8k_math.runner` +pipeline. Loads PR #159's 50-case unsealed GSM8K train sample, replays +each case through the existing parser → solver → verifier path, and +emits a deterministic ``report.json`` with per-case verdicts. + +Exit code reflects ADR-0126's gate: + + correct >= 10 AND wrong == 0 → exit 0 + otherwise → exit 1 + +The runner is measurement-only. It does not touch the parser, the +sealed holdout, or any decryption surface (CLAUDE.md trust boundary). +``answer_numeric`` is grade-on-value-only — train-sample cases carry no +unit annotation, so ``expected_unit`` is normalized to ``""`` which +:func:`evals.gsm8k_math.runner._score_one` already treats as +"skip unit comparison". +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +from evals.gsm8k_math.runner import _score_one + +_HERE = Path(__file__).resolve().parent +_CASES_PATH = _HERE / "cases.jsonl" +_REPORT_PATH = _HERE / "report.json" +_SAMPLE_REL = "evals/gsm8k_math/train_sample/v1/cases.jsonl" +_EXPECTED_COUNT = 50 +_CORRECT_MIN = 10 +_WRONG_MAX = 0 + + +def _load_cases(path: Path) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + with path.open("r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + records.append(json.loads(line)) + assert len(records) == _EXPECTED_COUNT, ( + f"train sample must contain exactly {_EXPECTED_COUNT} cases; " + f"found {len(records)} at {path}" + ) + return records + + +def _adapt(case: dict[str, Any]) -> dict[str, Any]: + """Reshape a train-sample record into the runner's expected schema. + + Train sample uses ``case_id`` / ``question`` / ``answer_numeric``; + the canonical runner expects ``id`` / ``problem`` / ``expected_answer`` + / ``expected_unit``. An empty ``expected_unit`` tells the runner to + grade on value only (cf. :func:`evals.gsm8k_math.runner._score_one`). + """ + return { + "id": case["case_id"], + "problem": case["question"], + "expected_answer": case["answer_numeric"], + "expected_unit": "", + } + + +def build_report(cases: list[dict[str, Any]]) -> dict[str, Any]: + per_case: list[dict[str, str]] = [] + counts = {"correct": 0, "wrong": 0, "refused": 0} + for raw in cases: + outcome = _score_one(_adapt(raw)) + counts[outcome.outcome] += 1 + per_case.append( + { + "case_id": outcome.case_id, + "verdict": outcome.outcome, + "reason": outcome.reason, + } + ) + passed = counts["correct"] >= _CORRECT_MIN and counts["wrong"] <= _WRONG_MAX + return { + "schema_version": 1, + "adr": "0126", + "sample_path": _SAMPLE_REL, + "sample_count": len(cases), + "counts": counts, + "exit_criterion": { + "correct_min": _CORRECT_MIN, + "wrong_max": _WRONG_MAX, + "passed": passed, + }, + "per_case": per_case, + } + + +def write_report(report: dict[str, Any], path: Path = _REPORT_PATH) -> None: + path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main() -> int: + cases = _load_cases(_CASES_PATH) + report = build_report(cases) + write_report(report) + return 0 if report["exit_criterion"]["passed"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_adr_0126_train_sample_runner.py b/tests/test_adr_0126_train_sample_runner.py new file mode 100644 index 00000000..97382fc5 --- /dev/null +++ b/tests/test_adr_0126_train_sample_runner.py @@ -0,0 +1,73 @@ +"""ADR-0126 P6 — train-sample runner contract tests. + +Pins the measurement harness around PR #159's 50-case GSM8K train +sample. Three load-bearing properties: + +1. Report shape conforms to the documented JSON schema. +2. Sample is exactly 50 cases with canonical ``gsm8k-train-sample-v1-NNNN`` + ids. +3. ``wrong == 0`` against the current parser — the pre-pivot baseline + that ADR-0126's candidate-graph parser must preserve. Any future + pipeline change that confabulates an answer (parser succeeds but + yields wrong value) will break this gate. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest + +from evals.gsm8k_math.train_sample.v1 import runner as r + +CASE_ID_RE = re.compile(r"^gsm8k-train-sample-v1-\d{4}$") + + +@pytest.fixture(scope="module") +def report() -> dict: + cases = r._load_cases(r._CASES_PATH) + return r.build_report(cases) + + +def test_report_has_documented_shape(report: dict) -> None: + assert report["schema_version"] == 1 + assert report["adr"] == "0126" + assert report["sample_path"] == "evals/gsm8k_math/train_sample/v1/cases.jsonl" + assert set(report["counts"]) == {"correct", "wrong", "refused"} + assert set(report["exit_criterion"]) == {"correct_min", "wrong_max", "passed"} + assert report["exit_criterion"]["correct_min"] == 10 + assert report["exit_criterion"]["wrong_max"] == 0 + assert isinstance(report["per_case"], list) + assert len(report["per_case"]) == report["sample_count"] + for entry in report["per_case"]: + assert set(entry) == {"case_id", "verdict", "reason"} + assert entry["verdict"] in {"correct", "wrong", "refused"} + + +def test_sample_count_and_case_id_pattern(report: dict) -> None: + assert report["sample_count"] == 50 + ids = [entry["case_id"] for entry in report["per_case"]] + assert len(ids) == 50 + for cid in ids: + assert CASE_ID_RE.match(cid), cid + + +def test_wrong_count_is_zero_baseline(report: dict) -> None: + """ADR-0114a Obligation #4: CORE refuses rather than confabulates. + + Holds against the current parser (everything refuses) AND must + continue to hold once ADR-0126's candidate-graph parser lands. + """ + assert report["counts"]["wrong"] == 0 + + +def test_runner_writes_report_to_disk(tmp_path: Path) -> None: + cases = r._load_cases(r._CASES_PATH) + report = r.build_report(cases) + target = tmp_path / "report.json" + r.write_report(report, target) + loaded = json.loads(target.read_text(encoding="utf-8")) + assert loaded["sample_count"] == 50 + assert loaded["counts"]["wrong"] == 0