core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feat(gsm8k): ADR-0126 P6 — train-sample runner + exit-criterion gate

Browse source 
Wraps existing math pipeline (parser -> solver -> verifier) against
PR #159's 50-case train sample. Emits deterministic report.json with
per-case verdicts. CLI exit code reflects exit criterion
(correct >= 10 AND wrong == 0).

Baseline against current parser: 0 correct / 0 wrong / 50 refused.
This baseline is the inner-loop gradient signal for ADR-0126's
candidate-graph parser (in flight on feat/adr-0126-candidate-graph).

Registers tests/test_adr_0126_train_sample_runner.py under
'core test --suite math' so the wrong == 0 invariant becomes a hard
CI gate per ADR-0114a Obligation #4 (refuse rather than confabulate).

Depends on PR #159 (gemini/adr-0126-train-sample). Rebase onto main
after #159 lands.
This commit is contained in:
Shay 2026-05-23 06:18:43 -07:00
parent 0eb923b89f
commit 9d19b8176f
6 changed files with 461 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
core/cli.py
View file

@ -129,6 +129,12 @@ _TEST_SUITES: dict[str, tuple[str, ...]] = {
"tests/test_phase5_corpus.py",
"tests/test_phase6_demo.py",
),
# ADR-0126 P6 — measurement harness for the GSM8K candidate-graph
# parser exit criterion. ``wrong == 0`` is a hard gate (Obligation
# #4: refuse rather than confabulate).
"math": (
"tests/test_adr_0126_train_sample_runner.py",
),
"full": ("tests/",),
}

0
evals/gsm8k_math/train_sample/__init__.py Normal file
View file

0
evals/gsm8k_math/train_sample/v1/__init__.py Normal file
View file

268
evals/gsm8k_math/train_sample/v1/report.json Normal file
View file

@ -0,0 +1,268 @@
{
"adr": "0126",
"counts": {
"correct": 0,
"refused": 50,
"wrong": 0
},
"exit_criterion": {
"correct_min": 10,
"passed": false,
"wrong_max": 0
},
"per_case": [
{
"case_id": "gsm8k-train-sample-v1-0001",
"reason": "parser: could not parse statement clause: 'Tina makes $18.00 an hour' (in sentence: 'Tina makes $18.00 an hour.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0002",
"reason": "parser: could not parse statement clause: 'Jan buys 1000 feet of cable' (in sentence: 'Jan buys 1000 feet of cable.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0003",
"reason": "parser: could not parse statement clause: 'The student council sells scented erasers in the morning before school starts to help raise money for school dances' (in sentence: 'The student council sells scented erasers in the morning before school starts to help raise money for school dances.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0004",
"reason": "parser: could not parse statement clause: 'There are some kids in camp' (in sentence: 'There are some kids in camp.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0005",
"reason": "parser: could not parse statement clause: \"In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature\" (in sentence: \"In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature.\")",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0006",
"reason": "parser: could not parse statement clause: 'Mandy started reading books with only 8 pages when she was 6 years old' (in sentence: 'Mandy started reading books with only 8 pages when she was 6 years old.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0007",
"reason": "parser: could not parse statement clause: 'Francine has five full boxes of crayons and 5 loose crayons' (in sentence: 'Francine has five full boxes of crayons and 5 loose crayons, and her friend has 27 loose crayons.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0008",
"reason": "parser: could not parse statement clause: 'Marnie makes bead bracelets' (in sentence: 'Marnie makes bead bracelets.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0009",
"reason": "parser: could not parse statement clause: 'Jen has 10 more ducks than four times the number of chickens' (in sentence: 'Jen has 10 more ducks than four times the number of chickens.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0010",
"reason": "parser: could not parse statement clause: 'Yun had 20 paperclips initially, but then lost 12' (in sentence: 'Yun had 20 paperclips initially, but then lost 12.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0011",
"reason": "parser: could not parse statement clause: 'Alexa has a lemonade stand where she sells lemonade for $2 for one cup' (in sentence: 'Alexa has a lemonade stand where she sells lemonade for $2 for one cup.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0012",
"reason": "parser: could not parse statement clause: 'Dennis collected 10 rocks' (in sentence: 'Dennis collected 10 rocks.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0013",
"reason": "parser: could not parse statement clause: 'Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel' (in sentence: 'Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0014",
"reason": "parser: could not parse statement clause: 'Bob can shuck 10 oysters in 5 minutes' (in sentence: 'Bob can shuck 10 oysters in 5 minutes.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0015",
"reason": "parser: could not parse statement clause: 'Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride' (in sentence: 'Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0016",
"reason": "parser: could not parse statement clause: \"On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs\" (in sentence: \"On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs.\")",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0017",
"reason": "parser: could not parse statement clause: 'Jason has a carriage house that he rents out' (in sentence: 'Jason has a carriage house that he rents out.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0018",
"reason": "parser: could not parse statement clause: 'Xavier plays football with his friends' (in sentence: 'Xavier plays football with his friends.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0019",
"reason": "parser: could not parse statement clause: 'John adopts a dog from a shelter' (in sentence: 'John adopts a dog from a shelter.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0020",
"reason": "parser: could not parse statement clause: 'Two puppies, two kittens' (in sentence: 'Two puppies, two kittens, and three parakeets were for sale at the pet shop.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0021",
"reason": "parser: could not parse statement clause: 'John is lifting weights' (in sentence: 'John is lifting weights.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0022",
"reason": "parser: could not parse statement clause: 'Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish' (in sentence: 'Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0023",
"reason": "parser: could not parse statement clause: 'Nicole collected 400 Pokemon cards' (in sentence: 'Nicole collected 400 Pokemon cards.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0024",
"reason": "parser: could not parse statement clause: 'Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday' (in sentence: 'Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday, and 50 on Thursday.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0025",
"reason": "parser: could not parse statement clause: 'Lilibeth and her friends go strawberry picking' (in sentence: 'Lilibeth and her friends go strawberry picking.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0026",
"reason": "parser: could not parse statement clause: 'Aaron and his brother Carson each saved up $40 to go to dinner' (in sentence: 'Aaron and his brother Carson each saved up $40 to go to dinner.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0027",
"reason": "parser: could not parse statement clause: 'Malcolm has 240 followers on Instagram and 500 followers on Facebook' (in sentence: 'Malcolm has 240 followers on Instagram and 500 followers on Facebook.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0028",
"reason": "parser: could not parse statement clause: 'Tom opens an amusement park' (in sentence: 'Tom opens an amusement park.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0029",
"reason": "parser: could not parse statement clause: 'Fabian bought a brand new computer mouse and keyboard to be able to work from home' (in sentence: 'Fabian bought a brand new computer mouse and keyboard to be able to work from home.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0030",
"reason": "parser: could not parse statement clause: 'Jake decides to go to the beach for a fun day' (in sentence: 'Jake decides to go to the beach for a fun day.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0031",
"reason": "parser: could not parse statement clause: 'Jeremie wants to go to an amusement park with 3 friends at the end of summer' (in sentence: 'Jeremie wants to go to an amusement park with 3 friends at the end of summer.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0032",
"reason": "parser: could not parse statement clause: 'John decides to take up illustration' (in sentence: 'John decides to take up illustration.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0033",
"reason": "parser: could not parse statement clause: 'Rachel is 12 years old' (in sentence: 'Rachel is 12 years old, and her grandfather is 7 times her age.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0034",
"reason": "parser: could not parse statement clause: 'Georgie is a varsity player on a football team' (in sentence: 'Georgie is a varsity player on a football team.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0035",
"reason": "parser: could not parse statement clause: 'She decided to split them among her friends' (in sentence: 'She decided to split them among her friends.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0036",
"reason": "parser: could not parse statement clause: 'Monica way studying for an exam' (in sentence: 'Monica way studying for an exam.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0037",
"reason": "parser: could not parse statement clause: 'Michael wants to lose 10 pounds by June' (in sentence: 'Michael wants to lose 10 pounds by June.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0038",
"reason": "parser: could not parse statement clause: 'In a building, there are a hundred ladies on the first-floor studying' (in sentence: 'In a building, there are a hundred ladies on the first-floor studying.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0039",
"reason": "parser: could not parse statement clause: 'At the family reunion, everyone ate too much food and gained weight' (in sentence: 'At the family reunion, everyone ate too much food and gained weight.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0040",
"reason": "parser: could not parse statement clause: 'Over several years, Daniel has adopted any stray animals he sees on the side of the road' (in sentence: 'Over several years, Daniel has adopted any stray animals he sees on the side of the road.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0041",
"reason": "parser: could not parse statement clause: 'Troy bakes 2 pans of brownies, cut into 16 pieces per pan' (in sentence: 'Troy bakes 2 pans of brownies, cut into 16 pieces per pan.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0042",
"reason": "parser: could not parse statement clause: 'Ella has 4 bags with 20 apples in each bag and six bags with 25 apples in each bag' (in sentence: 'Ella has 4 bags with 20 apples in each bag and six bags with 25 apples in each bag.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0043",
"reason": "parser: could not parse statement clause: 'Sandra wants to buy some sweets' (in sentence: 'Sandra wants to buy some sweets.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0044",
"reason": "parser: could not parse statement clause: 'John invests in a bank and gets 10% simple interest' (in sentence: 'John invests in a bank and gets 10% simple interest.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0045",
"reason": "parser: could not parse statement clause: 'Bart fills out surveys to earn money' (in sentence: 'Bart fills out surveys to earn money.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0046",
"reason": "parser: could not parse statement clause: 'A school has 100 students' (in sentence: 'A school has 100 students.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0047",
"reason": "parser: could not parse statement clause: 'John bakes 12 coconut macaroons, each weighing 5 ounces' (in sentence: 'John bakes 12 coconut macaroons, each weighing 5 ounces.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0048",
"reason": "parser: could not parse statement clause: 'Jed collects stamp cards' (in sentence: 'Jed collects stamp cards.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0049",
"reason": "parser: could not parse statement clause: 'Malcolm is trying to find the fastest walk to school and is currently comparing two routes' (in sentence: 'Malcolm is trying to find the fastest walk to school and is currently comparing two routes.')",
"verdict": "refused"
},
{
"case_id": "gsm8k-train-sample-v1-0050",
"reason": "parser: could not parse statement clause: 'Mark does a gig every other day for 2 weeks' (in sentence: 'Mark does a gig every other day for 2 weeks.')",
"verdict": "refused"
}
],
"sample_count": 50,
"sample_path": "evals/gsm8k_math/train_sample/v1/cases.jsonl",
"schema_version": 1
}

114
evals/gsm8k_math/train_sample/v1/runner.py Normal file
View file

@ -0,0 +1,114 @@
"""ADR-0126 P6 — train-sample exit-criterion runner.
Thin measurement harness around the canonical :mod:`evals.gsm8k_math.runner`
pipeline. Loads PR #159's 50-case unsealed GSM8K train sample, replays
each case through the existing parser → solver → verifier path, and
emits a deterministic ``report.json`` with per-case verdicts.
Exit code reflects ADR-0126's gate:
correct >= 10 AND wrong == 0 → exit 0
otherwise → exit 1
The runner is measurement-only. It does not touch the parser, the
sealed holdout, or any decryption surface (CLAUDE.md trust boundary).
``answer_numeric`` is grade-on-value-only — train-sample cases carry no
unit annotation, so ``expected_unit`` is normalized to ``""`` which
:func:`evals.gsm8k_math.runner._score_one` already treats as
"skip unit comparison".
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
from typing import Any
from evals.gsm8k_math.runner import _score_one
_HERE = Path(__file__).resolve().parent
_CASES_PATH = _HERE / "cases.jsonl"
_REPORT_PATH = _HERE / "report.json"
_SAMPLE_REL = "evals/gsm8k_math/train_sample/v1/cases.jsonl"
_EXPECTED_COUNT = 50
_CORRECT_MIN = 10
_WRONG_MAX = 0
def _load_cases(path: Path) -> list[dict[str, Any]]:
records: list[dict[str, Any]] = []
with path.open("r", encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if not line:
continue
records.append(json.loads(line))
assert len(records) == _EXPECTED_COUNT, (
f"train sample must contain exactly {_EXPECTED_COUNT} cases; "
f"found {len(records)} at {path}"
)
return records
def _adapt(case: dict[str, Any]) -> dict[str, Any]:
"""Reshape a train-sample record into the runner's expected schema.
Train sample uses ``case_id`` / ``question`` / ``answer_numeric``;
the canonical runner expects ``id`` / ``problem`` / ``expected_answer``
/ ``expected_unit``. An empty ``expected_unit`` tells the runner to
grade on value only (cf. :func:`evals.gsm8k_math.runner._score_one`).
"""
return {
"id": case["case_id"],
"problem": case["question"],
"expected_answer": case["answer_numeric"],
"expected_unit": "",
}
def build_report(cases: list[dict[str, Any]]) -> dict[str, Any]:
per_case: list[dict[str, str]] = []
counts = {"correct": 0, "wrong": 0, "refused": 0}
for raw in cases:
outcome = _score_one(_adapt(raw))
counts[outcome.outcome] += 1
per_case.append(
{
"case_id": outcome.case_id,
"verdict": outcome.outcome,
"reason": outcome.reason,
}
)
passed = counts["correct"] >= _CORRECT_MIN and counts["wrong"] <= _WRONG_MAX
return {
"schema_version": 1,
"adr": "0126",
"sample_path": _SAMPLE_REL,
"sample_count": len(cases),
"counts": counts,
"exit_criterion": {
"correct_min": _CORRECT_MIN,
"wrong_max": _WRONG_MAX,
"passed": passed,
},
"per_case": per_case,
}
def write_report(report: dict[str, Any], path: Path = _REPORT_PATH) -> None:
path.write_text(
json.dumps(report, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
def main() -> int:
cases = _load_cases(_CASES_PATH)
report = build_report(cases)
write_report(report)
return 0 if report["exit_criterion"]["passed"] else 1
if __name__ == "__main__":
sys.exit(main())

73
tests/test_adr_0126_train_sample_runner.py Normal file
View file

@ -0,0 +1,73 @@
"""ADR-0126 P6 — train-sample runner contract tests.
Pins the measurement harness around PR #159's 50-case GSM8K train
sample. Three load-bearing properties:
1. Report shape conforms to the documented JSON schema.
2. Sample is exactly 50 cases with canonical ``gsm8k-train-sample-v1-NNNN``
ids.
3. ``wrong == 0`` against the current parser — the pre-pivot baseline
that ADR-0126's candidate-graph parser must preserve. Any future
pipeline change that confabulates an answer (parser succeeds but
yields wrong value) will break this gate.
"""
from __future__ import annotations
import json
import re
from pathlib import Path
import pytest
from evals.gsm8k_math.train_sample.v1 import runner as r
CASE_ID_RE = re.compile(r"^gsm8k-train-sample-v1-\d{4}$")
@pytest.fixture(scope="module")
def report() -> dict:
cases = r._load_cases(r._CASES_PATH)
return r.build_report(cases)
def test_report_has_documented_shape(report: dict) -> None:
assert report["schema_version"] == 1
assert report["adr"] == "0126"
assert report["sample_path"] == "evals/gsm8k_math/train_sample/v1/cases.jsonl"
assert set(report["counts"]) == {"correct", "wrong", "refused"}
assert set(report["exit_criterion"]) == {"correct_min", "wrong_max", "passed"}
assert report["exit_criterion"]["correct_min"] == 10
assert report["exit_criterion"]["wrong_max"] == 0
assert isinstance(report["per_case"], list)
assert len(report["per_case"]) == report["sample_count"]
for entry in report["per_case"]:
assert set(entry) == {"case_id", "verdict", "reason"}
assert entry["verdict"] in {"correct", "wrong", "refused"}
def test_sample_count_and_case_id_pattern(report: dict) -> None:
assert report["sample_count"] == 50
ids = [entry["case_id"] for entry in report["per_case"]]
assert len(ids) == 50
for cid in ids:
assert CASE_ID_RE.match(cid), cid
def test_wrong_count_is_zero_baseline(report: dict) -> None:
"""ADR-0114a Obligation #4: CORE refuses rather than confabulates.
Holds against the current parser (everything refuses) AND must
continue to hold once ADR-0126's candidate-graph parser lands.
"""
assert report["counts"]["wrong"] == 0
def test_runner_writes_report_to_disk(tmp_path: Path) -> None:
cases = r._load_cases(r._CASES_PATH)
report = r.build_report(cases)
target = tmp_path / "report.json"
r.write_report(report, target)
loaded = json.loads(target.read_text(encoding="utf-8"))
assert loaded["sample_count"] == 50
assert loaded["counts"]["wrong"] == 0