From 5b193280a5fb71dcf9be9a07496ed026706e2d9a Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 6 Jun 2026 17:18:45 -0700 Subject: [PATCH] feat(setup-oracle): independent R1 gold + the reader refuses, never misreads (PR-5b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure-gold. Defines what "correct" means for the R1 (comparative / derived-symbol / multi-step) shapes the reader cannot read yet, and verifies the current reader REFUSES them rather than misreading them. NO frame extraction, no parser change, no serving. - evals/setup_oracle/r1_gold.jsonl — 10 INDEPENDENT, self-contained fixtures (id, text, relations, expected_units, query, notes) covering: twice/half-as-many (multiplicative), more/fewer-than -> total, a multi-step derived chain (jon=3*ivy; kim=jon+2), a derived subtotal reused downstream (total=lee+mae; per_box=total/3), an inverse target, and the hard negatives (ambiguous referent, missing base quantity, distractor quantity). The gold is the authority — each fixture is reviewable without joining files. - run_r1() scores the CURRENT reader against this gold. Investigative result (the deliverable): **10/10 setup_refused, setup_wrong=0, setup_correct=0.** The reader refuses every R1 shape with a TYPED reason (non_digit_quantity / no_quantity_template / unreadable_quantity_clause / admissibility_refused) — it NEVER misreads an R1 case as a simpler form. So the wrong=0 posture holds all the way down, and PR-5c (the R1 frame) starts from a clean foundation: every R1 case is currently a safe refusal, and the frame's job is refusal -> correct, with the oracle ensuring no setup_wrong is ever introduced. `python -m evals.setup_oracle r1` prints the auditable R1 report (exit 0 iff setup_wrong==0). The 15-case setup gate + relational_metric answer lane are untouched. --- evals/setup_oracle/__init__.py | 3 +- evals/setup_oracle/__main__.py | 12 +++-- evals/setup_oracle/r1_gold.jsonl | 10 ++++ evals/setup_oracle/runner.py | 79 +++++++++++++++++++++++++++++++- tests/test_setup_oracle.py | 21 +++++++++ 5 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 evals/setup_oracle/r1_gold.jsonl diff --git a/evals/setup_oracle/__init__.py b/evals/setup_oracle/__init__.py index 357cb2dd..282550aa 100644 --- a/evals/setup_oracle/__init__.py +++ b/evals/setup_oracle/__init__.py @@ -13,7 +13,7 @@ the problem. v1 grades structure (facts + equations + question target/state/form modelling stays covered by the admissibility tests (a documented signature extension). """ -from evals.setup_oracle.runner import run +from evals.setup_oracle.runner import run, run_r1 from evals.setup_oracle.signature import ( gold_unknown_signature, reader_symbol_units, @@ -28,5 +28,6 @@ __all__ = [ "reader_unknown_signature", "relation_signature", "run", + "run_r1", "symbol_unit_signature", ] diff --git a/evals/setup_oracle/__main__.py b/evals/setup_oracle/__main__.py index f1874400..6d591f8f 100644 --- a/evals/setup_oracle/__main__.py +++ b/evals/setup_oracle/__main__.py @@ -1,20 +1,24 @@ """CLI: print the setup-oracle report. - python -m evals.setup_oracle + python -m evals.setup_oracle # the 15-case relational-metric setup gold + python -m evals.setup_oracle r1 # the independent R1 gold (PR-5b) Exit 0 iff ``setup_wrong == 0`` — the gate the milestone rests on (a wrong reading must -never pass, and serving must not move while setup_wrong > 0). +never pass, and serving must not move while setup_wrong > 0). For ``r1`` the reader is +expected to REFUSE the unsupported shapes (setup_refused), never misread them. """ from __future__ import annotations import json +import sys -from evals.setup_oracle.runner import run +from evals.setup_oracle.runner import run, run_r1 def main() -> int: - report = run() + lane = sys.argv[1] if len(sys.argv) > 1 else "" + report = run_r1() if lane == "r1" else run() print(json.dumps(report, indent=2, default=str)) return 0 if report["setup_wrong"] == 0 else 1 diff --git a/evals/setup_oracle/r1_gold.jsonl b/evals/setup_oracle/r1_gold.jsonl new file mode 100644 index 00000000..e62623d5 --- /dev/null +++ b/evals/setup_oracle/r1_gold.jsonl @@ -0,0 +1,10 @@ +{"id": "r1-01-twice", "text": "Anna has 6 apples. Bella has twice as many apples as Anna. How many apples does Bella have?", "relations": [{"kind": "fact", "entity": "anna", "value": 6}, {"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2}], "expected_units": {"anna": "item", "bella": "item"}, "query": {"entity": "bella", "unit": "item"}, "notes": "Multiplicative: bella = 2 * anna. The reader has no multiplicative template AND 'twice' is non-digit -> must REFUSE, never misread as a fact/additive."} +{"id": "r1-02-half", "text": "Carl has 8 coins. Dora has half as many coins as Carl. How many coins does Dora have?", "relations": [{"kind": "fact", "entity": "carl", "value": 8}, {"kind": "times_as_many", "entity": "dora", "ref": "carl", "factor": 0.5}], "expected_units": {"carl": "item", "dora": "item"}, "query": {"entity": "dora", "unit": "item"}, "notes": "Multiplicative (factor 0.5). 'half' is non-digit -> must REFUSE."} +{"id": "r1-03-more-total", "text": "Finn has 10 books. Evan has 5 more books than Finn. How many books do Evan and Finn have altogether?", "relations": [{"kind": "fact", "entity": "finn", "value": 10}, {"kind": "more_than", "entity": "evan", "ref": "finn", "delta": 5}, {"kind": "sum_of", "entity": "total", "parts": ["evan", "finn"]}], "expected_units": {"finn": "item", "evan": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive + aggregate. 'altogether' (not the bare 'have') may fall outside the strict query template -> likely REFUSE; if read, structure must match exactly."} +{"id": "r1-04-fewer-total", "text": "Gail has 20 cards. Hank has 6 fewer cards than Gail. How many cards do Gail and Hank have in total?", "relations": [{"kind": "fact", "entity": "gail", "value": 20}, {"kind": "fewer_than", "entity": "hank", "ref": "gail", "delta": 6}, {"kind": "sum_of", "entity": "total", "parts": ["gail", "hank"]}], "expected_units": {"gail": "item", "hank": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive(fewer) + aggregate. 'in total' qualifier likely outside the query template -> REFUSE expected."} +{"id": "r1-05-chain", "text": "Ivy has 4 pens. Jon has 3 times as many pens as Ivy. Kim has 2 more pens than Jon. How many pens does Kim have?", "relations": [{"kind": "fact", "entity": "ivy", "value": 4}, {"kind": "times_as_many", "entity": "jon", "ref": "ivy", "factor": 3}, {"kind": "more_than", "entity": "kim", "ref": "jon", "delta": 2}], "expected_units": {"ivy": "item", "jon": "item", "kim": "item"}, "query": {"entity": "kim", "unit": "item"}, "notes": "Multi-step derived chain: jon=3*ivy (intermediate), kim=jon+2. The multiplicative middle step has no template -> must REFUSE the whole reading, never read a partial/wrong chain."} +{"id": "r1-06-subtotal-reused", "text": "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them equally into 3 boxes. How many hats are in each box?", "relations": [{"kind": "fact", "entity": "lee", "value": 5}, {"kind": "fact", "entity": "mae", "value": 7}, {"kind": "sum_of", "entity": "total", "parts": ["lee", "mae"]}, {"kind": "divide_by", "entity": "per_box", "ref": "total", "divisor": 3}], "expected_units": {"lee": "item", "mae": "item", "total": "item", "per_box": "item"}, "query": {"entity": "per_box", "unit": "item"}, "notes": "Derived subtotal reused downstream: total = lee + mae, per_box = total / 3. No partition template -> must REFUSE."} +{"id": "r1-07-inverse", "text": "Nia has 9 more beads than Omar. Nia has 15 beads. How many beads does Omar have?", "relations": [{"kind": "fact", "entity": "nia", "value": 15}, {"kind": "more_than", "entity": "nia", "ref": "omar", "delta": 9}, {"kind": "ask_base", "entity": "omar"}], "expected_units": {"nia": "item", "omar": "item"}, "query": {"entity": "omar", "unit": "item"}, "notes": "Inverse target: omar = nia - 9 = 6, asked via the base of a more_than. The reader may read the STRUCTURE (nia=15, nia=omar+9, ask omar) — that setup is correct; only the solver can't invert. So setup_correct OR refuse, but NEVER setup_wrong."} +{"id": "r1-08-ambiguous-referent", "text": "Pat has 5 marbles. He has 3 more than her. How many marbles does she have?", "relations": [], "expected_units": {}, "query": {"entity": "she", "unit": "item"}, "notes": "Ambiguous pronoun referents (he/her/she) with no grounded base for 'her'/'she' -> must REFUSE (no honest reading), never bind a guessed referent."} +{"id": "r1-09-missing-base", "text": "Quinn has twice as many toys as Rosa. How many toys does Quinn have?", "relations": [{"kind": "times_as_many", "entity": "quinn", "ref": "rosa", "factor": 2}], "expected_units": {"quinn": "item", "rosa": "item"}, "query": {"entity": "quinn", "unit": "item"}, "notes": "Missing base quantity: Rosa's count is never given, so Quinn is underdetermined -> must REFUSE."} +{"id": "r1-10-distractor", "text": "Sam has 7 pencils and 3 erasers. Tom has 4 more pencils than Sam. How many pencils does Tom have?", "relations": [{"kind": "fact", "entity": "sam", "value": 7}, {"kind": "more_than", "entity": "tom", "ref": "sam", "delta": 4}], "expected_units": {"sam": "item", "tom": "item"}, "query": {"entity": "tom", "unit": "item"}, "notes": "Distractor quantity (3 erasers) in a compound clause the 'X has N unit' template can't parse -> must REFUSE rather than mis-bind the distractor."} diff --git a/evals/setup_oracle/runner.py b/evals/setup_oracle/runner.py index b540ba68..c63edc4b 100644 --- a/evals/setup_oracle/runner.py +++ b/evals/setup_oracle/runner.py @@ -28,6 +28,7 @@ from generate.meaning_graph.reader import Refusal from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric _EXPECTED_UNITS_PATH = Path(__file__).resolve().parent / "expected_units.json" +_R1_GOLD_PATH = Path(__file__).resolve().parent / "r1_gold.jsonl" def _load_expected_units() -> dict[str, dict[str, str]]: @@ -35,6 +36,14 @@ def _load_expected_units() -> dict[str, dict[str, str]]: return {k: v for k, v in raw.items() if not k.startswith("_")} +def _load_r1_gold() -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in _R1_GOLD_PATH.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + def run() -> dict[str, Any]: """Score the reader's setup (structure + units + target) against the independent gold.""" cases = _load_cases() @@ -94,4 +103,72 @@ def run() -> dict[str, Any]: } -__all__ = ["run"] +def run_r1() -> dict[str, Any]: + """Score the CURRENT reader against the independent, self-contained R1 gold. + + The reader cannot read most R1 shapes (multiplicative, multi-step, partition) yet. + The SUCCESS CRITERION is **``setup_wrong == 0``**: it must REFUSE every unsupported + shape, NEVER misread it as a simpler form. ``setup_refused`` is the unsupported count; + ``setup_correct`` is any shape already faithfully supported. A ``setup_wrong`` is a + pre-existing wrong-reading hazard to fix BEFORE building the R1 frame (PR-5c). + """ + fixtures = _load_r1_gold() + setup_correct = setup_wrong = setup_refused = 0 + details: list[dict[str, Any]] = [] + + for fx in fixtures: + comp = comprehend_quantitative(fx["text"]) + if isinstance(comp, Refusal): + setup_refused += 1 + details.append({"id": fx["id"], "outcome": "refused", "reason": comp.reason}) + continue + projected = to_relational_metric(comp) + if projected is None: + setup_refused += 1 + details.append({"id": fx["id"], "outcome": "refused", "reason": "unprojectable"}) + continue + reader_relations, _ = projected + units = fx["expected_units"] + + reader_rel = relation_signature(reader_relations) + gold_rel = relation_signature(fx["relations"]) + reader_units = reader_symbol_units(comp.binding_graph) + gold_units = symbol_unit_signature(units) + reader_unk = reader_unknown_signature(comp.binding_graph) + gold_unk = gold_unknown_signature(fx["relations"], fx["query"], units) + + if reader_rel == gold_rel and reader_units == gold_units and reader_unk == gold_unk: + setup_correct += 1 + details.append({"id": fx["id"], "outcome": "correct"}) + else: + setup_wrong += 1 + details.append( + { + "id": fx["id"], + "outcome": "WRONG", + "relations_match": reader_rel == gold_rel, + "units_match": reader_units == gold_units, + "target_match": reader_unk == gold_unk, + "reader_relations": reader_rel, + "gold_relations": gold_rel, + "reader_target": reader_unk, + "gold_target": gold_unk, + } + ) + + return { + "lane": "setup_oracle_r1", + "total": len(fixtures), + "setup_correct": setup_correct, + "setup_wrong": setup_wrong, + "setup_refused": setup_refused, + "details": details, + "counts": { + "setup_correct": setup_correct, + "setup_wrong": setup_wrong, + "setup_refused": setup_refused, + }, + } + + +__all__ = ["run", "run_r1"] diff --git a/tests/test_setup_oracle.py b/tests/test_setup_oracle.py index 97459dd4..c375d9ca 100644 --- a/tests/test_setup_oracle.py +++ b/tests/test_setup_oracle.py @@ -140,3 +140,24 @@ def test_reader_units_read_from_the_binding_graph() -> None: ) assert reader_symbol_units(graph) == (("iris", "dollars"), ("jack", "dollars")) assert reader_unknown_signature(graph) == ("jack", "terminal", "count", "dollars") + + +# --------------------------------------------------------------------------- # +# PR-5b — independent R1 gold: the reader must REFUSE, never MISREAD +# --------------------------------------------------------------------------- # + + +def test_r1_reader_refuses_never_misreads() -> None: + from evals.setup_oracle import run_r1 + + r = run_r1() + assert r["total"] == 10 + # THE success criterion: the reader misreads NO unsupported R1 case. A wrong setup + # here would be a pre-existing hazard to fix before the R1 frame (PR-5c). + assert r["setup_wrong"] == 0 + # The reader cannot read these shapes yet -> every one is a safe (typed) refusal. + assert r["setup_refused"] == 10 + assert r["setup_correct"] == 0 + for d in r["details"]: + assert d["outcome"] == "refused" + assert d.get("reason") # a typed refusal reason, never a silent drop