diff --git a/evals/setup_oracle/__main__.py b/evals/setup_oracle/__main__.py index 6d591f8f..495c076d 100644 --- a/evals/setup_oracle/__main__.py +++ b/evals/setup_oracle/__main__.py @@ -1,11 +1,12 @@ """CLI: print the setup-oracle report. - python -m evals.setup_oracle # the 15-case relational-metric setup gold - python -m evals.setup_oracle r1 # the independent R1 gold (PR-5b) + python -m evals.setup_oracle # the 15-case relational-metric setup gold + python -m evals.setup_oracle r1 # the independent R1 setup gold (PR-5b) + python -m evals.setup_oracle r1-answers # off-serving R1 answer lane (PR-6b) -Exit 0 iff ``setup_wrong == 0`` — the gate the milestone rests on (a wrong reading must -never pass, and serving must not move while setup_wrong > 0). For ``r1`` the reader is -expected to REFUSE the unsupported shapes (setup_refused), never misread them. +Exit 0 iff the lane's wrong-critical counters are zero. For setup lanes that means +``setup_wrong == 0``. For ``r1-answers`` that means ``setup_wrong == wrong == +gold_error == 0``; unsupported R1 fixtures may still refuse honestly. """ from __future__ import annotations @@ -13,14 +14,22 @@ from __future__ import annotations import json import sys -from evals.setup_oracle.runner import run, run_r1 +from evals.setup_oracle.runner import run, run_r1, run_r1_answers def main() -> int: lane = sys.argv[1] if len(sys.argv) > 1 else "" - report = run_r1() if lane == "r1" else run() + if lane == "r1": + report = run_r1() + ok = report["setup_wrong"] == 0 + elif lane in ("r1-answers", "r1_answers"): + report = run_r1_answers() + ok = report["setup_wrong"] == 0 and report["wrong"] == 0 and report["gold_error"] == 0 + else: + report = run() + ok = report["setup_wrong"] == 0 print(json.dumps(report, indent=2, default=str)) - return 0 if report["setup_wrong"] == 0 else 1 + return 0 if ok else 1 if __name__ == "__main__": diff --git a/evals/setup_oracle/r1_gold.jsonl b/evals/setup_oracle/r1_gold.jsonl index e62623d5..ad8e80c7 100644 --- a/evals/setup_oracle/r1_gold.jsonl +++ b/evals/setup_oracle/r1_gold.jsonl @@ -1,8 +1,8 @@ -{"id": "r1-01-twice", "text": "Anna has 6 apples. Bella has twice as many apples as Anna. How many apples does Bella have?", "relations": [{"kind": "fact", "entity": "anna", "value": 6}, {"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2}], "expected_units": {"anna": "item", "bella": "item"}, "query": {"entity": "bella", "unit": "item"}, "notes": "Multiplicative: bella = 2 * anna. The reader has no multiplicative template AND 'twice' is non-digit -> must REFUSE, never misread as a fact/additive."} +{"id": "r1-01-twice", "text": "Anna has 6 apples. Bella has twice as many apples as Anna. How many apples does Bella have?", "relations": [{"kind": "fact", "entity": "anna", "value": 6}, {"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2}], "expected_units": {"anna": "item", "bella": "item"}, "query": {"entity": "bella", "unit": "item"}, "gold": 12, "notes": "Multiplicative: bella = 2 * anna. The reader has no multiplicative template AND 'twice' is non-digit -> must REFUSE, never misread as a fact/additive."} {"id": "r1-02-half", "text": "Carl has 8 coins. Dora has half as many coins as Carl. How many coins does Dora have?", "relations": [{"kind": "fact", "entity": "carl", "value": 8}, {"kind": "times_as_many", "entity": "dora", "ref": "carl", "factor": 0.5}], "expected_units": {"carl": "item", "dora": "item"}, "query": {"entity": "dora", "unit": "item"}, "notes": "Multiplicative (factor 0.5). 'half' is non-digit -> must REFUSE."} {"id": "r1-03-more-total", "text": "Finn has 10 books. Evan has 5 more books than Finn. How many books do Evan and Finn have altogether?", "relations": [{"kind": "fact", "entity": "finn", "value": 10}, {"kind": "more_than", "entity": "evan", "ref": "finn", "delta": 5}, {"kind": "sum_of", "entity": "total", "parts": ["evan", "finn"]}], "expected_units": {"finn": "item", "evan": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive + aggregate. 'altogether' (not the bare 'have') may fall outside the strict query template -> likely REFUSE; if read, structure must match exactly."} {"id": "r1-04-fewer-total", "text": "Gail has 20 cards. Hank has 6 fewer cards than Gail. How many cards do Gail and Hank have in total?", "relations": [{"kind": "fact", "entity": "gail", "value": 20}, {"kind": "fewer_than", "entity": "hank", "ref": "gail", "delta": 6}, {"kind": "sum_of", "entity": "total", "parts": ["gail", "hank"]}], "expected_units": {"gail": "item", "hank": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive(fewer) + aggregate. 'in total' qualifier likely outside the query template -> REFUSE expected."} -{"id": "r1-05-chain", "text": "Ivy has 4 pens. Jon has 3 times as many pens as Ivy. Kim has 2 more pens than Jon. How many pens does Kim have?", "relations": [{"kind": "fact", "entity": "ivy", "value": 4}, {"kind": "times_as_many", "entity": "jon", "ref": "ivy", "factor": 3}, {"kind": "more_than", "entity": "kim", "ref": "jon", "delta": 2}], "expected_units": {"ivy": "item", "jon": "item", "kim": "item"}, "query": {"entity": "kim", "unit": "item"}, "notes": "Multi-step derived chain: jon=3*ivy (intermediate), kim=jon+2. The multiplicative middle step has no template -> must REFUSE the whole reading, never read a partial/wrong chain."} +{"id": "r1-05-chain", "text": "Ivy has 4 pens. Jon has 3 times as many pens as Ivy. Kim has 2 more pens than Jon. How many pens does Kim have?", "relations": [{"kind": "fact", "entity": "ivy", "value": 4}, {"kind": "times_as_many", "entity": "jon", "ref": "ivy", "factor": 3}, {"kind": "more_than", "entity": "kim", "ref": "jon", "delta": 2}], "expected_units": {"ivy": "item", "jon": "item", "kim": "item"}, "query": {"entity": "kim", "unit": "item"}, "gold": 14, "notes": "Multi-step derived chain: jon=3*ivy (intermediate), kim=jon+2. The multiplicative middle step has no template -> must REFUSE the whole reading, never read a partial/wrong chain."} {"id": "r1-06-subtotal-reused", "text": "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them equally into 3 boxes. How many hats are in each box?", "relations": [{"kind": "fact", "entity": "lee", "value": 5}, {"kind": "fact", "entity": "mae", "value": 7}, {"kind": "sum_of", "entity": "total", "parts": ["lee", "mae"]}, {"kind": "divide_by", "entity": "per_box", "ref": "total", "divisor": 3}], "expected_units": {"lee": "item", "mae": "item", "total": "item", "per_box": "item"}, "query": {"entity": "per_box", "unit": "item"}, "notes": "Derived subtotal reused downstream: total = lee + mae, per_box = total / 3. No partition template -> must REFUSE."} {"id": "r1-07-inverse", "text": "Nia has 9 more beads than Omar. Nia has 15 beads. How many beads does Omar have?", "relations": [{"kind": "fact", "entity": "nia", "value": 15}, {"kind": "more_than", "entity": "nia", "ref": "omar", "delta": 9}, {"kind": "ask_base", "entity": "omar"}], "expected_units": {"nia": "item", "omar": "item"}, "query": {"entity": "omar", "unit": "item"}, "notes": "Inverse target: omar = nia - 9 = 6, asked via the base of a more_than. The reader may read the STRUCTURE (nia=15, nia=omar+9, ask omar) — that setup is correct; only the solver can't invert. So setup_correct OR refuse, but NEVER setup_wrong."} {"id": "r1-08-ambiguous-referent", "text": "Pat has 5 marbles. He has 3 more than her. How many marbles does she have?", "relations": [], "expected_units": {}, "query": {"entity": "she", "unit": "item"}, "notes": "Ambiguous pronoun referents (he/her/she) with no grounded base for 'her'/'she' -> must REFUSE (no honest reading), never bind a guessed referent."} diff --git a/tests/test_setup_oracle.py b/tests/test_setup_oracle.py index 99cc2ce6..cf7d8a91 100644 --- a/tests/test_setup_oracle.py +++ b/tests/test_setup_oracle.py @@ -10,12 +10,17 @@ Two obligations: from __future__ import annotations +import pytest + +from evals.relational_metric.oracle import OracleError, oracle_answer from evals.setup_oracle import ( gold_unknown_signature, reader_symbol_units, reader_unknown_signature, relation_signature, run, + run_r1, + run_r1_answers, symbol_unit_signature, ) from generate.binding_graph.model import ( @@ -148,8 +153,6 @@ def test_reader_units_read_from_the_binding_graph() -> None: def test_r1_multiplicative_supported_rest_refused_wrong_zero() -> None: - from evals.setup_oracle import run_r1 - r = run_r1() assert r["total"] == 10 # THE invariant through the first capability slice: NO R1 case is misread. Adding the @@ -167,3 +170,51 @@ def test_r1_multiplicative_supported_rest_refused_wrong_zero() -> None: assert d["outcome"] in ("correct", "refused") if d["outcome"] == "refused": assert d.get("reason") + + +# --------------------------------------------------------------------------- # +# PR-6b — off-serving answer oracle support for times_as_many +# --------------------------------------------------------------------------- # + + +def test_oracle_computes_times_as_many_forward_only() -> None: + assert oracle_answer( + [ + {"kind": "fact", "entity": "anna", "value": 6}, + {"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2}, + ], + {"entity": "bella"}, + ) == 12 + + +def test_oracle_rejects_invalid_times_factor_and_forward_ref() -> None: + with pytest.raises(OracleError): + oracle_answer( + [ + {"kind": "fact", "entity": "anna", "value": 6}, + {"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2.5}, + ], + {"entity": "bella"}, + ) + with pytest.raises(OracleError): + oracle_answer( + [{"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2}], + {"entity": "bella"}, + ) + + +def test_r1_answer_lane_scores_only_setup_correct_fixtures() -> None: + r = run_r1_answers() + assert r["total"] == 10 + assert r["setup_wrong"] == 0 + assert r["wrong"] == 0 + assert r["gold_error"] == 0 + assert r["correct"] == 2 + assert r["refused"] == 8 + by_id = {d["id"]: d for d in r["details"]} + assert by_id["r1-01-twice"] == {"id": "r1-01-twice", "outcome": "correct", "answer": 12} + assert by_id["r1-05-chain"] == {"id": "r1-05-chain", "outcome": "correct", "answer": 14} + for fixture_id, detail in by_id.items(): + if fixture_id not in {"r1-01-twice", "r1-05-chain"}: + assert detail["outcome"] == "refused" + assert detail.get("reason")