feat(setup-oracle): independent R1 gold + the reader refuses, never misreads (PR-5b)
Pure-gold. Defines what "correct" means for the R1 (comparative / derived-symbol / multi-step) shapes the reader cannot read yet, and verifies the current reader REFUSES them rather than misreading them. NO frame extraction, no parser change, no serving. - evals/setup_oracle/r1_gold.jsonl — 10 INDEPENDENT, self-contained fixtures (id, text, relations, expected_units, query, notes) covering: twice/half-as-many (multiplicative), more/fewer-than -> total, a multi-step derived chain (jon=3*ivy; kim=jon+2), a derived subtotal reused downstream (total=lee+mae; per_box=total/3), an inverse target, and the hard negatives (ambiguous referent, missing base quantity, distractor quantity). The gold is the authority — each fixture is reviewable without joining files. - run_r1() scores the CURRENT reader against this gold. Investigative result (the deliverable): **10/10 setup_refused, setup_wrong=0, setup_correct=0.** The reader refuses every R1 shape with a TYPED reason (non_digit_quantity / no_quantity_template / unreadable_quantity_clause / admissibility_refused) — it NEVER misreads an R1 case as a simpler form. So the wrong=0 posture holds all the way down, and PR-5c (the R1 frame) starts from a clean foundation: every R1 case is currently a safe refusal, and the frame's job is refusal -> correct, with the oracle ensuring no setup_wrong is ever introduced. `python -m evals.setup_oracle r1` prints the auditable R1 report (exit 0 iff setup_wrong==0). The 15-case setup gate + relational_metric answer lane are untouched.
This commit is contained in:
Shay
parent
6c8f2b8332
commit
5b193280a5
5 changed files with 119 additions and 6 deletions
|
|
@ -13,7 +13,7 @@ the problem. v1 grades structure (facts + equations + question target/state/form
|
|||
modelling stays covered by the admissibility tests (a documented signature extension).
|
||||
"""
|
||||
|
||||
from evals.setup_oracle.runner import run
|
||||
from evals.setup_oracle.runner import run, run_r1
|
||||
from evals.setup_oracle.signature import (
|
||||
gold_unknown_signature,
|
||||
reader_symbol_units,
|
||||
|
|
@ -28,5 +28,6 @@ __all__ = [
|
|||
"reader_unknown_signature",
|
||||
"relation_signature",
|
||||
"run",
|
||||
"run_r1",
|
||||
"symbol_unit_signature",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,20 +1,24 @@
|
|||
"""CLI: print the setup-oracle report.
|
||||
|
||||
python -m evals.setup_oracle
|
||||
python -m evals.setup_oracle # the 15-case relational-metric setup gold
|
||||
python -m evals.setup_oracle r1 # the independent R1 gold (PR-5b)
|
||||
|
||||
Exit 0 iff ``setup_wrong == 0`` — the gate the milestone rests on (a wrong reading must
|
||||
never pass, and serving must not move while setup_wrong > 0).
|
||||
never pass, and serving must not move while setup_wrong > 0). For ``r1`` the reader is
|
||||
expected to REFUSE the unsupported shapes (setup_refused), never misread them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
from evals.setup_oracle.runner import run
|
||||
from evals.setup_oracle.runner import run, run_r1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
report = run()
|
||||
lane = sys.argv[1] if len(sys.argv) > 1 else ""
|
||||
report = run_r1() if lane == "r1" else run()
|
||||
print(json.dumps(report, indent=2, default=str))
|
||||
return 0 if report["setup_wrong"] == 0 else 1
|
||||
|
||||
|
|
|
|||
10
evals/setup_oracle/r1_gold.jsonl
Normal file
10
evals/setup_oracle/r1_gold.jsonl
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
{"id": "r1-01-twice", "text": "Anna has 6 apples. Bella has twice as many apples as Anna. How many apples does Bella have?", "relations": [{"kind": "fact", "entity": "anna", "value": 6}, {"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2}], "expected_units": {"anna": "item", "bella": "item"}, "query": {"entity": "bella", "unit": "item"}, "notes": "Multiplicative: bella = 2 * anna. The reader has no multiplicative template AND 'twice' is non-digit -> must REFUSE, never misread as a fact/additive."}
|
||||
{"id": "r1-02-half", "text": "Carl has 8 coins. Dora has half as many coins as Carl. How many coins does Dora have?", "relations": [{"kind": "fact", "entity": "carl", "value": 8}, {"kind": "times_as_many", "entity": "dora", "ref": "carl", "factor": 0.5}], "expected_units": {"carl": "item", "dora": "item"}, "query": {"entity": "dora", "unit": "item"}, "notes": "Multiplicative (factor 0.5). 'half' is non-digit -> must REFUSE."}
|
||||
{"id": "r1-03-more-total", "text": "Finn has 10 books. Evan has 5 more books than Finn. How many books do Evan and Finn have altogether?", "relations": [{"kind": "fact", "entity": "finn", "value": 10}, {"kind": "more_than", "entity": "evan", "ref": "finn", "delta": 5}, {"kind": "sum_of", "entity": "total", "parts": ["evan", "finn"]}], "expected_units": {"finn": "item", "evan": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive + aggregate. 'altogether' (not the bare 'have') may fall outside the strict query template -> likely REFUSE; if read, structure must match exactly."}
|
||||
{"id": "r1-04-fewer-total", "text": "Gail has 20 cards. Hank has 6 fewer cards than Gail. How many cards do Gail and Hank have in total?", "relations": [{"kind": "fact", "entity": "gail", "value": 20}, {"kind": "fewer_than", "entity": "hank", "ref": "gail", "delta": 6}, {"kind": "sum_of", "entity": "total", "parts": ["gail", "hank"]}], "expected_units": {"gail": "item", "hank": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive(fewer) + aggregate. 'in total' qualifier likely outside the query template -> REFUSE expected."}
|
||||
{"id": "r1-05-chain", "text": "Ivy has 4 pens. Jon has 3 times as many pens as Ivy. Kim has 2 more pens than Jon. How many pens does Kim have?", "relations": [{"kind": "fact", "entity": "ivy", "value": 4}, {"kind": "times_as_many", "entity": "jon", "ref": "ivy", "factor": 3}, {"kind": "more_than", "entity": "kim", "ref": "jon", "delta": 2}], "expected_units": {"ivy": "item", "jon": "item", "kim": "item"}, "query": {"entity": "kim", "unit": "item"}, "notes": "Multi-step derived chain: jon=3*ivy (intermediate), kim=jon+2. The multiplicative middle step has no template -> must REFUSE the whole reading, never read a partial/wrong chain."}
|
||||
{"id": "r1-06-subtotal-reused", "text": "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them equally into 3 boxes. How many hats are in each box?", "relations": [{"kind": "fact", "entity": "lee", "value": 5}, {"kind": "fact", "entity": "mae", "value": 7}, {"kind": "sum_of", "entity": "total", "parts": ["lee", "mae"]}, {"kind": "divide_by", "entity": "per_box", "ref": "total", "divisor": 3}], "expected_units": {"lee": "item", "mae": "item", "total": "item", "per_box": "item"}, "query": {"entity": "per_box", "unit": "item"}, "notes": "Derived subtotal reused downstream: total = lee + mae, per_box = total / 3. No partition template -> must REFUSE."}
|
||||
{"id": "r1-07-inverse", "text": "Nia has 9 more beads than Omar. Nia has 15 beads. How many beads does Omar have?", "relations": [{"kind": "fact", "entity": "nia", "value": 15}, {"kind": "more_than", "entity": "nia", "ref": "omar", "delta": 9}, {"kind": "ask_base", "entity": "omar"}], "expected_units": {"nia": "item", "omar": "item"}, "query": {"entity": "omar", "unit": "item"}, "notes": "Inverse target: omar = nia - 9 = 6, asked via the base of a more_than. The reader may read the STRUCTURE (nia=15, nia=omar+9, ask omar) — that setup is correct; only the solver can't invert. So setup_correct OR refuse, but NEVER setup_wrong."}
|
||||
{"id": "r1-08-ambiguous-referent", "text": "Pat has 5 marbles. He has 3 more than her. How many marbles does she have?", "relations": [], "expected_units": {}, "query": {"entity": "she", "unit": "item"}, "notes": "Ambiguous pronoun referents (he/her/she) with no grounded base for 'her'/'she' -> must REFUSE (no honest reading), never bind a guessed referent."}
|
||||
{"id": "r1-09-missing-base", "text": "Quinn has twice as many toys as Rosa. How many toys does Quinn have?", "relations": [{"kind": "times_as_many", "entity": "quinn", "ref": "rosa", "factor": 2}], "expected_units": {"quinn": "item", "rosa": "item"}, "query": {"entity": "quinn", "unit": "item"}, "notes": "Missing base quantity: Rosa's count is never given, so Quinn is underdetermined -> must REFUSE."}
|
||||
{"id": "r1-10-distractor", "text": "Sam has 7 pencils and 3 erasers. Tom has 4 more pencils than Sam. How many pencils does Tom have?", "relations": [{"kind": "fact", "entity": "sam", "value": 7}, {"kind": "more_than", "entity": "tom", "ref": "sam", "delta": 4}], "expected_units": {"sam": "item", "tom": "item"}, "query": {"entity": "tom", "unit": "item"}, "notes": "Distractor quantity (3 erasers) in a compound clause the 'X has N unit' template can't parse -> must REFUSE rather than mis-bind the distractor."}
|
||||
|
|
@ -28,6 +28,7 @@ from generate.meaning_graph.reader import Refusal
|
|||
from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric
|
||||
|
||||
_EXPECTED_UNITS_PATH = Path(__file__).resolve().parent / "expected_units.json"
|
||||
_R1_GOLD_PATH = Path(__file__).resolve().parent / "r1_gold.jsonl"
|
||||
|
||||
|
||||
def _load_expected_units() -> dict[str, dict[str, str]]:
|
||||
|
|
@ -35,6 +36,14 @@ def _load_expected_units() -> dict[str, dict[str, str]]:
|
|||
return {k: v for k, v in raw.items() if not k.startswith("_")}
|
||||
|
||||
|
||||
def _load_r1_gold() -> list[dict[str, Any]]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in _R1_GOLD_PATH.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def run() -> dict[str, Any]:
|
||||
"""Score the reader's setup (structure + units + target) against the independent gold."""
|
||||
cases = _load_cases()
|
||||
|
|
@ -94,4 +103,72 @@ def run() -> dict[str, Any]:
|
|||
}
|
||||
|
||||
|
||||
__all__ = ["run"]
|
||||
def run_r1() -> dict[str, Any]:
|
||||
"""Score the CURRENT reader against the independent, self-contained R1 gold.
|
||||
|
||||
The reader cannot read most R1 shapes (multiplicative, multi-step, partition) yet.
|
||||
The SUCCESS CRITERION is **``setup_wrong == 0``**: it must REFUSE every unsupported
|
||||
shape, NEVER misread it as a simpler form. ``setup_refused`` is the unsupported count;
|
||||
``setup_correct`` is any shape already faithfully supported. A ``setup_wrong`` is a
|
||||
pre-existing wrong-reading hazard to fix BEFORE building the R1 frame (PR-5c).
|
||||
"""
|
||||
fixtures = _load_r1_gold()
|
||||
setup_correct = setup_wrong = setup_refused = 0
|
||||
details: list[dict[str, Any]] = []
|
||||
|
||||
for fx in fixtures:
|
||||
comp = comprehend_quantitative(fx["text"])
|
||||
if isinstance(comp, Refusal):
|
||||
setup_refused += 1
|
||||
details.append({"id": fx["id"], "outcome": "refused", "reason": comp.reason})
|
||||
continue
|
||||
projected = to_relational_metric(comp)
|
||||
if projected is None:
|
||||
setup_refused += 1
|
||||
details.append({"id": fx["id"], "outcome": "refused", "reason": "unprojectable"})
|
||||
continue
|
||||
reader_relations, _ = projected
|
||||
units = fx["expected_units"]
|
||||
|
||||
reader_rel = relation_signature(reader_relations)
|
||||
gold_rel = relation_signature(fx["relations"])
|
||||
reader_units = reader_symbol_units(comp.binding_graph)
|
||||
gold_units = symbol_unit_signature(units)
|
||||
reader_unk = reader_unknown_signature(comp.binding_graph)
|
||||
gold_unk = gold_unknown_signature(fx["relations"], fx["query"], units)
|
||||
|
||||
if reader_rel == gold_rel and reader_units == gold_units and reader_unk == gold_unk:
|
||||
setup_correct += 1
|
||||
details.append({"id": fx["id"], "outcome": "correct"})
|
||||
else:
|
||||
setup_wrong += 1
|
||||
details.append(
|
||||
{
|
||||
"id": fx["id"],
|
||||
"outcome": "WRONG",
|
||||
"relations_match": reader_rel == gold_rel,
|
||||
"units_match": reader_units == gold_units,
|
||||
"target_match": reader_unk == gold_unk,
|
||||
"reader_relations": reader_rel,
|
||||
"gold_relations": gold_rel,
|
||||
"reader_target": reader_unk,
|
||||
"gold_target": gold_unk,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"lane": "setup_oracle_r1",
|
||||
"total": len(fixtures),
|
||||
"setup_correct": setup_correct,
|
||||
"setup_wrong": setup_wrong,
|
||||
"setup_refused": setup_refused,
|
||||
"details": details,
|
||||
"counts": {
|
||||
"setup_correct": setup_correct,
|
||||
"setup_wrong": setup_wrong,
|
||||
"setup_refused": setup_refused,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["run", "run_r1"]
|
||||
|
|
|
|||
|
|
@ -140,3 +140,24 @@ def test_reader_units_read_from_the_binding_graph() -> None:
|
|||
)
|
||||
assert reader_symbol_units(graph) == (("iris", "dollars"), ("jack", "dollars"))
|
||||
assert reader_unknown_signature(graph) == ("jack", "terminal", "count", "dollars")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# PR-5b — independent R1 gold: the reader must REFUSE, never MISREAD
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_r1_reader_refuses_never_misreads() -> None:
|
||||
from evals.setup_oracle import run_r1
|
||||
|
||||
r = run_r1()
|
||||
assert r["total"] == 10
|
||||
# THE success criterion: the reader misreads NO unsupported R1 case. A wrong setup
|
||||
# here would be a pre-existing hazard to fix before the R1 frame (PR-5c).
|
||||
assert r["setup_wrong"] == 0
|
||||
# The reader cannot read these shapes yet -> every one is a safe (typed) refusal.
|
||||
assert r["setup_refused"] == 10
|
||||
assert r["setup_correct"] == 0
|
||||
for d in r["details"]:
|
||||
assert d["outcome"] == "refused"
|
||||
assert d.get("reason") # a typed refusal reason, never a silent drop
|
||||
|
|
|
|||
Loading…
Reference in a new issue