core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge pull request #614 from AssetOverflow/pr-6b-r1-answer-oracle-followup

Browse source 
test(comprehension): prove R1 times-as-many answer oracle (PR-6b follow-up)
This commit is contained in:
Shay 2026-06-06 19:35:57 -07:00 committed by GitHub
commit 5ac1526536
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 72 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
evals/setup_oracle/__main__.py
View file

@ -1,11 +1,12 @@
"""CLI: print the setup-oracle report.
python -m evals.setup_oracle # the 15-case relational-metric setup gold
python -m evals.setup_oracle r1 # the independent R1 gold (PR-5b)
python -m evals.setup_oracle r1 # the independent R1 setup gold (PR-5b)
python -m evals.setup_oracle r1-answers # off-serving R1 answer lane (PR-6b)
Exit 0 iff ``setup_wrong == 0`` — the gate the milestone rests on (a wrong reading must
never pass, and serving must not move while setup_wrong > 0). For ``r1`` the reader is
expected to REFUSE the unsupported shapes (setup_refused), never misread them.
Exit 0 iff the lane's wrong-critical counters are zero. For setup lanes that means
``setup_wrong == 0``. For ``r1-answers`` that means ``setup_wrong == wrong ==
gold_error == 0``; unsupported R1 fixtures may still refuse honestly.
"""
from __future__ import annotations
@ -13,14 +14,22 @@ from __future__ import annotations
import json
import sys
from evals.setup_oracle.runner import run, run_r1
from evals.setup_oracle.runner import run, run_r1, run_r1_answers
def main() -> int:
lane = sys.argv[1] if len(sys.argv) > 1 else ""
report = run_r1() if lane == "r1" else run()
if lane == "r1":
report = run_r1()
ok = report["setup_wrong"] == 0
elif lane in ("r1-answers", "r1_answers"):
report = run_r1_answers()
ok = report["setup_wrong"] == 0 and report["wrong"] == 0 and report["gold_error"] == 0
else:
report = run()
ok = report["setup_wrong"] == 0
print(json.dumps(report, indent=2, default=str))
return 0 if report["setup_wrong"] == 0 else 1
return 0 if ok else 1
if __name__ == "__main__":

4
evals/setup_oracle/r1_gold.jsonl
View file

@ -1,8 +1,8 @@
{"id": "r1-01-twice", "text": "Anna has 6 apples. Bella has twice as many apples as Anna. How many apples does Bella have?", "relations": [{"kind": "fact", "entity": "anna", "value": 6}, {"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2}], "expected_units": {"anna": "item", "bella": "item"}, "query": {"entity": "bella", "unit": "item"}, "notes": "Multiplicative: bella = 2 * anna. The reader has no multiplicative template AND 'twice' is non-digit -> must REFUSE, never misread as a fact/additive."}
{"id": "r1-01-twice", "text": "Anna has 6 apples. Bella has twice as many apples as Anna. How many apples does Bella have?", "relations": [{"kind": "fact", "entity": "anna", "value": 6}, {"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2}], "expected_units": {"anna": "item", "bella": "item"}, "query": {"entity": "bella", "unit": "item"}, "gold": 12, "notes": "Multiplicative: bella = 2 * anna. The reader has no multiplicative template AND 'twice' is non-digit -> must REFUSE, never misread as a fact/additive."}
{"id": "r1-02-half", "text": "Carl has 8 coins. Dora has half as many coins as Carl. How many coins does Dora have?", "relations": [{"kind": "fact", "entity": "carl", "value": 8}, {"kind": "times_as_many", "entity": "dora", "ref": "carl", "factor": 0.5}], "expected_units": {"carl": "item", "dora": "item"}, "query": {"entity": "dora", "unit": "item"}, "notes": "Multiplicative (factor 0.5). 'half' is non-digit -> must REFUSE."}
{"id": "r1-03-more-total", "text": "Finn has 10 books. Evan has 5 more books than Finn. How many books do Evan and Finn have altogether?", "relations": [{"kind": "fact", "entity": "finn", "value": 10}, {"kind": "more_than", "entity": "evan", "ref": "finn", "delta": 5}, {"kind": "sum_of", "entity": "total", "parts": ["evan", "finn"]}], "expected_units": {"finn": "item", "evan": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive + aggregate. 'altogether' (not the bare 'have') may fall outside the strict query template -> likely REFUSE; if read, structure must match exactly."}
{"id": "r1-04-fewer-total", "text": "Gail has 20 cards. Hank has 6 fewer cards than Gail. How many cards do Gail and Hank have in total?", "relations": [{"kind": "fact", "entity": "gail", "value": 20}, {"kind": "fewer_than", "entity": "hank", "ref": "gail", "delta": 6}, {"kind": "sum_of", "entity": "total", "parts": ["gail", "hank"]}], "expected_units": {"gail": "item", "hank": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive(fewer) + aggregate. 'in total' qualifier likely outside the query template -> REFUSE expected."}
{"id": "r1-05-chain", "text": "Ivy has 4 pens. Jon has 3 times as many pens as Ivy. Kim has 2 more pens than Jon. How many pens does Kim have?", "relations": [{"kind": "fact", "entity": "ivy", "value": 4}, {"kind": "times_as_many", "entity": "jon", "ref": "ivy", "factor": 3}, {"kind": "more_than", "entity": "kim", "ref": "jon", "delta": 2}], "expected_units": {"ivy": "item", "jon": "item", "kim": "item"}, "query": {"entity": "kim", "unit": "item"}, "notes": "Multi-step derived chain: jon=3*ivy (intermediate), kim=jon+2. The multiplicative middle step has no template -> must REFUSE the whole reading, never read a partial/wrong chain."}
{"id": "r1-05-chain", "text": "Ivy has 4 pens. Jon has 3 times as many pens as Ivy. Kim has 2 more pens than Jon. How many pens does Kim have?", "relations": [{"kind": "fact", "entity": "ivy", "value": 4}, {"kind": "times_as_many", "entity": "jon", "ref": "ivy", "factor": 3}, {"kind": "more_than", "entity": "kim", "ref": "jon", "delta": 2}], "expected_units": {"ivy": "item", "jon": "item", "kim": "item"}, "query": {"entity": "kim", "unit": "item"}, "gold": 14, "notes": "Multi-step derived chain: jon=3*ivy (intermediate), kim=jon+2. The multiplicative middle step has no template -> must REFUSE the whole reading, never read a partial/wrong chain."}
{"id": "r1-06-subtotal-reused", "text": "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them equally into 3 boxes. How many hats are in each box?", "relations": [{"kind": "fact", "entity": "lee", "value": 5}, {"kind": "fact", "entity": "mae", "value": 7}, {"kind": "sum_of", "entity": "total", "parts": ["lee", "mae"]}, {"kind": "divide_by", "entity": "per_box", "ref": "total", "divisor": 3}], "expected_units": {"lee": "item", "mae": "item", "total": "item", "per_box": "item"}, "query": {"entity": "per_box", "unit": "item"}, "notes": "Derived subtotal reused downstream: total = lee + mae, per_box = total / 3. No partition template -> must REFUSE."}
{"id": "r1-07-inverse", "text": "Nia has 9 more beads than Omar. Nia has 15 beads. How many beads does Omar have?", "relations": [{"kind": "fact", "entity": "nia", "value": 15}, {"kind": "more_than", "entity": "nia", "ref": "omar", "delta": 9}, {"kind": "ask_base", "entity": "omar"}], "expected_units": {"nia": "item", "omar": "item"}, "query": {"entity": "omar", "unit": "item"}, "notes": "Inverse target: omar = nia - 9 = 6, asked via the base of a more_than. The reader may read the STRUCTURE (nia=15, nia=omar+9, ask omar) — that setup is correct; only the solver can't invert. So setup_correct OR refuse, but NEVER setup_wrong."}
{"id": "r1-08-ambiguous-referent", "text": "Pat has 5 marbles. He has 3 more than her. How many marbles does she have?", "relations": [], "expected_units": {}, "query": {"entity": "she", "unit": "item"}, "notes": "Ambiguous pronoun referents (he/her/she) with no grounded base for 'her'/'she' -> must REFUSE (no honest reading), never bind a guessed referent."}

55
tests/test_setup_oracle.py
View file

@ -10,12 +10,17 @@ Two obligations:
from __future__ import annotations
import pytest
from evals.relational_metric.oracle import OracleError, oracle_answer
from evals.setup_oracle import (
gold_unknown_signature,
reader_symbol_units,
reader_unknown_signature,
relation_signature,
run,
run_r1,
run_r1_answers,
symbol_unit_signature,
)
from generate.binding_graph.model import (
@ -148,8 +153,6 @@ def test_reader_units_read_from_the_binding_graph() -> None:
def test_r1_multiplicative_supported_rest_refused_wrong_zero() -> None:
from evals.setup_oracle import run_r1
r = run_r1()
assert r["total"] == 10
# THE invariant through the first capability slice: NO R1 case is misread. Adding the
@ -167,3 +170,51 @@ def test_r1_multiplicative_supported_rest_refused_wrong_zero() -> None:
assert d["outcome"] in ("correct", "refused")
if d["outcome"] == "refused":
assert d.get("reason")
# --------------------------------------------------------------------------- #
# PR-6b — off-serving answer oracle support for times_as_many
# --------------------------------------------------------------------------- #
def test_oracle_computes_times_as_many_forward_only() -> None:
assert oracle_answer(
[
{"kind": "fact", "entity": "anna", "value": 6},
{"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2},
],
{"entity": "bella"},
) == 12
def test_oracle_rejects_invalid_times_factor_and_forward_ref() -> None:
with pytest.raises(OracleError):
oracle_answer(
[
{"kind": "fact", "entity": "anna", "value": 6},
{"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2.5},
],
{"entity": "bella"},
)
with pytest.raises(OracleError):
oracle_answer(
[{"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2}],
{"entity": "bella"},
)
def test_r1_answer_lane_scores_only_setup_correct_fixtures() -> None:
r = run_r1_answers()
assert r["total"] == 10
assert r["setup_wrong"] == 0
assert r["wrong"] == 0
assert r["gold_error"] == 0
assert r["correct"] == 2
assert r["refused"] == 8
by_id = {d["id"]: d for d in r["details"]}
assert by_id["r1-01-twice"] == {"id": "r1-01-twice", "outcome": "correct", "answer": 12}
assert by_id["r1-05-chain"] == {"id": "r1-05-chain", "outcome": "correct", "answer": 14}
for fixture_id, detail in by_id.items():
if fixture_id not in {"r1-01-twice", "r1-05-chain"}:
assert detail["outcome"] == "refused"
assert detail.get("reason")