core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

test(comprehension): expose R1 answer lane CLI

Browse source
This commit is contained in:
Shay 2026-06-06 18:31:41 -07:00
parent 56260629b8
commit 15b0b583aa
1 changed files with 17 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
evals/setup_oracle/__main__.py
View file

@ -1,11 +1,12 @@
"""CLI: print the setup-oracle report.
python -m evals.setup_oracle # the 15-case relational-metric setup gold
python -m evals.setup_oracle r1 # the independent R1 gold (PR-5b)
python -m evals.setup_oracle # the 15-case relational-metric setup gold
python -m evals.setup_oracle r1 # the independent R1 setup gold (PR-5b)
python -m evals.setup_oracle r1-answers # off-serving R1 answer lane (PR-6b)
Exit 0 iff ``setup_wrong == 0`` — the gate the milestone rests on (a wrong reading must
never pass, and serving must not move while setup_wrong > 0). For ``r1`` the reader is
expected to REFUSE the unsupported shapes (setup_refused), never misread them.
Exit 0 iff the lane's wrong-critical counters are zero. For setup lanes that means
``setup_wrong == 0``. For ``r1-answers`` that means ``setup_wrong == wrong ==
gold_error == 0``; unsupported R1 fixtures may still refuse honestly.
"""
from __future__ import annotations
@ -13,14 +14,22 @@ from __future__ import annotations
import json
import sys
from evals.setup_oracle.runner import run, run_r1
from evals.setup_oracle.runner import run, run_r1, run_r1_answers
def main() -> int:
lane = sys.argv[1] if len(sys.argv) > 1 else ""
report = run_r1() if lane == "r1" else run()
if lane == "r1":
report = run_r1()
ok = report["setup_wrong"] == 0
elif lane in ("r1-answers", "r1_answers"):
report = run_r1_answers()
ok = report["setup_wrong"] == 0 and report["wrong"] == 0 and report["gold_error"] == 0
else:
report = run()
ok = report["setup_wrong"] == 0
print(json.dumps(report, indent=2, default=str))
return 0 if report["setup_wrong"] == 0 else 1
return 0 if ok else 1
if __name__ == "__main__":