From 15b0b583aa5e897cbb0454d56d3ffab0a019ea90 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 6 Jun 2026 18:31:41 -0700 Subject: [PATCH] test(comprehension): expose R1 answer lane CLI --- evals/setup_oracle/__main__.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/evals/setup_oracle/__main__.py b/evals/setup_oracle/__main__.py index 6d591f8f..495c076d 100644 --- a/evals/setup_oracle/__main__.py +++ b/evals/setup_oracle/__main__.py @@ -1,11 +1,12 @@ """CLI: print the setup-oracle report. - python -m evals.setup_oracle # the 15-case relational-metric setup gold - python -m evals.setup_oracle r1 # the independent R1 gold (PR-5b) + python -m evals.setup_oracle # the 15-case relational-metric setup gold + python -m evals.setup_oracle r1 # the independent R1 setup gold (PR-5b) + python -m evals.setup_oracle r1-answers # off-serving R1 answer lane (PR-6b) -Exit 0 iff ``setup_wrong == 0`` — the gate the milestone rests on (a wrong reading must -never pass, and serving must not move while setup_wrong > 0). For ``r1`` the reader is -expected to REFUSE the unsupported shapes (setup_refused), never misread them. +Exit 0 iff the lane's wrong-critical counters are zero. For setup lanes that means +``setup_wrong == 0``. For ``r1-answers`` that means ``setup_wrong == wrong == +gold_error == 0``; unsupported R1 fixtures may still refuse honestly. """ from __future__ import annotations @@ -13,14 +14,22 @@ from __future__ import annotations import json import sys -from evals.setup_oracle.runner import run, run_r1 +from evals.setup_oracle.runner import run, run_r1, run_r1_answers def main() -> int: lane = sys.argv[1] if len(sys.argv) > 1 else "" - report = run_r1() if lane == "r1" else run() + if lane == "r1": + report = run_r1() + ok = report["setup_wrong"] == 0 + elif lane in ("r1-answers", "r1_answers"): + report = run_r1_answers() + ok = report["setup_wrong"] == 0 and report["wrong"] == 0 and report["gold_error"] == 0 + else: + report = run() + ok = report["setup_wrong"] == 0 print(json.dumps(report, indent=2, default=str)) - return 0 if report["setup_wrong"] == 0 else 1 + return 0 if ok else 1 if __name__ == "__main__":