From b1f12e1dce385734c0e5373b128aa7c238325287 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 3 Jun 2026 01:01:50 -0700 Subject: [PATCH] chore(evals): refresh stale committed reports G3_numerics report.json: refusal reason-string drift (PR #359 made recognized-but-uninjectable refusals more specific); verdicts and wrong=0 identical. train_sample coverage probe: admitted_solved 4->6 reflecting reader coverage gains earlier in the window; wrong=0, safety_rail_intact. Regenerated via their runners. --- .../v1/train_sample_coverage_report.json | 28 +++++++------------ .../G3_numerics/v1/report.json | 10 +++---- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json b/evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json index eacd2892..122bf57b 100644 --- a/evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json +++ b/evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json @@ -1,12 +1,12 @@ { "adr": "0131.G", "metrics": { - "admission_rate": 0.08, - "admitted_solved": 4, + "admission_rate": 0.12, + "admitted_solved": 6, "admitted_wrong": 0, "cases_total": 50, - "refused": 46, - "refused_rate": 0.92, + "refused": 44, + "refused_rate": 0.88, "safety_rail_intact": true, "wrong_count_is_zero": true, "wrong_rate": 0.0 @@ -35,14 +35,14 @@ "trace_hash": null }, { - "actual_answer": null, + "actual_answer": 864.0, "actual_unit": null, "case_id": "gsm8k-train-sample-v1-0003", "expected_answer": 864.0, "expected_unit": "", - "outcome": "refused", + "outcome": "correct", "realized_prose": null, - "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'The local bookstore donated 48 boxes of erasers.' (category=discrete_count_statement)", + "reason": "fast-path", "trace_hash": null }, { @@ -233,14 +233,14 @@ "trace_hash": null }, { - "actual_answer": null, + "actual_answer": 450.0, "actual_unit": null, "case_id": "gsm8k-train-sample-v1-0021", "expected_answer": 450.0, "expected_unit": "", - "outcome": "refused", + "outcome": "correct", "realized_prose": null, - "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'He bench presses 15 pounds for 10 reps and does 3 sets.' (category=discrete_count_statement)", + "reason": "fast-path", "trace_hash": null }, { @@ -645,10 +645,6 @@ "count": 1, "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Half of the students are girls, the other " }, - { - "count": 1, - "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'He bench presses 15 pounds for 10 reps and" - }, { "count": 1, "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'He can run 40 yards within 5 seconds.' (ca" @@ -725,10 +721,6 @@ "count": 1, "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'The guests eat all of 1 pan, and 75% of th" }, - { - "count": 1, - "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'The local bookstore donated 48 boxes of er" - }, { "count": 1, "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'The number of followers he has on Twitter " diff --git a/evals/math_capability_axes/G3_numerics/v1/report.json b/evals/math_capability_axes/G3_numerics/v1/report.json index 9269e2ed..a62f0685 100644 --- a/evals/math_capability_axes/G3_numerics/v1/report.json +++ b/evals/math_capability_axes/G3_numerics/v1/report.json @@ -270,7 +270,7 @@ "class": "refuse_money_precision", "expected_answer": 0, "expected_outcome": "refused", - "reason": "candidate_graph: no branch produced a solvable graph", + "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Bob has $40.000 in savings.' (category=currency_amount)", "trace_hash": null, "verdict": "refused" }, @@ -282,7 +282,7 @@ "class": "refuse_money_precision", "expected_answer": 0, "expected_outcome": "refused", - "reason": "candidate_graph: no branch produced a solvable graph", + "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Sarah has $1.2345 in change.' (category=currency_amount)", "trace_hash": null, "verdict": "refused" }, @@ -306,7 +306,7 @@ "class": "refuse_unknown_compound", "expected_answer": 0, "expected_outcome": "refused", - "reason": "candidate_graph: no branch produced a solvable graph", + "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Bob has five-and-a-half apples.' (category=descriptive_setup_no_quantity)", "trace_hash": null, "verdict": "refused" }, @@ -318,7 +318,7 @@ "class": "refuse_unknown_compound", "expected_answer": 0, "expected_outcome": "refused", - "reason": "candidate_graph: no branch produced a solvable graph", + "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Bob has gobbledy-gook apples.' (category=descriptive_setup_no_quantity)", "trace_hash": null, "verdict": "refused" }, @@ -330,7 +330,7 @@ "class": "refuse_percentage", "expected_answer": 0, "expected_outcome": "refused", - "reason": "candidate_graph: no branch produced a solvable graph", + "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Bob has 50% apples.' (category=discrete_count_statement)", "trace_hash": null, "verdict": "refused" }