core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

chore(evals): refresh stale committed reports

Browse source 
G3_numerics report.json: refusal reason-string drift (PR #359 made
recognized-but-uninjectable refusals more specific); verdicts and wrong=0
identical. train_sample coverage probe: admitted_solved 4->6 reflecting
reader coverage gains earlier in the window; wrong=0, safety_rail_intact.
Regenerated via their runners.
This commit is contained in:
Shay 2026-06-03 01:01:50 -07:00
parent 4c0fa30785
commit b1f12e1dce
2 changed files with 15 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json
View file

@ -1,12 +1,12 @@
{
"adr": "0131.G",
"metrics": {
"admission_rate": 0.08,
"admitted_solved": 4,
"admission_rate": 0.12,
"admitted_solved": 6,
"admitted_wrong": 0,
"cases_total": 50,
"refused": 46,
"refused_rate": 0.92,
"refused": 44,
"refused_rate": 0.88,
"safety_rail_intact": true,
"wrong_count_is_zero": true,
"wrong_rate": 0.0
@ -35,14 +35,14 @@
"trace_hash": null
},
{
"actual_answer": null,
"actual_answer": 864.0,
"actual_unit": null,
"case_id": "gsm8k-train-sample-v1-0003",
"expected_answer": 864.0,
"expected_unit": "",
"outcome": "refused",
"outcome": "correct",
"realized_prose": null,
"reason": "candidate_graph: recognizer matched but produced no injection for statement: 'The local bookstore donated 48 boxes of erasers.' (category=discrete_count_statement)",
"reason": "fast-path",
"trace_hash": null
},
{
@ -233,14 +233,14 @@
"trace_hash": null
},
{
"actual_answer": null,
"actual_answer": 450.0,
"actual_unit": null,
"case_id": "gsm8k-train-sample-v1-0021",
"expected_answer": 450.0,
"expected_unit": "",
"outcome": "refused",
"outcome": "correct",
"realized_prose": null,
"reason": "candidate_graph: recognizer matched but produced no injection for statement: 'He bench presses 15 pounds for 10 reps and does 3 sets.' (category=discrete_count_statement)",
"reason": "fast-path",
"trace_hash": null
},
{
@ -645,10 +645,6 @@
"count": 1,
"reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Half of the students are girls, the other "
},
{
"count": 1,
"reason": "candidate_graph: recognizer matched but produced no injection for statement: 'He bench presses 15 pounds for 10 reps and"
},
{
"count": 1,
"reason": "candidate_graph: recognizer matched but produced no injection for statement: 'He can run 40 yards within 5 seconds.' (ca"
@ -725,10 +721,6 @@
"count": 1,
"reason": "candidate_graph: recognizer matched but produced no injection for statement: 'The guests eat all of 1 pan, and 75% of th"
},
{
"count": 1,
"reason": "candidate_graph: recognizer matched but produced no injection for statement: 'The local bookstore donated 48 boxes of er"
},
{
"count": 1,
"reason": "candidate_graph: recognizer matched but produced no injection for statement: 'The number of followers he has on Twitter "

10
evals/math_capability_axes/G3_numerics/v1/report.json
View file

@ -270,7 +270,7 @@
"class": "refuse_money_precision",
"expected_answer": 0,
"expected_outcome": "refused",
"reason": "candidate_graph: no branch produced a solvable graph",
"reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Bob has $40.000 in savings.' (category=currency_amount)",
"trace_hash": null,
"verdict": "refused"
},
@ -282,7 +282,7 @@
"class": "refuse_money_precision",
"expected_answer": 0,
"expected_outcome": "refused",
"reason": "candidate_graph: no branch produced a solvable graph",
"reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Sarah has $1.2345 in change.' (category=currency_amount)",
"trace_hash": null,
"verdict": "refused"
},
@ -306,7 +306,7 @@
"class": "refuse_unknown_compound",
"expected_answer": 0,
"expected_outcome": "refused",
"reason": "candidate_graph: no branch produced a solvable graph",
"reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Bob has five-and-a-half apples.' (category=descriptive_setup_no_quantity)",
"trace_hash": null,
"verdict": "refused"
},
@ -318,7 +318,7 @@
"class": "refuse_unknown_compound",
"expected_answer": 0,
"expected_outcome": "refused",
"reason": "candidate_graph: no branch produced a solvable graph",
"reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Bob has gobbledy-gook apples.' (category=descriptive_setup_no_quantity)",
"trace_hash": null,
"verdict": "refused"
},
@ -330,7 +330,7 @@
"class": "refuse_percentage",
"expected_answer": 0,
"expected_outcome": "refused",
"reason": "candidate_graph: no branch produced a solvable graph",
"reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Bob has 50% apples.' (category=discrete_count_statement)",
"trace_hash": null,
"verdict": "refused"
}