{
  "adr": "0131.G.3.1",
  "axis": "numerics_extensions",
  "cases_path": "evals/math_capability_axes/G3_numerics/v1_1/cases.jsonl",
  "class_counts": {
    "fraction_of_unit": 5,
    "multi_currency": 5,
    "multi_word_cardinal": 5,
    "refuse_locale_separator": 2,
    "refuse_money_precision": 2,
    "refuse_percentage": 2,
    "refuse_scientific_notation": 2,
    "word_num_adjective": 5
  },
  "metrics": {
    "cases_total": 28,
    "correct_rate_on_positive_cases": 1.0,
    "overall_pass": true,
    "refused_as_expected": 8,
    "solved_correct": 20,
    "solved_wrong": 0,
    "wrong_count_is_zero": true
  },
  "per_case": [
    {
      "actual_answer": 0.5,
      "actual_outcome": "correct",
      "actual_unit": "cups",
      "case_id": "g31-frac-01",
      "class": "fraction_of_unit",
      "expected_answer": 0.5,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "b1b05ff2759cb6cf5d630d2396eb9f489d71043a3eb12a4c2cdd88062ad64848",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 0.75,
      "actual_outcome": "correct",
      "actual_unit": "bags",
      "case_id": "g31-frac-02",
      "class": "fraction_of_unit",
      "expected_answer": 0.75,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "f64e99dbc03193d143a151ac5c7e9e283ba9e4b7dfea676410cba2dcf92cba74",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 0.25,
      "actual_outcome": "correct",
      "actual_unit": "pies",
      "case_id": "g31-frac-03",
      "class": "fraction_of_unit",
      "expected_answer": 0.25,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "53c696733759f9c0c71d96fc004614d70494dabe0a63226ac73155b5942aabdb",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 1.5,
      "actual_outcome": "correct",
      "actual_unit": "liters",
      "case_id": "g31-frac-04",
      "class": "fraction_of_unit",
      "expected_answer": 1.5,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "9ff18f4ee0f7797e3972c9d5e088775b53acfaf28b622d65c8a88fa1309abc12",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 0.8,
      "actual_outcome": "correct",
      "actual_unit": "cups",
      "case_id": "g31-frac-05",
      "class": "fraction_of_unit",
      "expected_answer": 0.8,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "92e90758cc1ee3dab91dd8610dfed35b2c616e64bf95e2ed682467c9849eec07",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 50.0,
      "actual_outcome": "correct",
      "actual_unit": "cents",
      "case_id": "g31-cur-01",
      "class": "multi_currency",
      "expected_answer": 50.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "490bbc2f0384acc8f94a2740f0e37572a6d2036692edd8b7ff77de80785cbbff",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 20.0,
      "actual_outcome": "correct",
      "actual_unit": "euros",
      "case_id": "g31-cur-02",
      "class": "multi_currency",
      "expected_answer": 20.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "d1da3f9a8af64633231a2157593e4d571de973f815801172ef8dac1c40e4bb1f",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 100.0,
      "actual_outcome": "correct",
      "actual_unit": "yen",
      "case_id": "g31-cur-03",
      "class": "multi_currency",
      "expected_answer": 100.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "c9a074741863a38a4d58fe4bfc17c18ff220b4172cf8541b3cc20d36be2dba9d",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 200.0,
      "actual_outcome": "correct",
      "actual_unit": "pesos",
      "case_id": "g31-cur-04",
      "class": "multi_currency",
      "expected_answer": 200.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "38597a8bef5b6e08134e3ca2448eb4199daec93a24e0358da932133a02938c36",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 20.0,
      "actual_outcome": "correct",
      "actual_unit": "euros",
      "case_id": "g31-cur-05",
      "class": "multi_currency",
      "expected_answer": 20.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "c313f6ad6fe05d09425942fb4e5022014cfd78fdd3d297d6a2349cc719181ac8",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 100.0,
      "actual_outcome": "correct",
      "actual_unit": "apples",
      "case_id": "g31-mwc-01",
      "class": "multi_word_cardinal",
      "expected_answer": 100.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "0023bf2b389b79528556e91cb984a0d6d22b3b9b95b61d3ae73414fa93ead560",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 1000.0,
      "actual_outcome": "correct",
      "actual_unit": "books",
      "case_id": "g31-mwc-02",
      "class": "multi_word_cardinal",
      "expected_answer": 1000.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "7c44c06806a79fd3d797d42b4311688bab66d8a62d7dd09606457faacaa11364",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 300.0,
      "actual_outcome": "correct",
      "actual_unit": "cookies",
      "case_id": "g31-mwc-03",
      "class": "multi_word_cardinal",
      "expected_answer": 300.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "e7a76500137f6f7936a904f6b916dc310defbbd8bdfc87727e98cdf7685187c0",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 2500.0,
      "actual_outcome": "correct",
      "actual_unit": "marbles",
      "case_id": "g31-mwc-04",
      "class": "multi_word_cardinal",
      "expected_answer": 2500.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "30b22bade97f050d89549af89c9d7512e69facab1346c584bb22c1e6c25e2ef7",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 50000.0,
      "actual_outcome": "correct",
      "actual_unit": "cents",
      "case_id": "g31-mwc-05",
      "class": "multi_word_cardinal",
      "expected_answer": 50000.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "7ca5b0470f11ed2a806e95982211954202f99e48c707780431df2522f01b871f",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 5.0,
      "actual_outcome": "correct",
      "actual_unit": "boxes",
      "case_id": "g31-adj-01",
      "class": "word_num_adjective",
      "expected_answer": 5.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "80f43f99acfea39c9226f1a6bc46d56f950a799b698e4eb6bc4be086046828aa",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 3.0,
      "actual_outcome": "correct",
      "actual_unit": "crayons",
      "case_id": "g31-adj-02",
      "class": "word_num_adjective",
      "expected_answer": 3.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "46fcecab0fd1a1ab4db5b2bc3033a0d680f3919eef992714415cbdbcae5acc03",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 7.0,
      "actual_outcome": "correct",
      "actual_unit": "cans",
      "case_id": "g31-adj-03",
      "class": "word_num_adjective",
      "expected_answer": 7.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "0d10b1e8251b5298ed826c8c10526f0e28c54d443b6256c10c9be6f8190cab09",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 12.0,
      "actual_outcome": "correct",
      "actual_unit": "pies",
      "case_id": "g31-adj-04",
      "class": "word_num_adjective",
      "expected_answer": 12.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "74083e671f30421f0a968f1f8d4314f56b1a255854a757c6ab6f2eddafa74d0c",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": 8.0,
      "actual_outcome": "correct",
      "actual_unit": "books",
      "case_id": "g31-adj-05",
      "class": "word_num_adjective",
      "expected_answer": 8.0,
      "expected_outcome": "solved_correct",
      "reason": "",
      "trace_hash": "ba2b359492336fc2fcc71a0fde24f05a4fa446133c6530dc13bbce46ec577dc4",
      "verdict": "solved_correct"
    },
    {
      "actual_answer": null,
      "actual_outcome": "refused",
      "actual_unit": null,
      "case_id": "g31-ref-pct-01",
      "class": "refuse_percentage",
      "expected_answer": 0,
      "expected_outcome": "refused",
      "reason": "candidate_graph: no admissible candidate for statement: 'Bob has 50% apples.'",
      "trace_hash": null,
      "verdict": "refused"
    },
    {
      "actual_answer": null,
      "actual_outcome": "refused",
      "actual_unit": null,
      "case_id": "g31-ref-pct-02",
      "class": "refuse_percentage",
      "expected_answer": 0,
      "expected_outcome": "refused",
      "reason": "candidate_graph: no admissible candidate for statement: 'Sam has 75% of a pie.'",
      "trace_hash": null,
      "verdict": "refused"
    },
    {
      "actual_answer": null,
      "actual_outcome": "refused",
      "actual_unit": null,
      "case_id": "g31-ref-sci-01",
      "class": "refuse_scientific_notation",
      "expected_answer": 0,
      "expected_outcome": "refused",
      "reason": "candidate_graph: no admissible candidate for statement: 'Sam has 1e3 marbles.'",
      "trace_hash": null,
      "verdict": "refused"
    },
    {
      "actual_answer": null,
      "actual_outcome": "refused",
      "actual_unit": null,
      "case_id": "g31-ref-sci-02",
      "class": "refuse_scientific_notation",
      "expected_answer": 0,
      "expected_outcome": "refused",
      "reason": "candidate_graph: no admissible candidate for statement: 'Alice has 2.5e2 books.'",
      "trace_hash": null,
      "verdict": "refused"
    },
    {
      "actual_answer": null,
      "actual_outcome": "refused",
      "actual_unit": null,
      "case_id": "g31-ref-locale-01",
      "class": "refuse_locale_separator",
      "expected_answer": 0,
      "expected_outcome": "refused",
      "reason": "candidate_graph: no admissible candidate for statement: 'Alice has 1,000 pennies.'",
      "trace_hash": null,
      "verdict": "refused"
    },
    {
      "actual_answer": null,
      "actual_outcome": "refused",
      "actual_unit": null,
      "case_id": "g31-ref-locale-02",
      "class": "refuse_locale_separator",
      "expected_answer": 0,
      "expected_outcome": "refused",
      "reason": "candidate_graph: no admissible candidate for statement: 'Bob has 10,000 apples.'",
      "trace_hash": null,
      "verdict": "refused"
    },
    {
      "actual_answer": null,
      "actual_outcome": "refused",
      "actual_unit": null,
      "case_id": "g31-ref-3dec-01",
      "class": "refuse_money_precision",
      "expected_answer": 0,
      "expected_outcome": "refused",
      "reason": "candidate_graph: no admissible candidate for statement: 'Bob has $1.234.'",
      "trace_hash": null,
      "verdict": "refused"
    },
    {
      "actual_answer": null,
      "actual_outcome": "refused",
      "actual_unit": null,
      "case_id": "g31-ref-3dec-02",
      "class": "refuse_money_precision",
      "expected_answer": 0,
      "expected_outcome": "refused",
      "reason": "candidate_graph: no admissible candidate for statement: 'Sam has \u20ac5.678.'",
      "trace_hash": null,
      "verdict": "refused"
    }
  ],
  "schema_version": 1,
  "verdict_counts": {
    "refused": 8,
    "solved_correct": 20
  }
}
