{
  "adr": "0126",
  "counts": {
    "correct": 30,
    "refused": 20,
    "wrong": 0
  },
  "exit_criterion": {
    "correct_min": 10,
    "passed": true,
    "wrong_max": 0
  },
  "per_case": [
    {
      "case_id": "gsm8k-train-sample-v1-0001",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0002",
      "reason": "",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0003",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0004",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0005",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0006",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0007",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0008",
      "reason": "",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0009",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0010",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0011",
      "reason": "candidate_graph: no branch produced a solvable graph",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0012",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'He put all of them in his aquarium but his fish ate half of them.' (category=descriptive_setup_no_quantity)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0013",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0014",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0015",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0016",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0017",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0018",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0019",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'After the first appointment, John paid $100 for pet insurance that covers 80% of the subsequent visits.' (category=currency_amount)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0020",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Two puppies, two kittens, and three parakeets were for sale at the pet shop.' (category=discrete_count_statement)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0021",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0022",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'She goes out fishing today and catches twice as many fish as she caught in total in the past four months.' (category=discrete_count_statement)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0023",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: \"Cindy collected twice as many, and Rex collected half of Nicole and Cindy's combined total.\" (category=descriptive_setup_no_quantity)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0024",
      "reason": "",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0025",
      "reason": "",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0026",
      "reason": "candidate_graph: no admissible candidate for statement: 'Aaron and his brother Carson each saved up $40 to go to dinner.'",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0027",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0028",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'It cost $100,000 to open initially.' (category=currency_amount)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0029",
      "reason": "",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0030",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0031",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Jeremie wants to go to an amusement park with 3 friends at the end of summer.' (category=discrete_count_statement)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0032",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'He draws and colors 10 pictures.' (category=discrete_count_statement)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0033",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Rachel is 12 years old, and her grandfather is 7 times her age.' (category=discrete_count_statement)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0034",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0035",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0036",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'She studied for 2 hours on Wednesday and three times as long on Thursday.' (category=discrete_count_statement)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0037",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0038",
      "reason": "",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0039",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0040",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'He now has 2 horses, 5 dogs, 7 cats, 3 turtles, and 1 goat.' (category=discrete_count_statement)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0041",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'The guests eat all of 1 pan, and 75% of the 2nd pan.' (category=discrete_count_statement)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0042",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0043",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'She saved $10 for this purpose.' (category=currency_amount)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0044",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'John invests in a bank and gets 10% simple interest.' (category=discrete_count_statement)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0045",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0046",
      "reason": "fast-path",
      "verdict": "correct"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0047",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'He then packs an equal number of the macaroons in 4 different brown bags, ready for delivery.' (category=discrete_count_statement)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0048",
      "reason": "candidate_graph: no admissible candidate for statement: 'Every week, he gets 6 cards.'",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0049",
      "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'Malcolm is trying to find the fastest walk to school and is currently comparing two routes.' (category=discrete_count_statement)",
      "verdict": "refused"
    },
    {
      "case_id": "gsm8k-train-sample-v1-0050",
      "reason": "candidate_graph: no admissible candidate for statement: 'Mark does a gig every other day for 2 weeks.'",
      "verdict": "refused"
    }
  ],
  "sample_count": 50,
  "sample_path": "evals/gsm8k_math/train_sample/v1/cases.jsonl",
  "schema_version": 1
}
