From 7d0803b45713bfc8f1ff6faf811d769c9929dab0 Mon Sep 17 00:00:00 2001 From: Shay Date: Sun, 24 May 2026 06:57:23 -0700 Subject: [PATCH] chore(eval): mark candidate-graph runner aggregation as needing audit (#213) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a 3-line TODO comment above `_score_one_candidate_graph` in evals/gsm8k_math/runner.py. No behavior change. Flags that `report.json` metrics may not credit candidate-graph admissions routed through this branch (Stage 1 candidate-graph parse + internal solve path) the same way `_score_one` admissions are credited. Aggregation in calling code needs an audit before the canonical run.honest_runner.json artifact can be trusted for cross-phase comparison. This is Piece A of a three-piece hygiene split. The MEMORY.md compaction and worktree audit pieces are deferred — they need human judgment (re-shaping vs. truncating) and an OS-correct date predicate (BSD vs. GNU), neither of which fits a one-shot script pass. No tests run — this change is comment-only and has zero runtime effect. --- evals/gsm8k_math/runner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/evals/gsm8k_math/runner.py b/evals/gsm8k_math/runner.py index 175cf0e1..85506254 100644 --- a/evals/gsm8k_math/runner.py +++ b/evals/gsm8k_math/runner.py @@ -205,6 +205,9 @@ def _score_one(case: dict[str, Any]) -> CaseOutcome: ) +# TODO(ADR-future): report.json metrics may not credit candidate-graph admissions +# routed through this branch. Aggregation in calling code needs an audit before +# the canonical run.honest_runner.json artifact can be trusted for cross-phase comparison. def _score_one_candidate_graph(case: dict[str, Any]) -> CaseOutcome: """ADR-0126 P4 — score one case via the candidate-graph pipeline.