From cd0df2019f57a0fb1ebb7158ea12b294624216e5 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 18 Jul 2026 16:39:04 -0700 Subject: [PATCH] test(reader-arc): pin deterministic tune/measure split BEFORE grammar work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AMENDMENT 3 (PR #76): the reader grammar family is developed against the TUNE split only; increment deltas are measured on the disjoint MEASURE split only. Split = pure function of case id (measure iff sha256(id) even, else tune), so disjointness is enforced by construction and cannot drift. Fixed 2026-07-18, never re-seeded. Sizes: tune=261, measure=239. This commit lands BEFORE any grammar-tuning commit — the git history is the proof the split was pinned in advance (criterion 3). wrong=0 is verified across the full 500 regardless of split; only the coverage delta is attributed to the measure split. --- evals/gsm8k_math/holdout_dev/v1/split.py | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 evals/gsm8k_math/holdout_dev/v1/split.py diff --git a/evals/gsm8k_math/holdout_dev/v1/split.py b/evals/gsm8k_math/holdout_dev/v1/split.py new file mode 100644 index 00000000..31c9f4ca --- /dev/null +++ b/evals/gsm8k_math/holdout_dev/v1/split.py @@ -0,0 +1,25 @@ +"""Deterministic tune / measure split of the official holdout_dev/v1 (ADR-0250 reader arc). + +Pinned **before any grammar work** (compare increment plan §5, AMENDMENT 3): the reader grammar +family is developed against the **tune** split only; every increment's coverage delta is measured on +the disjoint **measure** split only. The assignment is a pure function of the case id, so it is +reproducible and cannot drift — disjointness is enforced by construction, not by discipline. + +Rule (fixed 2026-07-18, never re-seeded): a case is ``measure`` iff ``sha256(id)`` is even, else +``tune`` (≈ 50 / 50). `wrong=0` is verified across the **full 500** regardless of split; only the +*coverage delta* is attributed to the measure split. +""" +from __future__ import annotations + +import hashlib + +__all__ = ["split_of", "TUNE", "MEASURE"] + +TUNE = "tune" +MEASURE = "measure" + + +def split_of(case_id: str) -> str: + """Return ``"tune"`` or ``"measure"`` for ``case_id`` — deterministic, id-only.""" + digest = int(hashlib.sha256(case_id.encode("utf-8")).hexdigest(), 16) + return MEASURE if digest % 2 == 0 else TUNE