From 7451e7cd7403d9dab31d2fc64ca4075225c8a692 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 29 May 2026 10:21:58 -0700 Subject: [PATCH] feat(adr-0177-cp2a): cue-precision ledger training + measurement (+ unit hygiene) (#461) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CP-2a populates the CP-1 ledger from gold-labelled candidate readings and reports per-pattern reliability — the measurement the cue-precision thesis rests on. Plus the function-word unit filter, whose value this measurement makes concrete (clean unit_shape labelling). What landed (all sealed; serving 3/47/0 byte-identical): - generate/cue_precision/trainer.py — train_from_cases(cases, enumerators): folds gold-labelled candidate chains into the ledger via record_case. Decoupled (the candidate enumerators are injected, so the package still imports nothing from search). candidates_for dedupes a reading shared by two enumerators. - generate/derivation/multistep.py — extracted the enumeration half of search_chain into public candidate_chains(problem_text); search_chain now delegates (verified byte-identical: ms3 tests + practice counts unchanged). CP-2 needs the readings the search weighs, not just the one it resolves. - generate/derivation/extract.py — function-word unit filter (_NON_UNIT_WORDS): blanks spurious function-word units ($0.75 each -> "", 3/4 of -> "") that corrupt same-unit detection and unit_shape. Closed lexeme set, ADR-0165-safe. - evals/gsm8k_math/practice/v1/cue_precision_report.py — trains over 200 sealed cases (50 train_sample + 150 ADR-0163-F additive) with the real enumerators and prints the per-pattern reliability table. - tests/test_adr_0177_cp2a_training.py — trainer obligations (credit/dedupe/ determinism/empty) via synthetic enumerators; real-measurement well-formedness; search_chain parity. Load-bearing finding (recorded in ADR-0177): over 200 cases EVERY (cue,op,unit_shape) pattern floors at ~0.0 reliability (best: for-multiply-cross_unit 0.0116 at 2/34). The blunt product/sum-of-all readings are almost always wrong vs gold, so the conservative floor correctly trusts nothing. => CP-2b (trust reliable cues) is blocked on candidate GENERATION, not the ledger: candidate readings must get less crude (clause/referent structure, ADR-0178 GB-3b) before any cue earns reliability. Cue-precision and compositional structure are coupled; structure comes first. Verification: 107 targeted tests green (CP-2a/CP-1/extract/ms3/GB-1/2/3/MS-1/2) + architectural invariants; serving CLAIMS.md sha unchanged; practice 4/1/45 and 0/1/149 unchanged. Inert: trains/reports only, consulted by no search/gate. --- .../ADR-0177-cue-precision-learning.md | 21 ++++ .../practice/v1/cue_precision_report.py | 73 ++++++++++++++ generate/cue_precision/__init__.py | 20 +++- generate/cue_precision/trainer.py | 68 +++++++++++++ generate/derivation/extract.py | 33 ++++++- generate/derivation/multistep.py | 36 ++++--- tests/test_adr_0177_cp2a_training.py | 99 +++++++++++++++++++ 7 files changed, 333 insertions(+), 17 deletions(-) create mode 100644 evals/gsm8k_math/practice/v1/cue_precision_report.py create mode 100644 generate/cue_precision/trainer.py create mode 100644 tests/test_adr_0177_cp2a_training.py diff --git a/docs/decisions/ADR-0177-cue-precision-learning.md b/docs/decisions/ADR-0177-cue-precision-learning.md index a614bc9e..83d9e7a1 100644 --- a/docs/decisions/ADR-0177-cue-precision-learning.md +++ b/docs/decisions/ADR-0177-cue-precision-learning.md @@ -146,6 +146,27 @@ Each needs a failing-under-violation test (CLAUDE.md §Schema-Defined Proof Obli only if its patterns clear `θ`; the search orders/prunes by reliability. Tests: invariant #1 (cold ⇒ no proposals, no regression); U2 never causes a wrong=0 violation. + - **CP-2a — ledger training + measurement (landed).** The training step + (`generate/cue_precision/trainer.py`) folds gold-labelled candidate readings + from the real search enumerators (`search._sentence_candidates` + + `multistep.candidate_chains`) into the CP-1 ledger; the measurement + (`evals/gsm8k_math/practice/v1/cue_precision_report.py`) reports per-pattern + reliability over the 200 sealed cases (50 train_sample + 150 ADR-0163-F + additive). Inert: trains/reports only, consulted by nobody — serving `3/47/0` + byte-identical, practice counts unchanged. `search_chain` now delegates + enumeration to the public `candidate_chains` (verified byte-identical). + - **CP-2a finding (load-bearing): no cue is reliable yet — CP-2b is blocked on + candidate *generation*, not on the ledger.** Trained over 200 cases, **every** + `(cue, op, unit_shape)` pattern floors at ≈ 0.0 (best: `for·multiply·cross_unit` + = 0.0116 at 2/34; `each·multiply` ≈ 0.006; `times·multiply` 0/57, `total·add` + 0/47). The blunt product/sum-of-all readings the search proposes are almost + always wrong vs gold, so the conservative floor correctly trusts nothing. The + lever is therefore **not** "trust high-reliability cues" (there are none) — it + is that the candidate readings must get *less crude* (clause structure + + referent-awareness, i.e. **ADR-0178 GB-3b**) before any pattern earns reliability. + Cue-precision (CP-2b) and compositional structure (GB-3b) are **coupled, and + structure comes first.** This is the ADR-0177 §"bottleneck" honesty, now + measured rather than asserted. (Table reproducible via the report; deterministic.) - **CP-3 — disagreement resolution (U3), wrong=0-first.** Margin+θ-gated resolution; **prove ties refuse before enabling resolution.** Measure any coverage delta. - **CP-4 — measurement + scale dependency.** Per-pattern reliability table; the diff --git a/evals/gsm8k_math/practice/v1/cue_precision_report.py b/evals/gsm8k_math/practice/v1/cue_precision_report.py new file mode 100644 index 00000000..3f95e743 --- /dev/null +++ b/evals/gsm8k_math/practice/v1/cue_precision_report.py @@ -0,0 +1,73 @@ +"""ADR-0177 CP-2a — the cue-precision measurement. + +Trains the CP-1 ledger over the gold-labelled sealed cases (train_sample + the +ADR-0163-F additive practice set) using the *real* search candidate enumerators, +then reports each ``(cue, op, unit_shape)`` pattern's commitment precision via the +pinned conservative floor. + +This is the diagnostic the cue-precision thesis rests on: it shows which cues are +*reliable* (high floor over enough committed trials) versus which are noise. CP-2b +consults this table to decide which patterns a self-verifying chain may be trusted +on; until then it is a read-only measurement that changes no search/gate behaviour +(serving stays ``3/47/0``). + +Deterministic and replay-stable: fixed case order, fixed enumerator order, the +ledger's canonical sorted storage. +""" + +from __future__ import annotations + +from generate.cue_precision.ledger import CuePrecisionLedger +from generate.cue_precision.trainer import TrainingCase, train_from_cases +from generate.derivation.multistep import candidate_chains +from generate.derivation.search import _sentence_candidates +from evals.gsm8k_math.practice.v1.runner import _load_practice_cases +from evals.gsm8k_math.train_sample.v1.runner import _CASES_PATH, _load_cases + +# The real readings the search weighs: the multiplicative per-sentence products +# (the lane the practice scorer attempts) and the multistep principled chains +# (which carry the additive candidates the ADR-0163-F set exercises). Fixed order. +_ENUMERATORS = (_sentence_candidates, candidate_chains) + + +def _training_cases() -> list[TrainingCase]: + """All gold-labelled sealed cases, train_sample first then practice. Stable.""" + cases: list[TrainingCase] = [] + for record in _load_cases(_CASES_PATH): + cases.append((record["question"], float(record["answer_numeric"]))) + for record in _load_practice_cases(): + cases.append((record["question"], float(record["answer_numeric"]))) + return cases + + +def build_cue_precision_ledger() -> CuePrecisionLedger: + """Train the ledger over every sealed case with the real enumerators.""" + return train_from_cases(_training_cases(), _ENUMERATORS) + + +def format_reliability_table(ledger: CuePrecisionLedger) -> str: + """A deterministic Markdown table of every learned pattern, sorted canonically.""" + lines = [ + "| cue | op | unit_shape | correct | wrong | committed | reliability |", + "|-----|----|-----------|--------:|------:|----------:|------------:|", + ] + for tally in ledger.tallies: # already canonically sorted + p = tally.pattern + lines.append( + f"| {p.cue} | {p.op} | {p.unit_shape} | {tally.correct} | " + f"{tally.wrong} | {tally.committed} | {tally.reliability:.4f} |" + ) + return "\n".join(lines) + + +def main() -> int: + ledger = build_cue_precision_ledger() + cases = _training_cases() + print(f"cue-precision ledger trained over {len(cases)} sealed cases") + print(f"distinct (cue, op, unit_shape) patterns: {len(ledger.tallies)}") + print(format_reliability_table(ledger)) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/generate/cue_precision/__init__.py b/generate/cue_precision/__init__.py index 7dafee84..03dd5b23 100644 --- a/generate/cue_precision/__init__.py +++ b/generate/cue_precision/__init__.py @@ -1,8 +1,8 @@ -"""ADR-0177 CP-1 — cue-precision reliability ledger substrate. +"""ADR-0177 CP-1/CP-2a — cue-precision reliability ledger + training. -Standalone, deterministic, replay-stable. **Inert**: NOT wired into the gate, any -scorer, or the search (that is CP-2/CP-3). Imported by nothing outside its own -tests — like ``core/reliability_gate/`` before its consumer existed. +Standalone, deterministic, replay-stable. **Inert w.r.t. the runtime path**: NOT +wired into the gate, any scorer, or the search (trust/guidance is CP-2b/CP-3). The +CP-2a trainer is consumed only by the sealed eval measurement, never by serving. Public surface: - :class:`CuePattern` — the ``(cue, op, unit_shape)`` reading key. @@ -13,6 +13,8 @@ Public surface: precision via the pinned ADR-0175 conservative floor. - :class:`CuePrecisionLedger` — immutable pattern->tally map + credit assignment (``record_chain`` / ``record_case``). +- :func:`train_from_cases`, :func:`candidates_for` (CP-2a) — fold gold-labelled + candidate readings (from injected enumerators) into a trained ledger. """ from __future__ import annotations @@ -27,14 +29,24 @@ from generate.cue_precision.ledger import ( pattern_for_step, patterns_in_chain, ) +from generate.cue_precision.trainer import ( + CandidateEnumerator, + TrainingCase, + candidates_for, + train_from_cases, +) __all__ = [ "CROSS_UNIT", + "CandidateEnumerator", "CuePattern", "CuePrecisionLedger", "PatternTally", "SAME_UNIT", + "TrainingCase", "UNIT_SHAPES", + "candidates_for", "pattern_for_step", "patterns_in_chain", + "train_from_cases", ] diff --git a/generate/cue_precision/trainer.py b/generate/cue_precision/trainer.py new file mode 100644 index 00000000..76d09155 --- /dev/null +++ b/generate/cue_precision/trainer.py @@ -0,0 +1,68 @@ +"""ADR-0177 CP-2a — populate the cue-precision ledger from gold-labelled cases. + +The CP-1 ledger (:mod:`generate.cue_precision.ledger`) is the mechanism; this is +the **training step** that gives it signal. For each ``(problem_text, gold)`` case +it gathers the candidate readings the search would consider, labels each by gold, +and folds the per-step ``(cue, op, unit_shape)`` credit into the ledger +(``record_case``). The result is the per-pattern reliability table — the +*measurement* CP-2b/CP-3 will consult before trusting a cue. + +Decoupled by construction: the candidate *enumerators* are injected (callables +``problem_text -> Iterable[GroundedDerivation]``), so this module imports nothing +from :mod:`generate.derivation.search` / ``multistep`` and stays as inert and +replay-stable as CP-1. The eval side (:mod:`evals.gsm8k_math...`) wires the real +enumerators to the real cases. + +wrong=0 posture: training reads gold (Tier-1, available in the sealed practice +regime only) and writes counts. It changes no search/gate behaviour — the ledger +is still consulted by nobody until CP-2b. Serving stays ``3/47/0``. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable + +from generate.cue_precision.ledger import CuePrecisionLedger +from generate.derivation.model import GroundedDerivation + +# A candidate enumerator turns a problem into the readings the search considers. +CandidateEnumerator = Callable[[str], Iterable[GroundedDerivation]] + +# A training case: the problem text and its gold numeric answer. +TrainingCase = tuple[str, float] + + +def candidates_for( + problem_text: str, enumerators: Iterable[CandidateEnumerator] +) -> tuple[GroundedDerivation, ...]: + """The deduplicated union of every enumerator's candidates, in stable order. + + A reading produced by two enumerators is counted **once** per case (per-step + credit already counts each pattern occurrence within a chain; double-counting + the whole chain across enumerators would inflate the same evidence twice). + Dedup preserves first-seen order, so the fold is deterministic. + """ + seen: dict[GroundedDerivation, None] = {} + for enumerate_candidates in enumerators: + for candidate in enumerate_candidates(problem_text): + seen.setdefault(candidate, None) + return tuple(seen) + + +def train_from_cases( + cases: Iterable[TrainingCase], + enumerators: Iterable[CandidateEnumerator], +) -> CuePrecisionLedger: + """Fold every case's gold-labelled candidates into a fresh ledger. + + Deterministic in ``cases`` order, ``enumerators`` order, and each enumerator's + own candidate order. A case with no candidate contributes nothing (no refusal + penalty — the ledger only labels readings, ADR-0177). + """ + enumerator_tuple = tuple(enumerators) + ledger = CuePrecisionLedger() + for problem_text, gold in cases: + candidates = candidates_for(problem_text, enumerator_tuple) + if candidates: + ledger = ledger.record_case(candidates, float(gold)) + return ledger diff --git a/generate/derivation/extract.py b/generate/derivation/extract.py index 89871c3d..27751349 100644 --- a/generate/derivation/extract.py +++ b/generate/derivation/extract.py @@ -23,6 +23,13 @@ this module, so none of this can move the serving ``3/47/0``): * **EX-5 — sentence-final numbers.** A number with no following unit word (end of sentence/text or before terminal punctuation) extracts with an empty unit so it stays available to the completeness check without inventing a unit lexeme. +* **Unit hygiene (function-word filter).** When the token after a number is a + function word (``$0.75 each`` → ``each``, ``$40 to go`` → ``to``, ``3/4 of`` → + ``of``), the single-word unit pattern would tag it as the unit — a spurious unit + that corrupts same-unit detection (GB-2/GB-3) and CP-1's ``unit_shape``. Such + units are blanked (empty, like a sentence-final number): the value stays + grounded, the unit is honestly unknown. Closed lexeme set (``_NON_UNIT_WORDS``), + not a grammar template (ADR-0165). EX-3 (multi-word units) is deliberately **not** integrated. Two distinct traps defeat the tightest lookahead-anchored rule the brief admits: @@ -88,13 +95,35 @@ _FINAL_NUMBER_RE: Final[re.Pattern[str]] = re.compile( ) +# Function words that are never units. When the token immediately after a number +# is one of these (``$0.75 each``, ``$40 to go``, ``3/4 of``), the single-word unit +# pattern would otherwise tag the function word as the unit — a spurious unit that +# corrupts same-unit detection (GB-2/GB-3) and CP-1's unit_shape. Emitting an empty +# unit instead (like a sentence-final number) is honest: the value is grounded, the +# unit is simply unknown. Closed lexeme set (cf. ``WORD_NUMBERS``); ADR-0165-safe — +# it names tokens that are not unit nouns, it does not parse sentence structure. +_NON_UNIT_WORDS: Final[frozenset[str]] = frozenset( + { + "a", "an", "the", "of", "to", "for", "in", "on", "at", "as", "than", + "per", "each", "every", "and", "or", "with", "by", "from", "more", + "less", "about", "that", + } +) + + +def _clean_unit(unit: str) -> str: + """Lowercase a unit token; blank it if it is a non-unit function word.""" + lowered = unit.lower() + return "" if lowered in _NON_UNIT_WORDS else lowered + + def _quantity(value_token: str, unit: str) -> Quantity | None: """Build a quantity from an already-matched numeric token.""" try: value = float(value_token) except ValueError: # pragma: no cover - regex guarantees numeric return None - return Quantity(value=value, unit=unit.lower(), source_token=value_token) + return Quantity(value=value, unit=_clean_unit(unit), source_token=value_token) def _resolve_word_number(first: str, second: str | None) -> float | None: @@ -172,7 +201,7 @@ def extract_quantities(problem_text: str) -> tuple[Quantity, ...]: found.append( ( match.start(1), - Quantity(value=value, unit=match.group(3).lower(), source_token=source_token), + Quantity(value=value, unit=_clean_unit(match.group(3)), source_token=source_token), ) ) diff --git a/generate/derivation/multistep.py b/generate/derivation/multistep.py index e0e329bd..92558281 100644 --- a/generate/derivation/multistep.py +++ b/generate/derivation/multistep.py @@ -65,6 +65,27 @@ def _candidate_chains( return candidates +def candidate_chains( + problem_text: str, target: Target | None = None +) -> list[GroundedDerivation]: + """The bounded, deterministic candidate readings :func:`search_chain` weighs. + + The *enumeration* half of the search, with no gate applied: extract quantities, + refuse-on-overflow (``> MAX_QUANTITIES``) / too-few (``< 2``) by yielding no + candidates, derive the target, and build the principled set. Exposed for CP-2 + ledger training (ADR-0177), which must see every reading the search considers — + not just the one it resolves to. ``search_chain`` delegates here, so the two + can never drift. + """ + quantities = list(extract_quantities(problem_text)) + if not 2 <= len(quantities) <= MAX_QUANTITIES: + return [] # refuse-on-overflow / too few to compose + resolved: Target = target if target is not None else extract_target( + problem_text, known_units=tuple(q.unit for q in quantities) + ) + return _candidate_chains(quantities, problem_text, resolved) + + def search_chain(problem_text: str, target: Target | None = None) -> Resolution | None: """Target-guided bounded multi-step search over the problem's quantities. @@ -72,18 +93,11 @@ def search_chain(problem_text: str, target: Target | None = None) -> Resolution (and, when a target unit is known, matches it); refuses on no candidate, disagreement, or > :data:`MAX_QUANTITIES` quantities. Deterministic. """ - quantities = list(extract_quantities(problem_text)) - if not 2 <= len(quantities) <= MAX_QUANTITIES: - return None # refuse-on-overflow / too few to compose - - resolved: Target = target if target is not None else extract_target( - problem_text, known_units=tuple(q.unit for q in quantities) - ) - candidates = _candidate_chains(quantities, problem_text, resolved) # Target-UNIT matching is deferred: the model's answer_unit is the start # quantity's unit, which is wrong for cross-unit products (6 boxes x 50 apples # -> value 300 but unit "boxes"), so a unit gate would over-refuse correct # answers. The Target still prunes via its aggregation cue + supplies the - # question quantities (above). Unit matching returns once a result-unit model - # exists (a superordinate-units pack + product-unit inference). - return select_self_verified(candidates, problem_text, target_units=()) + # question quantities. Unit matching returns once a result-unit model exists. + return select_self_verified( + candidate_chains(problem_text, target), problem_text, target_units=() + ) diff --git a/tests/test_adr_0177_cp2a_training.py b/tests/test_adr_0177_cp2a_training.py new file mode 100644 index 00000000..2df63194 --- /dev/null +++ b/tests/test_adr_0177_cp2a_training.py @@ -0,0 +1,99 @@ +"""ADR-0177 CP-2a — ledger training + the cue-precision measurement. + +Covers the decoupled trainer (synthetic enumerators, so the obligations are proven +without relying on search internals) and an integration check that the real +measurement builds a non-empty, well-formed ledger over the sealed cases. +""" + +from __future__ import annotations + +from generate.cue_precision.ledger import CuePattern, CuePrecisionLedger +from generate.cue_precision.trainer import candidates_for, train_from_cases +from generate.derivation.model import GroundedDerivation, Quantity, Step + + +def _chain(start_val: float, start_unit: str, op: str, operand_val: float, operand_unit: str, cue: str) -> GroundedDerivation: + return GroundedDerivation( + start=Quantity(start_val, start_unit, str(start_val)), + steps=(Step(op=op, operand=Quantity(operand_val, operand_unit, str(operand_val)), cue=cue),), + ) + + +class TestTrainFromCases: + def test_matching_chain_credits_correct(self) -> None: + # 6 boxes * 4 (apples) = 24 == gold -> pattern (each, multiply, cross_unit) +correct. + chain = _chain(6.0, "boxes", "multiply", 4.0, "apples", "each") + ledger = train_from_cases([("6 boxes 4 apples each", 24.0)], [lambda _t: [chain]]) + tally = ledger.tally_for(CuePattern(cue="each", op="multiply", unit_shape="cross_unit")) + assert (tally.correct, tally.wrong) == (1, 0) + + def test_mismatching_chain_credits_wrong(self) -> None: + # same chain, gold != 24 -> the pattern is credited wrong (the elimination signal). + chain = _chain(6.0, "boxes", "multiply", 4.0, "apples", "each") + ledger = train_from_cases([("6 boxes 4 apples each", 999.0)], [lambda _t: [chain]]) + tally = ledger.tally_for(CuePattern(cue="each", op="multiply", unit_shape="cross_unit")) + assert (tally.correct, tally.wrong) == (0, 1) + + def test_same_chain_from_two_enumerators_counted_once(self) -> None: + chain = _chain(2.0, "x", "add", 3.0, "x", "and") + ledger = train_from_cases( + [("2 x and 3 x", 5.0)], + [lambda _t: [chain], lambda _t: [chain]], # two enumerators, same reading + ) + tally = ledger.tally_for(CuePattern(cue="and", op="add", unit_shape="same_unit")) + assert tally.committed == 1 # deduped, not 2 + + def test_deterministic(self) -> None: + chain = _chain(6.0, "boxes", "multiply", 4.0, "apples", "each") + cases = [("a", 24.0), ("b", 1.0)] + enums = [lambda _t: [chain]] + assert train_from_cases(cases, enums) == train_from_cases(cases, enums) + + def test_no_candidates_yields_empty_ledger(self) -> None: + ledger = train_from_cases([("nothing here", 0.0)], [lambda _t: []]) + assert ledger == CuePrecisionLedger() + + def test_candidates_for_dedupes_preserving_order(self) -> None: + a = _chain(1.0, "u", "add", 2.0, "u", "and") + b = _chain(1.0, "u", "multiply", 2.0, "u", "each") + got = candidates_for("t", [lambda _t: [a, b], lambda _t: [a]]) + assert got == (a, b) + + +class TestRealMeasurement: + def test_ledger_builds_nonempty_and_well_formed(self) -> None: + from evals.gsm8k_math.practice.v1.cue_precision_report import ( + build_cue_precision_ledger, + format_reliability_table, + ) + + ledger = build_cue_precision_ledger() + assert len(ledger.tallies) >= 1 # the search produces *some* labelled patterns + for tally in ledger.tallies: + assert 0.0 <= tally.reliability < 1.0 # conservative floor is in [0, 1) + assert tally.committed == tally.correct + tally.wrong + # deterministic render + assert format_reliability_table(ledger) == format_reliability_table( + build_cue_precision_ledger() + ) + + +class TestSearchChainParity: + """The candidate_chains refactor must not change search_chain behaviour.""" + + def test_decimal_product_still_resolves(self) -> None: + from generate.derivation.multistep import search_chain + + text = ( + "There are 48 boxes with 24 erasers in each box. " + "They sell the erasers for $0.75 each. How much money will they make?" + ) + res = search_chain(text) + assert res is not None and res.answer == 864.0 + + def test_candidate_chains_is_enumeration_only(self) -> None: + from generate.derivation.multistep import candidate_chains, search_chain + + # too-few quantities -> no candidates -> search refuses + assert candidate_chains("She has 6 apples.") == [] + assert search_chain("She has 6 apples.") is None