feat(adr-0177-cp1): cue-precision reliability ledger substrate (inert) (#458)
CP-1 of ADR-0177: the per-(cue, op, unit_shape) reliability ledger + credit
assignment mechanism. Mirrors the ADR-0175 per-class ledger discipline
(core/reliability_gate/ledger.py): counts-only integers, reliability via the
pinned conservative_floor, refusals never counted as commitments.
- generate/cue_precision/ledger.py
- CuePattern: (cue, op, unit_shape) key; op in VALID_OPS, unit_shape closed-set.
- pattern_for_step / patterns_in_chain: per-step extraction. unit_shape compares
the operand unit to the model's running (primary/start) unit; a dimensionless
comparative scalar scales within the dimension -> same_unit.
- PatternTally: counts-only (correct/wrong, no refused axis); reliability =
conservative_floor(correct, committed); 0.0 while cold/below N_MIN.
- CuePrecisionLedger: immutable pattern->tally map (canonical sorted tuple);
record_chain / record_case credit candidate chains by gold label, independent
of whether the search resolved or refused.
Inert substrate: not wired into the gate, any scorer, or the search (CP-2/CP-3).
Imported by nothing outside its own tests (asserted by a source-tree scan).
Tests (tests/test_adr_0177_cp1_ledger.py, 27 passing): pattern validation;
unit_shape classification; cold ledger -> 0 reliability; credit assignment;
refusals-not-counted; reliability earned by volume; determinism/replay;
immutability; inertness scan. Smoke suite green (67 passed).
This commit is contained in:
parent
5c1e9e7fe4
commit
1f559344ca
3 changed files with 587 additions and 0 deletions
40
generate/cue_precision/__init__.py
Normal file
40
generate/cue_precision/__init__.py
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
"""ADR-0177 CP-1 — cue-precision reliability ledger substrate.
|
||||
|
||||
Standalone, deterministic, replay-stable. **Inert**: NOT wired into the gate, any
|
||||
scorer, or the search (that is CP-2/CP-3). Imported by nothing outside its own
|
||||
tests — like ``core/reliability_gate/`` before its consumer existed.
|
||||
|
||||
Public surface:
|
||||
- :class:`CuePattern` — the ``(cue, op, unit_shape)`` reading key.
|
||||
- :data:`UNIT_SHAPES`, :data:`CROSS_UNIT`, :data:`SAME_UNIT` — the unit-shape set.
|
||||
- :func:`pattern_for_step`, :func:`patterns_in_chain` — extract patterns from a
|
||||
grounded derivation.
|
||||
- :class:`PatternTally` — per-pattern counted ledger; reliability = commitment
|
||||
precision via the pinned ADR-0175 conservative floor.
|
||||
- :class:`CuePrecisionLedger` — immutable pattern->tally map + credit assignment
|
||||
(``record_chain`` / ``record_case``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from generate.cue_precision.ledger import (
|
||||
CROSS_UNIT,
|
||||
SAME_UNIT,
|
||||
UNIT_SHAPES,
|
||||
CuePattern,
|
||||
CuePrecisionLedger,
|
||||
PatternTally,
|
||||
pattern_for_step,
|
||||
patterns_in_chain,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CROSS_UNIT",
|
||||
"CuePattern",
|
||||
"CuePrecisionLedger",
|
||||
"PatternTally",
|
||||
"SAME_UNIT",
|
||||
"UNIT_SHAPES",
|
||||
"pattern_for_step",
|
||||
"patterns_in_chain",
|
||||
]
|
||||
224
generate/cue_precision/ledger.py
Normal file
224
generate/cue_precision/ledger.py
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
"""ADR-0177 CP-1 — the per-cue-pattern reliability ledger + credit assignment.
|
||||
|
||||
A replayable tally of *counted* gold-labels per **cue-pattern**, mirroring the
|
||||
ADR-0175 per-class ledger (:mod:`core.reliability_gate.ledger`) but keyed on a
|
||||
``(cue, op, unit_shape)`` pattern string instead of a capability axis. Nothing
|
||||
learned, nothing stochastic — every figure is an integer count, and reliability
|
||||
is the same pinned :func:`conservative_floor` (commitment precision, earned by
|
||||
volume, never by a lucky streak).
|
||||
|
||||
This is **inert substrate** (ADR-0177 §"Recommended sequencing" CP-1): the
|
||||
mechanism + credit assignment only. It is **not** wired into the gate, any
|
||||
scorer, or the search (that is CP-2/CP-3). It is imported by nothing outside its
|
||||
own tests, exactly as ``core/reliability_gate/`` shipped before its consumer.
|
||||
|
||||
Credit assignment (ADR-0177 §"Credit assignment"): for a practice case the search
|
||||
emits candidate chains; each chain is labelled by gold (``answer == gold``); for
|
||||
**every step's pattern** in a chain we record ``+correct`` if the chain matched
|
||||
gold else ``+wrong``. Learning does **not** depend on the search *resolving* — it
|
||||
labels candidates, separate from the resolve/refuse decision (ADR-0177 §"The
|
||||
mechanism"). A case-level *refusal* is therefore never counted as a commitment:
|
||||
the ledger only ever sees gold-labelled candidate chains, and a pattern tally has
|
||||
no "refused" axis at all.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Final, Iterable
|
||||
|
||||
from core.reliability_gate.floor import conservative_floor
|
||||
from generate.derivation.model import VALID_OPS, GroundedDerivation, Step
|
||||
|
||||
# A step either stays within the running unit dimension or crosses into another.
|
||||
CROSS_UNIT: Final[str] = "cross_unit"
|
||||
SAME_UNIT: Final[str] = "same_unit"
|
||||
UNIT_SHAPES: Final[frozenset[str]] = frozenset({CROSS_UNIT, SAME_UNIT})
|
||||
|
||||
# Replay rounding for gold comparison — identical to the verify gate's notion of
|
||||
# "same answer" (generate/derivation/verify.py uses round(answer, 9)).
|
||||
_GOLD_DECIMALS: Final[int] = 9
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CuePattern:
|
||||
"""A ``(cue, op, unit_shape)`` reading the search asserts the text licenses.
|
||||
|
||||
``cue`` is the surface lexeme licensing ``op`` (e.g. ``"per"``); ``op`` is a
|
||||
:data:`generate.derivation.model.VALID_OPS` member; ``unit_shape`` records
|
||||
whether the operation crosses units (ADR-0177 §"Pattern key" — cross-unit
|
||||
multiplication is the aggregate signal).
|
||||
"""
|
||||
|
||||
cue: str
|
||||
op: str
|
||||
unit_shape: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.cue, str) or not self.cue:
|
||||
raise ValueError("cue must be a non-empty str")
|
||||
if self.op not in VALID_OPS:
|
||||
raise ValueError(f"op must be one of {sorted(VALID_OPS)}, got {self.op!r}")
|
||||
if self.unit_shape not in UNIT_SHAPES:
|
||||
raise ValueError(
|
||||
f"unit_shape must be one of {sorted(UNIT_SHAPES)}, got {self.unit_shape!r}"
|
||||
)
|
||||
|
||||
|
||||
def _unit_shape(running_unit: str, operand_unit: str) -> str:
|
||||
"""Classify a step's unit shape against the running (primary) unit.
|
||||
|
||||
The value model keeps the primary (``start``) unit through the whole fold
|
||||
(``GroundedDerivation.answer_unit == start.unit``), so the running unit is the
|
||||
start unit at every step. A dimensionless operand (a comparative scalar carries
|
||||
``unit == ""``) *scales within* the current dimension — ``twice as many apples``
|
||||
stays apples — so it reads :data:`SAME_UNIT`, not a cross-unit aggregate. The
|
||||
gate already forces add/subtract operands to share the primary unit, so only
|
||||
multiply/divide can ever be :data:`CROSS_UNIT`.
|
||||
"""
|
||||
if operand_unit == "" or operand_unit == running_unit:
|
||||
return SAME_UNIT
|
||||
return CROSS_UNIT
|
||||
|
||||
|
||||
def pattern_for_step(derivation: GroundedDerivation, step: Step) -> CuePattern:
|
||||
"""The :class:`CuePattern` a single step contributes within ``derivation``."""
|
||||
return CuePattern(
|
||||
cue=step.cue,
|
||||
op=step.op,
|
||||
unit_shape=_unit_shape(derivation.start.unit, step.operand.unit),
|
||||
)
|
||||
|
||||
|
||||
def patterns_in_chain(derivation: GroundedDerivation) -> tuple[CuePattern, ...]:
|
||||
"""Every step's pattern, in step order. Each *occurrence* counts (ADR-0177
|
||||
credit assignment is per-step, so a 3-step product-of-all credits its pattern
|
||||
three times — reliability is earned by clean appearances)."""
|
||||
return tuple(pattern_for_step(derivation, step) for step in derivation.steps)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PatternTally:
|
||||
"""Immutable per-pattern outcome counts.
|
||||
|
||||
Mirrors :class:`core.reliability_gate.ledger.ClassTally`: counts-only,
|
||||
reliability is commitment precision via the pinned conservative floor. There
|
||||
is **no** refused axis — a candidate chain is always a gold-labelled
|
||||
commitment; case-level refusals are never recorded here (ADR-0177).
|
||||
"""
|
||||
|
||||
pattern: CuePattern
|
||||
correct: int = 0
|
||||
wrong: int = 0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for value in (self.correct, self.wrong):
|
||||
if not isinstance(value, int) or value < 0:
|
||||
raise ValueError("tally counts must be non-negative ints")
|
||||
|
||||
@property
|
||||
def committed(self) -> int:
|
||||
"""Gold-labelled candidate-chain appearances of this pattern."""
|
||||
return self.correct + self.wrong
|
||||
|
||||
@property
|
||||
def reliability(self) -> float:
|
||||
"""Conservative lower bound on commitment precision (ADR-0175 §4a floor).
|
||||
|
||||
``0.0`` for a cold/low pattern (below ``N_MIN`` committed): a cold ledger
|
||||
trusts nothing, which is the wrong=0 safety property CP-2 will rely on.
|
||||
"""
|
||||
return conservative_floor(self.correct, self.committed)
|
||||
|
||||
def record(self, *, correct: int = 0, wrong: int = 0) -> "PatternTally":
|
||||
"""Return a new tally with the given outcomes added (immutable update)."""
|
||||
return PatternTally(
|
||||
pattern=self.pattern,
|
||||
correct=self.correct + correct,
|
||||
wrong=self.wrong + wrong,
|
||||
)
|
||||
|
||||
|
||||
def _sort_key(pattern: CuePattern) -> tuple[str, str, str]:
|
||||
return (pattern.cue, pattern.op, pattern.unit_shape)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CuePrecisionLedger:
|
||||
"""Immutable map of :class:`CuePattern` -> :class:`PatternTally`.
|
||||
|
||||
Canonical storage is a tuple sorted by pattern (deterministic, byte-stable
|
||||
across runs). Every ``record_*`` returns a new ledger (immutability rule);
|
||||
an absent pattern reads as an empty tally, so a cold ledger reports ``0.0``
|
||||
reliability for every pattern.
|
||||
"""
|
||||
|
||||
tallies: tuple[PatternTally, ...] = field(default_factory=tuple)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
seen: set[CuePattern] = set()
|
||||
for tally in self.tallies:
|
||||
if tally.pattern in seen:
|
||||
raise ValueError(f"duplicate pattern in ledger: {tally.pattern!r}")
|
||||
seen.add(tally.pattern)
|
||||
|
||||
def tally_for(self, pattern: CuePattern) -> PatternTally:
|
||||
"""The tally for ``pattern``, or an empty one if unseen (cold ⇒ 0)."""
|
||||
for tally in self.tallies:
|
||||
if tally.pattern == pattern:
|
||||
return tally
|
||||
return PatternTally(pattern=pattern)
|
||||
|
||||
def reliability(self, pattern: CuePattern) -> float:
|
||||
"""Conservative reliability of ``pattern`` (``0.0`` when cold/low)."""
|
||||
return self.tally_for(pattern).reliability
|
||||
|
||||
def _record_pattern(
|
||||
self, pattern: CuePattern, *, correct: int = 0, wrong: int = 0
|
||||
) -> "CuePrecisionLedger":
|
||||
index = {tally.pattern: tally for tally in self.tallies}
|
||||
base = index.get(pattern, PatternTally(pattern=pattern))
|
||||
index[pattern] = base.record(correct=correct, wrong=wrong)
|
||||
ordered = tuple(sorted(index.values(), key=lambda t: _sort_key(t.pattern)))
|
||||
return CuePrecisionLedger(tallies=ordered)
|
||||
|
||||
def record_chain(
|
||||
self, derivation: GroundedDerivation, *, matched_gold: bool
|
||||
) -> "CuePrecisionLedger":
|
||||
"""Credit every step's pattern in ``derivation`` by its gold label.
|
||||
|
||||
``+correct`` per step occurrence when the chain matched gold, else
|
||||
``+wrong``. A chain whose value cannot be computed (a divide-by-zero the
|
||||
gate would reject) is not a labelable reading and contributes nothing —
|
||||
a deliberate, documented skip, not a swallowed error.
|
||||
"""
|
||||
ledger = self
|
||||
for pattern in patterns_in_chain(derivation):
|
||||
if matched_gold:
|
||||
ledger = ledger._record_pattern(pattern, correct=1)
|
||||
else:
|
||||
ledger = ledger._record_pattern(pattern, wrong=1)
|
||||
return ledger
|
||||
|
||||
def record_case(
|
||||
self,
|
||||
candidate_chains: Iterable[GroundedDerivation],
|
||||
gold_answer: float,
|
||||
) -> "CuePrecisionLedger":
|
||||
"""Label each candidate chain by gold and credit its patterns.
|
||||
|
||||
Independent of whether the search *resolved* the case (ADR-0177): the
|
||||
ledger learns from labelling candidates, so a refused case still records
|
||||
only its candidates' gold labels — never a separate refusal penalty.
|
||||
"""
|
||||
ledger = self
|
||||
for derivation in candidate_chains:
|
||||
try:
|
||||
value = derivation.answer
|
||||
except ZeroDivisionError:
|
||||
# Non-computable chain: not a labelable reading (the verify gate
|
||||
# rejects divide-by-zero before .answer is relied upon). Skip it.
|
||||
continue
|
||||
matched = round(value, _GOLD_DECIMALS) == round(gold_answer, _GOLD_DECIMALS)
|
||||
ledger = ledger.record_chain(derivation, matched_gold=matched)
|
||||
return ledger
|
||||
323
tests/test_adr_0177_cp1_ledger.py
Normal file
323
tests/test_adr_0177_cp1_ledger.py
Normal file
|
|
@ -0,0 +1,323 @@
|
|||
"""ADR-0177 CP-1 — cue-precision ledger + credit assignment.
|
||||
|
||||
Proves the ``(cue, op, unit_shape)`` pattern key, the per-pattern counted
|
||||
:class:`PatternTally` (reusing the ADR-0175 conservative floor), and the
|
||||
credit-assignment mechanism (gold-labelled candidate chains -> per-pattern
|
||||
counts). Each property is exercised by a test that *fails* under the violation it
|
||||
names (CLAUDE.md §Schema-Defined Proof Obligations):
|
||||
|
||||
- cold ledger ⇒ no trust -> TestColdLedger
|
||||
- counts-only, refusals excluded -> TestCreditAssignment / TestRefusalsNotCounted
|
||||
- reliability earned by volume -> TestReliabilityEarnedByVolume
|
||||
- determinism / replay -> TestDeterminism
|
||||
- immutability -> TestImmutability
|
||||
|
||||
This substrate is **inert** — nothing outside this test imports it (ADR-0177 CP-1,
|
||||
"imported by nothing outside its own tests"); asserted in TestInertSubstrate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from core.reliability_gate.floor import N_MIN, conservative_floor
|
||||
from generate.cue_precision import (
|
||||
CROSS_UNIT,
|
||||
SAME_UNIT,
|
||||
UNIT_SHAPES,
|
||||
CuePattern,
|
||||
CuePrecisionLedger,
|
||||
PatternTally,
|
||||
pattern_for_step,
|
||||
patterns_in_chain,
|
||||
)
|
||||
from generate.derivation.model import GroundedDerivation, Quantity, Step
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _q(value: float, unit: str, token: str | None = None) -> Quantity:
|
||||
return Quantity(value=value, unit=unit, source_token=token or str(value))
|
||||
|
||||
|
||||
def _chain(start: Quantity, *steps: Step) -> GroundedDerivation:
|
||||
return GroundedDerivation(start=start, steps=tuple(steps))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CuePattern (the key)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestCuePattern:
|
||||
def test_valid_pattern(self) -> None:
|
||||
p = CuePattern(cue="per", op="multiply", unit_shape=CROSS_UNIT)
|
||||
assert (p.cue, p.op, p.unit_shape) == ("per", "multiply", CROSS_UNIT)
|
||||
|
||||
def test_empty_cue_rejected(self) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
CuePattern(cue="", op="multiply", unit_shape=SAME_UNIT)
|
||||
|
||||
def test_invalid_op_rejected(self) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
CuePattern(cue="per", op="exponentiate", unit_shape=SAME_UNIT)
|
||||
|
||||
def test_invalid_unit_shape_rejected(self) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
CuePattern(cue="per", op="multiply", unit_shape="mixed")
|
||||
|
||||
def test_unit_shapes_closed_set(self) -> None:
|
||||
assert UNIT_SHAPES == frozenset({CROSS_UNIT, SAME_UNIT})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pattern extraction (unit_shape classification)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPatternExtraction:
|
||||
def test_cross_unit_when_operand_differs_from_primary(self) -> None:
|
||||
# 6 boxes x 50 apples -> running unit stays "boxes", operand "apples".
|
||||
d = _chain(
|
||||
_q(6, "boxes"),
|
||||
Step(op="multiply", operand=_q(50, "apples"), cue="per"),
|
||||
)
|
||||
assert pattern_for_step(d, d.steps[0]) == CuePattern(
|
||||
cue="per", op="multiply", unit_shape=CROSS_UNIT
|
||||
)
|
||||
|
||||
def test_same_unit_when_operand_matches_primary(self) -> None:
|
||||
# 6 apples + 4 apples -> same unit.
|
||||
d = _chain(
|
||||
_q(6, "apples"),
|
||||
Step(op="add", operand=_q(4, "apples"), cue="and"),
|
||||
)
|
||||
assert pattern_for_step(d, d.steps[0]) == CuePattern(
|
||||
cue="and", op="add", unit_shape=SAME_UNIT
|
||||
)
|
||||
|
||||
def test_dimensionless_scalar_is_same_unit(self) -> None:
|
||||
# A comparative scalar (twice -> x2) carries unit "" and scales within the
|
||||
# current dimension; it is NOT a cross-unit aggregate.
|
||||
d = _chain(
|
||||
_q(5, "apples"),
|
||||
Step(op="multiply", operand=_q(2, "", "twice"), cue="twice", comparative=True),
|
||||
)
|
||||
assert pattern_for_step(d, d.steps[0]).unit_shape == SAME_UNIT
|
||||
|
||||
def test_patterns_in_chain_preserves_step_order_and_occurrences(self) -> None:
|
||||
d = _chain(
|
||||
_q(2, "boxes"),
|
||||
Step(op="multiply", operand=_q(3, "apples"), cue="per"),
|
||||
Step(op="multiply", operand=_q(4, "apples"), cue="per"),
|
||||
)
|
||||
patterns = patterns_in_chain(d)
|
||||
# Both steps share the same pattern -> two occurrences (per-step credit).
|
||||
assert len(patterns) == 2
|
||||
assert patterns[0] == patterns[1] == CuePattern(
|
||||
cue="per", op="multiply", unit_shape=CROSS_UNIT
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PatternTally (counts-only, conservative floor)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPatternTally:
|
||||
def _pat(self) -> CuePattern:
|
||||
return CuePattern(cue="per", op="multiply", unit_shape=CROSS_UNIT)
|
||||
|
||||
def test_negative_counts_rejected(self) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
PatternTally(pattern=self._pat(), correct=-1)
|
||||
|
||||
def test_committed_excludes_nothing_but_correct_and_wrong(self) -> None:
|
||||
t = PatternTally(pattern=self._pat(), correct=7, wrong=3)
|
||||
assert t.committed == 10
|
||||
|
||||
def test_no_refused_axis(self) -> None:
|
||||
# A tally is purely correct/wrong: there is no refusal field to count.
|
||||
assert set(PatternTally.__dataclass_fields__) == {"pattern", "correct", "wrong"}
|
||||
|
||||
def test_reliability_matches_conservative_floor(self) -> None:
|
||||
t = PatternTally(pattern=self._pat(), correct=10, wrong=0)
|
||||
assert t.reliability == conservative_floor(10, 10)
|
||||
|
||||
def test_record_is_immutable(self) -> None:
|
||||
t0 = PatternTally(pattern=self._pat())
|
||||
t1 = t0.record(correct=1)
|
||||
assert t0.correct == 0 and t1.correct == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cold ledger ⇒ no trust (the wrong=0 safety property CP-2 relies on)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestColdLedger:
|
||||
def test_empty_ledger_reliability_is_zero(self) -> None:
|
||||
ledger = CuePrecisionLedger()
|
||||
p = CuePattern(cue="per", op="multiply", unit_shape=CROSS_UNIT)
|
||||
assert ledger.reliability(p) == 0.0
|
||||
assert ledger.tally_for(p).committed == 0
|
||||
|
||||
def test_below_n_min_reliability_is_zero(self) -> None:
|
||||
p = CuePattern(cue="per", op="multiply", unit_shape=CROSS_UNIT)
|
||||
d = _chain(_q(2, "boxes"), Step(op="multiply", operand=_q(3, "apples"), cue="per"))
|
||||
ledger = CuePrecisionLedger()
|
||||
for _ in range(N_MIN - 1): # all correct but still under N_MIN
|
||||
ledger = ledger.record_chain(d, matched_gold=True)
|
||||
assert ledger.tally_for(p).committed == N_MIN - 1
|
||||
assert ledger.reliability(p) == 0.0 # earned by volume, not a streak
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Credit assignment (gold-labelled candidate chains)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestCreditAssignment:
|
||||
def test_matched_chain_credits_correct_per_step(self) -> None:
|
||||
d = _chain(
|
||||
_q(2, "boxes"),
|
||||
Step(op="multiply", operand=_q(3, "apples"), cue="per"),
|
||||
Step(op="multiply", operand=_q(4, "apples"), cue="per"),
|
||||
)
|
||||
ledger = CuePrecisionLedger().record_chain(d, matched_gold=True)
|
||||
p = CuePattern(cue="per", op="multiply", unit_shape=CROSS_UNIT)
|
||||
assert ledger.tally_for(p).correct == 2
|
||||
assert ledger.tally_for(p).wrong == 0
|
||||
|
||||
def test_unmatched_chain_credits_wrong_per_step(self) -> None:
|
||||
d = _chain(_q(2, "boxes"), Step(op="multiply", operand=_q(3, "apples"), cue="per"))
|
||||
ledger = CuePrecisionLedger().record_chain(d, matched_gold=False)
|
||||
p = CuePattern(cue="per", op="multiply", unit_shape=CROSS_UNIT)
|
||||
assert ledger.tally_for(p).correct == 0
|
||||
assert ledger.tally_for(p).wrong == 1
|
||||
|
||||
def test_record_case_labels_candidates_by_gold(self) -> None:
|
||||
# gold = 12. A correct product chain (2 x 6) and a wrong sum chain (2 + 6 = 8).
|
||||
good = _chain(_q(2, "boxes"), Step(op="multiply", operand=_q(6, "apples"), cue="per"))
|
||||
bad = _chain(_q(2, "apples"), Step(op="add", operand=_q(6, "apples"), cue="and"))
|
||||
ledger = CuePrecisionLedger().record_case([good, bad], gold_answer=12.0)
|
||||
mult = CuePattern(cue="per", op="multiply", unit_shape=CROSS_UNIT)
|
||||
add = CuePattern(cue="and", op="add", unit_shape=SAME_UNIT)
|
||||
assert ledger.tally_for(mult).correct == 1
|
||||
assert ledger.tally_for(mult).wrong == 0
|
||||
assert ledger.tally_for(add).correct == 0
|
||||
assert ledger.tally_for(add).wrong == 1
|
||||
|
||||
def test_divide_by_zero_chain_is_skipped(self) -> None:
|
||||
# A non-computable chain is not a labelable reading -> contributes nothing.
|
||||
bad = _chain(_q(6, "apples"), Step(op="divide", operand=_q(0, "apples"), cue="per"))
|
||||
ledger = CuePrecisionLedger().record_case([bad], gold_answer=0.0)
|
||||
assert ledger.tallies == ()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Refusals are never counted (independent of resolve/refuse)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestRefusalsNotCounted:
|
||||
def test_recording_independent_of_resolution(self) -> None:
|
||||
# Two disagreeing self-verifiable chains -> the search would REFUSE this
|
||||
# case, yet the ledger still records exactly the candidates' gold labels,
|
||||
# with no separate refusal penalty. committed == number of step occurrences.
|
||||
a = _chain(_q(2, "boxes"), Step(op="multiply", operand=_q(6, "apples"), cue="per"))
|
||||
b = _chain(_q(2, "apples"), Step(op="add", operand=_q(6, "apples"), cue="and"))
|
||||
ledger = CuePrecisionLedger().record_case([a, b], gold_answer=12.0)
|
||||
total_committed = sum(t.committed for t in ledger.tallies)
|
||||
assert total_committed == 2 # one step each; no phantom refusal count
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reliability earned by volume
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestReliabilityEarnedByVolume:
|
||||
def test_clean_record_below_then_at_n_min(self) -> None:
|
||||
p = CuePattern(cue="per", op="multiply", unit_shape=CROSS_UNIT)
|
||||
d = _chain(_q(2, "boxes"), Step(op="multiply", operand=_q(3, "apples"), cue="per"))
|
||||
ledger = CuePrecisionLedger()
|
||||
for _ in range(N_MIN):
|
||||
ledger = ledger.record_chain(d, matched_gold=True)
|
||||
assert ledger.tally_for(p).committed == N_MIN
|
||||
assert ledger.reliability(p) > 0.0
|
||||
assert ledger.reliability(p) == conservative_floor(N_MIN, N_MIN)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Determinism / replay
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestDeterminism:
|
||||
def _cases(self) -> list[tuple[list[GroundedDerivation], float]]:
|
||||
c1 = _chain(_q(2, "boxes"), Step(op="multiply", operand=_q(6, "apples"), cue="per"))
|
||||
c2 = _chain(_q(2, "apples"), Step(op="add", operand=_q(6, "apples"), cue="and"))
|
||||
c3 = _chain(_q(4, "apples"), Step(op="add", operand=_q(4, "apples"), cue="and"))
|
||||
return [([c1, c2], 12.0), ([c3], 8.0)]
|
||||
|
||||
def test_same_cases_same_order_byte_stable(self) -> None:
|
||||
def run() -> CuePrecisionLedger:
|
||||
ledger = CuePrecisionLedger()
|
||||
for chains, gold in self._cases():
|
||||
ledger = ledger.record_case(chains, gold)
|
||||
return ledger
|
||||
|
||||
assert run().tallies == run().tallies
|
||||
|
||||
def test_tallies_sorted_canonically(self) -> None:
|
||||
ledger = CuePrecisionLedger()
|
||||
for chains, gold in self._cases():
|
||||
ledger = ledger.record_case(chains, gold)
|
||||
keys = [(t.pattern.cue, t.pattern.op, t.pattern.unit_shape) for t in ledger.tallies]
|
||||
assert keys == sorted(keys)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Immutability
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestImmutability:
|
||||
def test_record_chain_returns_new_ledger(self) -> None:
|
||||
d = _chain(_q(2, "boxes"), Step(op="multiply", operand=_q(3, "apples"), cue="per"))
|
||||
ledger0 = CuePrecisionLedger()
|
||||
ledger1 = ledger0.record_chain(d, matched_gold=True)
|
||||
assert ledger0.tallies == ()
|
||||
assert ledger1.tallies != ()
|
||||
|
||||
def test_duplicate_pattern_rejected(self) -> None:
|
||||
p = CuePattern(cue="per", op="multiply", unit_shape=CROSS_UNIT)
|
||||
with pytest.raises(ValueError):
|
||||
CuePrecisionLedger(
|
||||
tallies=(PatternTally(pattern=p), PatternTally(pattern=p))
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inert substrate — imported by nothing outside its own tests (ADR-0177 CP-1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestInertSubstrate:
|
||||
def test_not_imported_outside_package_or_tests(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
# Mirror CLAUDE.md §Architectural Scan Exclusions.
|
||||
excluded = {
|
||||
".git", ".venv", "__pycache__", ".pytest_cache", ".hypothesis",
|
||||
".claude", "tests", "core-rs", "docs", "evals", "benchmarks",
|
||||
"scripts",
|
||||
}
|
||||
offenders: list[str] = []
|
||||
for dirpath, dirnames, filenames in os.walk(repo_root):
|
||||
dirnames[:] = [d for d in dirnames if d not in excluded]
|
||||
# Don't flag the package's own modules.
|
||||
if "cue_precision" in Path(dirpath).parts:
|
||||
continue
|
||||
for name in filenames:
|
||||
if not name.endswith(".py"):
|
||||
continue
|
||||
src = Path(dirpath, name).read_text(encoding="utf-8")
|
||||
if "cue_precision" in src:
|
||||
offenders.append(str(Path(dirpath, name).relative_to(repo_root)))
|
||||
assert offenders == [], f"cue_precision imported by serving/runtime: {offenders}"
|
||||
Loading…
Reference in a new issue