From 44c0aa2896891c574b1ccdced27f7b9381e78742 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 27 May 2026 15:15:11 -0700 Subject: [PATCH] feat(ADR-0169/CC-2+CC-3): CompositionClaim ratification handler + decomposer heuristic tightening (#393) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-β of the CompositionClaim wave (CC-2 + CC-3 bundled per the brief pack — CC-3's heuristic depends on CC-2's new change_kind Literal value). Mirrors the F1 / ADR-0168 FrameClaim template 1:1 with composition-specific substitutions. CC-2 — handler implementation - teaching/math_composition_proposal.py — MathCompositionClaimProposal adapter per ADR-0169.1 §"Data shape". Frozen dataclass, deterministic proposal_id / claim_signature, source="math_audit" pin at the proposal layer (rejects "corpus" laundering). - teaching/math_composition_ratification.py — apply_composition_claim() handler. SAFE_COMPOSITION_CATEGORIES = {multiplicative, additive, subtractive}_composition per ADR-0169 §"Initial safe category scope". New WrongCompositionCategory exception per ADR-0169.1 §"Trip-wires" #8. Writes only to language_packs/data/en_core_math_v1/compositions/{category}.jsonl; no solver / parser / decomposer / runtime mutation. - workbench/readers.py — _HANDLER_DISPATCH now routes composition_reclassification → CompositionClaim; suggested_cli branch added for both read_math_proposal and ratify_math_proposal. - teaching/math_contemplation_proposal.py — ChangeKind Literal + _VALID_CHANGE_KINDS frozenset extended with composition_reclassification. - language_packs/data/en_core_math_v1/compositions/.gitkeep — reviewed-pack scaffold. - tests/test_math_composition_ratification.py — 22 tests including case 0050 hazard pin, cross-process replay equivalence, queue-order independence, partition, no-corpus-laundering, dispatch wire, Literal acceptance, JSONL round-trip. - tests/test_adr_0172_w1_shape_proposal.py — parametrize round-trip over all 5 change_kinds. - core/cli.py — teaching suite tuple includes new test file. CC-3 — decomposer heuristic tightening - teaching/math_contemplation.py::_CHANGE_KIND_BY_PAIR: + (incomplete_operation, quantity_extraction) → composition_reclassification + (incomplete_operation, multi_quantity_composition) → composition_reclassification - (unexpected_category, multi_subject_sentence) demoted to injector_sub_shape (was frame_reclassification; FrameClaim SAFE_FRAME_CATEGORIES doesn't cover this — needs ReferenceClaim/CompositionClaim) - (unexpected_category, descriptive_frame_question) demoted to injector_sub_shape (was frame_reclassification; needs SlotClaim, not FrameClaim) Updated hypothesis-step justification text to reflect new dispatch table. - tests/test_adr_0172_w2_decomposer.py — distribution assertion tightened from "≥3 matcher, ≥2 frame" to exact counts: 3 matcher / 2 composition / 3 injector / 0 frame. New per-pair tests for the four CC-3 dispatch changes. Verification on real audit_brief_11.json (20 of 47 highest-leverage refusals now routable): 2 composition_reclassification (12 quantity_extraction + 8 multi_quantity_composition) 3 injector_sub_shape (2 multi_subject + 2 descriptive_frame + 4 unattached_quantity) 3 matcher_extension (9 pre_frame_filler + 4 fraction_percentage + 4 pronoun) 0 frame_reclassification (the two prior misroutes are gone) Workbench POST /math-proposals/{id}/ratify on either composition proposal now returns 200/routed with a real apply_composition_claim() command instead of 501. Suites green: - core test --suite teaching -q → 71 passed - core test --suite runtime -q → 20 passed Co-authored-by: Claude Opus 4.7 --- core/cli.py | 1 + .../en_core_math_v1/compositions/.gitkeep | 0 teaching/math_composition_proposal.py | 355 ++++++++ teaching/math_composition_ratification.py | 387 ++++++++ teaching/math_contemplation.py | 27 +- teaching/math_contemplation_proposal.py | 2 + tests/test_adr_0172_w1_shape_proposal.py | 5 +- tests/test_adr_0172_w2_decomposer.py | 118 ++- tests/test_math_composition_ratification.py | 849 ++++++++++++++++++ workbench/readers.py | 16 + 10 files changed, 1745 insertions(+), 15 deletions(-) create mode 100644 language_packs/data/en_core_math_v1/compositions/.gitkeep create mode 100644 teaching/math_composition_proposal.py create mode 100644 teaching/math_composition_ratification.py create mode 100644 tests/test_math_composition_ratification.py diff --git a/core/cli.py b/core/cli.py index 5550c9c1..a978dcfa 100644 --- a/core/cli.py +++ b/core/cli.py @@ -68,6 +68,7 @@ _TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_adr_0172_w2_decomposer.py", "tests/test_adr_0172_w5_inference_proposal.py", "tests/test_math_frame_ratification.py", + "tests/test_math_composition_ratification.py", ), "packs": ( "tests/test_core_semantic_seed_pack.py", diff --git a/language_packs/data/en_core_math_v1/compositions/.gitkeep b/language_packs/data/en_core_math_v1/compositions/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/teaching/math_composition_proposal.py b/teaching/math_composition_proposal.py new file mode 100644 index 00000000..22e7dd97 --- /dev/null +++ b/teaching/math_composition_proposal.py @@ -0,0 +1,355 @@ +"""ADR-0169 / ADR-0169.1 — MathCompositionClaimProposal adapter. + +Defines the math-domain proposal type carrying CompositionClaim evidence +pointers. Mirror of :mod:`teaching.math_frame_proposal` for the composition +admissibility sub-type. + +Per ADR-0169.1 §"Evidence floor", a :class:`MathCompositionClaimProposal` is +eligible only if every evidence pointer declares ``source="math_audit"``. +Audit evidence MUST NEVER be laundered as ``source="corpus"`` — that would +falsely impersonate ADR-0057's cognition corpus evidence floor. + +Per ADR-0169 §"Definition of a CompositionClaim", a composition claim's +``surface_pattern`` is a deterministic structural pattern over already-bound +slots (e.g. ``bound(count) bound(unit_cost)``), not a regex over raw text. +``composition_category`` is one entry of the +:data:`teaching.math_composition_ratification.SAFE_COMPOSITION_CATEGORIES` +allowlist. + +Trust boundary: schema-only module. No filesystem I/O, no teaching-store +writes, no runtime pipeline hooks. All validation lives in +:func:`build_composition_claim_proposal` and :func:`build_evidence_pointer`. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from typing import Any, Final, Literal + +from teaching.math_evidence import MathReaderRefusalEvidence + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +Polarity = Literal["affirms", "falsifies"] + +_VALID_POLARITIES: Final[frozenset[str]] = frozenset({"affirms", "falsifies"}) + +# Mirrors SAFE_COMPOSITION_CATEGORIES in +# :mod:`teaching.math_composition_ratification`. Held here as well so the +# proposal layer can reject illegal categories before the proposal is ever +# built — defense in depth at the schema boundary. +_ALLOWLISTED_COMPOSITION_CATEGORIES: Final[frozenset[str]] = frozenset( + { + "multiplicative_composition", + "additive_composition", + "subtractive_composition", + } +) + +# Forbidden evidence source values. ADR-0169.1 §"Evidence floor" requires +# math-audit evidence to carry ``source="math_audit"``. Any cognition-style +# value reaching this module indicates evidence laundering — fail loudly. +_FORBIDDEN_EVIDENCE_SOURCES: Final[frozenset[str]] = frozenset({"corpus"}) + + +# --------------------------------------------------------------------------- +# Evidence pointer +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class MathReaderRefusalEvidencePointer: + """One pointer to a math-domain refusal-row evidence record. + + Per ADR-0169.1, ``source`` is pinned to ``"math_audit"`` at construction. + The factory :func:`build_evidence_pointer` is the only sanctioned way to + create one — direct instantiation works but is discouraged because the + factory ensures ``source`` cannot drift to a cognition value. + """ + + source: Literal["math_audit"] + case_id: str + sentence_index: int + token_index: int + missing_operator: str + refusal_reason: str + evidence_hash: str + audit_row_digest: str + + +def _audit_row_digest(evidence: MathReaderRefusalEvidence) -> str: + """SHA-256 over the evidence record's canonical bytes (excludes hash itself). + + Provides the replay anchor: identical evidence yields an identical + digest across processes, runs, and reorderings. + """ + + return hashlib.sha256(evidence.to_canonical_bytes()).hexdigest() + + +def build_evidence_pointer( + evidence: MathReaderRefusalEvidence, +) -> MathReaderRefusalEvidencePointer: + """Build a math-audit evidence pointer from a refusal-row record. + + Pins ``source="math_audit"`` per ADR-0169.1 §"Evidence floor". + """ + + if evidence.missing_operator is None: + raise ValueError( + "MathCompositionClaim evidence must declare a missing_operator; got None" + ) + return MathReaderRefusalEvidencePointer( + source="math_audit", + case_id=evidence.case_id, + sentence_index=evidence.sentence_index, + token_index=evidence.token_index, + missing_operator=evidence.missing_operator, + refusal_reason=evidence.refusal_reason, + evidence_hash=evidence.evidence_hash, + audit_row_digest=_audit_row_digest(evidence), + ) + + +# --------------------------------------------------------------------------- +# Proposal +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class MathCompositionClaimProposal: + """A reviewed assertion that ``surface_pattern`` composes under ``composition_category``. + + Per ADR-0169.1 the proposal carries: + + - canonical identity (proposal_id, claim_signature) + - the structural claim (surface_pattern, composition_category, polarity) + - math-domain evidence pointers (source="math_audit" only) + - review state and operator note + + The proposal is *not* a runtime composition registry entry. Operator + acceptance does not auto-mutate the runtime — see + :mod:`teaching.math_composition_ratification` for the explicit mutation + boundary. + """ + + proposal_id: str + claim_signature: str + surface_pattern: str + composition_category: str + polarity: Polarity + evidence: tuple[MathReaderRefusalEvidencePointer, ...] + review_state: Literal["pending", "accepted", "rejected", "withdrawn"] + operator_note: str = "" + domain: Literal["math"] = "math" + sub_type: Literal["composition"] = "composition" + + +def _serialise_pointer(p: MathReaderRefusalEvidencePointer) -> dict[str, Any]: + return { + "source": p.source, + "case_id": p.case_id, + "sentence_index": p.sentence_index, + "token_index": p.token_index, + "missing_operator": p.missing_operator, + "refusal_reason": p.refusal_reason, + "evidence_hash": p.evidence_hash, + "audit_row_digest": p.audit_row_digest, + } + + +def compute_claim_signature( + *, + surface_pattern: str, + composition_category: str, + polarity: str, + evidence: tuple[MathReaderRefusalEvidencePointer, ...], +) -> str: + """Deterministic canonical signature for a composition claim. + + Per ADR-0169 §"Replay obligations" #1, equivalent refusals must yield + identical signatures. We hash over: + + - normalized surface pattern (lower, stripped) + - composition category (allowlisted string) + - polarity + - the sorted set of audit_row_digests from the evidence pointers + - refusal_reason and missing_operator coverage (drawn from evidence) + """ + + pattern_norm = surface_pattern.lower().strip() + digests = sorted(p.audit_row_digest for p in evidence) + refusal_reasons = sorted({p.refusal_reason for p in evidence}) + missing_operators = sorted({p.missing_operator for p in evidence}) + payload = json.dumps( + { + "surface_pattern": pattern_norm, + "composition_category": composition_category, + "polarity": polarity, + "audit_row_digests": digests, + "refusal_reasons": refusal_reasons, + "missing_operators": missing_operators, + }, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def compute_proposal_id( + *, + surface_pattern: str, + composition_category: str, + polarity: str, + evidence: tuple[MathReaderRefusalEvidencePointer, ...], +) -> str: + """Stable proposal_id from canonical claim identity (ADR-0169.1 §"Idempotency"). + + ``sha256(domain | subtype | surface_pattern | composition_category | + polarity | evidence_digest_set)`` — clock-time-independent and + identity-stable. + """ + + pattern_norm = surface_pattern.lower().strip() + digests = sorted(p.audit_row_digest for p in evidence) + payload = json.dumps( + { + "domain": "math", + "sub_type": "composition_claim", + "surface_pattern": pattern_norm, + "composition_category": composition_category, + "polarity": polarity, + "audit_row_digests": digests, + }, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def canonical_bytes(proposal: MathCompositionClaimProposal) -> bytes: + """Deterministic canonical bytes for replay / persistence. + + Excludes ``proposal_id`` (the hash function input) and ``review_state`` + (mutable transition surface). + """ + + payload: dict[str, Any] = { + "domain": proposal.domain, + "sub_type": proposal.sub_type, + "claim_signature": proposal.claim_signature, + "surface_pattern": proposal.surface_pattern, + "composition_category": proposal.composition_category, + "polarity": proposal.polarity, + "evidence": sorted( + (_serialise_pointer(p) for p in proposal.evidence), + key=lambda d: d["audit_row_digest"], + ), + "operator_note": proposal.operator_note, + } + return json.dumps( + payload, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + + +def build_composition_claim_proposal( + *, + surface_pattern: str, + composition_category: str, + polarity: str, + evidence: tuple[MathReaderRefusalEvidencePointer, ...], + operator_note: str = "", + review_state: Literal["pending", "accepted", "rejected", "withdrawn"] = "pending", +) -> MathCompositionClaimProposal: + """Build a :class:`MathCompositionClaimProposal` with all invariants enforced. + + Invariants (per ADR-0169 §"Decision" + ADR-0169.1 §"Evidence floor") + + 1. ``surface_pattern`` non-empty after normalization. + 2. ``composition_category`` in the ADR-0169 allowlist (also enforced by + the handler, but rejected here too for defense-in-depth). + 3. ``polarity in {"affirms", "falsifies"}``. + 4. ``len(evidence) >= 1`` — at least one audit/refusal evidence pointer. + 5. every evidence pointer carries ``source="math_audit"`` — corpus + evidence is rejected as schema-illegal. + 6. proposal_id and claim_signature are derived deterministically. + + Raises ``ValueError`` on any violation; the caller must fix the inputs. + """ + + normalized = surface_pattern.lower().strip() + if not normalized: + raise ValueError( + f"surface_pattern must be non-empty after normalization; got {surface_pattern!r}" + ) + + if composition_category not in _ALLOWLISTED_COMPOSITION_CATEGORIES: + raise ValueError( + f"composition_category {composition_category!r} is not in the ADR-0169 allowlist " + f"{sorted(_ALLOWLISTED_COMPOSITION_CATEGORIES)!r}" + ) + + if polarity not in _VALID_POLARITIES: + raise ValueError( + f"polarity must be one of {sorted(_VALID_POLARITIES)!r}; got {polarity!r}" + ) + + if len(evidence) < 1: + raise ValueError( + "MathCompositionClaimProposal requires at least one math-audit evidence pointer" + ) + + for idx, pointer in enumerate(evidence): + if pointer.source != "math_audit": + raise ValueError( + f"evidence[{idx}].source must be 'math_audit'; got {pointer.source!r} — " + "audit evidence MUST NOT be laundered as cognition corpus evidence " + "(ADR-0169.1 §'Evidence floor')" + ) + if pointer.source in _FORBIDDEN_EVIDENCE_SOURCES: + raise ValueError( + f"evidence[{idx}].source is forbidden: {pointer.source!r}" + ) + + claim_signature = compute_claim_signature( + surface_pattern=normalized, + composition_category=composition_category, + polarity=polarity, + evidence=evidence, + ) + proposal_id = compute_proposal_id( + surface_pattern=normalized, + composition_category=composition_category, + polarity=polarity, + evidence=evidence, + ) + + return MathCompositionClaimProposal( + proposal_id=proposal_id, + claim_signature=claim_signature, + surface_pattern=normalized, + composition_category=composition_category, + polarity=polarity, # type: ignore[arg-type] + evidence=tuple(evidence), + review_state=review_state, + operator_note=operator_note, + ) + + +__all__ = [ + "MathCompositionClaimProposal", + "MathReaderRefusalEvidencePointer", + "Polarity", + "build_composition_claim_proposal", + "build_evidence_pointer", + "canonical_bytes", + "compute_claim_signature", + "compute_proposal_id", +] diff --git a/teaching/math_composition_ratification.py b/teaching/math_composition_ratification.py new file mode 100644 index 00000000..c9de2a60 --- /dev/null +++ b/teaching/math_composition_ratification.py @@ -0,0 +1,387 @@ +"""ADR-0169 — CompositionClaim ratification into reviewed math composition mappings. + +This module is the explicit post-review mutation boundary for math-domain +composition-pattern evidence. It edits only per-category source files +under ``language_packs/data/en_core_math_v1/compositions/``; it does not +regenerate the compiled lexicon and therefore does not rewrite the pack +manifest checksum. + +Mirrors :mod:`teaching.math_frame_ratification` (F1) but lifts the safe +surface from bounded frame openers to allowlisted composition categories +over already-bound slot patterns (ADR-0169 §"Definition of a CompositionClaim"). + +Hard rules (ADR-0169 §"Decision"): + +- ``SAFE_COMPOSITION_CATEGORIES`` is the only sanctioned surface — no other + categories may be ratified through this handler. +- case 0050 hazard pin: after any composition ratification, GSM8K + train-sample case 0050 must still refuse (no speculative pre-frame-filler + admission). +- evidence pointers MUST carry ``source="math_audit"`` — never ``"corpus"``. +- ``polarity in {"affirms", "falsifies"}``; the falsifies branch records the + pattern as non-composing and never appends an affirmative composition row. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import string +from dataclasses import dataclass +from datetime import date +from pathlib import Path +from typing import Final + +from teaching.math_evidence import MathReaderRefusalEvidence, from_audit_row + + +# --------------------------------------------------------------------------- +# Allowlist + constants +# --------------------------------------------------------------------------- + +SAFE_COMPOSITION_CATEGORIES: Final[frozenset[str]] = frozenset( + { + "multiplicative_composition", + "additive_composition", + "subtractive_composition", + } +) + +_VALID_POLARITIES: Final[frozenset[str]] = frozenset({"affirms", "falsifies"}) +_REVIEWER_RE: Final[re.Pattern[str]] = re.compile(r"[^a-z0-9_-]+") +_TRIM_PUNCTUATION: Final[str] = string.punctuation + + +# --------------------------------------------------------------------------- +# Error hierarchy (mirror F1) +# --------------------------------------------------------------------------- + + +class RatificationError(ValueError): + """Base class for composition ratification rejections.""" + + +class WrongClaimSubType(RatificationError): + """Raised when a non-composition evidence record reaches this handler.""" + + +class WrongCompositionCategory(RatificationError): + """Raised when ``composition_category`` is outside :data:`SAFE_COMPOSITION_CATEGORIES`. + + Per ADR-0169.1 §"Trip-wires" #8, this is the explicit allowlist-enforcement + exception. Distinct from :class:`WrongZeroViolationCandidate` to give + operators a clearer remediation signal (the category is the problem; the + claim shape itself may be fine). + """ + + +class WrongZeroViolationCandidate(RatificationError): + """Raised when a ratification could open a wrong>0 admission path.""" + + +class AlreadyRatified(RatificationError): + """Raised when the target claim is already covered by the source file.""" + + +class EvidenceTampering(RatificationError): + """Raised when the evidence hash no longer matches canonical bytes.""" + + +class UnknownCategory(RatificationError): + """Raised when the requested category is not a known composition source.""" + + +class InvalidPolarity(RatificationError): + """Raised when polarity is not in :data:`_VALID_POLARITIES`.""" + + +class EvidenceLaundering(RatificationError): + """Raised when audit evidence is presented as cognition corpus evidence. + + ADR-0169.1 §"Evidence floor": math-audit evidence must never be + serialised with ``source="corpus"``; that would impersonate ADR-0057's + cognition corpus evidence floor. + """ + + +# --------------------------------------------------------------------------- +# Receipt +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class CompositionRatificationReceipt: + target_file: str + surface_pattern: str + composition_category: str + polarity: str + provenance: str + file_sha256_before: str + file_sha256_after: str + evidence_hash: str + is_duplicate_evidence: bool + + +# --------------------------------------------------------------------------- +# Path / file helpers +# --------------------------------------------------------------------------- + + +def _repo_root() -> Path: + here = Path(__file__).resolve() + for parent in here.parents: + if (parent / "pyproject.toml").exists() or (parent / "setup.cfg").exists(): + return parent + return here.parents[1] + + +def _default_pack_root() -> Path: + return _repo_root() / "language_packs" / "data" / "en_core_math_v1" + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _normalise_surface(surface: str) -> str: + return surface.lower().strip(_TRIM_PUNCTUATION) + + +def _reviewer_slug(reviewer: str) -> str: + slug = _REVIEWER_RE.sub("_", reviewer.lower()).strip("_-") + if not slug: + raise RatificationError("reviewer must contain at least one safe character") + return slug + + +def _read_entries(path: Path) -> list[dict[str, object]]: + entries: list[dict[str, object]] = [] + for line_number, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): + if not line.strip(): + continue + record = json.loads(line) + if not isinstance(record.get("surface_pattern"), str): + raise RatificationError( + f"{path}: line {line_number} missing string surface_pattern" + ) + if not isinstance(record.get("composition_category"), str): + raise RatificationError( + f"{path}: line {line_number} missing string composition_category" + ) + if not isinstance(record.get("polarity"), str): + raise RatificationError( + f"{path}: line {line_number} missing string polarity" + ) + entries.append(record) + return entries + + +def _write_entries(path: Path, entries: list[dict[str, object]]) -> None: + ordered = sorted(entries, key=lambda item: str(item["surface_pattern"])) + lines = [ + json.dumps(entry, ensure_ascii=False, sort_keys=False) for entry in ordered + ] + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +# --------------------------------------------------------------------------- +# Evidence validation +# --------------------------------------------------------------------------- + + +def _validate_evidence(claim: MathReaderRefusalEvidence) -> None: + """Verify sub_type and tamper-resistance of evidence record.""" + if claim.sub_type != "composition": + raise WrongClaimSubType( + f"Composition ratification requires sub_type='composition'; got {claim.sub_type!r}" + ) + recomputed = from_audit_row( + claim.audit_row, + claim.sub_type, + claim_signature=claim.claim_signature, + ) + if recomputed.evidence_hash != claim.evidence_hash: + raise EvidenceTampering( + "MathReaderRefusalEvidence hash does not match canonical audit row bytes" + ) + + +def _check_no_corpus_laundering(evidence_source: str) -> None: + """ADR-0169.1 §'Evidence floor' — audit evidence must declare math_audit. + + Fail loudly if a caller hands us an evidence pointer that claims to be + cognition corpus evidence; that would impersonate the ADR-0057 floor. + """ + if evidence_source == "corpus": + raise EvidenceLaundering( + "audit evidence MUST NOT be laundered as source='corpus'; " + "use source='math_audit' (ADR-0169.1 §'Evidence floor')" + ) + + +# --------------------------------------------------------------------------- +# Apply +# --------------------------------------------------------------------------- + + +def apply_composition_claim( + *, + claim: MathReaderRefusalEvidence, + composition_category: str, + polarity: str, + reviewer: str, + surface_pattern: str | None = None, + pack_root: Path | None = None, + evidence_source: str = "math_audit", +) -> CompositionRatificationReceipt: + """Apply a reviewed composition claim to a math composition source file. + + Preconditions: + + - ``claim.sub_type == "composition"`` (F1 analog of LexicalClaim sub_type pin). + - ``composition_category`` ∈ :data:`SAFE_COMPOSITION_CATEGORIES`. + - ``polarity`` ∈ ``{"affirms", "falsifies"}``. + - ``evidence_source == "math_audit"``; ``"corpus"`` raises + :class:`EvidenceLaundering` per ADR-0169.1. + - ``claim.evidence_hash`` matches a recomputation from ``claim.audit_row``. + + The write target is + ``{pack_root}/compositions/{composition_category}.jsonl``. Entries are + sorted alphabetically by ``surface_pattern``. Each entry records + ``(surface_pattern, composition_category, polarity, provenance, + evidence_hashes)`` so the polarity decision is auditable and + falsification can supersede an earlier affirmation only through an + explicit reviewed re-apply. + + ``surface_pattern`` defaults to the normalized audit-row token when not + supplied — useful for tests and minimal first ratifications. Production + callers should pass an explicit bound-slot pattern (see ADR-0169 + §"Definition of a CompositionClaim"). + + The compiled ``lexicon.jsonl`` and ``manifest.json`` are intentionally + not regenerated. + """ + # Trust-boundary checks (ADR-0169.1 §'Evidence floor' before any I/O). + _check_no_corpus_laundering(evidence_source) + if evidence_source != "math_audit": + raise EvidenceLaundering( + f"evidence_source must be 'math_audit'; got {evidence_source!r}" + ) + + if polarity not in _VALID_POLARITIES: + raise InvalidPolarity( + f"polarity must be one of {sorted(_VALID_POLARITIES)!r}; got {polarity!r}" + ) + + _validate_evidence(claim) + + if composition_category not in SAFE_COMPOSITION_CATEGORIES: + raise WrongCompositionCategory( + f"composition_category {composition_category!r} is outside SAFE_COMPOSITION_CATEGORIES=" + f"{sorted(SAFE_COMPOSITION_CATEGORIES)!r}; " + "CompositionClaim ratification is allowlist-only (ADR-0169 §'Initial safe category scope')" + ) + + root = (pack_root if pack_root is not None else _default_pack_root()).resolve() + source_dir = root / "compositions" + if not source_dir.exists(): + raise UnknownCategory( + f"composition source directory does not exist: {source_dir}; " + "CompositionClaim handler requires a reviewed compositions/ tree" + ) + + target_file = source_dir / f"{composition_category}.jsonl" + if not target_file.exists(): + # Initialise as empty source file rather than fail — the .gitkeep + # scaffold guarantees the parent directory; first ratification for a + # never-seen safe category seeds the file deterministically. + target_file.write_text("", encoding="utf-8") + + if surface_pattern is None: + normalized_pattern = _normalise_surface(claim.audit_row.token_text) + else: + normalized_pattern = surface_pattern.lower().strip() + if not normalized_pattern: + raise WrongZeroViolationCandidate( + "empty composition surface_pattern cannot be ratified" + ) + + provenance = ( + f"adr_0169_composition_ratified_{_reviewer_slug(reviewer)}_" + f"{date.today().isoformat()}" + ) + before = _sha256_file(target_file) + entries = _read_entries(target_file) + target_relative = ( + str(target_file.relative_to(_repo_root())) + if target_file.is_relative_to(_repo_root()) + else str(target_file) + ) + + # Idempotency: identical (surface_pattern, composition_category, polarity, + # evidence_hash) is a no-op AlreadyRatified. Evidence-hash-only + # duplication (same claim ratified for a second time by *different* + # evidence) appends evidence per ADR-0169.1 §"Idempotency" path #1. + matching = [ + e + for e in entries + if str(e["surface_pattern"]) == normalized_pattern + and str(e["composition_category"]) == composition_category + and str(e["polarity"]) == polarity + ] + is_duplicate_evidence = False + if matching: + existing = matching[0] + existing_evidence: list[str] = list(existing.get("evidence_hashes", [])) # type: ignore[arg-type] + if claim.evidence_hash in existing_evidence: + raise AlreadyRatified( + f"composition claim ({normalized_pattern!r}, {composition_category!r}, " + f"{polarity!r}) is already ratified by evidence_hash={claim.evidence_hash}" + ) + existing_evidence.append(claim.evidence_hash) + existing["evidence_hashes"] = sorted(set(existing_evidence)) + is_duplicate_evidence = True + else: + entries.append( + { + "surface_pattern": normalized_pattern, + "composition_category": composition_category, + "polarity": polarity, + "provenance": provenance, + "evidence_hashes": [claim.evidence_hash], + } + ) + + _write_entries(target_file, entries) + after = _sha256_file(target_file) + + return CompositionRatificationReceipt( + target_file=target_relative, + surface_pattern=normalized_pattern, + composition_category=composition_category, + polarity=polarity, + provenance=provenance, + file_sha256_before=before, + file_sha256_after=after, + evidence_hash=claim.evidence_hash, + is_duplicate_evidence=is_duplicate_evidence, + ) + + +__all__ = [ + "AlreadyRatified", + "CompositionRatificationReceipt", + "EvidenceLaundering", + "EvidenceTampering", + "InvalidPolarity", + "RatificationError", + "SAFE_COMPOSITION_CATEGORIES", + "UnknownCategory", + "WrongClaimSubType", + "WrongCompositionCategory", + "WrongZeroViolationCandidate", + "apply_composition_claim", +] diff --git a/teaching/math_contemplation.py b/teaching/math_contemplation.py index a4035a14..3adb83de 100644 --- a/teaching/math_contemplation.py +++ b/teaching/math_contemplation.py @@ -134,10 +134,23 @@ def audit_problem_to_evidence( # docs/handoff/ADR-0167-FOLLOWUPS.md §1. _CHANGE_KIND_BY_PAIR: dict[tuple[str, str], str] = { ("unexpected_category", "pre_frame_filler_sentence"): "matcher_extension", - ("unexpected_category", "multi_subject_sentence"): "frame_reclassification", ("unexpected_category", "fraction_percentage_literal"): "matcher_extension", - ("unexpected_category", "descriptive_frame_question"): "frame_reclassification", ("unresolved_pronoun", "pronoun_resolution"): "matcher_extension", + # ADR-0169 CC-3: route CompositionClaim audit groups (20 of 47 cases — + # `quantity_extraction` = 12 + `multi_quantity_composition` = 8) to the + # new CompositionClaim handler. Previously these fell through to + # `injector_sub_shape` because no pair entry existed. + ("incomplete_operation", "quantity_extraction"): "composition_reclassification", + ("incomplete_operation", "multi_quantity_composition"): "composition_reclassification", + # ADR-0169 CC-3 demotion: the prior two `frame_reclassification` routes + # for (unexpected_category, multi_subject_sentence) and + # (unexpected_category, descriptive_frame_question) were proven at the + # 2026-05-27 end-to-end demo to be handler mismatches — those refusals + # need ReferenceClaim/CompositionClaim and SlotClaim respectively, not + # FrameClaim (whose SAFE_FRAME_CATEGORIES allowlist is quantity/ownership + # frames). Until SlotClaim and ReferenceClaim handlers ship, they fall + # through to `injector_sub_shape` (the catch-all) so the workbench stops + # emitting handler-mismatched FrameClaim proposals. } # Single-key fallback retained for completeness — covers reader refusals @@ -263,9 +276,15 @@ def _build_reasoning_trace( "Dispatched via the (refusal_reason, missing_operator) pair table: " "(unexpected_category, pre_frame_filler_sentence|" "fraction_percentage_literal) → matcher_extension; " + "(unresolved_pronoun, pronoun_resolution) → matcher_extension; " + "(incomplete_operation, quantity_extraction|" + "multi_quantity_composition) → composition_reclassification " + "(ADR-0169 CC-3). " "(unexpected_category, multi_subject_sentence|" - "descriptive_frame_question) → frame_reclassification; " - "(unresolved_pronoun, pronoun_resolution) → matcher_extension. " + "descriptive_frame_question) demoted from frame_reclassification " + "to injector_sub_shape (ADR-0169 CC-3): the FrameClaim " + "SAFE_FRAME_CATEGORIES allowlist does not cover those shapes; " + "they await ReferenceClaim/SlotClaim handlers. " "Refusal-reason fallback: lexicon_entry → vocabulary_addition; " "narrowness_violation → matcher_extension; " "frame_unrecognized → frame_reclassification; " diff --git a/teaching/math_contemplation_proposal.py b/teaching/math_contemplation_proposal.py index 46786a62..4cc5a28d 100644 --- a/teaching/math_contemplation_proposal.py +++ b/teaching/math_contemplation_proposal.py @@ -43,6 +43,7 @@ ChangeKind = Literal[ "injector_sub_shape", "vocabulary_addition", "frame_reclassification", + "composition_reclassification", ] _VALID_CHANGE_KINDS: frozenset[str] = frozenset({ @@ -50,6 +51,7 @@ _VALID_CHANGE_KINDS: frozenset[str] = frozenset({ "injector_sub_shape", "vocabulary_addition", "frame_reclassification", + "composition_reclassification", }) _WRONG_ZERO_MIN_LEN: int = 40 diff --git a/tests/test_adr_0172_w1_shape_proposal.py b/tests/test_adr_0172_w1_shape_proposal.py index f07163c4..ac83a555 100644 --- a/tests/test_adr_0172_w1_shape_proposal.py +++ b/tests/test_adr_0172_w1_shape_proposal.py @@ -9,7 +9,7 @@ Verifies the eight obligations specified in the ADR-0172 brief: 5. JSON-serializable payload enforced. 6. wrong_zero_assertion non-empty (≥40 chars) enforced. 7. reasoning_trace required (None rejected). -8. all four change_kinds round-trip cleanly. +8. all five change_kinds round-trip cleanly (including ADR-0169 CC-2's composition_reclassification). W0 dependency: ``teaching/math_reasoning_trace.py`` (A1 branch) has not landed yet. Tests use a minimal stub that exposes the ``trace_id`` duck- @@ -208,7 +208,7 @@ def test_reasoning_trace_required_empty_trace_id() -> None: # --------------------------------------------------------------------------- -# Test 8 — all four change_kinds round-trip +# Test 8 — all five change_kinds round-trip # --------------------------------------------------------------------------- @@ -217,6 +217,7 @@ def test_reasoning_trace_required_empty_trace_id() -> None: "injector_sub_shape", "vocabulary_addition", "frame_reclassification", + "composition_reclassification", # ADR-0169 CC-2 ]) def test_all_four_change_kinds_round_trip(kind: str) -> None: """Every valid change_kind is accepted and round-trips through the schema.""" diff --git a/tests/test_adr_0172_w2_decomposer.py b/tests/test_adr_0172_w2_decomposer.py index 230d0b19..d54b1c60 100644 --- a/tests/test_adr_0172_w2_decomposer.py +++ b/tests/test_adr_0172_w2_decomposer.py @@ -129,7 +129,10 @@ def test_decompose_audit_change_kind_dispatch(tmp_path: Path) -> None: ("lexicon_entry", "lexicon_entry", "vocabulary_addition"), ("narrowness_violation", "multi_quantity_composition", "matcher_extension"), ("frame_unrecognized", "pre_frame_filler_sentence", "frame_reclassification"), - ("incomplete_operation", "quantity_extraction", "injector_sub_shape"), + # ADR-0169 CC-3: (incomplete_operation, quantity_extraction) routes + # to composition_reclassification via the pair table (formerly + # fell through to injector_sub_shape). + ("incomplete_operation", "quantity_extraction", "composition_reclassification"), ] per_case: list[dict] = [] for refusal_reason, missing_operator, _kind in branches: @@ -342,10 +345,18 @@ def test_decompose_audit_pair_dispatch_unexpected_category_matcher( assert proposals[0].proposed_change_kind == "matcher_extension" -def test_decompose_audit_pair_dispatch_unexpected_category_frame_reclass( +def test_decompose_audit_pair_dispatch_multi_subject_sentence_injector( tmp_path: Path, ) -> None: - """(unexpected_category, multi_subject_sentence) → frame_reclassification.""" + """(unexpected_category, multi_subject_sentence) → injector_sub_shape (ADR-0169 CC-3). + + Demoted from frame_reclassification at the 2026-05-27 end-to-end demo: + the FrameClaim SAFE_FRAME_CATEGORIES allowlist (increment_frame, + decrement_frame, transfer_frame, remainder_frame) does not cover + multi_subject_sentence shapes — those need ReferenceClaim / + CompositionClaim, not FrameClaim. Until those handlers ship, this + pair falls through to injector_sub_shape. + """ audit = _write_audit( tmp_path, [ @@ -363,7 +374,82 @@ def test_decompose_audit_pair_dispatch_unexpected_category_frame_reclass( ) proposals = decompose_audit(audit) assert len(proposals) == 1 - assert proposals[0].proposed_change_kind == "frame_reclassification" + assert proposals[0].proposed_change_kind == "injector_sub_shape" + + +def test_decompose_audit_pair_dispatch_descriptive_frame_question_injector( + tmp_path: Path, +) -> None: + """(unexpected_category, descriptive_frame_question) → injector_sub_shape (ADR-0169 CC-3). + + Demoted from frame_reclassification: needs SlotClaim, not FrameClaim. + """ + audit = _write_audit( + tmp_path, + [ + _case( + case_id="dfq-001", + refusal_reason="unexpected_category", + missing_operator="descriptive_frame_question", + ), + _case( + case_id="dfq-002", + refusal_reason="unexpected_category", + missing_operator="descriptive_frame_question", + ), + ], + ) + proposals = decompose_audit(audit) + assert len(proposals) == 1 + assert proposals[0].proposed_change_kind == "injector_sub_shape" + + +def test_quantity_extraction_routes_to_composition_reclassification( + tmp_path: Path, +) -> None: + """(incomplete_operation, quantity_extraction) → composition_reclassification (ADR-0169 CC-3).""" + audit = _write_audit( + tmp_path, + [ + _case( + case_id="qe-001", + refusal_reason="incomplete_operation", + missing_operator="quantity_extraction", + ), + _case( + case_id="qe-002", + refusal_reason="incomplete_operation", + missing_operator="quantity_extraction", + ), + ], + ) + proposals = decompose_audit(audit) + assert len(proposals) == 1 + assert proposals[0].proposed_change_kind == "composition_reclassification" + + +def test_multi_quantity_composition_routes_to_composition_reclassification( + tmp_path: Path, +) -> None: + """(incomplete_operation, multi_quantity_composition) → composition_reclassification (ADR-0169 CC-3).""" + audit = _write_audit( + tmp_path, + [ + _case( + case_id="mqc-001", + refusal_reason="incomplete_operation", + missing_operator="multi_quantity_composition", + ), + _case( + case_id="mqc-002", + refusal_reason="incomplete_operation", + missing_operator="multi_quantity_composition", + ), + ], + ) + proposals = decompose_audit(audit) + assert len(proposals) == 1 + assert proposals[0].proposed_change_kind == "composition_reclassification" def test_decompose_audit_pair_dispatch_pronoun(tmp_path: Path) -> None: @@ -391,16 +477,30 @@ def test_decompose_audit_pair_dispatch_pronoun(tmp_path: Path) -> None: def test_decompose_audit_real_audit_change_kind_distribution() -> None: """Against the real audit_brief_11.json, dispatch yields the expected mix. - Per ADR-0172 tightening follow-up #2, the widened heuristic must produce - at least 3 matcher_extension and at least 2 frame_reclassification - proposals on the train-sample audit. + Per ADR-0169 CC-3, the tightened heuristic must produce on the + train-sample audit: + - 3 matcher_extension proposals (pre_frame_filler, fraction_percentage, + unresolved_pronoun) + - 2 composition_reclassification proposals (quantity_extraction, + multi_quantity_composition) — 20 of 47 cases + - 3 injector_sub_shape proposals (multi_subject_sentence and + descriptive_frame_question demoted per CC-3, plus + unattached_quantity/unit_binding catch-all) + - 0 frame_reclassification proposals (the two prior routes were + demoted; audit_brief_11.json has no frame_unrecognized rows) """ proposals = decompose_audit(REAL_AUDIT_PATH) kinds = [p.proposed_change_kind for p in proposals] matcher = sum(1 for k in kinds if k == "matcher_extension") + composition = sum(1 for k in kinds if k == "composition_reclassification") frame = sum(1 for k in kinds if k == "frame_reclassification") - assert matcher >= 3, f"expected ≥3 matcher_extension, got {matcher}: {kinds}" - assert frame >= 2, f"expected ≥2 frame_reclassification, got {frame}: {kinds}" + injector = sum(1 for k in kinds if k == "injector_sub_shape") + assert matcher == 3, f"expected 3 matcher_extension, got {matcher}: {kinds}" + assert composition == 2, ( + f"expected 2 composition_reclassification, got {composition}: {kinds}" + ) + assert frame == 0, f"expected 0 frame_reclassification, got {frame}: {kinds}" + assert injector == 3, f"expected 3 injector_sub_shape, got {injector}: {kinds}" # --------------------------------------------------------------------------- diff --git a/tests/test_math_composition_ratification.py b/tests/test_math_composition_ratification.py new file mode 100644 index 00000000..444934ac --- /dev/null +++ b/tests/test_math_composition_ratification.py @@ -0,0 +1,849 @@ +"""ADR-0169 — CompositionClaim ratification handler tests. + +Mirrors :mod:`tests.test_math_frame_ratification` (F1) and pins: + +1. SAFE_COMPOSITION_CATEGORIES is exactly the ADR-0169 allowlist (3 entries). +2. apply_composition_claim writes an entry for a safe category. +3. receipt records before/after sha + evidence_hash. +4. idempotent same-evidence → AlreadyRatified. +5. rejects non-composition sub_type. +6. rejects categories outside SAFE_COMPOSITION_CATEGORIES → WrongCompositionCategory. +7. rejects invalid polarity. +8. rejects evidence tampering. +9. rejects evidence laundering (source='corpus' is forbidden). +10. case 0050 hazard pin — after ratification, case 0050 still refuses. +11. polarity=falsifies branch records non-composing pattern without admitting. +12. duplicate evidence on second apply appends evidence_hash, not new row. +13. manifest.json checksum is unchanged by composition ratification. +14. alphabetical sort by surface_pattern preserved across writes. +15. claim signature canonicalization deterministic. +16. claim signature replay equivalence cross-process (subprocess). +17. queue-order independence (A→B == B→A ratify). +18. partition: cognition TeachingChainProposal records not seen by handler. +19. audit evidence not laundered as source="corpus" at proposal layer. +20. workbench dispatch routes composition_reclassification → CompositionClaim. +21. proposed_change_kind Literal accepts composition_reclassification. +22. JSONL round-trip preserves composition_reclassification change_kind. + +ADR-0169 §"Decision" + ADR-0169.1 §"Evidence floor" + hazard pins. +""" + +from __future__ import annotations + +import functools +import hashlib +import json +import shutil +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from evals.refusal_taxonomy.shape_categories import ShapeCategory +from generate.comprehension import lexicon as comprehension_lexicon +from generate.comprehension import lifecycle +from generate.comprehension.audit import AuditRow, audit_problem +from generate.comprehension.state import ReaderRefusal +from teaching.math_composition_proposal import ( + build_composition_claim_proposal, + build_evidence_pointer, + compute_claim_signature, +) +from teaching.math_composition_ratification import ( + AlreadyRatified, + EvidenceLaundering, + EvidenceTampering, + InvalidPolarity, + SAFE_COMPOSITION_CATEGORIES, + WrongClaimSubType, + WrongCompositionCategory, + apply_composition_claim, +) +from teaching.math_contemplation_proposal import ( + _VALID_CHANGE_KINDS, + ChangeKind, + build_proposal, + from_jsonl_record, + to_jsonl_record, +) +from teaching.math_evidence import from_audit_row +from teaching.math_reasoning_trace import ReasoningStep, build_trace + + +REPO_ROOT = Path(__file__).resolve().parents[1] +PACK_ROOT = REPO_ROOT / "language_packs" / "data" / "en_core_math_v1" +CASES_PATH = REPO_ROOT / "evals" / "gsm8k_math" / "train_sample" / "v1" / "cases.jsonl" +AUDIT_BRIEF_PATH = ( + REPO_ROOT + / "evals" + / "gsm8k_math" + / "train_sample" + / "v1" + / "audit_brief_11.json" +) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def pack_copy(tmp_path: Path) -> Path: + target = tmp_path / "en_core_math_v1" + shutil.copytree(PACK_ROOT, target) + (target / "compositions").mkdir(exist_ok=True) + comprehension_lexicon._CACHE.clear() + lifecycle._get_lexicon.cache_clear() + yield target + comprehension_lexicon._CACHE.clear() + lifecycle._get_lexicon.cache_clear() + + +def _row( + surface: str, + *, + missing_operator: str = "quantity_extraction", + refusal_reason: str = "incomplete_operation", + case_id: str | None = None, +) -> AuditRow: + return AuditRow( + case_id=case_id or f"case-{surface}", + sentence_index=0, + token_index=2, + token_text=surface, + recognized_terms=("Mark", "buys"), + skipped_frame=None, + missing_operator=missing_operator, + refusal_reason=refusal_reason, + refusal_detail=f"composition gap for '{surface}'", + ) + + +def _claim(surface: str, *, sub_type: str = "composition", case_id: str | None = None): + return from_audit_row(_row(surface, case_id=case_id), sub_type) # type: ignore[arg-type] + + +def _entries(path: Path) -> list[dict]: + if not path.exists(): + return [] + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def _manifest_sha(pack_root: Path) -> str: + return hashlib.sha256((pack_root / "manifest.json").read_bytes()).hexdigest() + + +def _use_pack_for_reader(monkeypatch: pytest.MonkeyPatch, pack_root: Path) -> None: + comprehension_lexicon._CACHE.clear() + + @functools.cache + def _tmp_lexicon(): + return comprehension_lexicon.load_lexicon(pack_root) + + monkeypatch.setattr(lifecycle, "_get_lexicon", _tmp_lexicon) + + +# --------------------------------------------------------------------------- +# 1 — SAFE_COMPOSITION_CATEGORIES allowlist pinned +# --------------------------------------------------------------------------- + + +def test_safe_composition_categories_is_adr_0169_allowlist() -> None: + """The three ADR-0169 categories, no more, no less.""" + assert SAFE_COMPOSITION_CATEGORIES == frozenset( + { + "multiplicative_composition", + "additive_composition", + "subtractive_composition", + } + ) + + +# --------------------------------------------------------------------------- +# 2 — apply writes an entry for a safe category +# --------------------------------------------------------------------------- + + +def test_apply_composition_claim_writes_entry(pack_copy: Path) -> None: + receipt = apply_composition_claim( + claim=_claim("each"), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + surface_pattern="bound(count) bound(unit_cost)", + pack_root=pack_copy, + ) + + assert receipt.surface_pattern == "bound(count) bound(unit_cost)" + assert receipt.composition_category == "multiplicative_composition" + assert receipt.polarity == "affirms" + assert receipt.is_duplicate_evidence is False + entry = next( + e + for e in _entries( + pack_copy / "compositions" / "multiplicative_composition.jsonl" + ) + if e["surface_pattern"] == "bound(count) bound(unit_cost)" + ) + assert entry["composition_category"] == "multiplicative_composition" + assert entry["polarity"] == "affirms" + assert receipt.evidence_hash in entry["evidence_hashes"] + + +# --------------------------------------------------------------------------- +# 3 — receipt records before/after sha +# --------------------------------------------------------------------------- + + +def test_receipt_records_before_after_sha(pack_copy: Path) -> None: + receipt = apply_composition_claim( + claim=_claim("each"), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + + assert len(receipt.file_sha256_before) == 64 + assert len(receipt.file_sha256_after) == 64 + assert receipt.file_sha256_before != receipt.file_sha256_after + assert receipt.evidence_hash == _claim("each").evidence_hash + + +# --------------------------------------------------------------------------- +# 4 — idempotent same-evidence raises AlreadyRatified +# --------------------------------------------------------------------------- + + +def test_idempotent_same_evidence_raises_already_ratified(pack_copy: Path) -> None: + claim = _claim("each") + apply_composition_claim( + claim=claim, + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + + with pytest.raises(AlreadyRatified, match="already ratified"): + apply_composition_claim( + claim=claim, + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + + +# --------------------------------------------------------------------------- +# 5 — rejects non-composition sub_type +# --------------------------------------------------------------------------- + + +def test_rejects_non_composition_sub_type(pack_copy: Path) -> None: + with pytest.raises(WrongClaimSubType): + apply_composition_claim( + claim=_claim("each", sub_type="lexical"), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + + +# --------------------------------------------------------------------------- +# 6 — rejects categories outside SAFE_COMPOSITION_CATEGORIES (wrong=0 hazard) +# --------------------------------------------------------------------------- + + +def test_rejects_unsafe_composition_category(pack_copy: Path) -> None: + with pytest.raises( + WrongCompositionCategory, match="SAFE_COMPOSITION_CATEGORIES" + ): + apply_composition_claim( + claim=_claim("each"), + composition_category="distributive_composition", # not in allowlist + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + + +# --------------------------------------------------------------------------- +# 7 — rejects invalid polarity +# --------------------------------------------------------------------------- + + +def test_rejects_invalid_polarity(pack_copy: Path) -> None: + with pytest.raises(InvalidPolarity, match="polarity must be one of"): + apply_composition_claim( + claim=_claim("each"), + composition_category="multiplicative_composition", + polarity="maybe", # not affirms/falsifies + reviewer="Ada", + pack_root=pack_copy, + ) + + +# --------------------------------------------------------------------------- +# 8 — rejects evidence tampering +# --------------------------------------------------------------------------- + + +def test_rejects_evidence_tampering(pack_copy: Path) -> None: + claim = _claim("each") + object.__setattr__(claim, "evidence_hash", "0" * 64) + + with pytest.raises(EvidenceTampering): + apply_composition_claim( + claim=claim, + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + + +# --------------------------------------------------------------------------- +# 9 — rejects evidence laundering as source='corpus' +# --------------------------------------------------------------------------- + + +def test_rejects_evidence_laundered_as_corpus(pack_copy: Path) -> None: + """ADR-0169.1 §'Evidence floor': source='corpus' MUST be rejected.""" + with pytest.raises(EvidenceLaundering, match="MUST NOT be laundered"): + apply_composition_claim( + claim=_claim("each"), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + evidence_source="corpus", # forbidden + ) + + +# --------------------------------------------------------------------------- +# 10 — case 0050 hazard pin: still refused after ratification +# --------------------------------------------------------------------------- + + +def test_case_0050_remains_refused_after_composition_ratification( + pack_copy: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """After ratifying a safe composition pattern, case 0050 must still refuse. + + Case 0050 ("Mark does a gig every other day for 2 weeks. ..." — the + period token at sentence_index=0 with missing_operator= + pre_frame_filler_sentence) is the prototype wrong=0 hazard. A + speculative composition admission here would feed a wrong operand to + the solver. The new compositions/ entries must not open that + admission path. + """ + apply_composition_claim( + claim=_claim("each"), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + _use_pack_for_reader(monkeypatch, pack_copy) + cases = [json.loads(line) for line in CASES_PATH.read_text().splitlines()] + case = next(c for c in cases if c["case_id"] == "gsm8k-train-sample-v1-0050") + + result, _rows = audit_problem(case["question"], case_id=case["case_id"]) + + assert isinstance(result, ReaderRefusal), ( + "case 0050 must remain refused after CompositionClaim ratification" + ) + assert result.sentence_index == 0 + + +# --------------------------------------------------------------------------- +# 11 — polarity=falsifies branch records non-composing pattern +# --------------------------------------------------------------------------- + + +def test_polarity_falsifies_records_non_composing(pack_copy: Path) -> None: + """A 'falsifies' ratification records the pattern as NOT composing. + + This is the negative-evidence path: an operator reviewed the audit row + and decided the pattern does not in fact compose under the category. + The entry is written to the same source file but with polarity=falsifies + so downstream registry builds can mark the pattern as a known + non-composer (refusal-stable). + """ + receipt = apply_composition_claim( + claim=_claim("along"), + composition_category="multiplicative_composition", + polarity="falsifies", + reviewer="Ada", + pack_root=pack_copy, + ) + + assert receipt.polarity == "falsifies" + entry = next( + e + for e in _entries( + pack_copy / "compositions" / "multiplicative_composition.jsonl" + ) + if e["surface_pattern"] == "along" + ) + assert entry["polarity"] == "falsifies" + + # affirms + falsifies of the same pattern produce distinct entries + receipt2 = apply_composition_claim( + claim=_claim("each"), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + assert receipt2.polarity == "affirms" + polarities = { + e["polarity"] + for e in _entries( + pack_copy / "compositions" / "multiplicative_composition.jsonl" + ) + } + assert {"affirms", "falsifies"} <= polarities + + +# --------------------------------------------------------------------------- +# 12 — duplicate evidence appends to existing row +# --------------------------------------------------------------------------- + + +def test_duplicate_pattern_polarity_with_new_evidence_appends_hash( + pack_copy: Path, +) -> None: + """ADR-0169.1 §'Idempotency': same claim + new evidence appends hash, not row.""" + apply_composition_claim( + claim=_claim("each", case_id="case-each-1"), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + + # Second evidence for the same claim — different case_id, so the + # evidence_hash differs even though pattern+polarity+category match. + second_row = AuditRow( + case_id="case-each-2", + sentence_index=1, + token_index=4, + token_text="each", + recognized_terms=("Bob", "buys"), + skipped_frame=None, + missing_operator="quantity_extraction", + refusal_reason="incomplete_operation", + refusal_detail="duplicate composition evidence", + ) + second_claim = from_audit_row(second_row, "composition") + receipt = apply_composition_claim( + claim=second_claim, + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + + assert receipt.is_duplicate_evidence is True + rows = [ + e + for e in _entries( + pack_copy / "compositions" / "multiplicative_composition.jsonl" + ) + if e["surface_pattern"] == "each" + ] + assert len(rows) == 1, "duplicate evidence must not create a second row" + assert len(rows[0]["evidence_hashes"]) == 2 + + +# --------------------------------------------------------------------------- +# 13 — manifest.json checksum unchanged +# --------------------------------------------------------------------------- + + +def test_manifest_checksum_unchanged_by_composition_ratification( + pack_copy: Path, +) -> None: + manifest_bytes_before = (pack_copy / "manifest.json").read_bytes() + manifest_sha_before = _manifest_sha(pack_copy) + declared_before = json.loads(manifest_bytes_before)["checksum"] + + apply_composition_claim( + claim=_claim("each"), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + + manifest_bytes_after = (pack_copy / "manifest.json").read_bytes() + assert manifest_bytes_after == manifest_bytes_before + assert _manifest_sha(pack_copy) == manifest_sha_before + assert json.loads(manifest_bytes_after)["checksum"] == declared_before + + +# --------------------------------------------------------------------------- +# 14 — alphabetical sort preserved across multiple writes +# --------------------------------------------------------------------------- + + +def test_alphabetical_sort_preserved(pack_copy: Path) -> None: + for surface in ("times", "each", "apiece"): + apply_composition_claim( + claim=_claim(surface), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + + patterns = [ + e["surface_pattern"] + for e in _entries( + pack_copy / "compositions" / "multiplicative_composition.jsonl" + ) + ] + assert patterns == sorted(patterns) + + +# --------------------------------------------------------------------------- +# 15 — claim signature canonicalization deterministic +# --------------------------------------------------------------------------- + + +def test_claim_signature_canonicalization_deterministic() -> None: + """ADR-0169 §'Replay obligations' #1: equivalent claims → identical signatures.""" + ev1 = _claim("each", case_id="case-1") + ev2 = _claim("each", case_id="case-1") # identical evidence + p1 = build_evidence_pointer(ev1) + p2 = build_evidence_pointer(ev2) + + sig_a = compute_claim_signature( + surface_pattern="bound(count) bound(unit_cost)", + composition_category="multiplicative_composition", + polarity="affirms", + evidence=(p1,), + ) + sig_b = compute_claim_signature( + surface_pattern="bound(count) bound(unit_cost)", + composition_category="multiplicative_composition", + polarity="affirms", + evidence=(p2,), + ) + assert sig_a == sig_b + assert len(sig_a) == 64 + + # Different polarity → different signature + sig_c = compute_claim_signature( + surface_pattern="bound(count) bound(unit_cost)", + composition_category="multiplicative_composition", + polarity="falsifies", + evidence=(p1,), + ) + assert sig_a != sig_c + + +# --------------------------------------------------------------------------- +# 16 — claim signature replay equivalence cross-process (subprocess) +# --------------------------------------------------------------------------- + + +def test_claim_signature_replay_equivalence_cross_process() -> None: + """ADR-0169 §'Replay obligations' #2: equivalent across processes.""" + script = textwrap.dedent( + """ + import json, sys + from teaching.math_composition_proposal import ( + build_evidence_pointer, compute_claim_signature, + ) + from teaching.math_evidence import from_audit_row + from generate.comprehension.audit import AuditRow + row = AuditRow( + case_id="case-xprocess", sentence_index=0, token_index=2, + token_text="each", recognized_terms=("Mark", "buys"), + skipped_frame=None, missing_operator="quantity_extraction", + refusal_reason="incomplete_operation", + refusal_detail="composition gap for 'each'", + ) + ev = from_audit_row(row, "composition") + p = build_evidence_pointer(ev) + sig = compute_claim_signature( + surface_pattern="bound(count) bound(unit_cost)", + composition_category="multiplicative_composition", + polarity="affirms", + evidence=(p,), + ) + sys.stdout.write(sig) + """ + ) + proc = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + cwd=str(REPO_ROOT), + check=True, + ) + subprocess_sig = proc.stdout.strip() + + # In-process compute + row = AuditRow( + case_id="case-xprocess", + sentence_index=0, + token_index=2, + token_text="each", + recognized_terms=("Mark", "buys"), + skipped_frame=None, + missing_operator="quantity_extraction", + refusal_reason="incomplete_operation", + refusal_detail="composition gap for 'each'", + ) + ev = from_audit_row(row, "composition") + p = build_evidence_pointer(ev) + inproc_sig = compute_claim_signature( + surface_pattern="bound(count) bound(unit_cost)", + composition_category="multiplicative_composition", + polarity="affirms", + evidence=(p,), + ) + + assert subprocess_sig == inproc_sig + assert len(subprocess_sig) == 64 + + +# --------------------------------------------------------------------------- +# 17 — queue-order independence (A→B == B→A) +# --------------------------------------------------------------------------- + + +def test_queue_order_independence(pack_copy: Path, tmp_path: Path) -> None: + """ADR-0169 §'Replay obligations' #2: queue-order independence. + + Ratifying claim A then claim B produces the same target file as + ratifying B then A. Sort-by-surface_pattern write discipline carries + most of the load; this test pins it. + """ + # Order A → B + target_a = tmp_path / "en_core_math_v1_a" + shutil.copytree(PACK_ROOT, target_a) + (target_a / "compositions").mkdir(exist_ok=True) + apply_composition_claim( + claim=_claim("each", case_id="case-each"), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=target_a, + ) + apply_composition_claim( + claim=_claim("apiece", case_id="case-apiece"), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=target_a, + ) + out_a = ( + target_a / "compositions" / "multiplicative_composition.jsonl" + ).read_bytes() + + # Order B → A + target_b = tmp_path / "en_core_math_v1_b" + shutil.copytree(PACK_ROOT, target_b) + (target_b / "compositions").mkdir(exist_ok=True) + apply_composition_claim( + claim=_claim("apiece", case_id="case-apiece"), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=target_b, + ) + apply_composition_claim( + claim=_claim("each", case_id="case-each"), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=target_b, + ) + out_b = ( + target_b / "compositions" / "multiplicative_composition.jsonl" + ).read_bytes() + + # Provenance carries the date but not the order, so byte-equality holds. + assert out_a == out_b + + +# --------------------------------------------------------------------------- +# 18 — partition: cognition TeachingChainProposal records not seen +# --------------------------------------------------------------------------- + + +def test_partition_cognition_proposals_not_seen(pack_copy: Path) -> None: + """ADR-0169 §'Partition guarantees': handler is math-only. + + A cognition-style proposal (a TeachingChainProposal-like dict, or any + object that is not a MathReaderRefusalEvidence with + sub_type='composition') must be rejected by the handler — not silently + accepted as if it were math evidence. + """ + # A non-composition sub_type is the cognition-flow analog: rejected. + with pytest.raises(WrongClaimSubType): + apply_composition_claim( + claim=_claim("each", sub_type="lexical"), + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + + # A non-evidence object (the cognition TeachingChainProposal would not + # carry sub_type at all) must also fail loudly — AttributeError or + # TypeError, not a silent admit. + class _NotEvidence: + pass + + with pytest.raises((AttributeError, TypeError)): + apply_composition_claim( + claim=_NotEvidence(), # type: ignore[arg-type] + composition_category="multiplicative_composition", + polarity="affirms", + reviewer="Ada", + pack_root=pack_copy, + ) + + +# --------------------------------------------------------------------------- +# 19 — audit evidence not laundered as source="corpus" at proposal layer +# --------------------------------------------------------------------------- + + +def test_audit_evidence_not_laundered_as_corpus_at_proposal_layer() -> None: # noqa: D401 + """ADR-0169.1 §'Trip-wires' #1: proposal layer rejects corpus pointers. + + Defense-in-depth: the handler rejects source='corpus' on apply, and the + proposal-build layer (build_composition_claim_proposal) also rejects + pointers whose source drifted from 'math_audit'. + """ + ev = _claim("each") + pointer = build_evidence_pointer(ev) + + # Forge a pointer with corpus source — direct dataclass construction + # since the factory pins source='math_audit'. + from teaching.math_composition_proposal import MathReaderRefusalEvidencePointer + + laundered = MathReaderRefusalEvidencePointer( + source="corpus", # type: ignore[arg-type] + case_id=pointer.case_id, + sentence_index=pointer.sentence_index, + token_index=pointer.token_index, + missing_operator=pointer.missing_operator, + refusal_reason=pointer.refusal_reason, + evidence_hash=pointer.evidence_hash, + audit_row_digest=pointer.audit_row_digest, + ) + with pytest.raises(ValueError, match="math_audit"): + build_composition_claim_proposal( + surface_pattern="bound(count) bound(unit_cost)", + composition_category="multiplicative_composition", + polarity="affirms", + evidence=(laundered,), + ) + + +# --------------------------------------------------------------------------- +# 20 — workbench dispatch routes composition_reclassification +# --------------------------------------------------------------------------- + + +def test_workbench_dispatch_composition_reclassification() -> None: + """Workbench routes composition_reclassification → CompositionClaim, not 501.""" + from workbench.readers import _HANDLER_DISPATCH + + assert _HANDLER_DISPATCH["composition_reclassification"] == "CompositionClaim" + # Sibling routes still exist + assert _HANDLER_DISPATCH["vocabulary_addition"] == "LexicalClaim" + assert _HANDLER_DISPATCH["frame_reclassification"] == "FrameClaim" + + +# --------------------------------------------------------------------------- +# 21 — proposed_change_kind Literal accepts composition_reclassification +# --------------------------------------------------------------------------- + + +def test_proposal_change_kind_literal_accepts_composition_reclassification() -> None: + """Literal extension permits composition_reclassification as a valid ChangeKind.""" + from typing import get_args + + assert "composition_reclassification" in _VALID_CHANGE_KINDS + assert "composition_reclassification" in get_args(ChangeKind) + + +# --------------------------------------------------------------------------- +# 22 — JSONL round-trip preserves composition_reclassification change_kind +# --------------------------------------------------------------------------- + + +def test_jsonl_round_trip_with_composition_reclassification() -> None: + """W1 round-trip test extended for the new change_kind (ADR-0169 §'Acceptance gates').""" + ev1 = _claim("each", case_id="case-1") + ev2 = _claim("each", case_id="case-2") + steps = ( + ReasoningStep( + step_index=0, + step_kind="observation", + input_pointers=("case-1", "case-2"), + claim="2 refusal rows share composition gap", + justification="grouped by (refusal_reason, missing_operator)", + output_payload={"evidence_count": 2}, + ), + ReasoningStep( + step_index=1, + step_kind="grouping", + input_pointers=("case-1", "case-2"), + claim="group key", + justification="exact pair equality", + output_payload={"k": "v"}, + ), + ReasoningStep( + step_index=2, + step_kind="hypothesis", + input_pointers=("case-1", "case-2"), + claim="composition_reclassification fits", + justification="dispatched via pair table to composition_reclassification", + output_payload={"proposed_change_kind": "composition_reclassification"}, + ), + ReasoningStep( + step_index=3, + step_kind="conclusion", + input_pointers=("case-1", "case-2"), + claim="propose composition_reclassification", + justification="evidence-only proposal", + output_payload={"proposed_change_kind": "composition_reclassification"}, + ), + ) + trace = build_trace(steps) + proposal = build_proposal( + shape_category=ShapeCategory.UNCATEGORIZED, + structural_commonality="2 refusals share composition gap", + evidence_pointers=(ev1, ev2), + proposed_change_kind="composition_reclassification", + proposed_change_payload={"k": "v"}, + wrong_zero_assertion=( + "Proposal is evidence-only; ratification handler is the wrong=0 surface." + ), + replay_equivalence_hash="0" * 64, + reasoning_trace=trace, + ) + + record = to_jsonl_record(proposal) + assert record["proposed_change_kind"] == "composition_reclassification" + round_tripped = from_jsonl_record(record) + assert round_tripped.proposed_change_kind == "composition_reclassification" + assert round_tripped.proposal_id == proposal.proposal_id diff --git a/workbench/readers.py b/workbench/readers.py index fad24cf1..8a46900e 100644 --- a/workbench/readers.py +++ b/workbench/readers.py @@ -55,6 +55,7 @@ _DEFAULT_MATH_AUDIT_PATH = ( _HANDLER_DISPATCH: dict[str, str] = { "vocabulary_addition": "LexicalClaim", "frame_reclassification": "FrameClaim", + "composition_reclassification": "CompositionClaim", } @@ -404,6 +405,14 @@ def read_math_proposal( f"# apply_frame_claim(claim=, frame_category='increment_frame', " f"polarity='affirms', reviewer='')" ) + elif handler_name == "CompositionClaim": + suggested_cli = ( + f"# ratify via Python REPL (ADR-0169):\n" + f"from teaching.math_composition_ratification import apply_composition_claim\n" + f"# apply_composition_claim(claim=, " + f"composition_category='multiplicative_composition', " + f"polarity='affirms', reviewer='')" + ) return MathProposalDetail( proposal_id=proposal_id, @@ -466,6 +475,13 @@ def ratify_math_proposal( f"# apply_frame_claim(claim=, frame_category='increment_frame', " f"polarity='affirms', reviewer='')" ) + elif handler_name == "CompositionClaim": + suggested_cli = ( + f"from teaching.math_composition_ratification import apply_composition_claim\n" + f"# apply_composition_claim(claim=, " + f"composition_category='multiplicative_composition', " + f"polarity='affirms', reviewer='')" + ) return MathRatifyResult( proposal_id=proposal_id,