diff --git a/core/cli.py b/core/cli.py index fa18c800..df4eebed 100644 --- a/core/cli.py +++ b/core/cli.py @@ -86,6 +86,7 @@ _TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_me2_case_0019_admits.py", "tests/test_me3_additive_composition.py", "tests/test_me4_subtractive_composition.py", + "tests/test_me5_all_categories_integration.py", ), "algebra": ( "tests/test_versor_closure.py", diff --git a/docs/handoff/ME1-ME5-MILESTONE.md b/docs/handoff/ME1-ME5-MILESTONE.md new file mode 100644 index 00000000..311528d8 --- /dev/null +++ b/docs/handoff/ME1-ME5-MILESTONE.md @@ -0,0 +1,132 @@ +# Matcher Extensions Milestone — ME-1 through ME-5 + +**Date:** 2026-05-27 +**Author:** Shay +**Stack:** PRs #398 → #400 → #401 → #402 → #403 → #404 +**Status:** All five PRs open and stacked; matcher half of the math +composition compounding loop is operational end-to-end. + +--- + +## What landed + +The matcher-extension wave delivers a runtime consumer for every entry +in `SAFE_COMPOSITION_CATEGORIES`. When a recognizer match publishes a +`composition_shape` key in `parsed_anchors` AND the +composition_registry carries an `affirms` entry for that shape, +`inject_from_match` emits a pre-composed `CandidateInitial` whose +value is the arithmetically composed quantity. + +| PR | Wave | Scope | New tests | +|---|---|---|---| +| #398 | CW-1 + CW-2 | Consumption infrastructure: compile_frames, compile_compositions, frame_registry, composition_registry, manifest checksum extensions, inject_from_match consult | 38 | +| #400 | ME-1 | Currency-per-unit composition (multiplicative; Maria canary; Option A subject binding) | 21 | +| #401 | ME-2 | Cross-sentence subject binding (case 0019 real; threads prior_subject through match() + math_candidate_graph) | 19 | +| #402 | ME-3 | Additive composition (two-quantity same-unit "and" connective) | 15 | +| #403 | ME-4 | Subtractive composition (initial-then-removal; non-negative remainder discipline) | 17 | +| #404 | ME-5 | Integration smoke across all three SAFE categories + milestone | 4 | + +**Total new tests:** 114, all green. +**Packs suite:** 127 passed. +**`core eval gsm8k_math --split public`:** 150/150, `wrong=0`. + +--- + +## Architecture + +``` +audit refusal + → core eval math-contemplation (decomposer dispatch) + → MathCompositionClaimProposal + → HITL ratify via apply_composition_claim() + → compositions/{category}.jsonl append + → pack-compile (CW-2) → compositions.jsonl + manifest.composition_checksum + │ + ▼ + load_composition_registry() + │ +─── runtime path ─────────────────┼──────────────────────────────── + ▼ +recognizer matches statement + → matcher extension (ME-1..ME-4) publishes composition_shape + + composed_initial in parsed_anchors + → inject_from_match per-category injector returns () + → _consult_composition_registry sees composition_shape + → registry: affirms entry for the shape + → CandidateInitial admitted + → candidate graph admits the statement + → previously-refused audit case admits +``` + +--- + +## SAFE_COMPOSITION_CATEGORIES — matcher coverage matrix + +| Category | Surface pattern | Matcher | Dispatch via spec | Subject binding | +|---|---|---|---|---| +| `multiplicative_composition` | `bound(count) × bound(unit_cost)` | `_match_rate_with_currency` | `anchor_kind=\"currency_per_unit_composition\"` | Option A (ME-1) + Option C (ME-2 via `prior_subject`) | +| `additive_composition` | `bound(qty_a) + bound(qty_b)` | `_match_multiplicative_aggregation` | `anchor_kind=\"additive_quantity_composition\"` | Option A only | +| `subtractive_composition` | `bound(initial) − bound(removed)` | `_match_multiplicative_aggregation` | `anchor_kind=\"subtractive_quantity_composition\"` | Option A only | + +--- + +## Invariants preserved across the entire stack + +- `wrong == 0` on `core eval gsm8k_math --split public` (150/150) +- Case 0050 hazard pin (parametrized over all three allowlist categories) +- ADR-0166 — no new eval lanes +- ADR-0167 partition — no cognition imports in any new module +- ADR-0169 mutation boundary — registry is a gate, never an + arithmetic primitive; matcher owns the math +- All matcher detection paths byte-identical (regression pins per PR) +- `engine_state/*` never committed +- `SAFE_COMPOSITION_CATEGORIES` enforced at write AND load +- Polarity `falsifies` honored uniformly across all matchers +- Refusal-preferring: pronoun / determiner / cross-unit / unobserved + / unknown verb / zero count / negative remainder all refuse + +--- + +## Scope boundary — what does NOT fire yet + +The wiring is end-to-end correct (verified by ME-5's integration +test that stages a synthetic pack and admits all three canaries). +**Live `train_sample` admission** requires operator-seeded +ratifications: + +1. A `RatifiedRecognizer` proposal in the proposal log carrying: + - `shape_category = RATE_WITH_CURRENCY` (or `MULTIPLICATIVE_AGGREGATION`) + - `canonical_pattern.anchor_kind = "currency_per_unit_composition"` + (or `additive_quantity_composition` / `subtractive_quantity_composition`) + - the right `observed_*` sets +2. A composition_registry entry compiled into the canonical pack: + - `surface_pattern = "bound(count) × bound(unit_cost)"` + (or the additive / subtractive shapes) + - `composition_category` in the allowlist + - `polarity = "affirms"` + +These are operator decisions, not automated. When seeded, the live +train_sample run admits the 20 audit cases (12 quantity_extraction + +8 multi_quantity_composition) covered by this wave. + +--- + +## Recommended next dispatch + +**RAT-1:** Operator-workflow PR / runbook that seeds the necessary +RecognizerSpec proposals + composition_registry entries to fire the +live eval-delta truth test. With RAT-1 merged, `train_sample` moves +from 3/47 → ≥4/46 (with case 0050 still refused, `wrong=0` +preserved). The flywheel produces its first `ratify → admit` event +on the canonical pack. + +--- + +## Memory pointers + +- [[milestone-me1-me5-matcher-extensions-complete]] — this milestone +- [[project-ratification-consumption-gap-2026-05-27]] — original finding +- [[feedback-ratify-vs-consume-loop-closure]] — general pattern +- [[milestone-adr-0172-tier1-2026-05-27]] — Tier 1 parent +- [[adr-0167-audit-as-evidence-wave]] — parent corridor +- [[feedback-wrong-zero-hazard-case-0050]] — preserved across the stack diff --git a/tests/test_me5_all_categories_integration.py b/tests/test_me5_all_categories_integration.py new file mode 100644 index 00000000..c19d4751 --- /dev/null +++ b/tests/test_me5_all_categories_integration.py @@ -0,0 +1,330 @@ +"""ME-5 — integration smoke test across all three SAFE_COMPOSITION_CATEGORIES. + +Exercises the complete matcher → registry → injector → admission chain +for each of the three SAFE categories in one test run. Verifies the +ME-1 through ME-4 stack composes cleanly when all three category +ratifications are present in the same pack. + +This is the milestone canary: when this test passes, the math +composition flywheel's matcher half is operational end-to-end across +all admissible categories. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping + +from evals.refusal_taxonomy.shape_categories import ShapeCategory +from generate.comprehension.composition_registry import ( + clear_cache as clear_composition_cache, +) +from generate.math_candidate_parser import CandidateInitial +from generate.recognizer_anchor_inject import inject_from_match +from generate.recognizer_match import ( + RecognizerMatch, + _match_multiplicative_aggregation, + _match_rate_with_currency, +) +from language_packs.compile_compositions import compile_compositions +from teaching.math_composition_ratification import SAFE_COMPOSITION_CATEGORIES + + +def setup_function(_): + clear_composition_cache() + + +def _stage_pack_all_three(tmp_path: Path) -> Path: + pack = tmp_path / "en_core_math_v1" + comp_dir = pack / "compositions" + comp_dir.mkdir(parents=True) + + entries = [ + ( + "multiplicative_composition.jsonl", + { + "surface_pattern": "bound(count) × bound(unit_cost)", + "composition_category": "multiplicative_composition", + "polarity": "affirms", + "provenance": "test_me5", + "evidence_hashes": [], + }, + ), + ( + "additive_composition.jsonl", + { + "surface_pattern": "bound(qty_a) + bound(qty_b)", + "composition_category": "additive_composition", + "polarity": "affirms", + "provenance": "test_me5", + "evidence_hashes": [], + }, + ), + ( + "subtractive_composition.jsonl", + { + "surface_pattern": "bound(initial) − bound(removed)", + "composition_category": "subtractive_composition", + "polarity": "affirms", + "provenance": "test_me5", + "evidence_hashes": [], + }, + ), + ] + for filename, entry in entries: + (comp_dir / filename).write_text(json.dumps(entry) + "\n", encoding="utf-8") + + _, sha = compile_compositions(pack) + (pack / "manifest.json").write_text( + json.dumps( + { + "pack_id": "en_core_math_v1", + "checksum": "x", + "composition_checksum": sha, + } + ), + encoding="utf-8", + ) + return pack + + +def _patch_pack_root(monkeypatch, pack_path: Path) -> None: + from generate.comprehension import composition_registry as cr + + monkeypatch.setattr(cr, "_DEFAULT_PACK_RELPATH", pack_path) + monkeypatch.setattr(cr, "_repo_root", lambda: Path("/")) + + +def _make_match(anchors, category: ShapeCategory) -> RecognizerMatch: + class _R: + spec_id = "test_me5" + + return RecognizerMatch( + recognizer=_R(), # type: ignore[arg-type] + category=category, + outcome="admissible", + graph_intent="rate", + parsed_anchors=anchors, + ) + + +def test_all_three_canaries_admit_through_full_pipeline(monkeypatch, tmp_path): + """End-to-end: one canary per SAFE category, all admit.""" + pack = _stage_pack_all_three(tmp_path) + _patch_pack_root(monkeypatch, pack) + + # Multiplicative canary (ME-1) + mult_spec: Mapping[str, Any] = { + "anchor_kind": "currency_per_unit_composition", + "observed_currency_symbols": ["$"], + "observed_per_units": ["each", "apiece"], + } + mult_stmt = "Maria bought 3 vet appointments at $400 each." + mult_result = _match_rate_with_currency(mult_stmt, mult_spec) + assert mult_result is not None + mult_match = _make_match(mult_result[0], ShapeCategory.RATE_WITH_CURRENCY) + mult_emit = inject_from_match(mult_match, mult_stmt) + assert len(mult_emit) == 1 + assert isinstance(mult_emit[0], CandidateInitial) + assert mult_emit[0].initial.entity == "Maria" + assert mult_emit[0].initial.quantity.value == 1200 + assert mult_emit[0].initial.quantity.unit == "dollars" + + # Additive canary (ME-3) + add_spec: Mapping[str, Any] = { + "anchor_kind": "additive_quantity_composition", + "observed_units": ["dollars"], + } + add_stmt = "Sam earned 100 dollars and 50 dollars." + add_result = _match_multiplicative_aggregation(add_stmt, add_spec) + assert add_result is not None + add_match = _make_match(add_result[0], ShapeCategory.MULTIPLICATIVE_AGGREGATION) + add_emit = inject_from_match(add_match, add_stmt) + assert len(add_emit) == 1 + assert add_emit[0].initial.entity == "Sam" + assert add_emit[0].initial.quantity.value == 150 + assert add_emit[0].initial.quantity.unit == "dollars" + + # Subtractive canary (ME-4) + sub_spec: Mapping[str, Any] = { + "anchor_kind": "subtractive_quantity_composition", + "observed_units": ["apples", "apple"], + } + sub_stmt = "Tom had 10 apples and lost 4 apples." + sub_result = _match_multiplicative_aggregation(sub_stmt, sub_spec) + assert sub_result is not None + sub_match = _make_match(sub_result[0], ShapeCategory.MULTIPLICATIVE_AGGREGATION) + sub_emit = inject_from_match(sub_match, sub_stmt) + assert len(sub_emit) == 1 + assert sub_emit[0].initial.entity == "Tom" + assert sub_emit[0].initial.quantity.value == 6 + assert sub_emit[0].initial.quantity.unit == "apples" + + +def test_partial_pack_only_admits_present_categories(monkeypatch, tmp_path): + """If only multiplicative is ratified, additive + subtractive refuse.""" + pack = tmp_path / "en_core_math_v1" + comp_dir = pack / "compositions" + comp_dir.mkdir(parents=True) + (comp_dir / "multiplicative_composition.jsonl").write_text( + json.dumps( + { + "surface_pattern": "bound(count) × bound(unit_cost)", + "composition_category": "multiplicative_composition", + "polarity": "affirms", + "provenance": "test_me5_partial", + "evidence_hashes": [], + } + ) + + "\n", + encoding="utf-8", + ) + _, sha = compile_compositions(pack) + (pack / "manifest.json").write_text( + json.dumps( + { + "pack_id": "en_core_math_v1", + "checksum": "x", + "composition_checksum": sha, + } + ), + encoding="utf-8", + ) + _patch_pack_root(monkeypatch, pack) + + # Multiplicative admits... + mult_result = _match_rate_with_currency( + "Maria bought 3 vet appointments at $400 each.", + { + "anchor_kind": "currency_per_unit_composition", + "observed_currency_symbols": ["$"], + "observed_per_units": ["each"], + }, + ) + assert mult_result is not None + mult_match = _make_match(mult_result[0], ShapeCategory.RATE_WITH_CURRENCY) + mult_emit = inject_from_match(mult_match, "Maria bought 3 vet appointments at $400 each.") + assert len(mult_emit) == 1 + + # ...but additive refuses because no additive entry in this pack. + add_result = _match_multiplicative_aggregation( + "Sam earned 100 dollars and 50 dollars.", + { + "anchor_kind": "additive_quantity_composition", + "observed_units": ["dollars"], + }, + ) + assert add_result is not None + add_match = _make_match(add_result[0], ShapeCategory.MULTIPLICATIVE_AGGREGATION) + add_emit = inject_from_match(add_match, "Sam earned 100 dollars and 50 dollars.") + assert add_emit == () # refusal-preferring + + +def test_all_safe_categories_have_extension_admission(): + """Every entry in SAFE_COMPOSITION_CATEGORIES has a corresponding + matcher extension path. This pin breaks if a future ADR widens the + allowlist without also shipping a matcher extension — operator must + decide consciously.""" + expected = { + "multiplicative_composition", + "additive_composition", + "subtractive_composition", + } + assert SAFE_COMPOSITION_CATEGORIES == frozenset(expected), ( + "ME-5 covers exactly these three categories — widening " + "SAFE_COMPOSITION_CATEGORIES requires adding a matcher extension " + "and updating this pin." + ) + + +def test_falsifies_uniformly_suppresses_across_categories(monkeypatch, tmp_path): + """polarity='falsifies' suppresses for any category.""" + pack = tmp_path / "en_core_math_v1" + comp_dir = pack / "compositions" + comp_dir.mkdir(parents=True) + for filename, category, shape in [ + ( + "multiplicative_composition.jsonl", + "multiplicative_composition", + "bound(count) × bound(unit_cost)", + ), + ( + "additive_composition.jsonl", + "additive_composition", + "bound(qty_a) + bound(qty_b)", + ), + ( + "subtractive_composition.jsonl", + "subtractive_composition", + "bound(initial) − bound(removed)", + ), + ]: + (comp_dir / filename).write_text( + json.dumps( + { + "surface_pattern": shape, + "composition_category": category, + "polarity": "falsifies", + "provenance": "test_me5_falsifies", + "evidence_hashes": [], + } + ) + + "\n", + encoding="utf-8", + ) + _, sha = compile_compositions(pack) + (pack / "manifest.json").write_text( + json.dumps( + {"pack_id": "en_core_math_v1", "checksum": "x", "composition_checksum": sha} + ), + encoding="utf-8", + ) + _patch_pack_root(monkeypatch, pack) + + # All three matchers extract anchors but inject_from_match suppresses. + mult = _match_rate_with_currency( + "Maria bought 3 vet appointments at $400 each.", + { + "anchor_kind": "currency_per_unit_composition", + "observed_currency_symbols": ["$"], + "observed_per_units": ["each"], + }, + ) + assert mult is not None + assert ( + inject_from_match( + _make_match(mult[0], ShapeCategory.RATE_WITH_CURRENCY), + "Maria bought 3 vet appointments at $400 each.", + ) + == () + ) + + add = _match_multiplicative_aggregation( + "Sam earned 100 dollars and 50 dollars.", + {"anchor_kind": "additive_quantity_composition", "observed_units": ["dollars"]}, + ) + assert add is not None + assert ( + inject_from_match( + _make_match(add[0], ShapeCategory.MULTIPLICATIVE_AGGREGATION), + "Sam earned 100 dollars and 50 dollars.", + ) + == () + ) + + sub = _match_multiplicative_aggregation( + "Tom had 10 apples and lost 4 apples.", + { + "anchor_kind": "subtractive_quantity_composition", + "observed_units": ["apples"], + }, + ) + assert sub is not None + assert ( + inject_from_match( + _make_match(sub[0], ShapeCategory.MULTIPLICATIVE_AGGREGATION), + "Tom had 10 apples and lost 4 apples.", + ) + == () + )