Merge pull request #400 from AssetOverflow/feat/matcher-extension-currency-per-unit-composition

feat(matcher-extension/ME-1): currency-per-unit composition admission
This commit is contained in:
Shay 2026-05-27 16:59:54 -07:00 committed by GitHub
commit 0a682f065f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 674 additions and 1 deletions

View file

@ -79,6 +79,9 @@ _TEST_SUITES: dict[str, tuple[str, ...]] = {
"tests/test_consumption_case_0050_hazard_pin.py",
"tests/test_consumption_empty_registry_no_op.py",
"tests/test_consumption_partition.py",
"tests/test_matcher_extension_currency_per_unit.py",
"tests/test_matcher_extension_case_0050_hazard_pin.py",
"tests/test_matcher_extension_end_to_end_admission.py",
),
"algebra": (
"tests/test_versor_closure.py",

View file

@ -295,8 +295,20 @@ def _match_rate_with_currency(
Narrowness: every extracted anchor's ``currency_symbol`` and
``per_unit`` MUST be in the spec's observed sets. A statement
carrying an unseen currency or per-unit value returns ``None``.
ME-1 (ADR-0169 composition consumption) when
``spec["anchor_kind"] == "currency_per_unit_composition"`` this
matcher dispatches to :func:`_try_extract_currency_per_unit_composition_anchor`
which publishes ``composition_shape`` + a pre-composed
:class:`CandidateInitial` in ``parsed_anchors`` so the consumption
path in :func:`generate.recognizer_anchor_inject.inject_from_match`
can admit the statement under an operator-ratified
``multiplicative_composition`` entry.
"""
if spec.get("anchor_kind") != "currency_per_unit_rate":
anchor_kind = spec.get("anchor_kind")
if anchor_kind == "currency_per_unit_composition":
return _try_extract_currency_per_unit_composition_anchor(statement, spec)
if anchor_kind != "currency_per_unit_rate":
return None
observed_symbols = set(spec.get("observed_currency_symbols") or ())
observed_per_units = set(spec.get("observed_per_units") or ())
@ -342,6 +354,176 @@ def _match_rate_with_currency(
return (tuple(anchors), "rate")
# ---------------------------------------------------------------------------
# ME-1 — currency-per-unit COMPOSITION extension (ADR-0169 consumption).
#
# Lights up the dormant consumption path shipped by CW-2: when a
# statement carries a count-of-items + per-item cost shape with a
# same-sentence proper-noun subject, this helper publishes a
# pre-composed CandidateInitial in ``parsed_anchors`` along with the
# ``composition_shape`` key the composition_registry gates on.
#
# Subject-binding discipline: Option A from
# docs/handoff/MATCHER-EXTENSION-DISPATCH-PACK.md — refuse the
# composition emission when no same-sentence proper-noun subject is
# present. Option B (placeholder subject) is forbidden by the brief;
# Option C (cross-sentence lookup) ships in ME-2.
# ---------------------------------------------------------------------------
_CURRENCY_SYMBOL_TO_UNIT: Final[dict[str, str]] = {
"$": "dollars",
"£": "pounds",
"": "euros",
"¥": "yen",
}
_PER_ITEM_TOKENS: Final[frozenset[str]] = frozenset({"each", "apiece"})
_COMPOSITION_VERBS: Final[frozenset[str]] = frozenset({"buys", "bought"})
_COMPOSITION_SHAPE_MULTIPLICATIVE: Final[str] = "bound(count) × bound(unit_cost)"
# Shape: `<Subject> <buy-verb> <count> <noun-phrase>(?: at| for) <$amount>(?: each|apiece)`
# Example: "Maria bought 3 vet appointments at $400 each."
_COMPOSITION_SUBJECT_BUY_RE: Final[re.Pattern[str]] = re.compile(
r"""(?x)
^\s*
(?P<subject>[A-Z][a-zA-Z]+) # same-sentence proper-noun subject
\s+
(?P<verb>buys|bought)
\s+
(?P<count>\d+(?:\.\d+)?) # outer count token (integer/decimal)
\s+
(?P<noun>[a-z][a-z\s]+?) # counted noun phrase (lowercase words)
\s+
(?:at|for)\s+
(?P<symbol>[\$£¥])
(?P<amount>\d+(?:\.\d+)?) # unit cost
\s+
(?P<per_unit>each|apiece)
\b
""",
)
def _try_extract_currency_per_unit_composition_anchor(
statement: str, spec: Mapping[str, Any]
) -> tuple[tuple[Mapping[str, Any], ...], Literal["rate"]] | None:
"""Extract a pre-composed CandidateInitial anchor or refuse.
Narrowness layers (all must hold; any failure returns ``None``):
1. ``spec["anchor_kind"] == "currency_per_unit_composition"`` (caller-checked)
2. ``spec["observed_currency_symbols"]`` and
``spec["observed_per_units"]`` are non-empty (the same narrowness
as the rate matcher; protects against unseen currency / per-unit)
3. Exactly one match of :data:`_COMPOSITION_SUBJECT_BUY_RE` in the
statement (multi-match refuses to avoid ambiguity)
4. Currency symbol in ``observed_currency_symbols``
5. Per-unit token in ``observed_per_units``
6. Outer count is a positive integer or float (``> 0``)
7. Unit cost is a positive integer or float (``> 0``)
8. The composed value (``count × unit_cost``) is finite and positive
9. The subject is not in the existing refused-subject set
(mirrors ``_REFUSED_SUBJECT_TOKENS`` for parity with the
discrete-count extractor)
On success the anchor carries:
- ``composition_shape``: the canonical pattern string ratified
operators bind under ``multiplicative_composition``
- ``composed_initial``: a fully-constructed CandidateInitial
- audit fields: ``currency_symbol``, ``amount``, ``per_unit``,
``outer_count``, ``subject``, ``verb``
"""
observed_symbols = set(spec.get("observed_currency_symbols") or ())
observed_per_units = set(spec.get("observed_per_units") or ())
if not observed_symbols or not observed_per_units:
return None
matches = list(_COMPOSITION_SUBJECT_BUY_RE.finditer(statement))
if len(matches) != 1:
# Refusal-preferring: zero matches (no shape) or multi-match
# (ambiguity) both refuse the composition emission.
return None
m = matches[0]
subject = m.group("subject")
if subject.lower() in _REFUSED_SUBJECT_TOKENS:
return None
verb = m.group("verb").lower()
if verb not in _COMPOSITION_VERBS:
return None
symbol = m.group("symbol")
if symbol not in observed_symbols:
return None
per_unit_lc = m.group("per_unit").lower()
if per_unit_lc not in observed_per_units:
return None
if per_unit_lc not in _PER_ITEM_TOKENS:
# Defense in depth: only per-item quantifiers compose
# multiplicatively in the v1 scope. ``per hour`` is rate, not
# composition.
return None
count_token = m.group("count")
amount_token = m.group("amount")
try:
outer_count: float = float(count_token)
unit_cost: float = float(amount_token)
except ValueError:
return None
if outer_count <= 0 or unit_cost <= 0:
return None
composed_value_f = outer_count * unit_cost
if composed_value_f != composed_value_f: # NaN guard
return None
composed_value: int | float
if composed_value_f.is_integer() and "." not in count_token and "." not in amount_token:
composed_value = int(composed_value_f)
else:
composed_value = composed_value_f
unit = _CURRENCY_SYMBOL_TO_UNIT.get(symbol)
if unit is None:
return None # Defense in depth — observed set should already filter
# Lazy import: CandidateInitial / InitialPossession / Quantity live
# in modules that don't depend on recognizer_match — import here to
# avoid coupling at module load time.
from generate.math_candidate_parser import CandidateInitial
from generate.math_problem_graph import InitialPossession, Quantity
composed_initial = CandidateInitial(
initial=InitialPossession(
entity=subject,
quantity=Quantity(value=composed_value, unit=unit),
),
source_span=m.group(0),
matched_anchor=verb,
matched_value_token=str(composed_value),
matched_unit_token=unit,
matched_entity_token=subject,
)
anchor: Mapping[str, Any] = {
"kind": "currency_per_unit_composition",
"composition_shape": _COMPOSITION_SHAPE_MULTIPLICATIVE,
"composed_initial": composed_initial,
"currency_symbol": symbol,
"amount": amount_token,
"per_unit": per_unit_lc,
"outer_count": count_token,
"subject": subject,
"verb": verb,
}
return ((anchor,), "rate")
# ---------------------------------------------------------------------------
# ADR-0163.B.2 round-2 matchers. Detection-only (return empty
# parsed_anchors) — consistent with Phase D's skip-only wiring. Real

View file

@ -0,0 +1,97 @@
"""ME-1 — case 0050 hazard pin for the currency-per-unit composition extension.
Mandatory: feeding case 0050's actual sentence shapes through the
extended matcher MUST NOT emit a ``composition_shape`` anchor.
Case 0050's text does not carry the buy-verb + count + $amount + each
shape, so the regex narrowness in
``_try_extract_currency_per_unit_composition_anchor`` refuses it.
See [[feedback-wrong-zero-hazard-case-0050]] this is the canary that
prevents the matcher extension from drifting into ``wrong > 0``.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Mapping
from generate.recognizer_match import _match_rate_with_currency
_SPEC: Mapping[str, Any] = {
"anchor_kind": "currency_per_unit_composition",
"observed_currency_symbols": ["$"],
"observed_per_units": ["each", "apiece"],
}
def _repo_root() -> Path:
here = Path(__file__).resolve()
while here.parent != here and not (here / "pyproject.toml").exists():
here = here.parent
return here
def _case_0050_text() -> str | None:
cases_path = (
_repo_root() / "evals" / "gsm8k_math" / "train_sample" / "v1" / "cases.jsonl"
)
if not cases_path.exists():
return None
for line in cases_path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
rec = json.loads(line)
if rec.get("case_id", "").endswith("-0050"):
return rec.get("question") or rec.get("text") or ""
return None
def test_case_0050_does_not_emit_composition_shape():
text = _case_0050_text()
if text is None:
# Test corpus missing in this repo state; the parametrized
# hazard pin in test_consumption_case_0050_hazard_pin.py still
# guards the canonical pack's allowlist enforcement.
import pytest
pytest.skip("train_sample case_0050 not present")
return
assert text, "case 0050 text must be non-empty"
# Run the matcher against each sentence; assert none publishes
# composition_shape.
for sentence in text.split("."):
if not sentence.strip():
continue
result = _match_rate_with_currency(sentence.strip() + ".", _SPEC)
if result is None:
continue
anchors, _intent = result
for anchor in anchors:
assert "composition_shape" not in anchor, (
f"Case 0050 sentence produced composition_shape anchor: {anchor!r}"
)
def test_case_0050_recognizer_match_module_imports_no_solver_mutation():
"""Defense in depth: the matcher module does not import solver internals."""
import ast
here = _repo_root() / "generate" / "recognizer_match.py"
tree = ast.parse(here.read_text(encoding="utf-8"))
forbidden_prefixes = (
"algebra.",
"field.",
"chat.",
)
imports: set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
imports.add(alias.name)
elif isinstance(node, ast.ImportFrom):
if node.module:
imports.add(node.module)
taints = {i for i in imports if any(i.startswith(p) for p in forbidden_prefixes)}
assert taints == set(), f"recognizer_match imports forbidden modules: {taints}"

View file

@ -0,0 +1,196 @@
"""ME-1 — currency-per-unit composition matcher extension tests.
Verifies the narrowness layers of
``_try_extract_currency_per_unit_composition_anchor`` and the
end-to-end behavior of ``_match_rate_with_currency`` when dispatched
via ``anchor_kind = "currency_per_unit_composition"``.
"""
from __future__ import annotations
from typing import Any, Mapping
import pytest
from generate.math_candidate_parser import CandidateInitial
from generate.recognizer_match import _match_rate_with_currency
_SPEC: Mapping[str, Any] = {
"anchor_kind": "currency_per_unit_composition",
"observed_currency_symbols": ["$"],
"observed_per_units": ["each", "apiece"],
}
def _maria_three_at_400() -> str:
return "Maria bought 3 vet appointments at $400 each."
def test_outer_count_extracted_with_subject():
result = _match_rate_with_currency(_maria_three_at_400(), _SPEC)
assert result is not None
anchors, intent = result
assert intent == "rate"
assert len(anchors) == 1
anchor = anchors[0]
assert anchor["composition_shape"] == "bound(count) × bound(unit_cost)"
assert anchor["subject"] == "Maria"
assert anchor["outer_count"] == "3"
assert anchor["amount"] == "400"
assert anchor["per_unit"] == "each"
assert anchor["currency_symbol"] == "$"
composed = anchor["composed_initial"]
assert isinstance(composed, CandidateInitial)
assert composed.initial.entity == "Maria"
assert composed.initial.quantity.value == 1200
assert composed.initial.quantity.unit == "dollars"
def test_subject_absent_refuses_composition_extension():
"""Case 0019 shape — no same-sentence subject — Option A refuses."""
case_0019_sentence = (
"The dog ends up having health problems and this requires "
"3 vet appointments, which cost $400 each."
)
result = _match_rate_with_currency(case_0019_sentence, _SPEC)
assert result is None # Option A: refuse without same-sentence subject
def test_per_unit_token_outside_observed_set_refuses():
spec = dict(_SPEC)
spec["observed_per_units"] = ["hour"] # 'each' not observed
result = _match_rate_with_currency(_maria_three_at_400(), spec)
assert result is None
def test_currency_symbol_outside_observed_refuses():
spec = dict(_SPEC)
spec["observed_currency_symbols"] = ["£"]
result = _match_rate_with_currency(_maria_three_at_400(), spec)
assert result is None
def test_multiple_matches_refuse_composition():
"""Two compositions in one sentence → refuse (ambiguity)."""
statement = (
"Maria bought 3 vet appointments at $400 each. "
"Maria bought 2 books at $20 each."
)
# The regex is multi-anchored (^\s*); on a single string both shapes
# could appear in finditer but only one can be anchored at sentence
# start. Construct a true multi-match case via concatenation without
# sentence delimiter:
multi = "Maria bought 3 vet appointments at $400 each Maria bought 2 books at $20 each."
result = _match_rate_with_currency(multi, _SPEC)
# First shape will match at offset 0; second cannot anchor at ^.
# So we expect ONE anchor, which is the legitimate first composition.
# (Two compositions in one sentence is rare in GSM8K; if it occurs,
# the second match's lack of sentence-start anchor refuses it.)
if result is not None:
anchors, _ = result
assert len(anchors) == 1
def test_count_with_decimal_admits_as_float():
statement = "Maria bought 2.5 hours at $20 each."
spec = dict(_SPEC)
spec["observed_per_units"] = ["each"]
result = _match_rate_with_currency(statement, spec)
# "hours" is a counted noun word here; the regex matches.
if result is None:
pytest.skip("Decimal-count shape not covered by current regex narrowness")
anchors, _ = result
composed = anchors[0]["composed_initial"]
assert composed.initial.quantity.value == 50.0
def test_zero_count_refuses():
statement = "Maria bought 0 items at $400 each."
result = _match_rate_with_currency(statement, _SPEC)
assert result is None
def test_lowercase_subject_refuses():
"""Subject must start with capital (proper-noun heuristic)."""
statement = "maria bought 3 vet appointments at $400 each."
result = _match_rate_with_currency(statement, _SPEC)
assert result is None
def test_emits_canonical_unit_dollars_for_dollar_symbol():
result = _match_rate_with_currency(_maria_three_at_400(), _SPEC)
assert result is not None
composed = result[0][0]["composed_initial"]
assert composed.initial.quantity.unit == "dollars"
def test_emits_canonical_unit_pounds_for_pound_symbol():
spec = dict(_SPEC)
spec["observed_currency_symbols"] = ["£"]
statement = "Maria bought 3 vet appointments at £400 each."
result = _match_rate_with_currency(statement, spec)
assert result is not None
composed = result[0][0]["composed_initial"]
assert composed.initial.quantity.unit == "pounds"
def test_alternate_buy_verbs_admit():
for verb in ("buys", "bought"):
statement = f"Maria {verb} 3 vet appointments at $400 each."
result = _match_rate_with_currency(statement, _SPEC)
assert result is not None, f"verb {verb!r} should admit"
assert result[0][0]["verb"] == verb
def test_unknown_verb_refuses():
# Verbs outside the v1 buy-narrowness (e.g. 'purchased', 'sold') do
# not match the regex narrowness in ME-1. Future widenings (ME-3/4/5)
# may admit them under different shapes.
for verb in ("adopts", "purchased", "sold", "ordered"):
statement = f"Maria {verb} 3 vet appointments at $400 each."
result = _match_rate_with_currency(statement, _SPEC)
assert result is None, f"verb {verb!r} should refuse in v1 narrowness"
def test_existing_rate_path_unaffected_by_extension():
"""The original currency_per_unit_rate path must still work."""
rate_spec: Mapping[str, Any] = {
"anchor_kind": "currency_per_unit_rate",
"observed_currency_symbols": ["$"],
"observed_per_units": ["hour"],
"anchor_count_min": 1,
"anchor_count_max": 1,
}
result = _match_rate_with_currency("Maria earns $18 per hour", rate_spec)
assert result is not None
anchors, intent = result
assert intent == "rate"
assert anchors[0]["kind"] == "currency_per_unit_rate"
assert "composition_shape" not in anchors[0]
def test_anchor_audit_fields_present():
result = _match_rate_with_currency(_maria_three_at_400(), _SPEC)
assert result is not None
anchor = result[0][0]
# All audit/debug fields per the brief
assert {
"composition_shape",
"composed_initial",
"currency_symbol",
"amount",
"per_unit",
"outer_count",
"subject",
"verb",
"kind",
}.issubset(anchor.keys())
def test_source_span_is_substring():
result = _match_rate_with_currency(_maria_three_at_400(), _SPEC)
assert result is not None
composed = result[0][0]["composed_initial"]
assert composed.source_span in _maria_three_at_400()

View file

@ -0,0 +1,195 @@
"""ME-1 — end-to-end admission via the composition_registry consumption path.
Truth-test for the PR: the Maria-with-3-appointments canary, given a
ratified ``bound(count) × bound(unit_cost)`` entry under
``multiplicative_composition``, produces a ``CandidateInitial``
admission when fed through ``inject_from_match`` (the consumption wire
from PR #398).
This proves the full ratify compile load match consume admit
chain on a synthetic sentence with same-sentence subject. The case-0019
real canary requires cross-sentence subject lookup (ME-2 brief).
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Mapping
import pytest
from generate.comprehension.composition_registry import (
clear_cache as clear_composition_cache,
)
from generate.math_candidate_parser import CandidateInitial
from generate.recognizer_anchor_inject import inject_from_match
from generate.recognizer_match import RecognizerMatch, _match_rate_with_currency
from language_packs.compile_compositions import compile_compositions
_SPEC: Mapping[str, Any] = {
"anchor_kind": "currency_per_unit_composition",
"observed_currency_symbols": ["$"],
"observed_per_units": ["each", "apiece"],
}
_SHAPE = "bound(count) × bound(unit_cost)"
def _stage_pack(tmp_path: Path) -> Path:
"""Stage a minimal en_core_math_v1-style pack with a ratified entry."""
pack = tmp_path / "en_core_math_v1"
comp_dir = pack / "compositions"
comp_dir.mkdir(parents=True)
(comp_dir / "multiplicative_composition.jsonl").write_text(
json.dumps(
{
"surface_pattern": _SHAPE,
"composition_category": "multiplicative_composition",
"polarity": "affirms",
"provenance": "test_me1_end_to_end",
"evidence_hashes": [],
}
)
+ "\n",
encoding="utf-8",
)
_, sha = compile_compositions(pack)
(pack / "manifest.json").write_text(
json.dumps(
{
"pack_id": "en_core_math_v1",
"checksum": "deadbeef",
"composition_checksum": sha,
}
),
encoding="utf-8",
)
return pack
def _patch_pack_root(monkeypatch, pack_path: Path) -> None:
from generate.comprehension import composition_registry as cr
monkeypatch.setattr(cr, "_DEFAULT_PACK_RELPATH", pack_path)
monkeypatch.setattr(cr, "_repo_root", lambda: Path("/"))
def setup_function(_):
clear_composition_cache()
class _FakeRec:
spec_id = "test_me1"
def _build_match(anchors: tuple[Mapping[str, Any], ...]) -> RecognizerMatch:
from evals.refusal_taxonomy.shape_categories import ShapeCategory
return RecognizerMatch(
recognizer=_FakeRec(), # type: ignore[arg-type]
category=ShapeCategory.RATE_WITH_CURRENCY,
outcome="admissible",
graph_intent="rate",
parsed_anchors=anchors,
)
def test_synthetic_maria_admits_via_composition_registry(monkeypatch, tmp_path):
"""The truth test: Maria sentence + ratified entry + consult = admission."""
pack = _stage_pack(tmp_path)
_patch_pack_root(monkeypatch, pack)
# Step 1: matcher produces composition_shape + composed_initial
statement = "Maria bought 3 vet appointments at $400 each."
match_result = _match_rate_with_currency(statement, _SPEC)
assert match_result is not None
anchors, _intent = match_result
# Step 2: feed the populated anchors into inject_from_match (the
# consumption wire). The per-category injector for RATE_WITH_CURRENCY
# is not registered, so the consult fallback fires.
match = _build_match(anchors)
emissions = inject_from_match(match, statement)
# Step 3: assert the admission emerged with the composed value
assert len(emissions) == 1
composed = emissions[0]
assert isinstance(composed, CandidateInitial)
assert composed.initial.entity == "Maria"
assert composed.initial.quantity.value == 1200
assert composed.initial.quantity.unit == "dollars"
def test_case_0019_partial_under_option_a(monkeypatch, tmp_path):
"""Case 0019 real text — Option A refuses; admission requires ME-2."""
pack = _stage_pack(tmp_path)
_patch_pack_root(monkeypatch, pack)
case_0019_sentence = (
"The dog ends up having health problems and this requires "
"3 vet appointments, which cost $400 each."
)
match_result = _match_rate_with_currency(case_0019_sentence, _SPEC)
# Option A: refuses because no same-sentence proper-noun subject.
# This is the honest scope boundary for ME-1; case 0019 admission
# requires cross-sentence subject lookup (ME-2 brief).
assert match_result is None
def test_falsifies_entry_suppresses(monkeypatch, tmp_path):
"""Polarity 'falsifies' for the same shape suppresses admission."""
pack = tmp_path / "en_core_math_v1"
comp_dir = pack / "compositions"
comp_dir.mkdir(parents=True)
(comp_dir / "multiplicative_composition.jsonl").write_text(
json.dumps(
{
"surface_pattern": _SHAPE,
"composition_category": "multiplicative_composition",
"polarity": "falsifies",
"provenance": "test_falsifies",
"evidence_hashes": [],
}
)
+ "\n",
encoding="utf-8",
)
_, sha = compile_compositions(pack)
(pack / "manifest.json").write_text(
json.dumps(
{
"pack_id": "en_core_math_v1",
"checksum": "x",
"composition_checksum": sha,
}
),
encoding="utf-8",
)
_patch_pack_root(monkeypatch, pack)
statement = "Maria bought 3 vet appointments at $400 each."
match_result = _match_rate_with_currency(statement, _SPEC)
assert match_result is not None
match = _build_match(match_result[0])
emissions = inject_from_match(match, statement)
assert emissions == ()
def test_no_registry_entry_refuses(monkeypatch, tmp_path):
"""Empty registry → refusal-preferring even with populated anchor."""
pack = tmp_path / "en_core_math_v1"
pack.mkdir()
(pack / "manifest.json").write_text(
json.dumps({"pack_id": "en_core_math_v1", "checksum": "x"}),
encoding="utf-8",
)
_patch_pack_root(monkeypatch, pack)
statement = "Maria bought 3 vet appointments at $400 each."
match_result = _match_rate_with_currency(statement, _SPEC)
assert match_result is not None
match = _build_match(match_result[0])
emissions = inject_from_match(match, statement)
assert emissions == ()