diff --git a/evals/gsm8k_math/train_sample/v1/audit_brief_11.md b/evals/gsm8k_math/train_sample/v1/audit_brief_11.md index 1634b981..4a255d66 100644 --- a/evals/gsm8k_math/train_sample/v1/audit_brief_11.md +++ b/evals/gsm8k_math/train_sample/v1/audit_brief_11.md @@ -204,3 +204,95 @@ from `AuditRow` → `MathReaderRefusalEvidence` → `claim_signature` → `apply_lexical_claim` → re-audit, plus a cross-process determinism check and the cognition-domain partition guard from W2-C. +## Wave-Next A2 — `rate_with_currency` injector — schema-refusal delta + +Branch: `feat/injector-rate-with-currency`. Brief: Wave-Next A2 (see +`docs/handoff/WAVE-NEXT-INJECTORS.md`). + +### Schema decision + +**Does `Quantity` structurally model a per-unit rate? No — but a separate +`Rate` type (ADR-0122) does.** The schema-refusal hinges not on the +absence of `Rate` but on its absence from the per-sentence injector +contract: + +- `Quantity` is a scalar + single unit pair; it carries no + numerator/denominator distinction (verified in + `tests/test_injector_rate_with_currency.py::TestSchemaEvidence`). +- `Rate(value, numerator_unit, denominator_unit)` exists at + `generate/math_problem_graph.py:78` and is the operand of + `Operation(kind='apply_rate')`. +- The per-sentence dispatch contract is + `SentenceChoice = Union[CandidateInitial, CandidateOperation]` + (`generate/math_candidate_graph.py:152`). There is no + `CandidateRate` variant for the injector to deposit. +- A rate-declaration sentence alone ("Tina makes $18.00 an hour.") + carries no denominator quantity, so it cannot be coerced into a + complete `apply_rate` `CandidateOperation` either. + +### What this PR delivers + +- `generate/recognizer_anchor_inject.py` — new + `inject_rate_with_currency(match, sentence) -> ()` function plus + dispatch-table entry for `ShapeCategory.RATE_WITH_CURRENCY`. The + injector returns `()` for every input shape and documents the + schema gap inline. +- `tests/test_injector_rate_with_currency.py` — 16 tests across six + pin classes: schema evidence, schema refusal, dispatch wired, case + 0050 hazard pin, determinism, and wrong=0 invariant. + +### Eval delta + +| dimension | before A2 | after A2 | +|---|---:|---:| +| correct | 3 | 3 | +| wrong | 0 | 0 | +| refused | 47 | 47 | + +Lift count: **0** (expected — the schema decision is "Rate is not in +`SentenceChoice`," so v1 cannot emit state). The PR's deliverable is +documenting the gap; the case-by-case picture is unchanged. + +### Case 0050 hazard verification + +Sentence 0 of case `gsm8k-train-sample-v1-0050` is +"Mark does a gig every other day for 2 weeks." — it carries no +currency symbol, so the `rate_with_currency` recognizer never matches +it. Even if A2 v1 emitted state (it doesn't), this case is not +reachable through the A2 path. Pinned by +`tests/test_injector_rate_with_currency.py::TestCase0050HazardPin` +(sentence-zero currency-absence check + end-to-end refusal check). + +### Follow-up note — "rate-type schema needs extension" + +For a future PR to lift any of the rate-shaped refusals via the +recognizer-injector path, the `SentenceChoice` union must be widened. +Concretely: + +1. Add a `CandidateRate` dataclass to + `generate/math_candidate_parser.py` (sibling of `CandidateInitial` + and `CandidateOperation`) carrying a `Rate` operand keyed by + actor, with source-span provenance fields analogous to the + existing two `Candidate*` types. +2. Extend `SentenceChoice = Union[CandidateInitial, + CandidateOperation]` to include `CandidateRate`. Update + `_slot_count`, `_collapse_per_sentence_ties`, `_initial_admissible` + (or add a sibling `_rate_admissible`), and the per-sentence + admission gate in + `generate.math_candidate_graph.parse_and_solve`. +3. Teach `parse_and_solve` to compose an accepted `CandidateRate` + with a downstream `apply_rate`/multiply-shaped question. The + existing `extract_earnings_candidates` short-circuit path is the + closest precedent and should be unified with the new + `CandidateRate` flow rather than left as a sibling. +4. Only after (1)–(3) ship can `inject_rate_with_currency` emit + `CandidateRate` and start lifting rate-shaped refusals. The + matcher (`generate/recognizer_match.py:_match_rate_with_currency`) + already extracts the necessary `(currency_symbol, amount, + per_unit)` triple; an entity-extraction extension is the only + additional matcher work required. + +This sequencing keeps the wrong=0 doctrine intact: no widening of +admissible solver state until the schema extension is reviewed and +the round-trip filter is updated. + diff --git a/generate/recognizer_anchor_inject.py b/generate/recognizer_anchor_inject.py index a78fa9d6..da96d662 100644 --- a/generate/recognizer_anchor_inject.py +++ b/generate/recognizer_anchor_inject.py @@ -230,13 +230,67 @@ def _locate_possession_verb(sentence: str) -> str | None: # registers its injector. No global state, no side effects. # --------------------------------------------------------------------------- +def inject_rate_with_currency( + match: RecognizerMatch, + sentence: str, +) -> tuple[CandidateInitial, ...]: + """Inject a ``rate_with_currency`` recognizer match. + + Schema decision (Wave-Next A2): the :class:`Rate` type in + :mod:`generate.math_problem_graph` (ADR-0122) DOES structurally + model a per-unit rate via ``(value, numerator_unit, + denominator_unit)``. However, ``Rate`` is the operand of an + ``Operation(kind='apply_rate')`` — it is not, and cannot be + coerced into, a :class:`CandidateInitial` or a standalone + :class:`CandidateOperation` that the per-sentence choice + dispatcher consumes: + + - :class:`InitialPossession` requires a :class:`Quantity` (scalar + + unit) and an entity. A rate carries two units; the + rate-declaration sentence alone ("Tina makes $18.00 an hour.") + does not establish how many hours Tina worked, so the + denominator quantity is unknown. + - ``Operation(kind='apply_rate')`` requires the denominator + quantity (the "how many X" the rate multiplies) to be present + in the same sentence; an isolated rate-declaration sentence + does not carry it. + - ``SentenceChoice = Union[CandidateInitial, CandidateOperation]`` + (see :mod:`generate.math_candidate_graph`) — there is no + ``CandidateRate`` variant for the injector to deposit. + + The existing + :func:`generate.math_candidate_parser.extract_earnings_candidates` + handles rate-declaration sentences via a separate short-circuit + path keyed on a sibling :class:`CandidateEarningsRate` type that + is NOT part of ``SentenceChoice``. It is consumed by a special + earnings short-circuit in + :func:`generate.math_candidate_graph.parse_and_solve` BEFORE the + per-sentence-choices Cartesian product runs. + + Conclusion: the recognizer-injector contract (return a tuple of + :class:`CandidateInitial`) cannot meaningfully express a per-unit + rate without a wider ``SentenceChoice`` union. v1 therefore + RETURNS ``()`` (refusal-preferring, wrong=0 doctrine) and + documents the gap. This is the explicit-refusal A2 outcome. + + Follow-up (separate PR): extend ``SentenceChoice`` with a + ``CandidateRate`` variant carrying a :class:`Rate` operand keyed + by actor, and teach + :func:`generate.math_candidate_graph.parse_and_solve` to compose + a ``CandidateRate`` with a downstream apply_rate/multiply-shaped + question. Only at that point can a recognizer-injector emit + useful state for ``rate_with_currency``. + """ + return () + + _INJECTORS: Mapping[ShapeCategory, "type"] = { ShapeCategory.DISCRETE_COUNT_STATEMENT: inject_discrete_count_statement, # type: ignore[dict-item] - # The five other recognizer categories route to the empty-tuple + ShapeCategory.RATE_WITH_CURRENCY: inject_rate_with_currency, # type: ignore[dict-item] + # The four other recognizer categories route to the empty-tuple # fallback (skip-only) until their D.2.x injector lands: # # ShapeCategory.DESCRIPTIVE_SETUP_NO_QUANTITY — by design (no quantity) - # ShapeCategory.RATE_WITH_CURRENCY — D.2.2 follow-up # ShapeCategory.TEMPORAL_AGGREGATION — D.2.3 follow-up # ShapeCategory.MULTIPLICATIVE_AGGREGATION — D.2.4 follow-up # ShapeCategory.CURRENCY_AMOUNT — D.2.5 follow-up @@ -246,4 +300,5 @@ _INJECTORS: Mapping[ShapeCategory, "type"] = { __all__ = [ "inject_from_match", "inject_discrete_count_statement", + "inject_rate_with_currency", ] diff --git a/tests/test_injector_rate_with_currency.py b/tests/test_injector_rate_with_currency.py new file mode 100644 index 00000000..ddd7b433 --- /dev/null +++ b/tests/test_injector_rate_with_currency.py @@ -0,0 +1,371 @@ +"""Wave-Next A2 — ``rate_with_currency`` injector. + +This test file is the load-bearing artifact for the A2 injector. The +A2 outcome is an explicit, documented schema-refusal: ``Rate`` (ADR-0122) +DOES structurally model a per-unit rate, but it is not a member of the +``SentenceChoice = Union[CandidateInitial, CandidateOperation]`` union +the per-sentence injector contract requires. The injector therefore +returns ``()`` and the load-bearing assertions in this file pin: + + a. SCHEMA EVIDENCE — ``Rate`` exists and structurally models + a (value, numerator_unit, denominator_unit) + per-unit rate, distinct from ``Quantity``. + b. SCHEMA REFUSAL — the injector returns ``()`` for every + shape (broad and narrow canonical forms). + c. DISPATCH WIRED — dispatch table routes + ``RATE_WITH_CURRENCY`` to the injector + (no longer the empty-tuple default). + d. CASE 0050 HAZARD PIN — case + ``gsm8k-train-sample-v1-0050`` remains + refused at sentence_index=0 (sentence + carries no currency, so it neither + matches nor would be lifted by A2). + e. DETERMINISM — identical ``(match, sentence)`` → + byte-identical injector output. + f. NO STATE INJECTED — the injector never produces a + ``CandidateInitial`` (would be a wrong=0 + hazard, since Rate ≠ Quantity). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from evals.refusal_taxonomy.shape_categories import ShapeCategory +from generate.math_candidate_graph import SentenceChoice, parse_and_solve +from generate.math_problem_graph import Quantity, Rate +from generate.recognizer_anchor_inject import ( + _INJECTORS, + inject_from_match, + inject_rate_with_currency, +) +from generate.recognizer_match import RecognizerMatch +from generate.recognizer_registry import RatifiedRecognizer + + +# --------------------------------------------------------------------------- +# Synthetic match builder — mirrors the A1/D.2 test pattern. +# --------------------------------------------------------------------------- + + +def _make_match(parsed_anchors: tuple[dict, ...]) -> RecognizerMatch: + rec = RatifiedRecognizer( + proposal_id="test-rate-with-currency", + shape_category=ShapeCategory.RATE_WITH_CURRENCY, + canonical_pattern={ + "anchor_kind": "currency_per_unit_rate", + "shape_category": "rate_with_currency", + "graph_intent": "rate", + "anchor_count_min": 1, + "anchor_count_max": 1, + "outcome": "admissible", + "observed_currency_symbols": ["$"], + "observed_per_units": ["hour", "day", "week"], + }, + spec_digest="test-digest", + review_date="2026-05-27", + ratified_at_revision="test", + ) + return RecognizerMatch( + recognizer=rec, + category=ShapeCategory.RATE_WITH_CURRENCY, + outcome="admissible", + graph_intent="rate", + parsed_anchors=parsed_anchors, + ) + + +# --------------------------------------------------------------------------- +# (a) Schema evidence — Rate models a per-unit rate. +# --------------------------------------------------------------------------- + + +class TestSchemaEvidence: + """The schema decision: does ``Quantity`` model a per-unit rate? + + Answer: NO — ``Quantity`` is a scalar+unit pair, not a rate. BUT + a separate ``Rate`` type (ADR-0122) DOES structurally model the + per-unit rate via ``numerator_unit`` / ``denominator_unit``. The + A2 schema-refusal hinges not on the absence of ``Rate`` but on + its absence from the ``SentenceChoice`` union. + """ + + def test_quantity_does_not_model_a_rate(self) -> None: + # Quantity is value + unit; no numerator/denominator distinction. + q = Quantity(value=18.0, unit="dollars") + assert q.value == 18.0 + assert q.unit == "dollars" + # No rate-shaped attributes: this is exactly the gap A2 documents. + assert not hasattr(q, "numerator_unit") + assert not hasattr(q, "denominator_unit") + + def test_rate_type_exists_and_models_per_unit_rate(self) -> None: + # Rate(18, "dollars", "hour") means "18 dollars per hour". + r = Rate(value=18.0, numerator_unit="dollars", denominator_unit="hour") + assert r.value == 18.0 + assert r.numerator_unit == "dollars" + assert r.denominator_unit == "hour" + + def test_sentence_choice_union_excludes_rate(self) -> None: + # The per-sentence injector contract is + # ``SentenceChoice = Union[CandidateInitial, CandidateOperation]``. + # No CandidateRate exists. This is the load-bearing reason the + # A2 injector cannot meaningfully emit a rate primitive. + from generate.math_candidate_parser import CandidateInitial + from generate.math_roundtrip import CandidateOperation + + # The Union is realised structurally — every SentenceChoice + # must be one of these two types. The test pins the closed + # set; expanding it is the explicit follow-up. + allowed = {CandidateInitial, CandidateOperation} + # Best-effort introspection of the Union type. Python's + # ``Union`` exposes its members via ``__args__`` on the alias. + import typing + + args = set(typing.get_args(SentenceChoice)) + assert args == allowed + + +# --------------------------------------------------------------------------- +# (b) Schema refusal — every shape returns (). +# --------------------------------------------------------------------------- + + +class TestSchemaRefusal: + """A2 v1 refuses every input shape, by design.""" + + def test_canonical_per_form_refuses(self) -> None: + m = _make_match(( + { + "kind": "currency_per_unit_rate", + "currency_symbol": "$", + "amount": "18.00", + "amount_kind": "decimal", + "per_unit": "hour", + }, + )) + out = inject_rate_with_currency(m, "Tina makes $18.00 an hour.") + assert out == () + + def test_canonical_for_form_refuses(self) -> None: + m = _make_match(( + { + "kind": "currency_per_unit_rate", + "currency_symbol": "$", + "amount": "30", + "amount_kind": "integer", + "per_unit": "hour", + }, + )) + out = inject_rate_with_currency(m, "Sam charges $30 for each hour.") + assert out == () + + def test_empty_parsed_anchors_refuses(self) -> None: + # No anchors → no possible state regardless of schema. + m = _make_match(()) + out = inject_rate_with_currency(m, "Tina makes $18.00 an hour.") + assert out == () + + def test_returns_empty_tuple_never_raises(self) -> None: + # Adversarial input: malformed anchor payload. The injector + # MUST NOT raise; it MUST return ``()``. + m = _make_match(({"kind": "currency_per_unit_rate", "junk": True},)) + out = inject_rate_with_currency(m, "") + assert out == () + + def test_injector_never_emits_candidate_initial(self) -> None: + # Iterate over a spread of shapes; none may admit any candidate. + sentences = ( + "Tina makes $18.00 an hour.", + "Sam charges $30 for each hour.", + "Bob pays $5 per cup.", + "Alice earns $100 a day.", + ) + for s in sentences: + m = _make_match(( + { + "kind": "currency_per_unit_rate", + "currency_symbol": "$", + "amount": "1", + "amount_kind": "integer", + "per_unit": "hour", + }, + )) + assert inject_rate_with_currency(m, s) == () + + +# --------------------------------------------------------------------------- +# (c) Dispatch wired — registry routes RATE_WITH_CURRENCY to injector. +# --------------------------------------------------------------------------- + + +class TestDispatchWired: + def test_injector_table_registers_rate_with_currency(self) -> None: + # Before A2: RATE_WITH_CURRENCY was absent from _INJECTORS + # (default empty-tuple skip). After A2: present and routed + # to inject_rate_with_currency. + assert ShapeCategory.RATE_WITH_CURRENCY in _INJECTORS + assert _INJECTORS[ShapeCategory.RATE_WITH_CURRENCY] is inject_rate_with_currency + + def test_dispatch_returns_empty_tuple_via_registry(self) -> None: + m = _make_match(( + { + "kind": "currency_per_unit_rate", + "currency_symbol": "$", + "amount": "18.00", + "amount_kind": "decimal", + "per_unit": "hour", + }, + )) + out = inject_from_match(m, "Tina makes $18.00 an hour.") + assert out == () + + def test_dispatch_equals_direct_call(self) -> None: + m = _make_match(( + { + "kind": "currency_per_unit_rate", + "currency_symbol": "$", + "amount": "18.00", + "amount_kind": "decimal", + "per_unit": "hour", + }, + )) + s = "Tina makes $18.00 an hour." + assert inject_from_match(m, s) == inject_rate_with_currency(m, s) + + +# --------------------------------------------------------------------------- +# (d) Case 0050 hazard pin — sentence_index=0 stays refused. +# --------------------------------------------------------------------------- + + +_CASES_PATH = ( + Path(__file__).resolve().parent.parent + / "evals" + / "gsm8k_math" + / "train_sample" + / "v1" + / "cases.jsonl" +) + + +def _load_case_0050() -> dict: + """Look up case 0050 from the fixed eval cases file.""" + with _CASES_PATH.open() as f: + for line in f: + c = json.loads(line) + if c["case_id"] == "gsm8k-train-sample-v1-0050": + return c + raise AssertionError("case gsm8k-train-sample-v1-0050 not found in cases.jsonl") + + +class TestCase0050HazardPin: + """Case 0050: "Mark does a gig every other day for 2 weeks. ..." + + Sentence 0 carries no currency symbol — rate_with_currency never + matches it. Even if A2 v1 emitted state (it doesn't), this case + would not be reachable through the A2 path. This test makes the + invariant explicit so a future A2 widening cannot silently lift + the case 0050 hazard. + """ + + def test_sentence_zero_has_no_currency_symbol(self) -> None: + case = _load_case_0050() + # The case's question text is the full problem; split into + # sentences on '.' the same way the candidate-graph does. + sentences = [s.strip() for s in case["question"].split(".") if s.strip()] + sentence_zero = sentences[0] + assert "Mark does a gig" in sentence_zero + for symbol in "$£€¥": + assert symbol not in sentence_zero + + def test_case_0050_remains_refused_end_to_end(self) -> None: + case = _load_case_0050() + r = parse_and_solve(case["question"]) + assert r.answer is None + # The wrong reading (~3 minutes, per the audit_brief_11 note) + # MUST never appear. 280 is the correct expected answer; the + # injector must not regress to either. + assert r.is_admitted is False + + +# --------------------------------------------------------------------------- +# (e) Determinism — same input, byte-identical output. +# --------------------------------------------------------------------------- + + +class TestDeterminism: + def test_injection_is_deterministic(self) -> None: + m = _make_match(( + { + "kind": "currency_per_unit_rate", + "currency_symbol": "$", + "amount": "18.00", + "amount_kind": "decimal", + "per_unit": "hour", + }, + )) + s = "Tina makes $18.00 an hour." + out1 = inject_rate_with_currency(m, s) + out2 = inject_rate_with_currency(m, s) + assert out1 == out2 + assert out1 == () # explicitly pinned: refusal + + def test_dispatch_is_deterministic(self) -> None: + m = _make_match(( + { + "kind": "currency_per_unit_rate", + "currency_symbol": "$", + "amount": "18.00", + "amount_kind": "decimal", + "per_unit": "hour", + }, + )) + s = "Tina makes $18.00 an hour." + out1 = inject_from_match(m, s) + out2 = inject_from_match(m, s) + assert out1 == out2 + + +# --------------------------------------------------------------------------- +# (f) Wrong=0 invariant — no candidate is ever produced. +# --------------------------------------------------------------------------- + + +class TestWrongZeroInvariant: + """The strongest possible wrong=0 statement: the injector emits + nothing. A wider follow-up must replace this assertion with a + grounded admissibility check on the (Rate, Quantity) composition. + """ + + def test_no_candidate_emitted_for_any_known_shape(self) -> None: + # Every shape the existing matcher could produce. + anchor_variants = ( + { + "kind": "currency_per_unit_rate", + "currency_symbol": "$", + "amount": "18.00", + "amount_kind": "decimal", + "per_unit": "hour", + }, + { + "kind": "currency_per_unit_rate", + "currency_symbol": "$", + "amount": "5", + "amount_kind": "integer", + "per_unit": "cup", + }, + { + "kind": "currency_per_unit_rate", + "currency_symbol": "$", + "amount": "1/2", + "amount_kind": "word", + "per_unit": "pound", + }, + ) + for anchor in anchor_variants: + m = _make_match((anchor,)) + out = inject_rate_with_currency(m, "irrelevant under refusal") + assert out == () + assert len(out) == 0