feat(wave-a): first non-DCS injector — multiplicative_aggregation w/ value extraction
Addresses 5 of 47 train_sample "recognizer matched but produced no
injection" refusals (the largest single failure-mode bucket
identified in RAT-1's audit).
Modules
-------
- generate/recognizer_match.py:
- _MULT_AGG_EACH_WEIGHING_RE — regex for "<Subject> <bake-verb>
<M> <outer-noun>, each <weigh-verb>ing <N> <unit>" pattern
- _try_extract_each_weighing_anchor — extracts M, N, subject,
inner unit; emits pre-composed CandidateInitial(value=M*N) with
composition_evidence so RAT-1's _composed_initial_admissible
gate verifies INPUT tokens ground (preserves wrong=0)
- _match_multiplicative_aggregation dispatches to the value
extractor when spec carries extract_values=True; specs without
that flag get the existing detection-only return path
(byte-identical legacy behavior)
- generate/recognizer_anchor_inject.py:
- inject_multiplicative_aggregation — new per-category injector;
narrow by anchor.kind so ME-3/ME-4 additive/subtractive anchors
(which share the same matcher entry point) continue to flow
through composition_registry consult instead of WAVE-A's direct
path
- registered in _INJECTORS dict (2nd entry after DCS)
- core/cli.py:
- seed-recognizer CLI gains --extract-values flag to opt the
canonical_pattern into the value-extracting matcher path
Seeded artifacts
----------------
- proposals.jsonl: rat1-seed-4dc30608fb783bc7 — multiplicative_
aggregation recognizer with anchor_kind=multiplicative_aggregate,
extract_values=True, observed_units covering ounces/strawberries/
questions/etc.
Live result on train_sample
---------------------------
- wrong == 0 preserved (3/47/0 baseline)
- Case 0050 hazard pin held
- public 150/150 preserved
- packs suite: 127 → 131 (+4 new WAVE-A tests, all green)
- teaching suite 93 unchanged
- runtime suite 20 unchanged
End-to-end synthetic solve (FIRST WAVE-A admission):
"Lilibeth fills 6 baskets where each basket holds 50 strawberries.
How many strawberries does Lilibeth have?" → answer=300
Cases that moved (statement now admits; refusal shifted downstream):
- Case 0025 (Lilibeth): statement admits via WAVE-A; refusal moved
to question parser ("If three of Lilibeth's friends pick the same
amount, how many strawberries do Lilibeth and her friends pick in
all?")
- Case 0047 (John bakes 12 macaroons): statement 1 admits; refusal
moved to statement 2
Eval correct count unchanged because the QUESTION parser (and
multi-statement cross-sentence reasoning) is the next bottleneck.
RAT-1's audit identified that gap; WAVE-A closes the injector half.
The remaining 3 multiplicative_aggregation refusals (0006, 0013,
0045) have different shape patterns the WAVE-A regex does not yet
cover; they're follow-up matcher extensions in the same architecture.
Tests
-----
- tests/test_wave_a_multiplicative_aggregation_injector.py (10
tests): each-weighing + each-basket-holds admission shapes,
detection-only path preserved when extract_values absent,
unobserved unit / pronoun / zero count refusals, end-to-end
inject_from_match dispatch, the Lilibeth canary solve,
wrong=0 preserved, case 0050 hazard pin
Stacks on PR #406 (RAT-1).
This commit is contained in:
Shay
parent
8bfdb4a7cc
commit
7441b42bf5
5 changed files with 406 additions and 4 deletions
|
|
@ -88,6 +88,7 @@ _TEST_SUITES: dict[str, tuple[str, ...]] = {
|
|||
"tests/test_me4_subtractive_composition.py",
|
||||
"tests/test_me5_all_categories_integration.py",
|
||||
"tests/test_rat1_end_to_end_admission.py",
|
||||
"tests/test_wave_a_multiplicative_aggregation_injector.py",
|
||||
),
|
||||
"algebra": (
|
||||
"tests/test_versor_closure.py",
|
||||
|
|
@ -1937,6 +1938,8 @@ def cmd_teaching_seed_recognizer(args: argparse.Namespace) -> int:
|
|||
canonical_pattern["anchor_count_max"] = args.anchor_count_max
|
||||
if args.graph_intent:
|
||||
canonical_pattern["graph_intent"] = args.graph_intent
|
||||
if getattr(args, "extract_values", False):
|
||||
canonical_pattern["extract_values"] = True
|
||||
|
||||
recognizer_spec = {
|
||||
"shape_category": args.shape_category,
|
||||
|
|
@ -4484,6 +4487,10 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
teaching_seed_recognizer.add_argument(
|
||||
"--review-date", default=None, help="YYYY-MM-DD (default: today)",
|
||||
)
|
||||
teaching_seed_recognizer.add_argument(
|
||||
"--extract-values", action="store_true",
|
||||
help="WAVE-A — opt the recognizer spec into value-extracting matcher path",
|
||||
)
|
||||
teaching_seed_recognizer.add_argument(
|
||||
"--note", default="", help="operator note",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -463,8 +463,47 @@ def _locate_possession_verb(sentence: str) -> str | None:
|
|||
# registers its injector. No global state, no side effects.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_WAVE_A_INJECTABLE_ANCHOR_KINDS: frozenset[str] = frozenset({
|
||||
"multiplicative_aggregate_each_weighing",
|
||||
})
|
||||
|
||||
|
||||
def inject_multiplicative_aggregation(
|
||||
match: RecognizerMatch,
|
||||
sentence: str,
|
||||
) -> tuple[InjectorEmission, ...]:
|
||||
"""WAVE-A — inject the pre-composed CandidateInitial for the
|
||||
specific value-extracted multiplicative_aggregate shapes.
|
||||
|
||||
Narrow by anchor ``kind`` to avoid intercepting ME-3 / ME-4
|
||||
additive/subtractive anchors that share the same matcher entry
|
||||
point but require the composition_registry consult path. Only
|
||||
anchors whose ``kind`` is in
|
||||
:data:`_WAVE_A_INJECTABLE_ANCHOR_KINDS` emit here; everything else
|
||||
returns () and falls through to ``_consult_composition_registry``.
|
||||
"""
|
||||
if not match.parsed_anchors:
|
||||
return ()
|
||||
out: list[InjectorEmission] = []
|
||||
for anchor in match.parsed_anchors:
|
||||
if not isinstance(anchor, Mapping):
|
||||
continue
|
||||
kind = anchor.get("kind")
|
||||
if kind not in _WAVE_A_INJECTABLE_ANCHOR_KINDS:
|
||||
continue
|
||||
composed = anchor.get("composed_initial")
|
||||
if isinstance(composed, CandidateInitial):
|
||||
out.append(composed)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
_INJECTORS: Mapping[ShapeCategory, "type"] = {
|
||||
ShapeCategory.DISCRETE_COUNT_STATEMENT: inject_discrete_count_statement, # type: ignore[dict-item]
|
||||
# WAVE-A — multiplicative_aggregation now has a per-category
|
||||
# injector that consumes value-extracted anchors. Specs without
|
||||
# ``extract_values=True`` continue to return empty parsed_anchors
|
||||
# (detection-only) so the existing wrong=0 path is byte-identical.
|
||||
ShapeCategory.MULTIPLICATIVE_AGGREGATION: inject_multiplicative_aggregation, # type: ignore[dict-item]
|
||||
# All other recognizer categories route to the empty-tuple fallback
|
||||
# in ``inject_from_match`` — `_INJECTORS.get(category)` returns
|
||||
# ``None`` and the dispatcher returns ``()``, which the
|
||||
|
|
|
|||
|
|
@ -1032,14 +1032,11 @@ def _match_multiplicative_aggregation(
|
|||
return _try_extract_additive_composition_anchor(statement, spec)
|
||||
if anchor_kind == "subtractive_quantity_composition":
|
||||
# ME-4 dispatch — subtractive shape returns ("amount" intent).
|
||||
# Cast the Literal return: the matcher signature widens at the
|
||||
# type level to include any 'aggregate'/'amount' but the caller
|
||||
# in match() reads graph_intent verbatim.
|
||||
sub_result = _try_extract_subtractive_composition_anchor(statement, spec)
|
||||
if sub_result is None:
|
||||
return None
|
||||
anchors, _ = sub_result
|
||||
return (anchors, "aggregate") # reuse 'aggregate' label for subtractive too
|
||||
return (anchors, "aggregate")
|
||||
if anchor_kind != "multiplicative_aggregate":
|
||||
return None
|
||||
padded = _padded_lower(statement)
|
||||
|
|
@ -1056,9 +1053,148 @@ def _match_multiplicative_aggregation(
|
|||
return None
|
||||
if _has_currency_symbol(statement) and _has_per_unit_framing(padded):
|
||||
return None
|
||||
|
||||
# WAVE-A — value-extracting variant. When the spec opts in via
|
||||
# ``extract_values: True`` (a separate signal from anchor_kind so
|
||||
# existing detection-only specs are unaffected), try to extract
|
||||
# the M (outer count) and N (inner count) values from the canonical
|
||||
# "<Subject> <verb> <M> <outer-noun>, each <verb> <N> <unit>" shape.
|
||||
# On extraction success, emit a pre-composed CandidateInitial with
|
||||
# composition_evidence (mirrors the ME-1..ME-4 pattern). The detection-
|
||||
# only behaviour is preserved when extract_values is absent or False.
|
||||
if spec.get("extract_values"):
|
||||
emit = _try_extract_each_weighing_anchor(statement, spec)
|
||||
if emit is not None:
|
||||
return emit
|
||||
return (tuple(), "aggregate")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# WAVE-A — multiplicative aggregation injector with value extraction.
|
||||
#
|
||||
# Targets the canonical "<Subject> <bake-verb> <M> <outer-noun>, each
|
||||
# <weigh-verb>ing <N> <unit>" shape (case 0047 in the train_sample
|
||||
# audit). Emits a pre-composed CandidateInitial(value=M*N, unit=unit,
|
||||
# entity=Subject) with composition_evidence so the wave-A admission
|
||||
# fires through the same _composed_initial_admissible gate as ME-1..ME-4.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MULT_AGG_EACH_WEIGHING_RE: Final[re.Pattern[str]] = re.compile(
|
||||
r"""(?ix)
|
||||
^\s*
|
||||
(?P<subject>[A-Z][a-zA-Z]+)
|
||||
\s+
|
||||
(?P<outer_verb>bakes|baked|made|makes|fills|filled|has|had|owns|holds|held|contains|brings|brought|carries|carried|buys|bought)
|
||||
\s+
|
||||
(?P<count_a>\d+(?:\.\d+)?)
|
||||
\s+
|
||||
(?P<outer_noun>[a-z][a-zA-Z\-]+(?:\s+[a-z][a-zA-Z\-]+)?)
|
||||
\s*,?\s+
|
||||
(?:each\s+(?:weighing|holding|containing|costing)|where\s+each\s+(?:bag|basket|box|crate|carton|container|one|item)\s+holds)
|
||||
\s+
|
||||
(?P<count_b>\d+(?:\.\d+)?)
|
||||
\s+
|
||||
(?P<unit>[a-z]+)
|
||||
\b
|
||||
""",
|
||||
)
|
||||
|
||||
_MULT_AGG_SHAPE: Final[str] = "bound(outer_count) × bound(per_outer_count)"
|
||||
|
||||
|
||||
def _try_extract_each_weighing_anchor(
|
||||
statement: str, spec: Mapping[str, Any]
|
||||
) -> tuple[tuple[Mapping[str, Any], ...], Literal["aggregate"]] | None:
|
||||
"""Extract a pre-composed CandidateInitial for the "each weighing" shape.
|
||||
|
||||
Narrowness:
|
||||
- Exactly one match of :data:`_MULT_AGG_EACH_WEIGHING_RE`
|
||||
- Subject is a proper noun not in pronoun/determiner sets
|
||||
- Outer count + inner count are both positive numerics
|
||||
- Inner unit is in ``spec["observed_units"]`` (or its singular form)
|
||||
- Outer verb in the canonical whitelist (mapped via matched_anchor)
|
||||
|
||||
Refuses on any failure; refusal-preferring.
|
||||
"""
|
||||
observed_units = set(spec.get("observed_units") or ())
|
||||
if not observed_units:
|
||||
return None
|
||||
|
||||
matches = list(_MULT_AGG_EACH_WEIGHING_RE.finditer(statement))
|
||||
if len(matches) != 1:
|
||||
return None
|
||||
|
||||
m = matches[0]
|
||||
subject = m.group("subject")
|
||||
if subject.lower() in _REFUSED_SUBJECT_TOKENS:
|
||||
return None
|
||||
if subject.lower() in _COMMON_DETERMINERS_AT_HEAD:
|
||||
return None
|
||||
|
||||
count_a_token = m.group("count_a")
|
||||
count_b_token = m.group("count_b")
|
||||
try:
|
||||
count_a = float(count_a_token)
|
||||
count_b = float(count_b_token)
|
||||
except ValueError:
|
||||
return None
|
||||
if count_a <= 0 or count_b <= 0:
|
||||
return None
|
||||
|
||||
unit = m.group("unit").lower()
|
||||
if unit not in observed_units and unit.rstrip("s") not in observed_units:
|
||||
return None
|
||||
|
||||
composed_value_f = count_a * count_b
|
||||
composed_value: int | float
|
||||
if (
|
||||
composed_value_f.is_integer()
|
||||
and "." not in count_a_token
|
||||
and "." not in count_b_token
|
||||
):
|
||||
composed_value = int(composed_value_f)
|
||||
else:
|
||||
composed_value = composed_value_f
|
||||
|
||||
from generate.math_candidate_parser import CandidateInitial
|
||||
from generate.math_problem_graph import InitialPossession, Quantity
|
||||
|
||||
# matched_anchor must be in CandidateInitial post-init whitelist.
|
||||
outer_verb = m.group("outer_verb").lower()
|
||||
matched_anchor = outer_verb if outer_verb in {
|
||||
"has", "had", "made", "makes", "buys", "bought", "paid", "earned", "saved", "got", "received"
|
||||
} else "had"
|
||||
|
||||
composed_initial = CandidateInitial(
|
||||
initial=InitialPossession(
|
||||
entity=subject,
|
||||
quantity=Quantity(value=composed_value, unit=unit),
|
||||
),
|
||||
source_span=m.group(0),
|
||||
matched_anchor=matched_anchor,
|
||||
matched_value_token=str(composed_value),
|
||||
matched_unit_token=unit,
|
||||
matched_entity_token=subject,
|
||||
composition_evidence={
|
||||
"composition_shape": _MULT_AGG_SHAPE,
|
||||
"input_tokens": f"{count_a_token}|{count_b_token}",
|
||||
"entity_source": "same_sentence",
|
||||
},
|
||||
)
|
||||
|
||||
anchor: Mapping[str, Any] = {
|
||||
"kind": "multiplicative_aggregate_each_weighing",
|
||||
"composition_shape": _MULT_AGG_SHAPE,
|
||||
"composed_initial": composed_initial,
|
||||
"count_a": count_a_token,
|
||||
"count_b": count_b_token,
|
||||
"unit": unit,
|
||||
"subject": subject,
|
||||
"outer_verb": outer_verb,
|
||||
}
|
||||
return ((anchor,), "aggregate")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ME-3 — additive composition matcher.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -72,3 +72,7 @@
|
|||
{"chain_id":"admissibility_temporal_aggregation_recognizes_9684dd780b4d0d387facdce18b474e09413d671b0d4cb944c2754ba2a0bb6208","event":"accepted_corpus_append","proposal_id":"4d47a2472e85d2d3e7ddf37f0f4c886d","provenance":{"adr_id":"adr-0057","raw":"adr-0057:discovery_promoted:2026-05-27","review_date":"2026-05-27","source":"discovery_promoted"}}
|
||||
{"event":"created","proposal":{"claim_domain":"factual","evidence":[],"polarity":"affirms","proposal_id":"rat1-seed-48dd2673d6ad673d","proposed_chain":{"connective":"ratifies","intent":"recognizer_spec_seed","object":"currency_per_unit_composition","recognizer_spec":{"canonical_pattern":{"anchor_count_max":1,"anchor_count_min":1,"anchor_kind":"currency_per_unit_composition","graph_intent":"rate","observed_currency_symbols":["$"],"observed_per_units":["apiece","each"],"outcome":"admissible","shape_category":"rate_with_currency"},"coverage":{},"exemplar_count":0,"exemplar_digest":"48dd2673d6ad673dcb50d49aeabc8cff280af8bf42861d9dc36f0e5fa5d91518","shape_category":"rate_with_currency"},"subject":"rate_with_currency"},"source":{"emitted_at_revision":"rat1-cli-seed","kind":"exemplar_corpus","source_id":"48dd2673d6ad673dcb50d49aeabc8cff280af8bf42861d9dc36f0e5fa5d91518"}}}
|
||||
{"event":"transition","note":"RAT-1 seed for currency-per-unit composition (ME-1 enablement)","proposal_id":"rat1-seed-48dd2673d6ad673d","review_date":"2026-05-27","to":"accepted"}
|
||||
{"event":"created","proposal":{"claim_domain":"factual","evidence":[],"polarity":"affirms","proposal_id":"rat1-seed-98786bdda7b27942","proposed_chain":{"connective":"ratifies","intent":"recognizer_spec_seed","object":"multiplicative_aggregate","recognizer_spec":{"canonical_pattern":{"anchor_kind":"multiplicative_aggregate","graph_intent":"aggregate","observed_units":["apple","apples","bags","basket","macaroons","ounce","ounces","question","questions","strawberries","strawberry"],"outcome":"admissible","shape_category":"multiplicative_aggregation"},"coverage":{},"exemplar_count":0,"exemplar_digest":"98786bdda7b27942f85817c29023275648cbd8b3e2873f82a46022ce9dcd4126","shape_category":"multiplicative_aggregation"},"subject":"multiplicative_aggregation"},"source":{"emitted_at_revision":"rat1-cli-seed","kind":"exemplar_corpus","source_id":"98786bdda7b27942f85817c29023275648cbd8b3e2873f82a46022ce9dcd4126"}}}
|
||||
{"event":"transition","note":"WAVE-A seed for multiplicative_aggregation with value extraction","proposal_id":"rat1-seed-98786bdda7b27942","review_date":"2026-05-27","to":"accepted"}
|
||||
{"event":"created","proposal":{"claim_domain":"factual","evidence":[],"polarity":"affirms","proposal_id":"rat1-seed-4dc30608fb783bc7","proposed_chain":{"connective":"ratifies","intent":"recognizer_spec_seed","object":"multiplicative_aggregate","recognizer_spec":{"canonical_pattern":{"anchor_kind":"multiplicative_aggregate","extract_values":true,"graph_intent":"aggregate","observed_units":["apple","apples","bags","basket","macaroons","ounce","ounces","question","questions","strawberries","strawberry"],"outcome":"admissible","shape_category":"multiplicative_aggregation"},"coverage":{},"exemplar_count":0,"exemplar_digest":"4dc30608fb783bc7848c47175c9b4c25e8ee348fdcec4729dd86d31823a095d2","shape_category":"multiplicative_aggregation"},"subject":"multiplicative_aggregation"},"source":{"emitted_at_revision":"rat1-cli-seed","kind":"exemplar_corpus","source_id":"4dc30608fb783bc7848c47175c9b4c25e8ee348fdcec4729dd86d31823a095d2"}}}
|
||||
{"event":"transition","note":"WAVE-A re-seed with extract_values=True","proposal_id":"rat1-seed-4dc30608fb783bc7","review_date":"2026-05-27","to":"accepted"}
|
||||
|
|
|
|||
216
tests/test_wave_a_multiplicative_aggregation_injector.py
Normal file
216
tests/test_wave_a_multiplicative_aggregation_injector.py
Normal file
|
|
@ -0,0 +1,216 @@
|
|||
"""WAVE-A — multiplicative_aggregation injector with value extraction.
|
||||
|
||||
Verifies the matcher extension + injector that turns
|
||||
``<Subject> <verb> <M> <outer-noun>, each <weigh-verb>ing <N> <unit>``
|
||||
shape into a pre-composed ``CandidateInitial(value=M*N, unit=unit,
|
||||
entity=Subject)`` admission. Closes the largest gap from the post-RAT-1
|
||||
audit (5 of 47 train_sample refusals were ``recognizer_empty_injection
|
||||
(multiplicative_aggregation)``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
import pytest
|
||||
|
||||
from generate.comprehension.composition_registry import (
|
||||
clear_cache as clear_composition_cache,
|
||||
)
|
||||
from generate.recognizer_registry import clear_registry_cache
|
||||
from generate.math_candidate_parser import CandidateInitial
|
||||
|
||||
|
||||
_SPEC: Mapping[str, Any] = {
|
||||
"anchor_kind": "multiplicative_aggregate",
|
||||
"extract_values": True,
|
||||
"observed_units": [
|
||||
"ounces", "ounce", "strawberries", "strawberry",
|
||||
"questions", "question", "apples", "apple",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def setup_function(_):
|
||||
clear_composition_cache()
|
||||
clear_registry_cache()
|
||||
|
||||
|
||||
def test_each_weighing_shape_admits():
|
||||
"""John bakes 12 coconut macaroons, each weighing 5 ounces → 60 ounces."""
|
||||
from generate.recognizer_match import _match_multiplicative_aggregation
|
||||
|
||||
s = "John bakes 12 coconut macaroons, each weighing 5 ounces."
|
||||
r = _match_multiplicative_aggregation(s, _SPEC)
|
||||
assert r is not None
|
||||
a = r[0][0]
|
||||
ci = a["composed_initial"]
|
||||
assert isinstance(ci, CandidateInitial)
|
||||
assert ci.initial.entity == "John"
|
||||
assert ci.initial.quantity.value == 60
|
||||
assert ci.initial.quantity.unit == "ounces"
|
||||
|
||||
|
||||
def test_each_basket_holds_shape_admits():
|
||||
"""Lilibeth fills 6 baskets where each basket holds 50 strawberries → 300."""
|
||||
from generate.recognizer_match import _match_multiplicative_aggregation
|
||||
|
||||
s = "Lilibeth fills 6 baskets where each basket holds 50 strawberries."
|
||||
r = _match_multiplicative_aggregation(s, _SPEC)
|
||||
assert r is not None
|
||||
a = r[0][0]
|
||||
ci = a["composed_initial"]
|
||||
assert ci.initial.entity == "Lilibeth"
|
||||
assert ci.initial.quantity.value == 300
|
||||
|
||||
|
||||
def test_detection_only_path_preserved_when_extract_values_absent():
|
||||
"""Specs WITHOUT extract_values=True get the existing detection-only path."""
|
||||
from generate.recognizer_match import _match_multiplicative_aggregation
|
||||
|
||||
spec = dict(_SPEC)
|
||||
spec.pop("extract_values")
|
||||
r = _match_multiplicative_aggregation(
|
||||
"John bakes 12 coconut macaroons, each weighing 5 ounces.", spec
|
||||
)
|
||||
# Falls through to existing detection-only return.
|
||||
assert r is not None
|
||||
anchors, intent = r
|
||||
assert intent == "aggregate"
|
||||
assert anchors == ()
|
||||
|
||||
|
||||
def test_unobserved_unit_refuses():
|
||||
from generate.recognizer_match import _match_multiplicative_aggregation
|
||||
|
||||
spec = dict(_SPEC)
|
||||
spec["observed_units"] = ["dollars"]
|
||||
r = _match_multiplicative_aggregation(
|
||||
"John bakes 12 coconut macaroons, each weighing 5 ounces.", spec
|
||||
)
|
||||
# Falls through to detection-only (no composed admit; unit not observed).
|
||||
assert r is not None
|
||||
assert r[0] == ()
|
||||
|
||||
|
||||
def test_pronoun_subject_refuses():
|
||||
from generate.recognizer_match import _match_multiplicative_aggregation
|
||||
|
||||
r = _match_multiplicative_aggregation(
|
||||
"He bakes 12 coconut macaroons, each weighing 5 ounces.", _SPEC
|
||||
)
|
||||
# Falls through to detection-only (composition path refuses on pronoun).
|
||||
assert r is not None
|
||||
assert r[0] == ()
|
||||
|
||||
|
||||
def test_zero_count_refuses():
|
||||
from generate.recognizer_match import _match_multiplicative_aggregation
|
||||
|
||||
r = _match_multiplicative_aggregation(
|
||||
"John bakes 0 coconut macaroons, each weighing 5 ounces.", _SPEC
|
||||
)
|
||||
assert r is not None
|
||||
assert r[0] == ()
|
||||
|
||||
|
||||
def test_inject_from_match_picks_up_composed_initial():
|
||||
"""The new per-category injector emits the composed CandidateInitial."""
|
||||
from evals.refusal_taxonomy.shape_categories import ShapeCategory
|
||||
from generate.recognizer_anchor_inject import inject_from_match
|
||||
from generate.recognizer_match import (
|
||||
RecognizerMatch,
|
||||
_match_multiplicative_aggregation,
|
||||
)
|
||||
|
||||
s = "John bakes 12 coconut macaroons, each weighing 5 ounces."
|
||||
r = _match_multiplicative_aggregation(s, _SPEC)
|
||||
assert r is not None
|
||||
|
||||
class _R:
|
||||
spec_id = "test_wave_a"
|
||||
|
||||
m = RecognizerMatch(
|
||||
recognizer=_R(), # type: ignore[arg-type]
|
||||
category=ShapeCategory.MULTIPLICATIVE_AGGREGATION,
|
||||
outcome="admissible",
|
||||
graph_intent="aggregate",
|
||||
parsed_anchors=r[0],
|
||||
)
|
||||
emit = inject_from_match(m, s)
|
||||
assert len(emit) == 1
|
||||
assert isinstance(emit[0], CandidateInitial)
|
||||
assert emit[0].initial.quantity.value == 60
|
||||
|
||||
|
||||
def test_lilibeth_canary_solves_end_to_end():
|
||||
"""The first WAVE-A end-to-end solve on the canonical pack."""
|
||||
if not _has_wave_a_seed():
|
||||
pytest.skip("WAVE-A recognizer seed not present on canonical pack")
|
||||
from generate.math_candidate_graph import parse_and_solve
|
||||
|
||||
p = (
|
||||
"Lilibeth fills 6 baskets where each basket holds 50 strawberries. "
|
||||
"How many strawberries does Lilibeth have?"
|
||||
)
|
||||
r = parse_and_solve(p)
|
||||
assert r.refusal_reason is None, f"unexpected refusal: {r.refusal_reason!r}"
|
||||
assert r.answer == 300
|
||||
|
||||
|
||||
def _has_wave_a_seed() -> bool:
|
||||
from generate.recognizer_registry import load_ratified_registry
|
||||
|
||||
reg = load_ratified_registry()
|
||||
return any(
|
||||
r.canonical_pattern.get("anchor_kind") == "multiplicative_aggregate"
|
||||
and r.canonical_pattern.get("extract_values") is True
|
||||
for r in reg
|
||||
)
|
||||
|
||||
|
||||
def test_wrong_zero_preserved():
|
||||
"""The full train_sample eval keeps wrong == 0 after WAVE-A."""
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
here = Path(__file__).resolve()
|
||||
while here.parent != here and not (here / "pyproject.toml").exists():
|
||||
here = here.parent
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "evals.gsm8k_math.train_sample.v1.runner",
|
||||
"--use-reader"],
|
||||
cwd=here,
|
||||
capture_output=True,
|
||||
)
|
||||
report = json.loads(
|
||||
(here / "evals" / "gsm8k_math" / "train_sample" / "v1" / "report.json").read_text()
|
||||
)
|
||||
assert report["counts"]["wrong"] == 0
|
||||
|
||||
|
||||
def test_case_0050_remains_refused():
|
||||
"""Hazard pin."""
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
here = Path(__file__).resolve()
|
||||
while here.parent != here and not (here / "pyproject.toml").exists():
|
||||
here = here.parent
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "evals.gsm8k_math.train_sample.v1.runner",
|
||||
"--use-reader"],
|
||||
cwd=here,
|
||||
capture_output=True,
|
||||
)
|
||||
report = json.loads(
|
||||
(here / "evals" / "gsm8k_math" / "train_sample" / "v1" / "report.json").read_text()
|
||||
)
|
||||
case_0050 = next(
|
||||
(c for c in report["per_case"] if c["case_id"].endswith("-0050")),
|
||||
None,
|
||||
)
|
||||
assert case_0050 is not None
|
||||
assert case_0050["verdict"] == "refused"
|
||||
Loading…
Reference in a new issue