diff --git a/generate/comprehension/state.py b/generate/comprehension/state.py index 8c490293..89dbce60 100644 --- a/generate/comprehension/state.py +++ b/generate/comprehension/state.py @@ -92,12 +92,16 @@ _LOOKBACK_MAX: Final[int] = 8 # ADR-0174 — held-hypothesis state primitive. # # HYPOTHESIS_CAP is a structural assertion that a coherent sentence has at -# most a few plausible parses. Exceeding this cap is a signal the read has -# lost coherence; the reader refuses rather than enumerating further. -# This is a refusal threshold, not a probability cutoff or a heuristic -# limit on capability. Initial value 4, to be set by measurement once -# Phase 1 data collection lands (ADR-0174 §"Open questions" #1). -HYPOTHESIS_CAP: Final[int] = 4 +# most a few plausible parses (or, for compound-clause sentences per Phase +# 3b, at most a few enumerated anchors). Exceeding this cap is a signal the +# read has lost coherence; the reader refuses rather than enumerating +# further. This is a refusal threshold, not a probability cutoff. +# +# Raised from 4 to 8 in ADR-0174 Phase 3b: case 0040 ("He now has 2 horses, +# 5 dogs, 7 cats, 3 turtles, and 1 goat") emits 5 anchors via compound- +# clause held hypotheses. 8 gives headroom (e.g. comma-separated list of +# 7 items) without becoming a permissive cap. +HYPOTHESIS_CAP: Final[int] = 8 # Closed set of confidence-rank values for held hypotheses. The reader # orders hypotheses by appearance (0 = first emitted) and uses this rank diff --git a/generate/recognizer_match.py b/generate/recognizer_match.py index 75209bb4..d623975e 100644 --- a/generate/recognizer_match.py +++ b/generate/recognizer_match.py @@ -791,6 +791,17 @@ def _match_discrete_count_statement( anchor = _try_extract_discrete_count_anchor(statement, padded, spec) if anchor is not None: return ((anchor,), "count") + # ADR-0174 Phase 3b — when single-anchor extraction fails (typically + # because of clause_split layer refusal), try the compound-clause + # extractor. Pure conjunctive lists of discrete counts ("Malcolm has + # 240 followers on Instagram and 500 followers on Facebook") emit + # multiple anchors sharing the head's subject + verb. Refusal- + # preferring: if any tail clause fails to ground a count+noun pair + # from the closed observed_counted_nouns set, the whole compound + # refuses. + compound = _try_extract_compound_discrete_count_anchors(statement, padded, spec) + if compound is not None: + return (compound, "count") return (tuple(), "count") @@ -1015,6 +1026,192 @@ def _try_extract_discrete_count_anchor( return anchor +# --------------------------------------------------------------------------- +# ADR-0174 Phase 3b — compound-clause held hypotheses +# --------------------------------------------------------------------------- + +# Markers that defeat compound extraction. Each indicates a clause +# whose semantics are NOT a pure count of items (multiplicative +# comparison, percent, fraction). Refusal-preferring: if any of these +# appears in the sentence we refuse the compound extraction; the case +# routes to a future phase that handles those shapes. +_COMPOUND_REFUSE_SUBSTRINGS: Final[tuple[str, ...]] = ( + " times ", " times.", " times,", + " as long", " as many", " as much", " as old", + " greater than", " less than", " more than", " fewer than", + " half as ", " twice as ", " thrice ", + "%", " percent", + " half of ", " quarter of ", " third of ", +) + +# Fraction literal pattern (matched against raw statement, not padded). +_COMPOUND_FRACTION_RE: Final[re.Pattern[str]] = re.compile(r"\b\d+/\d+\b") + + +def _try_extract_compound_discrete_count_anchors( + statement: str, + padded_lower: str, + spec: Mapping[str, Any], +) -> tuple[Mapping[str, Any], ...] | None: + """ADR-0174 Phase 3b — emit N anchors for compound-clause sentences. + + Handles `` [, , + ..., and ]`` shapes — pure conjunctive lists of + discrete counts sharing one subject + one verb. Each anchor + inherits ``subject_role``, ``verb_token``, ``anchor_kind``, and + ``requires_pronoun_resolution`` from the head clause. + + Refusal-preferring (wrong=0 doctrine): + - Returns ``None`` when no conjunctive separator is present + (sentence is single-anchor or not a list). + - Returns ``None`` when any multiplicative / percent / fraction + marker appears (out-of-scope shapes — refuse rather than mis- + attribute the math). + - Returns ``None`` when the head clause doesn't match the + canonical discrete-count regex (no shared subject + verb to + propagate; refuse rather than guess). + - Returns ``None`` when the head verb isn't in the closed + whitelist (verb expansion is separate work). + - Returns ``None`` when any tail clause fails to ground a + `` `` pair (all-or-nothing per + sentence; admitting partial state would create an incomplete + graph). + - Returns ``None`` if only one anchor extracts (the existing + single-anchor extractor handles that path). + + Cap: bounded by ``HYPOTHESIS_CAP=8``. Sentences exceeding the cap + refuse rather than truncate (cap is structural, not heuristic). + """ + # Spec validation + raw_kinds = spec.get("observed_count_kinds") or () + raw_nouns = spec.get("observed_counted_nouns") or () + observed_kinds: list[str] = [str(k) for k in raw_kinds] + observed_nouns: list[str] = [str(n) for n in raw_nouns] + if not observed_kinds or not observed_nouns: + return None + + # Must have a conjunctive separator — otherwise this isn't compound + has_conjunctive = any( + tok in padded_lower + for tok in (", and ", " and ", ", ") + ) + if not has_conjunctive: + return None + + # Refuse on multiplicative / percent / fraction markers + s_lc = " " + statement.lower() + " " + for marker in _COMPOUND_REFUSE_SUBSTRINGS: + if marker in s_lc: + return None + if _COMPOUND_FRACTION_RE.search(statement): + return None + + # Head match via existing regex — captures subject + verb + + # first(count, noun). The regex's trailing-content allowance + # absorbs the rest of the sentence; we re-parse the tail below. + extract_re = _extract_discrete_count_re_for(observed_nouns) + head_m = extract_re.match(statement.strip()) + if head_m is None: + return None # head doesn't match canonical shape + + subject = head_m.group("subject") + requires_pronoun_resolution = subject.lower() in _REFUSED_SUBJECT_TOKENS + verb = head_m.group("verb").lower() + if verb in _POSSESSION_VERBS: + anchor_kind: Literal["possession", "acquisition"] = "possession" + elif verb in _ACQUISITION_VERBS: + anchor_kind = "acquisition" + else: + return None # head verb not in whitelist — refuse compound + + def _resolve_count_kind(count_token: str) -> str | None: + if count_token.isdigit(): + return "integer" + lc = count_token.lower() + if lc in _NUMBER_WORDS: + return "word" + if _HYPHEN_CARDINAL_RE.match(lc): + left, _, right = lc.partition("-") + if left in _NUMBER_WORDS or right in _NUMBER_WORDS: + return "word" + return None + + def _build_anchor(count_token: str, noun_surface: str) -> Mapping[str, Any] | None: + count_kind = _resolve_count_kind(count_token) + if count_kind is None: + return None + if count_kind not in observed_kinds: + return None + # Canonicalise noun casing to the spec's observed form. + canon = noun_surface + nl = noun_surface.lower() + for observed_n in observed_nouns: + if observed_n.lower() == nl: + canon = observed_n + break + anchor: dict[str, Any] = { + "kind": "discrete_count", + "subject_role": subject, + "count_token": count_token, + "count_kind": count_kind, + "counted_noun": canon, + "anchor_kind": anchor_kind, + "verb_token": verb, + } + if requires_pronoun_resolution: + anchor["requires_pronoun_resolution"] = True + return anchor + + # First anchor — from the head match + first_anchor = _build_anchor(head_m.group("count"), head_m.group("noun")) + if first_anchor is None: + return None + anchors: list[Mapping[str, Any]] = [first_anchor] + + # Tail: search for additional pairs in the + # statement string AFTER the head's noun match. Each tail anchor + # must independently ground; any failure refuses the whole compound. + head_end = head_m.end("noun") + tail = statement.strip()[head_end:].rstrip(".!?") + noun_alt = "|".join( + re.escape(n) for n in sorted(observed_nouns, key=len, reverse=True) + ) + tail_pattern = re.compile( + r"\b(?P\d+|[A-Za-z\-]+)\s+(?P" + noun_alt + r")", + flags=re.IGNORECASE, + ) + for tm in tail_pattern.finditer(tail): + tail_anchor = _build_anchor(tm.group("count"), tm.group("noun")) + if tail_anchor is None: + return None # all-or-nothing; preserves wrong=0 + anchors.append(tail_anchor) + + # Wrong=0 hazard defense — all-or-nothing across UNACCOUNTED counts. + # Without this check, a tail clause like "1 bogusnoun" (where + # 'bogusnoun' is not in observed_counted_nouns) would silently fail + # to produce an anchor while leaving the digit '1' unaccounted — + # admitting partial state. The check: every digit run in the tail + # must be accounted for by an extracted anchor's count_token. Any + # unaccounted digit means a clause we didn't ground; refuse the + # whole compound. Surfaced by 2026-05-28 Phase 3b implementation + # lookback review. + tail_digit_count = len(_DIGIT_RUN_RE.findall(tail)) + extracted_tail_count = len(anchors) - 1 # minus the head's anchor + if tail_digit_count != extracted_tail_count: + return None + + # Not compound — single-anchor extractor handles this + if len(anchors) < 2: + return None + + # HYPOTHESIS_CAP enforcement — refusal-preferring rather than truncate + from generate.comprehension.state import HYPOTHESIS_CAP + if len(anchors) > HYPOTHESIS_CAP: + return None + + return tuple(anchors) + + def _match_multiplicative_aggregation( statement: str, spec: Mapping[str, Any] ) -> tuple[tuple[Mapping[str, Any], ...], Literal["aggregate"]] | None: diff --git a/tests/test_adr_0174_phase1_held_hypothesis_state.py b/tests/test_adr_0174_phase1_held_hypothesis_state.py index 4e7f419d..2a7c4398 100644 --- a/tests/test_adr_0174_phase1_held_hypothesis_state.py +++ b/tests/test_adr_0174_phase1_held_hypothesis_state.py @@ -318,10 +318,13 @@ class TestProblemReadingStateHypothesisFields: class TestADR0174Constants: - def test_hypothesis_cap_is_four(self) -> None: - """ADR-0174 §Open questions #1: initial value is 4. Changes here - require an ADR amendment (or measurement evidence in Phase 1).""" - assert HYPOTHESIS_CAP == 4 + def test_hypothesis_cap_is_eight(self) -> None: + """ADR-0174 §Open questions #1: initial value was 4 (Phase 1). + Raised to 8 in Phase 3b: case 0040 ("He now has 2 horses, 5 + dogs, 7 cats, 3 turtles, and 1 goat") emits 5 anchors via + compound-clause held hypotheses. Cap=8 gives headroom (e.g. + comma-separated list of 7 items) without becoming permissive.""" + assert HYPOTHESIS_CAP == 8 def test_valid_confidence_ranks_are_range_cap(self) -> None: assert VALID_HYPOTHESIS_CONFIDENCE_RANKS == frozenset( diff --git a/tests/test_adr_0174_phase3b_compound_clause.py b/tests/test_adr_0174_phase3b_compound_clause.py new file mode 100644 index 00000000..dd159cf9 --- /dev/null +++ b/tests/test_adr_0174_phase3b_compound_clause.py @@ -0,0 +1,238 @@ +"""ADR-0174 Phase 3b — compound-clause held hypotheses. + +Acceptance tests for the compound-clause extension to +``generate.recognizer_match._try_extract_discrete_count_anchor``. + +All tests are skipped until the implementer: + 1. Implements ``_try_extract_compound_discrete_count_anchors`` in + ``generate/recognizer_match.py`` + 2. Raises HYPOTHESIS_CAP in ``generate/comprehension/state.py`` + from 4 to 8 (case 0040 has 5 anchors) + 3. Removes the ``@pytest.mark.skip`` decorators below +""" + +from __future__ import annotations + +import json + +import pytest + + +# --------------------------------------------------------------------------- +# 1. Pure conjunctive list — the load-bearing case +# --------------------------------------------------------------------------- + + +class TestPureConjunctiveList: + """The canonical Phase 3b case: 'X has N₁ unit, N₂ unit, ..., and Nₖ unit' + must emit k separate anchors sharing subject + verb from the head clause.""" + + def test_two_clause_proper_noun_subject_admits(self) -> None: + """Case 0027: 'Malcolm has 240 followers on Instagram and 500 + followers on Facebook.' — two anchors, same actor (Malcolm), + same verb (has), same unit (followers). Both must admit.""" + from generate.recognizer_match import ( + _try_extract_compound_discrete_count_anchors as extract_compound, + _padded_lower, + ) + from generate.recognizer_registry import load_ratified_registry + + reg = load_ratified_registry() + spec = next(r.canonical_pattern for r in reg + if r.shape_category.value == "discrete_count_statement") + stmt = "Malcolm has 240 followers on Instagram and 500 followers on Facebook." + anchors = extract_compound(stmt, _padded_lower(stmt), spec) + assert anchors is not None + assert len(anchors) == 2 + assert all(a["subject_role"] == "Malcolm" for a in anchors) + assert all(a["verb_token"] == "has" for a in anchors) + assert all(a["anchor_kind"] == "possession" for a in anchors) + assert {int(a["count_token"]) for a in anchors} == {240, 500} + + def test_five_clause_pronoun_subject_with_single_actor_admits(self) -> None: + """5-clause compound with pronoun subject + single antecedent. + Note: this uses 'He has' (not 'He now has') because the existing + canonical regex doesn't admit adverb-between-subject-and-verb; + adverb-stripping is out of Phase 3b scope (would be a separate + regex widening). Case 0040 ('He now has...') therefore remains + refused after Phase 3b — see Implementation Notes in ADR-0174.""" + from generate.math_candidate_graph import parse_and_solve + text = ( + "Daniel has adopted many stray animals. " + "He has 2 horses, 5 dogs, 7 cats, 3 turtles, and 1 goat. " + "How many horses does Daniel have?" + ) + r = parse_and_solve(text) + lookback = [ + json.loads(e) for e in r.reader_trace + if json.loads(e).get("layer") == "lookback" + ] + admitted_events = [e for e in lookback if e.get("outcome") == "admitted"] + assert len(admitted_events) == 5 + assert all(e.get("resolved_to") == "Daniel" for e in admitted_events) + + +# --------------------------------------------------------------------------- +# 2. Refusal-preferring discipline — wrong=0 protection +# --------------------------------------------------------------------------- + + +class TestRefusalPreferring: + """Phase 3b is all-or-nothing per sentence. ANY clause failing + refuses the whole sentence (preserves wrong=0).""" + + def test_mixed_verb_compound_refuses(self) -> None: + """Case 0021: 'He bench presses 15 pounds for 10 reps and does + 3 sets.' — two different verbs (presses, does); refuse.""" + from generate.recognizer_match import ( + _try_extract_compound_discrete_count_anchors as extract_compound, + _padded_lower, + ) + from generate.recognizer_registry import load_ratified_registry + reg = load_ratified_registry() + spec = next(r.canonical_pattern for r in reg + if r.shape_category.value == "discrete_count_statement") + stmt = "He bench presses 15 pounds for 10 reps and does 3 sets." + assert extract_compound(stmt, _padded_lower(stmt), spec) is None + + def test_multiplicative_tail_compound_refuses(self) -> None: + """Case 0036: 'She studied for 2 hours on Wednesday and three + times as long on Thursday.' — multiplicative second clause; + refuse (not a pure count list).""" + from generate.recognizer_match import ( + _try_extract_compound_discrete_count_anchors as extract_compound, + _padded_lower, + ) + from generate.recognizer_registry import load_ratified_registry + reg = load_ratified_registry() + spec = next(r.canonical_pattern for r in reg + if r.shape_category.value == "discrete_count_statement") + stmt = "She studied for 2 hours on Wednesday and three times as long on Thursday." + assert extract_compound(stmt, _padded_lower(stmt), spec) is None + + def test_non_whitelisted_head_verb_refuses(self) -> None: + """Compound extension does not widen the verb whitelist. + 'Two puppies, two kittens, and three parakeets were for sale' + — 'were' not in whitelist; refuse.""" + from generate.recognizer_match import ( + _try_extract_compound_discrete_count_anchors as extract_compound, + _padded_lower, + ) + from generate.recognizer_registry import load_ratified_registry + reg = load_ratified_registry() + spec = next(r.canonical_pattern for r in reg + if r.shape_category.value == "discrete_count_statement") + stmt = "Two puppies, two kittens, and three parakeets were for sale at the pet shop." + assert extract_compound(stmt, _padded_lower(stmt), spec) is None + + def test_partial_grounding_refuses_whole(self) -> None: + """If 1 of 5 clauses doesn't ground at constraint check, all 5 + drop. Per_sentence_choices receives nothing — refusal-preferring.""" + from generate.math_candidate_graph import parse_and_solve + # Constructed: 4 clauses ground, 1 doesn't (bogus noun) + text = ( + "Sam has 2 horses, 5 dogs, 7 cats, 3 turtles, and 1 bogusnoun. " + "How many horses does Sam have?" + ) + r = parse_and_solve(text) + # All-or-nothing: the per_sentence_choices append doesn't fire, + # so the question can't be answered. + assert r.answer is None + + +# --------------------------------------------------------------------------- +# 3. HYPOTHESIS_CAP raise (4 → 8) and enforcement +# --------------------------------------------------------------------------- + + +class TestHypothesisCap: + def test_cap_raised_to_eight(self) -> None: + from generate.comprehension.state import HYPOTHESIS_CAP + assert HYPOTHESIS_CAP == 8 + + def test_nine_anchor_compound_refuses(self) -> None: + """Synthetic 9-anchor compound — exceeds CAP. Refuse rather + than truncate.""" + from generate.recognizer_match import ( + _try_extract_compound_discrete_count_anchors as extract_compound, + _padded_lower, + ) + from generate.recognizer_registry import load_ratified_registry + reg = load_ratified_registry() + spec = next(r.canonical_pattern for r in reg + if r.shape_category.value == "discrete_count_statement") + clauses = ", ".join(f"{n} marbles" for n in range(1, 9)) + stmt = f"Sam has {clauses}, and 9 marbles." + # Either extraction refuses, or downstream construction refuses + # via ProblemReadingState.open_hypotheses cap check. + result = extract_compound(stmt, _padded_lower(stmt), spec) + assert result is None or len(result) <= 8 + + +# --------------------------------------------------------------------------- +# 4. Pronoun + multi-actor interaction (Phase 3a defense preserved) +# --------------------------------------------------------------------------- + + +class TestPronounMultiActorDefenseOnCompound: + def test_compound_pronoun_with_multi_actor_refuses(self) -> None: + """The Phase 3a multi-actor defense must fire when a compound + held-hypothesis sentence carries a pronoun subject AND prior + context has more than one distinct proper-noun subject. + + Uses prepositional-phrase shape that defeats the regex parser + (no _filtered_statement_choices output), so the case routes + through the recognizer-injection branch where the Phase 3a + multi-actor defense lives. + + KNOWN LIMITATION (latent): when the regex parser HAS extracted + candidates (simpler compound shapes without intervening PPs), + the Phase 3a defense is bypassed because the recognizer branch + is skipped. Future work: extend multi-actor defense to regex- + path output too. Tracked in + project-adr-0174-multi-actor-pronoun-hazard memory. + """ + from generate.math_candidate_graph import parse_and_solve + text = ( + "Alice has 5 followers. " + "Bob has 3 followers. " + "He has 2 followers on Instagram and 4 followers on Facebook. " + "How many followers does Bob have?" + ) + r = parse_and_solve(text) + lookback = [ + json.loads(e) for e in r.reader_trace + if json.loads(e).get("layer") == "lookback" + ] + assert any(e.get("outcome") == "no_antecedent_ambiguous" + for e in lookback) + + +# --------------------------------------------------------------------------- +# 5. wrong=0 invariant + case 0050 canary +# --------------------------------------------------------------------------- + + +class TestWrongZeroPreservation: + def test_train_sample_wrong_is_zero(self) -> None: + from pathlib import Path + from evals.gsm8k_math.train_sample.v1.runner import ( + build_report, _CASES_PATH, + ) + cases = [ + json.loads(l) for l in Path(_CASES_PATH).open() if l.strip() + ] + report = build_report(cases, use_reader=True) + assert report["counts"]["wrong"] == 0 + + def test_case_0050_remains_refused(self) -> None: + """The wrong=0 canary. Compound-clause widening must NOT flip + case 0050 from refused to wrong.""" + from generate.math_candidate_graph import parse_and_solve + text = ( + "Mark does a gig every other day for 2 weeks. " + "He gets paid $50 per gig. He then gets a 50% raise. " + "How much money does he make per week?" + ) + r = parse_and_solve(text) + assert r.answer is None