feat(ADR-0136.S.0): context-sentence classifier — skip no-digit sentences, gsm8k-0018 admits (#202)
- Add classify_sentence() + has_numeric_token() to math_candidate_parser.py.
Rule: sentence with no digit and no word-number cannot introduce parseable
numeric state — classify as "context" and skip safely (wrong==0 preserved).
- Add pre-pass in parse_and_solve() (math_candidate_graph.py): strips context
sentences before extraction; falls through to refusal if none remain numeric.
- Extend capacity patterns for gsm8k-0018:
- _CAPACITY_INVERTED_RE: "During M <time-unit> <Actor> can <verb> N <unit>"
- _CAPACITY_Q2_RE: "How many <unit> [on average] is <Actor> able to <verb>,
when the <event> lasted for T <time-unit>?"
- GSM8K: 1/50 -> 2/50 (gsm8k-0018 admits with answer 16.0); admitted_wrong==0.
- Tests: 47/47 pass (12 new for classifier, inverted patterns, 0018 end-to-end).
This commit is contained in:
Shay
GitHub
parent
52f2bf6f4c
commit
19ac7f94b9
3 changed files with 156 additions and 11 deletions
|
|
@ -41,6 +41,7 @@ from typing import Final, Union
|
|||
from generate.math_candidate_parser import (
|
||||
CandidateInitial,
|
||||
CandidateUnknown,
|
||||
classify_sentence,
|
||||
extract_capacity_candidates,
|
||||
extract_capacity_question_candidates,
|
||||
extract_earnings_candidates,
|
||||
|
|
@ -315,6 +316,15 @@ def parse_and_solve(text: str) -> CandidateGraphResult:
|
|||
question_sentences = [s for s in sentences if s.rstrip().endswith("?")]
|
||||
statement_sentences = [s for s in sentences if not s.rstrip().endswith("?")]
|
||||
|
||||
# ADR-0136.S.0 — Strip context-filler sentences before any extraction.
|
||||
# A sentence with no digit and no word-number cannot introduce parseable
|
||||
# numeric state; skipping it is provably safe for wrong == 0.
|
||||
numeric_statement_sentences = [
|
||||
s for s in statement_sentences if classify_sentence(s) == "numeric_state"
|
||||
]
|
||||
if numeric_statement_sentences or not statement_sentences:
|
||||
statement_sentences = numeric_statement_sentences
|
||||
|
||||
if len(question_sentences) != 1:
|
||||
return CandidateGraphResult(
|
||||
answer=None, selected_graph=None,
|
||||
|
|
|
|||
|
|
@ -1571,6 +1571,38 @@ def _build_conj_embedded_sum(
|
|||
return []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ADR-0136.S.0 — Context-sentence classifier
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_WORD_NUMBER_RE: Final[re.Pattern[str]] = re.compile(
|
||||
r"\b(?:" + "|".join(re.escape(w) for w in sorted(WORD_NUMBERS, key=len, reverse=True)) + r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def has_numeric_token(sentence: str) -> bool:
|
||||
"""Return True if *sentence* contains any digit or closed-set word-number.
|
||||
|
||||
A sentence with no numeric token cannot introduce quantified initial state,
|
||||
so it is safe to classify as a context filler and skip it.
|
||||
"""
|
||||
if re.search(r"\d", sentence):
|
||||
return True
|
||||
return bool(_WORD_NUMBER_RE.search(sentence))
|
||||
|
||||
|
||||
def classify_sentence(sentence: str) -> Literal["context", "numeric_state"]:
|
||||
"""Classify a statement sentence as skippable context or numeric-state-bearing.
|
||||
|
||||
Rule: if the sentence contains no digit and no word-number from the closed
|
||||
set, it cannot introduce any parseable numeric state — classify as
|
||||
``"context"`` (safe to skip). All other sentences are ``"numeric_state"``
|
||||
and must either parse successfully or cause a refusal.
|
||||
"""
|
||||
return "context" if not has_numeric_token(sentence) else "numeric_state"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ADR-0136.S.1 — Rate/event statement extractors (capacity + earnings)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -1626,6 +1658,7 @@ class CandidateCapacity:
|
|||
source_span: str
|
||||
|
||||
|
||||
# Shape A1 (canonical): "<Actor> can <verb> N <unit> in M <time-unit>."
|
||||
_CAPACITY_RE: Final[re.Pattern[str]] = re.compile(
|
||||
rf"^(?P<actor>{_ENTITY})\s+can\s+"
|
||||
rf"(?P<verb>{_CAPACITY_VERB_PATTERN})\s+"
|
||||
|
|
@ -1633,16 +1666,27 @@ _CAPACITY_RE: Final[re.Pattern[str]] = re.compile(
|
|||
rf"(?P<unit>\w+)\s+in\s+"
|
||||
rf"(?P<per_count>\d+(?:\.\d+)?)\s+"
|
||||
rf"(?P<per_unit>{_TIME_UNIT_SET})"
|
||||
r"(?:\s+on\s+average)?"
|
||||
r"\s*\.?\s*$",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Shape A2 (inverted): "During M <time-unit> <Actor> can <verb> N <unit> [on average]."
|
||||
_CAPACITY_INVERTED_RE: Final[re.Pattern[str]] = re.compile(
|
||||
rf"^[Dd]uring\s+"
|
||||
rf"(?P<per_count>\d+(?:\.\d+)?)\s+"
|
||||
rf"(?P<per_unit>{_TIME_UNIT_SET})\s+"
|
||||
rf"(?P<actor>{_ENTITY})\s+can\s+"
|
||||
rf"(?P<verb>{_CAPACITY_VERB_PATTERN})\s+"
|
||||
rf"(?P<count>\d+(?:\.\d+)?)\s+"
|
||||
rf"(?P<unit>\w+)"
|
||||
r"(?:\s+on\s+average)?"
|
||||
r"\s*\.?\s*$",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def extract_capacity_candidates(sentence: str) -> list[CandidateCapacity]:
|
||||
s = sentence.strip()
|
||||
m = _CAPACITY_RE.match(s)
|
||||
if m is None:
|
||||
return []
|
||||
def _capacity_from_match(m: re.Match[str], sentence: str) -> list[CandidateCapacity]:
|
||||
verb = m.group("verb").lower()
|
||||
if verb not in _CAPACITY_VERBS:
|
||||
return []
|
||||
|
|
@ -1662,6 +1706,17 @@ def extract_capacity_candidates(sentence: str) -> list[CandidateCapacity]:
|
|||
]
|
||||
|
||||
|
||||
def extract_capacity_candidates(sentence: str) -> list[CandidateCapacity]:
|
||||
s = sentence.strip()
|
||||
m = _CAPACITY_RE.match(s)
|
||||
if m is not None:
|
||||
return _capacity_from_match(m, sentence)
|
||||
m2 = _CAPACITY_INVERTED_RE.match(s)
|
||||
if m2 is not None:
|
||||
return _capacity_from_match(m2, sentence)
|
||||
return []
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CandidateCapacityQuestion:
|
||||
actor: str | None
|
||||
|
|
@ -1673,6 +1728,7 @@ class CandidateCapacityQuestion:
|
|||
|
||||
_PRONOUN_SET: Final[str] = r"(?:he|she|they|it)"
|
||||
|
||||
# Q1 (canonical): "How many <unit> can <actor> <verb> in T <time-unit>?"
|
||||
_CAPACITY_Q_RE: Final[re.Pattern[str]] = re.compile(
|
||||
r"^How\s+many\s+(?P<unit>\w+)\s+can\s+"
|
||||
rf"(?P<actor>{_ENTITY}|{_PRONOUN_SET})\s+"
|
||||
|
|
@ -1683,14 +1739,23 @@ _CAPACITY_Q_RE: Final[re.Pattern[str]] = re.compile(
|
|||
flags=re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Q2 (able-form): "How many <unit> [on average] is <actor> able to <verb>,
|
||||
# when the [match/game/session] lasted for T <time-unit>?"
|
||||
_CAPACITY_Q2_RE: Final[re.Pattern[str]] = re.compile(
|
||||
r"^How\s+many\s+(?P<unit>\w+)(?:\s+on\s+average)?\s+is\s+"
|
||||
rf"(?P<actor>{_ENTITY}|{_PRONOUN_SET})\s+able\s+to\s+"
|
||||
rf"(?P<verb>{_CAPACITY_VERB_PATTERN}),?\s+"
|
||||
r"when\s+the\s+\w+\s+lasted\s+for\s+"
|
||||
rf"(?P<per_count>\d+(?:\.\d+)?)\s+"
|
||||
rf"(?P<per_unit>{_TIME_UNIT_SET})"
|
||||
r"\s*\??\s*$",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
|
||||
def extract_capacity_question_candidates(
|
||||
sentence: str,
|
||||
|
||||
def _capacity_question_from_match(
|
||||
m: re.Match[str], sentence: str
|
||||
) -> list[CandidateCapacityQuestion]:
|
||||
s = sentence.strip()
|
||||
m = _CAPACITY_Q_RE.match(s)
|
||||
if m is None:
|
||||
return []
|
||||
verb = m.group("verb").lower()
|
||||
if verb not in _CAPACITY_VERBS:
|
||||
return []
|
||||
|
|
@ -1712,6 +1777,19 @@ def extract_capacity_question_candidates(
|
|||
]
|
||||
|
||||
|
||||
def extract_capacity_question_candidates(
|
||||
sentence: str,
|
||||
) -> list[CandidateCapacityQuestion]:
|
||||
s = sentence.strip()
|
||||
m = _CAPACITY_Q_RE.match(s)
|
||||
if m is not None:
|
||||
return _capacity_question_from_match(m, sentence)
|
||||
m2 = _CAPACITY_Q2_RE.match(s)
|
||||
if m2 is not None:
|
||||
return _capacity_question_from_match(m2, sentence)
|
||||
return []
|
||||
|
||||
|
||||
# --- Shape B: earnings rate ---
|
||||
|
||||
_EARNINGS_VERBS: Final[frozenset[str]] = frozenset({
|
||||
|
|
|
|||
|
|
@ -20,10 +20,12 @@ from generate.math_candidate_parser import (
|
|||
_EARNINGS_RE,
|
||||
_EARNINGS_VERBS,
|
||||
_to_seconds,
|
||||
classify_sentence,
|
||||
extract_capacity_candidates,
|
||||
extract_capacity_question_candidates,
|
||||
extract_earnings_candidates,
|
||||
extract_earnings_question_candidates,
|
||||
has_numeric_token,
|
||||
)
|
||||
|
||||
_REPO = Path(__file__).resolve().parent.parent
|
||||
|
|
@ -230,3 +232,58 @@ def test_gsm8k_post_s1_admission_honest() -> None:
|
|||
)
|
||||
assert len(admitted) >= 1, "gsm8k-0014 should admit"
|
||||
assert "gsm8k-train-sample-v1-0014" in admitted
|
||||
|
||||
|
||||
# ── ADR-0136.S.0 — Context-sentence classifier ───────────────────────
|
||||
|
||||
|
||||
class TestContextClassifier:
|
||||
@pytest.mark.parametrize("sentence", [
|
||||
"Jason has a carriage house that he rents out.",
|
||||
"Xavier plays football with his friends.",
|
||||
"Marnie makes bead bracelets.",
|
||||
"John decides to take up illustration.",
|
||||
"Sandra wants to buy some sweets.",
|
||||
])
|
||||
def test_no_digit_sentences_classified_context(self, sentence: str) -> None:
|
||||
assert not has_numeric_token(sentence)
|
||||
assert classify_sentence(sentence) == "context"
|
||||
|
||||
@pytest.mark.parametrize("sentence", [
|
||||
"Bob can shuck 10 oysters in 5 minutes.",
|
||||
"During 15 minutes Xavier can score 2 goals on average.",
|
||||
"Francine has five full boxes of crayons and 5 loose crayons.",
|
||||
"Tina makes $18.00 an hour.",
|
||||
])
|
||||
def test_numeric_sentences_classified_numeric_state(self, sentence: str) -> None:
|
||||
assert has_numeric_token(sentence)
|
||||
assert classify_sentence(sentence) == "numeric_state"
|
||||
|
||||
def test_gsm8k_0018_context_sentence_skipped_admits(self) -> None:
|
||||
"""Context gate removed: gsm8k-0018 admits with answer 16."""
|
||||
q = (
|
||||
"Xavier plays football with his friends. "
|
||||
"During 15 minutes Xavier can score 2 goals on average. "
|
||||
"How many goals on average is Xavier able to score, "
|
||||
"when the match lasted for 2 hours?"
|
||||
)
|
||||
r = parse_and_solve(q)
|
||||
assert r.answer == 16.0, f"expected 16.0 got {r.answer} ({r.refusal_reason})"
|
||||
|
||||
def test_inverted_capacity_pattern_matches(self) -> None:
|
||||
"""Shape A2: 'During M <time-unit> <Actor> can <verb> N <unit>'."""
|
||||
cands = extract_capacity_candidates(
|
||||
"During 15 minutes Xavier can score 2 goals on average."
|
||||
)
|
||||
assert len(cands) == 1
|
||||
assert cands[0].count == 2.0
|
||||
assert cands[0].per_count == 15.0
|
||||
assert cands[0].per_unit == "minutes"
|
||||
|
||||
def test_all_context_sentences_refused_when_no_numeric_follows(self) -> None:
|
||||
"""A problem with only context sentences and no numeric state refuses."""
|
||||
r = parse_and_solve(
|
||||
"Jason has a carriage house. How many houses does Jason rent?"
|
||||
)
|
||||
assert r.answer is None
|
||||
assert r.refusal_reason is not None
|
||||
|
|
|
|||
Loading…
Reference in a new issue