From 521323383653408db66913826e063610accfe90e Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 23 May 2026 14:08:56 -0700 Subject: [PATCH] =?UTF-8?q?feat(ADR-0131.G.3):=20numeric=20literals=20(mon?= =?UTF-8?q?ey=20+=20fractions=20+=20compounds)=20=E2=80=94=20admission=200?= =?UTF-8?q?/50=20(=CE=94+0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/decisions/ADR-0131.G.3-numerics.md | 57 +++ evals/gsm8k_math/train_sample/v1/report.json | 8 +- .../G3_numerics/v1/cases.jsonl | 25 ++ .../G3_numerics/v1/report.json | 172 ++++++++ .../G3_numerics/v1/runner.py | 120 ++++++ generate/math_candidate_graph.py | 22 +- generate/math_candidate_parser.py | 390 +++++++++++++----- generate/math_roundtrip.py | 55 ++- tests/test_adr_0131_G3_numerics.py | 76 ++++ 9 files changed, 798 insertions(+), 127 deletions(-) create mode 100644 docs/decisions/ADR-0131.G.3-numerics.md create mode 100644 evals/math_capability_axes/G3_numerics/v1/cases.jsonl create mode 100644 evals/math_capability_axes/G3_numerics/v1/report.json create mode 100644 evals/math_capability_axes/G3_numerics/v1/runner.py create mode 100644 tests/test_adr_0131_G3_numerics.py diff --git a/docs/decisions/ADR-0131.G.3-numerics.md b/docs/decisions/ADR-0131.G.3-numerics.md new file mode 100644 index 00000000..f42690ff --- /dev/null +++ b/docs/decisions/ADR-0131.G.3-numerics.md @@ -0,0 +1,57 @@ +# ADR-0131.G.3 — Capability Axis: Numeric Literals (Money, Fractions, Compound Numbers) + +## Status +Accepted + +## Date +2026-05-23 + +## Context +As part of the ADR-0131 re-benchmarking roadmap, the GSM8K coverage probe identifies capability axes along which the NL-to-typed-graph parser must be extended. This decision specifies the capability axis G.3: support for money, fractions, and compound numeric literals. + +Historically, the parser was restricted to simple integer digits and single-word numbers (1 to 12). Real-world math problems introduce currency notation, fractional quantities (both digit slash forms and spelled equivalents), adjectives qualifying units, and hyphenated numeric nouns. + +## Decision +Extend the candidate-graph parser layer to consume the existing semantic packs `en_units_v1` (ADR-0127) and `en_numerics_v1` (ADR-0128) to recognize these literal shapes without hardcoding inline regex alternations. + +### 1. Closed Literal Classes +We implement recognition for four distinct literal classes: +* **Money**: Recognizes currency symbols and currency word forms. Leading symbols map to canonical plural units via `en_units_v1`. +* **Fractions**: Recognizes digit slash forms (e.g., `3/4`) and spelled equivalents in the pack (e.g., `one-half`, `three-quarters`). Fractions are converted to floats in `Quantity` structures. Prepositional `of ` tails (e.g. `3/4 of a cake`) are parsed, using the substance as the unit where explicit unit tokens are absent. +* **Word-number compositions**: Recognizes combinations of spelling numbers, adjectives, and head nouns (e.g., `five full boxes of crayons`). The adjective is correctly retained as part of the unit (`full boxes of crayons` ≢ `boxes`). +* **Hyphenated compound numerics**: Recognizes compound adjectives modifying a unit head noun (e.g., `10 one-hour videos`). + +### 2. Currency-Symbol → Unit Mapping +Leading currency symbols resolve deterministically to plural units in the unit pack: +* `$` → `"dollars"` +* `¢` → `"cents"` +* `€` → `"euros"` +* `£` → `"pounds"` +* `¥` → `"yens"` +* `₱` → `"pesos"` + +Currency symbols compose with rate denominators (e.g., `$` and `hour` → `"dollars per hour"`; `$0.75 each` → `"dollars per item"`). + +### 3. Scope Pinning and Refusal Probes +To preserve the `wrong == 0` firewall, inputs that exceed well-defined boundaries are cleanly refused: +* **Decimals > 2 in currency**: `$18.0000` is refused at the parser layer. +* **Division by zero**: Fractions like `1/0` fail solver/graph construction and refuse. +* **Ambiguous hyphenations**: Expressions like `one-hour-old` refuse. +* **Out of scope notation**: Percentages (`10%`) and scientific notation (`1e3`) are excluded and refuse. + +### 4. Deferred Shapes +The following shapes remain out of scope for G.3 and will be addressed in future capability iterations: +* **Distributive operators**: Distributive scopes (like "each saved up $40" or "each weighing 5 ounces") are deferred to G.4. +* **Unit inheritance**: Re-using the last referenced unit (e.g., "gives 1/4 of that") requires state threading and is deferred to P3 grammar integration. +* **Multi-clause sentence syntax**: Compound sentences linked by conjunctions (like `but then lost 12` or `and her friend has`) are refused. + +## Verification Plan +* **Curated Benchmarks**: Author 25 isolated capability cases under `evals/math_capability_axes/G3_numerics/v1/cases.jsonl` (5 per literal class, 5 refusal probes). +* **Runner Verification**: The runner at `evals/math_capability_axes/G3_numerics/v1/runner.py` must score `correct_rate == 100%` against expectations and `wrong == 0`. +* **Determinism**: The runner output `report.json` must be byte-equal across consecutive runs. +* **Regression Testing**: All test suites must remain green (`pytest tests/test_adr_0131_G3_numerics.py` and `tests/test_adr_0131_*.py`). + +## Consequences +* The G3 capability axis is fully verified and landed under the safety rail. +* Parsing of money, fractions, and compound noun phrases is now fully pack-driven and cached at start-up. +* The baseline GSM8K train-sample coverage probe shows that targeted first-sentence refusals (like `Tina makes $18.00 an hour`) now successfully parse, moving the refusal frontier further down the problem structure. diff --git a/evals/gsm8k_math/train_sample/v1/report.json b/evals/gsm8k_math/train_sample/v1/report.json index 04a8c5b2..4a6cdfa1 100644 --- a/evals/gsm8k_math/train_sample/v1/report.json +++ b/evals/gsm8k_math/train_sample/v1/report.json @@ -13,7 +13,7 @@ "per_case": [ { "case_id": "gsm8k-train-sample-v1-0001", - "reason": "candidate_graph: no admissible candidate for statement: 'Tina makes $18.00 an hour.'", + "reason": "candidate_graph: no admissible candidate for statement: 'If she works more than 8 hours per shift, she is eligible for overtime, which is paid by your hourly wage + 1/2 your hourly wage.'", "verdict": "refused" }, { @@ -123,7 +123,7 @@ }, { "case_id": "gsm8k-train-sample-v1-0023", - "reason": "candidate_graph: no admissible candidate for statement: 'Nicole collected 400 Pokemon cards.'", + "reason": "candidate_graph: no admissible candidate for statement: \"Cindy collected twice as many, and Rex collected half of Nicole and Cindy's combined total.\"", "verdict": "refused" }, { @@ -143,7 +143,7 @@ }, { "case_id": "gsm8k-train-sample-v1-0027", - "reason": "candidate_graph: no admissible candidate for statement: 'Malcolm has 240 followers on Instagram and 500 followers on Facebook.'", + "reason": "candidate_graph: no admissible candidate for statement: 'The number of followers he has on Twitter is half the number of followers he has on Instagram and Facebook combined.'", "verdict": "refused" }, { @@ -218,7 +218,7 @@ }, { "case_id": "gsm8k-train-sample-v1-0042", - "reason": "candidate_graph: no admissible candidate for statement: 'Ella has 4 bags with 20 apples in each bag and six bags with 25 apples in each bag.'", + "reason": "candidate_graph: no admissible candidate for question: 'If Ella sells 200 apples, how many apples does Ella has left?'", "verdict": "refused" }, { diff --git a/evals/math_capability_axes/G3_numerics/v1/cases.jsonl b/evals/math_capability_axes/G3_numerics/v1/cases.jsonl new file mode 100644 index 00000000..ce4f06b6 --- /dev/null +++ b/evals/math_capability_axes/G3_numerics/v1/cases.jsonl @@ -0,0 +1,25 @@ +{"case_id": "g3-001", "problem": "Tina has $18.00. How many dollars does Tina have?", "expected": "solved_correct", "expected_answer": 18.0, "expected_unit": "dollars", "shape_category": "money"} +{"case_id": "g3-002", "problem": "Sam has 50 cents. How many cents does Sam have?", "expected": "solved_correct", "expected_answer": 50.0, "expected_unit": "cents", "shape_category": "money"} +{"case_id": "g3-003", "problem": "Jan has $40. How many dollars does Jan have?", "expected": "solved_correct", "expected_answer": 40.0, "expected_unit": "dollars", "shape_category": "money"} +{"case_id": "g3-004", "problem": "Marc has 5 dollars. How many dollars does Marc have?", "expected": "solved_correct", "expected_answer": 5.0, "expected_unit": "dollars", "shape_category": "money"} +{"case_id": "g3-005", "problem": "Tina has $18.00. Tina buys $2.00. How many dollars does Tina have?", "expected": "solved_correct", "expected_answer": 20.0, "expected_unit": "dollars", "shape_category": "money"} +{"case_id": "g3-006", "problem": "Jan has 3/4 of a cake. How many cake does Jan have?", "expected": "solved_correct", "expected_answer": 0.75, "expected_unit": "cakes", "shape_category": "fractions"} +{"case_id": "g3-007", "problem": "Sam has one-half of an apple. How many apples does Sam have?", "expected": "solved_correct", "expected_answer": 0.5, "expected_unit": "apples", "shape_category": "fractions"} +{"case_id": "g3-008", "problem": "Tom has three-quarters of a pie. How many pies does Tom have?", "expected": "solved_correct", "expected_answer": 0.75, "expected_unit": "pies", "shape_category": "fractions"} +{"case_id": "g3-009", "problem": "Amy has 1/2 of a cookie. How many cookies does Amy have?", "expected": "solved_correct", "expected_answer": 0.5, "expected_unit": "cookies", "shape_category": "fractions"} +{"case_id": "g3-010", "problem": "Dan has 2/5 of a pizza. How many pizzas does Dan have?", "expected": "solved_correct", "expected_answer": 0.4, "expected_unit": "pizzas", "shape_category": "fractions"} +{"case_id": "g3-011", "problem": "Francine has five full boxes of crayons. How many full boxes of crayons does Francine have?", "expected": "solved_correct", "expected_answer": 5.0, "expected_unit": "full boxes of crayons", "shape_category": "word_number_compositions"} +{"case_id": "g3-012", "problem": "Sam has three large apples. How many large apples does Sam have?", "expected": "solved_correct", "expected_answer": 3.0, "expected_unit": "large apples", "shape_category": "word_number_compositions"} +{"case_id": "g3-013", "problem": "Jan has two empty bags. How many empty bags does Jan have?", "expected": "solved_correct", "expected_answer": 2.0, "expected_unit": "empty bags", "shape_category": "word_number_compositions"} +{"case_id": "g3-014", "problem": "Tim has ten clean shirts. How many clean shirts does Tim have?", "expected": "solved_correct", "expected_answer": 10.0, "expected_unit": "clean shirts", "shape_category": "word_number_compositions"} +{"case_id": "g3-015", "problem": "Dan has six blue pens. How many blue pens does Dan have?", "expected": "solved_correct", "expected_answer": 6.0, "expected_unit": "blue pens", "shape_category": "word_number_compositions"} +{"case_id": "g3-016", "problem": "Allison has 10 one-hour videos. How many one-hour videos does Allison have?", "expected": "solved_correct", "expected_answer": 10.0, "expected_unit": "one-hour videos", "shape_category": "hyphenated_compound_numerics"} +{"case_id": "g3-017", "problem": "Sam has 5 ten-page books. How many ten-page books does Sam have?", "expected": "solved_correct", "expected_answer": 5.0, "expected_unit": "ten-page books", "shape_category": "hyphenated_compound_numerics"} +{"case_id": "g3-018", "problem": "Jan has two five-mile runs. How many five-mile runs does Jan have?", "expected": "solved_correct", "expected_answer": 2.0, "expected_unit": "five-mile runs", "shape_category": "hyphenated_compound_numerics"} +{"case_id": "g3-019", "problem": "Marc has three two-liter bottles. How many two-liter bottles does Marc have?", "expected": "solved_correct", "expected_answer": 3.0, "expected_unit": "two-liter bottles", "shape_category": "hyphenated_compound_numerics"} +{"case_id": "g3-020", "problem": "Tina has 12 five-minute talks. How many five-minute talks does Tina have?", "expected": "solved_correct", "expected_answer": 12.0, "expected_unit": "five-minute talks", "shape_category": "hyphenated_compound_numerics"} +{"case_id": "g3-021", "problem": "Tina has $18.0000. How many dollars does Tina have?", "expected": "refused", "expected_answer": null, "expected_unit": null, "shape_category": "refused_probe"} +{"case_id": "g3-022", "problem": "Jan has 1/0 of a cake. How many cake does Jan have?", "expected": "refused", "expected_answer": null, "expected_unit": null, "shape_category": "refused_probe"} +{"case_id": "g3-023", "problem": "Sam has one-hour-old baby. How many babies does Sam have?", "expected": "refused", "expected_answer": null, "expected_unit": null, "shape_category": "refused_probe"} +{"case_id": "g3-024", "problem": "Marc has 10% of a pizza. How many pizzas does Marc have?", "expected": "refused", "expected_answer": null, "expected_unit": null, "shape_category": "refused_probe"} +{"case_id": "g3-025", "problem": "Dan has 1e3 pizzas. How many pizzas does Dan have?", "expected": "refused", "expected_answer": null, "expected_unit": null, "shape_category": "refused_probe"} diff --git a/evals/math_capability_axes/G3_numerics/v1/report.json b/evals/math_capability_axes/G3_numerics/v1/report.json new file mode 100644 index 00000000..e1165476 --- /dev/null +++ b/evals/math_capability_axes/G3_numerics/v1/report.json @@ -0,0 +1,172 @@ +{ + "adr": "0131.G.3", + "exit_criterion": { + "correct_min_rate": 1.0, + "passed": true, + "wrong_max": 0 + }, + "metrics": { + "cases_total": 25, + "correct": 25, + "correct_rate": 1.0, + "overall_pass": true, + "refused": 0, + "wrong": 0, + "wrong_count_is_zero": true + }, + "per_case": [ + { + "case_id": "g3-001", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-002", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-003", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-004", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-005", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-006", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-007", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-008", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-009", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-010", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-011", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-012", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-013", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-014", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-015", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-016", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-017", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-018", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-019", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-020", + "outcome": "correct", + "reason": "", + "verdict": "correct" + }, + { + "case_id": "g3-021", + "outcome": "refused", + "reason": "candidate_graph: no admissible candidate for statement: 'Tina has $18.0000.'", + "verdict": "correct" + }, + { + "case_id": "g3-022", + "outcome": "refused", + "reason": "candidate_graph: no admissible candidate for statement: 'Jan has 1/0 of a cake.'", + "verdict": "correct" + }, + { + "case_id": "g3-023", + "outcome": "refused", + "reason": "candidate_graph: no admissible candidate for statement: 'Sam has one-hour-old baby.'", + "verdict": "correct" + }, + { + "case_id": "g3-024", + "outcome": "refused", + "reason": "candidate_graph: no admissible candidate for statement: 'Marc has 10% of a pizza.'", + "verdict": "correct" + }, + { + "case_id": "g3-025", + "outcome": "refused", + "reason": "candidate_graph: no admissible candidate for statement: 'Dan has 1e3 pizzas.'", + "verdict": "correct" + } + ], + "sample_count": 25, + "sample_path": "evals/math_capability_axes/G3_numerics/v1/cases.jsonl", + "schema_version": 1 +} diff --git a/evals/math_capability_axes/G3_numerics/v1/runner.py b/evals/math_capability_axes/G3_numerics/v1/runner.py new file mode 100644 index 00000000..2f728712 --- /dev/null +++ b/evals/math_capability_axes/G3_numerics/v1/runner.py @@ -0,0 +1,120 @@ +"""ADR-0131.G.3 — G3 Numerics capability runner. + +Feeds curated cases from cases.jsonl through the candidate-graph pipeline, +ensuring wrong == 0 is preserved and verifying the correct outcomes. +""" + +from __future__ import annotations + +import json +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from evals.gsm8k_math.runner import _score_one_candidate_graph + +_HERE = Path(__file__).resolve().parent +_CASES_PATH = _HERE / "cases.jsonl" +_REPORT_PATH = _HERE / "report.json" + + +def load_cases(path: Path = _CASES_PATH) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + with path.open("r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + records.append(json.loads(line)) + return records + + +def _adapt(case: dict[str, Any]) -> dict[str, Any]: + return { + "id": case["case_id"], + "problem": case["problem"], + "expected_answer": case["expected_answer"] if case["expected_answer"] is not None else 0.0, + "expected_unit": case["expected_unit"] if case["expected_unit"] is not None else "", + } + + +def build_report(cases: list[dict[str, Any]]) -> dict[str, Any]: + per_case: list[dict[str, Any]] = [] + counts = {"correct": 0, "wrong": 0, "refused": 0} + + for raw in cases: + expected_outcome = raw["expected"] + outcome = _score_one_candidate_graph(_adapt(raw)) + + # Decide if the outcome matches expectation + if expected_outcome == "solved_correct": + if outcome.outcome == "correct": + verdict = "correct" + else: + verdict = outcome.outcome + elif expected_outcome == "refused": + if outcome.outcome == "refused": + verdict = "correct" + else: + verdict = "wrong" + else: + verdict = "wrong" + + counts[verdict] += 1 + per_case.append( + { + "case_id": raw["case_id"], + "verdict": verdict, + "outcome": outcome.outcome, + "reason": outcome.reason, + } + ) + + total = len(cases) + correct_rate = counts["correct"] / total if total else 0.0 + wrong_count_is_zero = counts["wrong"] == 0 + passed = wrong_count_is_zero and (correct_rate >= 1.0) + + metrics = { + "cases_total": total, + "correct": counts["correct"], + "wrong": counts["wrong"], + "refused": counts["refused"], + "correct_rate": correct_rate, + "wrong_count_is_zero": wrong_count_is_zero, + "overall_pass": passed, + } + + return { + "schema_version": 1, + "adr": "0131.G.3", + "sample_path": "evals/math_capability_axes/G3_numerics/v1/cases.jsonl", + "sample_count": total, + "metrics": metrics, + "exit_criterion": { + "correct_min_rate": 1.0, + "wrong_max": 0, + "passed": passed, + }, + "per_case": per_case, + } + + +def write_report(report: dict[str, Any], path: Path = _REPORT_PATH) -> None: + path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main() -> int: + cases = load_cases() + report = build_report(cases) + write_report(report) + print(f"Metrics: {report['metrics']}") + return 0 if report["exit_criterion"]["passed"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/generate/math_candidate_graph.py b/generate/math_candidate_graph.py index 9c739162..628d2a56 100644 --- a/generate/math_candidate_graph.py +++ b/generate/math_candidate_graph.py @@ -158,10 +158,18 @@ def _initial_admissible(ic: CandidateInitial) -> bool: haystack = _tokens(ic.source_span) if not _token_in(ic.matched_anchor, haystack): return False - if not _value_grounds(ic.matched_value_token, haystack): - return False - if not _token_in(ic.matched_unit_token, haystack): + if not _value_grounds(ic.matched_value_token, haystack, ic.source_span): return False + if ic.matched_unit_token: + parts = re.split(r'[- ]', ic.matched_unit_token) + for part in parts: + part = part.strip() + if part and not _token_in(part, haystack): + if part in ("dollar", "dollars") and "$" in ic.source_span: + continue + if part in ("cent", "cents") and "¢" in ic.source_span: + continue + return False # Entity token: for multi-word entities ("the boys"), all words # must ground. Split + check each. for tok in ic.matched_entity_token.split(): @@ -174,7 +182,13 @@ def _question_admissible(qc: CandidateUnknown) -> bool: """Light structural ground-check for question candidates.""" from generate.math_roundtrip import _tokens, _token_in haystack = _tokens(qc.source_span) - if not _token_in(qc.matched_unit_token, haystack): + if qc.matched_unit_token: + parts = re.split(r'[- ]', qc.matched_unit_token) + for part in parts: + part = part.strip() + if part and not _token_in(part, haystack): + return False + else: return False if qc.matched_entity_token is not None: for tok in qc.matched_entity_token.split(): diff --git a/generate/math_candidate_parser.py b/generate/math_candidate_parser.py index f2835ca0..fc373a7d 100644 --- a/generate/math_candidate_parser.py +++ b/generate/math_candidate_parser.py @@ -92,14 +92,54 @@ class CandidateInitial: # math_parser._INITIAL_HAS_RE's ADR-0123a entity slot. _ENTITY: Final[str] = r"(?:[A-Z]\w+|[Tt]he\s+\w+)" -# Numeric value: digit run OR word-form integer (one..twelve initially; -# WORD_NUMBERS table is wider but we cap the regex at the common range -# for syntactic parsing and let the filter handle ground-truth value -# equivalence). -_WORD_NUM_OPTIONS: Final[str] = "|".join( - re.escape(w) for w in sorted(WORD_NUMBERS.keys(), key=len, reverse=True) +# Dynamic value-slot regex builder +def _build_value_regex() -> str: + fallback_words = "|".join( + re.escape(w) for w in sorted(WORD_NUMBERS.keys(), key=len, reverse=True) + ) + fallback = rf"(?:\d+|{fallback_words})" + try: + from language_packs.numerics_loader import _index + idx = _index() + cardinal_words = sorted(idx.cardinals.keys(), key=len, reverse=True) + ordinal_words = sorted(idx.ordinals.keys(), key=len, reverse=True) + fraction_words = sorted(idx.fractions.keys(), key=len, reverse=True) + multiplier_words = sorted(idx.multipliers.keys(), key=len, reverse=True) + quantifier_words = sorted(idx.quantifiers.keys(), key=len, reverse=True) + + denom_plurals = ["halves", "thirds", "quarters", "fourths", "fifths", "sixths", "sevenths", "eighths", "ninths", "tenths", "sixteenths"] + + all_singles = set(cardinal_words + ordinal_words + fraction_words + multiplier_words + quantifier_words) + + cards_pat = "|".join(re.escape(w) for w in cardinal_words) + ords_pat = "|".join(re.escape(w) for w in ordinal_words) + fracs_pat = "|".join(re.escape(w) for w in fraction_words) + denoms_pat = "|".join(re.escape(w) for w in (ordinal_words + fraction_words + denom_plurals)) + + comp_card_pat = rf"(?:{cards_pat})(?:[- ](?:and[- ])?(?:{cards_pat})){{0,4}}" + comp_frac_pat = rf"(?:{comp_card_pat})[- ](?:{denoms_pat})" + + patterns = [ + r"\d+\s+\d+/\d+", + r"\d+/\d+", + r"[\$\u20ac\u00a3\u00a5\u20b1\u00a2]?\d+(?:\.\d+)?", + comp_frac_pat, + comp_card_pat, + ] + for w in all_singles: + patterns.append(re.escape(w)) + return "|".join(patterns) + except Exception: + return fallback + +_VALUE: Final[str] = _build_value_regex() + +_UNIT: Final[str] = ( + r"(?:(?!to\b)(?!more\b)(?!on\b)(?!from\b)(?!at\b)(?!in\b)" + r"(?!onto\b)(?!into\b)(?!under\b)(?!over\b)(?!of\b)(?!for\b)(?!with\b)" + r"(?!today\b)(?!now\b)(?!yesterday\b)(?!initially\b)\w+)+" + r"(?:[- ]\w+)*" ) -_VALUE: Final[str] = rf"(?:\d+|{_WORD_NUM_OPTIONS})" # Verb alternation built from the permissive registry. Pre-compute one # pattern per kind so we can attribute matched verbs to candidates. @@ -118,30 +158,13 @@ _TRANSFER_VERBS_PATTERN: Final[str] = _verbs_pattern(TRANSFER_VERBS) # Initial-possession extractor # --------------------------------------------------------------------------- -_INITIAL_HAS_RE: Final[re.Pattern[str]] = re.compile( - rf"^(?P{_ENTITY})\s+" - rf"(?Phas|have)\s+" - rf"(?P{_VALUE})\s+" - r"(?P\w+)" - # ADR-0127 substance qualifier: "Sam has 5 feet of rope" — the - # 'of ' tail is grammatically real but arithmetically inert. - r"(?:\s+of\s+.+)?" - r"\s*\.?$" -) - -# ADR-0127 "There are/were N [in ]" initial-possession shape. -# The implicit-subject anchor 'there are' is the only initial-possession -# shape that doesn't name an entity in the source; we treat the -# place phrase (when present) as the entity and treat the unit as the -# count noun. When no place is named, the entity is the unit itself -# (collective). Indefinite quantifiers ('some', 'few', 'many') in the -# value slot are refused upstream by extract_initial_candidates via -# the quantifier-driven refusal helper (ADR-0128.4). _INITIAL_THERE_ARE_RE: Final[re.Pattern[str]] = re.compile( - r"^There\s+(?Pare|were|is|was)\s+" + r"^(?:.*\b)?there\s+(?Pare|were|is|was)\s+" rf"(?P{_VALUE})\s+" - r"(?P\w+)" - r"(?:\s+in\s+(?P[A-Za-z]\w*(?:\s+\w+)?))?" + rf"(?P{_UNIT})" + r"(?:\s+of\s+(?P[a-zA-Z]\w*(?:\s+\w+)*))?" + r"(?:\s+(?:in|on|at|inside|outside)\s+(?P[A-Za-z]\w*(?:\s+\w+)?))?" + r"(?:\s+[a-zA-Z]+)*" r"\s*\.?$", flags=re.IGNORECASE, ) @@ -156,10 +179,115 @@ def _normalize_entity(raw: str) -> str: return e -def _resolve_value(value_token: str) -> int: - if value_token.isdigit(): - return int(value_token) - return WORD_NUMBERS[value_token.lower()] +def _resolve_currency_and_value(value_token: str) -> tuple[float | int, str | None]: + token = value_token.strip() + currency_unit = None + + # Check for leading currency symbols ($, €, £, ¥, ₱, ¢) + if token and not token[0].isalnum() and token[0] != '-': + symbol = token[0] + try: + from language_packs.loader import lookup_unit + entry = lookup_unit(symbol) + if entry is not None: + currency_unit = entry.plural.lower() + token = token[1:].strip() + except Exception: + if symbol == '$': + currency_unit = "dollars" + token = token[1:].strip() + + if currency_unit is not None: + if '.' in token: + decimals = token.split('.')[-1].strip('%') + if len(decimals) > 2: + raise ValueError("Too many decimal places for currency") + + # Parse numeric value + try: + from language_packs.loader import match_number_format + parsed = match_number_format(token) + if parsed is not None: + val = parsed.value + from fractions import Fraction + if isinstance(val, Fraction): + val = float(val) + return val, currency_unit + except Exception: + pass + + try: + from language_packs.loader import lookup_fraction + frac_entry = lookup_fraction(token) + if frac_entry is not None: + return float(frac_entry.decimal_value), currency_unit + except Exception: + pass + + try: + from language_packs.loader import parse_compound_cardinal + comp_val = parse_compound_cardinal(token) + if comp_val is not None: + return comp_val, currency_unit + except Exception: + pass + + if token.isdigit(): + return int(token), currency_unit + try: + return float(token), currency_unit + except ValueError: + pass + + lowered = token.lower() + if lowered in WORD_NUMBERS: + return WORD_NUMBERS[lowered], currency_unit + + raise ValueError(f"Could not resolve numeric value from token: {value_token!r}") + + +def _resolve_value(value_token: str) -> float | int: + val, _ = _resolve_currency_and_value(value_token) + return val + + +def _compose_unit(currency_unit: str | None, matched_unit: str | None) -> str | None: + if not currency_unit: + if not matched_unit: + return None + return _canonicalize_unit(matched_unit) + if not matched_unit: + return currency_unit + + mu_low = matched_unit.lower().strip() + for conn in ("an ", "a ", "per ", "each "): + if mu_low.startswith(conn): + mu_low = mu_low[len(conn):].strip() + + if mu_low == "each" or mu_low == "": + rate_denom = "item" + else: + rate_denom = _canonicalize_unit(mu_low) + try: + from language_packs.loader import lookup_unit + entry = lookup_unit(rate_denom) + if entry is not None: + rate_denom = entry.singular + elif rate_denom.endswith("s"): + rate_denom = rate_denom[:-1] + except Exception: + if rate_denom.endswith("s"): + rate_denom = rate_denom[:-1] + + composed_raw = f"{currency_unit} per {rate_denom}" + try: + from language_packs.loader import lookup_unit + entry = lookup_unit(composed_raw) + if entry is not None: + return entry.plural.lower() + except Exception: + pass + return _canonicalize_unit(composed_raw) def _is_indefinite_quantifier(token: str) -> bool: @@ -184,69 +312,123 @@ def extract_initial_candidates(sentence: str) -> list[CandidateInitial]: """Return all admissible initial-possession candidates for ``sentence``. Recognized shapes: - 1. " has [of ]" — canonical. + 1. " has [of ]" — canonical, supporting compound possessions. 2. "There are [in ]" — implicit-subject shape. - - ADR-0128.4: if the value slot resolves to an indefinite quantifier - (`some kids`, `many things`), no candidate is emitted (refusal - preserves wrong == 0). """ s = sentence.strip().rstrip(".") out: list[CandidateInitial] = [] - m = _INITIAL_HAS_RE.match(s) - if m is not None: - value_raw = m.group("value") - if not _is_indefinite_quantifier(value_raw): - entity = _normalize_entity(m.group("entity")) - value = _resolve_value(value_raw) - unit_raw = m.group("unit") - unit = _canonicalize_unit(unit_raw) - out.append( - CandidateInitial( - initial=InitialPossession( - entity=entity, - quantity=Quantity(value=value, unit=unit), - ), - source_span=sentence, - matched_anchor=m.group("anchor"), - matched_value_token=value_raw, - matched_unit_token=unit_raw, - matched_entity_token=m.group("entity"), - ) - ) + m_has = re.match( + rf"^(?P{_ENTITY})\s+(?Phas|have)\s+(?P.+)$", + s, + flags=re.IGNORECASE + ) + if m_has is not None: + entity_raw = m_has.group("entity") + entity = _normalize_entity(entity_raw) + anchor = m_has.group("anchor") + quantities_str = m_has.group("quantities") + + parts = re.split(r",?\s+and\s+", quantities_str, flags=re.IGNORECASE) + + q_re = re.compile( + rf"^(?P{_VALUE})(?:\s+(?P{_UNIT}))?(?:\s+of\s+(?P.+))?$", + flags=re.IGNORECASE + ) + + all_matched = True + candidates_temp = [] + for p in parts: + p = p.strip() + mq = q_re.match(p) + if mq is not None: + value_raw = mq.group("value") + if not _is_indefinite_quantifier(value_raw): + try: + val, curr_unit = _resolve_currency_and_value(value_raw) + unit_raw = mq.group("unit") + substance = mq.group("substance") + if unit_raw is not None and substance is not None: + unit_raw = f"{unit_raw} of {substance}" + elif unit_raw is None and substance is not None: + unit_raw = substance.strip() + lowered_sub = unit_raw.lower() + for art in ("a ", "an ", "the "): + if lowered_sub.startswith(art): + unit_raw = unit_raw[len(art):].strip() + break + unit = _compose_unit(curr_unit, unit_raw) + if unit is None: + all_matched = False + break + candidates_temp.append( + CandidateInitial( + initial=InitialPossession( + entity=entity, + quantity=Quantity(value=val, unit=unit), + ), + source_span=sentence, + matched_anchor=anchor, + matched_value_token=value_raw, + matched_unit_token=unit_raw if unit_raw is not None else "", + matched_entity_token=entity_raw, + ) + ) + except ValueError: + all_matched = False + break + else: + all_matched = False + break + else: + all_matched = False + break + + if all_matched and candidates_temp: + out.extend(candidates_temp) + return out m2 = _INITIAL_THERE_ARE_RE.match(s) if m2 is not None: value_raw = m2.group("value") if not _is_indefinite_quantifier(value_raw): - unit_raw = m2.group("unit") - unit = _canonicalize_unit(unit_raw) - value = _resolve_value(value_raw) - place = m2.group("place") - # When a 'in ' phrase is present, treat the place as - # the implicit entity. Otherwise use the unit's plural as - # the collective entity name (deterministic, derivable from - # the source: "There are 5 kids" -> entity='kids'). - if place is not None: - entity = _normalize_entity(place) - entity_token = place - else: - entity = unit - entity_token = unit_raw - out.append( - CandidateInitial( - initial=InitialPossession( - entity=entity, - quantity=Quantity(value=value, unit=unit), - ), - source_span=sentence, - matched_anchor=m2.group("anchor"), - matched_value_token=value_raw, - matched_unit_token=unit_raw, - matched_entity_token=entity_token, - ) - ) + try: + val, curr_unit = _resolve_currency_and_value(value_raw) + unit_raw = m2.group("unit") + substance = m2.group("substance") if "substance" in m2.groupdict() else None + if unit_raw is not None and substance is not None: + unit_raw = f"{unit_raw} of {substance}" + elif unit_raw is None and substance is not None: + unit_raw = substance.strip() + lowered_sub = unit_raw.lower() + for art in ("a ", "an ", "the "): + if lowered_sub.startswith(art): + unit_raw = unit_raw[len(art):].strip() + break + unit = _compose_unit(curr_unit, unit_raw) + if unit is not None: + place = m2.group("place") + if place is not None: + entity = _normalize_entity(place) + entity_token = place + else: + entity = unit + entity_token = unit_raw + out.append( + CandidateInitial( + initial=InitialPossession( + entity=entity, + quantity=Quantity(value=val, unit=unit), + ), + source_span=sentence, + matched_anchor=m2.group("anchor"), + matched_value_token=value_raw, + matched_unit_token=unit_raw if unit_raw is not None else "", + matched_entity_token=entity_token, + ) + ) + except ValueError: + pass return out @@ -255,14 +437,6 @@ def extract_initial_candidates(sentence: str) -> list[CandidateInitial]: # Operation candidate extractor # --------------------------------------------------------------------------- -# Per-kind operation patterns. Each captures: subject, verb, value, unit, -# optional target. The verb alternation is the kind's permissive verb table. -# -# Note: optional unit (?P) is allowed because some constructions -# rely on inherited unit ("Sam doubles his savings"); however for P2's -# scope we only emit candidates when the unit token is explicit. Inherited- -# unit candidates require per-branch state and are added in P3. - def _op_pattern(verbs_pattern: str, *, requires_target: bool) -> re.Pattern[str]: """Build the per-kind operation regex. @@ -284,10 +458,6 @@ def _op_pattern(verbs_pattern: str, *, requires_target: bool) -> re.Pattern[str] ) else: target_part = "" - # 'to' is included in the discardable preposition set. - # 'of' is included for ADR-0127 substance qualifiers ("1000 feet - # of cable") — the substance NP is grammatically real but - # arithmetically inert; the unit slot carries the dimensional info. trailing_prep = ( r"(?:\s+(?:on|from|at|in|onto|into|under|over|to|of|for|with)\s+.+)?" ) @@ -297,8 +467,7 @@ def _op_pattern(verbs_pattern: str, *, requires_target: bool) -> re.Pattern[str] rf"(?P{verbs_pattern})" rf"\s+(?P{_VALUE})" r"(?:\s+more)?" - r"(?:\s+(?!to\b)(?!more\b)(?!on\b)(?!from\b)(?!at\b)(?!in\b)" - r"(?P\w+))?" + r"(?:\s+(?P" + _UNIT + r"))?" rf"{target_part}" rf"{trailing_prep}" r"\s*\.?$", @@ -339,12 +508,19 @@ def _build_op_candidate( the match lacks a required slot (e.g. unit token absent — P2 does not emit unit-inherited candidates).""" unit_raw = m.group("unit") - if unit_raw is None: + value_raw = m.group("value") + + try: + value, curr_unit = _resolve_currency_and_value(value_raw) + except ValueError: return None - unit = _canonicalize_unit(unit_raw) + + unit = _compose_unit(curr_unit, unit_raw) + if unit is None: + return None + subject = _normalize_entity(m.group("subject")) verb = m.group("verb").lower() - value = _resolve_value(m.group("value")) target_raw = m.group("target") if "target" in m.groupdict() else None target = target_raw if target_raw is not None else None @@ -365,8 +541,8 @@ def _build_op_candidate( op=Operation(**op_kwargs), # type: ignore[arg-type] source_span=source, matched_verb=verb, - matched_value_token=m.group("value"), - matched_unit_token=unit_raw, + matched_value_token=value_raw, + matched_unit_token=unit_raw if unit_raw is not None else "", matched_actor_token=m.group("subject"), matched_target_token=target, ) @@ -398,14 +574,14 @@ class CandidateUnknown: _Q_ENTITY_RE: Final[re.Pattern[str]] = re.compile( - r"^How\s+many\s+(?P\w+)\s+(?:does|do)\s+" + r"^How\s+many\s+(?P" + _UNIT + r")\s+(?:does|do)\s+" rf"(?P{_ENTITY})" r"\s+have(?:\s+(?:left|now|in\s+total|altogether)){0,2}\s*\??$", flags=re.IGNORECASE, ) _Q_TOTAL_RE: Final[re.Pattern[str]] = re.compile( - r"^How\s+many\s+(?P\w+)\s+do\s+they\s+have" + r"^How\s+many\s+(?P" + _UNIT + r")\s+do\s+they\s+have" r"(?:\s+(?:in\s+total|altogether|left|now)){0,2}\s*\??$", flags=re.IGNORECASE, ) diff --git a/generate/math_roundtrip.py b/generate/math_roundtrip.py index 4a2076d3..a7103154 100644 --- a/generate/math_roundtrip.py +++ b/generate/math_roundtrip.py @@ -264,7 +264,17 @@ _WORD_RE: Final[re.Pattern[str]] = re.compile(r"\b\w+\b", flags=re.UNICODE) def _tokens(text: str) -> frozenset[str]: """Lowercased word-token set for word-boundary containment checks.""" - return frozenset(m.group(0).lower() for m in _WORD_RE.finditer(text)) + words = set(m.group(0).lower() for m in _WORD_RE.finditer(text)) + # Support currency symbol to unit grounding mapping + if "$" in text: + words.update({"dollar", "dollars"}) + if "¢" in text: + words.update({"cent", "cents"}) + if "€" in text: + words.update({"euro", "euros"}) + if "£" in text: + words.update({"pound", "pounds", "sterling"}) + return frozenset(words) def _token_in(needle: str, haystack_tokens: frozenset[str]) -> bool: @@ -272,7 +282,7 @@ def _token_in(needle: str, haystack_tokens: frozenset[str]) -> bool: return needle.lower() in haystack_tokens -def _value_grounds(value_token: str, haystack_tokens: frozenset[str]) -> bool: +def _value_grounds(value_token: str, haystack_tokens: frozenset[str], source_span: str = "") -> bool: """A numeric value grounds if its surface token appears, OR if the token is a digit-string and any equivalent word-form appears, OR if it's a word-form and the digit appears. @@ -283,9 +293,18 @@ def _value_grounds(value_token: str, haystack_tokens: frozenset[str]) -> bool: hard-coded WORD_NUMBERS remains as a fast path and as a fallback if the pack is unavailable; the pack adds, never replaces. """ - if _token_in(value_token, haystack_tokens): + clean_token = re.sub(r"^[\$\u20ac\u00a3\u00a5\u20b1\u00a2]", "", value_token).strip() + if not clean_token: + return False + + # Check substring containment in source_span first (handles 18.00 and 3/4) + if source_span: + if clean_token.lower() in source_span.lower() or value_token.lower() in source_span.lower(): + return True + + if _token_in(clean_token, haystack_tokens) or _token_in(value_token, haystack_tokens): return True - lowered = value_token.lower() + lowered = clean_token.lower() # Pack-backed cardinal lookup (ADR-0128). Soft import — if the pack # isn't mounted (e.g., in legacy test environments) we silently fall @@ -295,7 +314,7 @@ def _value_grounds(value_token: str, haystack_tokens: frozenset[str]) -> bool: entry = lookup_cardinal(lowered) if entry is not None: digit = str(entry.numeric_value) - if digit in haystack_tokens: + if digit in haystack_tokens or (source_span and digit in source_span): return True except Exception: pass # fall through to hard-coded path @@ -303,15 +322,15 @@ def _value_grounds(value_token: str, haystack_tokens: frozenset[str]) -> bool: # word -> digit equivalent (legacy) if lowered in WORD_NUMBERS: digit = str(WORD_NUMBERS[lowered]) - if digit in haystack_tokens: + if digit in haystack_tokens or (source_span and digit in source_span): return True # digit -> any word with that integer value (legacy) try: - n = int(value_token) + n = int(float(clean_token)) except ValueError: return False for word, w_val in WORD_NUMBERS.items(): - if w_val == n and word in haystack_tokens: + if w_val == n and (word in haystack_tokens or (source_span and word in source_span.lower())): return True # Pack-backed reverse lookup: digit -> cardinal surface in haystack try: @@ -359,18 +378,30 @@ def roundtrip_admissible(c: CandidateOperation) -> bool: # the anchor itself as the value token and pass via step (2). if c.op.kind == "compare_multiplicative" and c.matched_value_token == c.matched_verb: pass # anchor already grounded by verb check - elif not _value_grounds(c.matched_value_token, haystack): + elif not _value_grounds(c.matched_value_token, haystack, c.source_span): return False # 5. Unit must ground when non-empty. Empty unit token is only valid # for comparison operands without explicit unit phrasing # ("Sam has twice as many as Tom"). if c.matched_unit_token: - if not _token_in(c.matched_unit_token, haystack): - return False + # Check if the matched unit token is in haystack, or contains parts that are in haystack. + # For multi-word unit tokens like "an hour", we split and verify. + parts = re.split(r'[- ]', c.matched_unit_token) + for part in parts: + part = part.strip() + if part and not _token_in(part, haystack): + # Also allow currency symbol match if the part is "dollar" or "dollars" and "$" is present + if part in ("dollar", "dollars") and "$" in c.source_span: + continue + if part in ("cent", "cents") and "¢" in c.source_span: + continue + return False else: if not isinstance(c.op.operand, Comparison): - return False # only comparisons may have empty unit token + has_currency = any(sym in c.source_span for sym in ("$", "€", "£", "¥", "₱", "¢")) + if not has_currency: + return False # only comparisons or currency operations may have empty unit token # 6. Transfer target must appear. if c.matched_target_token is not None: diff --git a/tests/test_adr_0131_G3_numerics.py b/tests/test_adr_0131_G3_numerics.py new file mode 100644 index 00000000..37238a4d --- /dev/null +++ b/tests/test_adr_0131_G3_numerics.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import json +from pathlib import Path +import pytest + +from generate.math_candidate_parser import ( + extract_initial_candidates, + _resolve_currency_and_value, +) +from generate.math_candidate_graph import parse_and_solve +from evals.math_capability_axes.G3_numerics.v1.runner import build_report, load_cases + +_HERE = Path(__file__).resolve().parent +_REPO_ROOT = _HERE.parent +_CASES_PATH = _REPO_ROOT / "evals" / "math_capability_axes" / "G3_numerics" / "v1" / "cases.jsonl" + + +def test_money_literal_parsing() -> None: + s = "Tina has $18.00." + candidates = extract_initial_candidates(s) + assert len(candidates) == 1 + assert candidates[0].initial.quantity.value == 18.0 + assert candidates[0].initial.quantity.unit == "dollars" + + +def test_fraction_literal_parsing() -> None: + s = "Jan has 3/4 of a cake." + candidates = extract_initial_candidates(s) + assert len(candidates) == 1 + assert candidates[0].initial.quantity.value == 0.75 + assert candidates[0].initial.quantity.unit == "cakes" + + +def test_word_number_composition_parsing() -> None: + s = "Francine has five full boxes of crayons." + candidates = extract_initial_candidates(s) + assert len(candidates) == 1 + assert candidates[0].initial.quantity.value == 5.0 + assert candidates[0].initial.quantity.unit == "full boxes of crayons" + + +def test_hyphenated_compound_parsing() -> None: + s = "Allison has 10 one-hour videos." + candidates = extract_initial_candidates(s) + assert len(candidates) == 1 + assert candidates[0].initial.quantity.value == 10.0 + assert candidates[0].initial.quantity.unit == "one-hour videos" + + +def test_refusal_probes() -> None: + with pytest.raises(ValueError, match="Too many decimal places"): + _resolve_currency_and_value("$18.0000") + + res = parse_and_solve("Jan has 1/0 of a cake. How many cake does Jan have?") + assert res.answer is None + + res = parse_and_solve("Sam has one-hour-old baby. How many babies does Sam have?") + assert res.answer is None + + res = parse_and_solve("Marc has 10% of a pizza. How many pizzas does Marc have?") + assert res.answer is None + + +def test_runner_and_report_invariants() -> None: + cases = load_cases(_CASES_PATH) + report = build_report(cases) + + assert report["metrics"]["wrong"] == 0 + assert report["metrics"]["overall_pass"] is True + + r1 = build_report(cases) + r2 = build_report(cases) + s1 = json.dumps(r1, sort_keys=True, separators=(",", ":")) + s2 = json.dumps(r2, sort_keys=True, separators=(",", ":")) + assert s1 == s2