From 5b193280a5fb71dcf9be9a07496ed026706e2d9a Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 6 Jun 2026 17:18:45 -0700 Subject: [PATCH 1/2] feat(setup-oracle): independent R1 gold + the reader refuses, never misreads (PR-5b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure-gold. Defines what "correct" means for the R1 (comparative / derived-symbol / multi-step) shapes the reader cannot read yet, and verifies the current reader REFUSES them rather than misreading them. NO frame extraction, no parser change, no serving. - evals/setup_oracle/r1_gold.jsonl — 10 INDEPENDENT, self-contained fixtures (id, text, relations, expected_units, query, notes) covering: twice/half-as-many (multiplicative), more/fewer-than -> total, a multi-step derived chain (jon=3*ivy; kim=jon+2), a derived subtotal reused downstream (total=lee+mae; per_box=total/3), an inverse target, and the hard negatives (ambiguous referent, missing base quantity, distractor quantity). The gold is the authority — each fixture is reviewable without joining files. - run_r1() scores the CURRENT reader against this gold. Investigative result (the deliverable): **10/10 setup_refused, setup_wrong=0, setup_correct=0.** The reader refuses every R1 shape with a TYPED reason (non_digit_quantity / no_quantity_template / unreadable_quantity_clause / admissibility_refused) — it NEVER misreads an R1 case as a simpler form. So the wrong=0 posture holds all the way down, and PR-5c (the R1 frame) starts from a clean foundation: every R1 case is currently a safe refusal, and the frame's job is refusal -> correct, with the oracle ensuring no setup_wrong is ever introduced. `python -m evals.setup_oracle r1` prints the auditable R1 report (exit 0 iff setup_wrong==0). The 15-case setup gate + relational_metric answer lane are untouched. --- evals/setup_oracle/__init__.py | 3 +- evals/setup_oracle/__main__.py | 12 +++-- evals/setup_oracle/r1_gold.jsonl | 10 ++++ evals/setup_oracle/runner.py | 79 +++++++++++++++++++++++++++++++- tests/test_setup_oracle.py | 21 +++++++++ 5 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 evals/setup_oracle/r1_gold.jsonl diff --git a/evals/setup_oracle/__init__.py b/evals/setup_oracle/__init__.py index 357cb2dd..282550aa 100644 --- a/evals/setup_oracle/__init__.py +++ b/evals/setup_oracle/__init__.py @@ -13,7 +13,7 @@ the problem. v1 grades structure (facts + equations + question target/state/form modelling stays covered by the admissibility tests (a documented signature extension). """ -from evals.setup_oracle.runner import run +from evals.setup_oracle.runner import run, run_r1 from evals.setup_oracle.signature import ( gold_unknown_signature, reader_symbol_units, @@ -28,5 +28,6 @@ __all__ = [ "reader_unknown_signature", "relation_signature", "run", + "run_r1", "symbol_unit_signature", ] diff --git a/evals/setup_oracle/__main__.py b/evals/setup_oracle/__main__.py index f1874400..6d591f8f 100644 --- a/evals/setup_oracle/__main__.py +++ b/evals/setup_oracle/__main__.py @@ -1,20 +1,24 @@ """CLI: print the setup-oracle report. - python -m evals.setup_oracle + python -m evals.setup_oracle # the 15-case relational-metric setup gold + python -m evals.setup_oracle r1 # the independent R1 gold (PR-5b) Exit 0 iff ``setup_wrong == 0`` — the gate the milestone rests on (a wrong reading must -never pass, and serving must not move while setup_wrong > 0). +never pass, and serving must not move while setup_wrong > 0). For ``r1`` the reader is +expected to REFUSE the unsupported shapes (setup_refused), never misread them. """ from __future__ import annotations import json +import sys -from evals.setup_oracle.runner import run +from evals.setup_oracle.runner import run, run_r1 def main() -> int: - report = run() + lane = sys.argv[1] if len(sys.argv) > 1 else "" + report = run_r1() if lane == "r1" else run() print(json.dumps(report, indent=2, default=str)) return 0 if report["setup_wrong"] == 0 else 1 diff --git a/evals/setup_oracle/r1_gold.jsonl b/evals/setup_oracle/r1_gold.jsonl new file mode 100644 index 00000000..e62623d5 --- /dev/null +++ b/evals/setup_oracle/r1_gold.jsonl @@ -0,0 +1,10 @@ +{"id": "r1-01-twice", "text": "Anna has 6 apples. Bella has twice as many apples as Anna. How many apples does Bella have?", "relations": [{"kind": "fact", "entity": "anna", "value": 6}, {"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2}], "expected_units": {"anna": "item", "bella": "item"}, "query": {"entity": "bella", "unit": "item"}, "notes": "Multiplicative: bella = 2 * anna. The reader has no multiplicative template AND 'twice' is non-digit -> must REFUSE, never misread as a fact/additive."} +{"id": "r1-02-half", "text": "Carl has 8 coins. Dora has half as many coins as Carl. How many coins does Dora have?", "relations": [{"kind": "fact", "entity": "carl", "value": 8}, {"kind": "times_as_many", "entity": "dora", "ref": "carl", "factor": 0.5}], "expected_units": {"carl": "item", "dora": "item"}, "query": {"entity": "dora", "unit": "item"}, "notes": "Multiplicative (factor 0.5). 'half' is non-digit -> must REFUSE."} +{"id": "r1-03-more-total", "text": "Finn has 10 books. Evan has 5 more books than Finn. How many books do Evan and Finn have altogether?", "relations": [{"kind": "fact", "entity": "finn", "value": 10}, {"kind": "more_than", "entity": "evan", "ref": "finn", "delta": 5}, {"kind": "sum_of", "entity": "total", "parts": ["evan", "finn"]}], "expected_units": {"finn": "item", "evan": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive + aggregate. 'altogether' (not the bare 'have') may fall outside the strict query template -> likely REFUSE; if read, structure must match exactly."} +{"id": "r1-04-fewer-total", "text": "Gail has 20 cards. Hank has 6 fewer cards than Gail. How many cards do Gail and Hank have in total?", "relations": [{"kind": "fact", "entity": "gail", "value": 20}, {"kind": "fewer_than", "entity": "hank", "ref": "gail", "delta": 6}, {"kind": "sum_of", "entity": "total", "parts": ["gail", "hank"]}], "expected_units": {"gail": "item", "hank": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive(fewer) + aggregate. 'in total' qualifier likely outside the query template -> REFUSE expected."} +{"id": "r1-05-chain", "text": "Ivy has 4 pens. Jon has 3 times as many pens as Ivy. Kim has 2 more pens than Jon. How many pens does Kim have?", "relations": [{"kind": "fact", "entity": "ivy", "value": 4}, {"kind": "times_as_many", "entity": "jon", "ref": "ivy", "factor": 3}, {"kind": "more_than", "entity": "kim", "ref": "jon", "delta": 2}], "expected_units": {"ivy": "item", "jon": "item", "kim": "item"}, "query": {"entity": "kim", "unit": "item"}, "notes": "Multi-step derived chain: jon=3*ivy (intermediate), kim=jon+2. The multiplicative middle step has no template -> must REFUSE the whole reading, never read a partial/wrong chain."} +{"id": "r1-06-subtotal-reused", "text": "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them equally into 3 boxes. How many hats are in each box?", "relations": [{"kind": "fact", "entity": "lee", "value": 5}, {"kind": "fact", "entity": "mae", "value": 7}, {"kind": "sum_of", "entity": "total", "parts": ["lee", "mae"]}, {"kind": "divide_by", "entity": "per_box", "ref": "total", "divisor": 3}], "expected_units": {"lee": "item", "mae": "item", "total": "item", "per_box": "item"}, "query": {"entity": "per_box", "unit": "item"}, "notes": "Derived subtotal reused downstream: total = lee + mae, per_box = total / 3. No partition template -> must REFUSE."} +{"id": "r1-07-inverse", "text": "Nia has 9 more beads than Omar. Nia has 15 beads. How many beads does Omar have?", "relations": [{"kind": "fact", "entity": "nia", "value": 15}, {"kind": "more_than", "entity": "nia", "ref": "omar", "delta": 9}, {"kind": "ask_base", "entity": "omar"}], "expected_units": {"nia": "item", "omar": "item"}, "query": {"entity": "omar", "unit": "item"}, "notes": "Inverse target: omar = nia - 9 = 6, asked via the base of a more_than. The reader may read the STRUCTURE (nia=15, nia=omar+9, ask omar) — that setup is correct; only the solver can't invert. So setup_correct OR refuse, but NEVER setup_wrong."} +{"id": "r1-08-ambiguous-referent", "text": "Pat has 5 marbles. He has 3 more than her. How many marbles does she have?", "relations": [], "expected_units": {}, "query": {"entity": "she", "unit": "item"}, "notes": "Ambiguous pronoun referents (he/her/she) with no grounded base for 'her'/'she' -> must REFUSE (no honest reading), never bind a guessed referent."} +{"id": "r1-09-missing-base", "text": "Quinn has twice as many toys as Rosa. How many toys does Quinn have?", "relations": [{"kind": "times_as_many", "entity": "quinn", "ref": "rosa", "factor": 2}], "expected_units": {"quinn": "item", "rosa": "item"}, "query": {"entity": "quinn", "unit": "item"}, "notes": "Missing base quantity: Rosa's count is never given, so Quinn is underdetermined -> must REFUSE."} +{"id": "r1-10-distractor", "text": "Sam has 7 pencils and 3 erasers. Tom has 4 more pencils than Sam. How many pencils does Tom have?", "relations": [{"kind": "fact", "entity": "sam", "value": 7}, {"kind": "more_than", "entity": "tom", "ref": "sam", "delta": 4}], "expected_units": {"sam": "item", "tom": "item"}, "query": {"entity": "tom", "unit": "item"}, "notes": "Distractor quantity (3 erasers) in a compound clause the 'X has N unit' template can't parse -> must REFUSE rather than mis-bind the distractor."} diff --git a/evals/setup_oracle/runner.py b/evals/setup_oracle/runner.py index b540ba68..c63edc4b 100644 --- a/evals/setup_oracle/runner.py +++ b/evals/setup_oracle/runner.py @@ -28,6 +28,7 @@ from generate.meaning_graph.reader import Refusal from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric _EXPECTED_UNITS_PATH = Path(__file__).resolve().parent / "expected_units.json" +_R1_GOLD_PATH = Path(__file__).resolve().parent / "r1_gold.jsonl" def _load_expected_units() -> dict[str, dict[str, str]]: @@ -35,6 +36,14 @@ def _load_expected_units() -> dict[str, dict[str, str]]: return {k: v for k, v in raw.items() if not k.startswith("_")} +def _load_r1_gold() -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in _R1_GOLD_PATH.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + def run() -> dict[str, Any]: """Score the reader's setup (structure + units + target) against the independent gold.""" cases = _load_cases() @@ -94,4 +103,72 @@ def run() -> dict[str, Any]: } -__all__ = ["run"] +def run_r1() -> dict[str, Any]: + """Score the CURRENT reader against the independent, self-contained R1 gold. + + The reader cannot read most R1 shapes (multiplicative, multi-step, partition) yet. + The SUCCESS CRITERION is **``setup_wrong == 0``**: it must REFUSE every unsupported + shape, NEVER misread it as a simpler form. ``setup_refused`` is the unsupported count; + ``setup_correct`` is any shape already faithfully supported. A ``setup_wrong`` is a + pre-existing wrong-reading hazard to fix BEFORE building the R1 frame (PR-5c). + """ + fixtures = _load_r1_gold() + setup_correct = setup_wrong = setup_refused = 0 + details: list[dict[str, Any]] = [] + + for fx in fixtures: + comp = comprehend_quantitative(fx["text"]) + if isinstance(comp, Refusal): + setup_refused += 1 + details.append({"id": fx["id"], "outcome": "refused", "reason": comp.reason}) + continue + projected = to_relational_metric(comp) + if projected is None: + setup_refused += 1 + details.append({"id": fx["id"], "outcome": "refused", "reason": "unprojectable"}) + continue + reader_relations, _ = projected + units = fx["expected_units"] + + reader_rel = relation_signature(reader_relations) + gold_rel = relation_signature(fx["relations"]) + reader_units = reader_symbol_units(comp.binding_graph) + gold_units = symbol_unit_signature(units) + reader_unk = reader_unknown_signature(comp.binding_graph) + gold_unk = gold_unknown_signature(fx["relations"], fx["query"], units) + + if reader_rel == gold_rel and reader_units == gold_units and reader_unk == gold_unk: + setup_correct += 1 + details.append({"id": fx["id"], "outcome": "correct"}) + else: + setup_wrong += 1 + details.append( + { + "id": fx["id"], + "outcome": "WRONG", + "relations_match": reader_rel == gold_rel, + "units_match": reader_units == gold_units, + "target_match": reader_unk == gold_unk, + "reader_relations": reader_rel, + "gold_relations": gold_rel, + "reader_target": reader_unk, + "gold_target": gold_unk, + } + ) + + return { + "lane": "setup_oracle_r1", + "total": len(fixtures), + "setup_correct": setup_correct, + "setup_wrong": setup_wrong, + "setup_refused": setup_refused, + "details": details, + "counts": { + "setup_correct": setup_correct, + "setup_wrong": setup_wrong, + "setup_refused": setup_refused, + }, + } + + +__all__ = ["run", "run_r1"] diff --git a/tests/test_setup_oracle.py b/tests/test_setup_oracle.py index 97459dd4..c375d9ca 100644 --- a/tests/test_setup_oracle.py +++ b/tests/test_setup_oracle.py @@ -140,3 +140,24 @@ def test_reader_units_read_from_the_binding_graph() -> None: ) assert reader_symbol_units(graph) == (("iris", "dollars"), ("jack", "dollars")) assert reader_unknown_signature(graph) == ("jack", "terminal", "count", "dollars") + + +# --------------------------------------------------------------------------- # +# PR-5b — independent R1 gold: the reader must REFUSE, never MISREAD +# --------------------------------------------------------------------------- # + + +def test_r1_reader_refuses_never_misreads() -> None: + from evals.setup_oracle import run_r1 + + r = run_r1() + assert r["total"] == 10 + # THE success criterion: the reader misreads NO unsupported R1 case. A wrong setup + # here would be a pre-existing hazard to fix before the R1 frame (PR-5c). + assert r["setup_wrong"] == 0 + # The reader cannot read these shapes yet -> every one is a safe (typed) refusal. + assert r["setup_refused"] == 10 + assert r["setup_correct"] == 0 + for d in r["details"]: + assert d["outcome"] == "refused" + assert d.get("reason") # a typed refusal reason, never a silent drop From e9cbe65d77b7d6d033b69e2df57d467f15390411 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 6 Jun 2026 17:29:23 -0700 Subject: [PATCH 2/2] =?UTF-8?q?feat(comprehension):=20the=20multiplicative?= =?UTF-8?q?=20comparative=20frame=20=E2=80=94=20first=20R1=20capability=20?= =?UTF-8?q?(PR-5c)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first capability slice on the R1 arc, gated by the setup-oracle: turn the "twice / N times as many" reading from REFUSED into a correct setup, without a single misread. Builds on the typed IR (PR-4) and the R1 gold (PR-5b). - IR: a Mul(symbol, literal-factor) node — to_canonical_string "ref * factor", operation_kind "multiply", dependencies {ref}, to_relation -> times_as_many. The product keeps the symbol's unit (count * scalar = count), admitted by the REAL check_admissibility multiply path (the literal factor is dimensionless, not a dep). - Reader: a multiplicative template "Y has as many as X" (factor word: twice/double/triple/quadruple) and "Y has times as many as X", checked BEFORE the digit gate (the factor may be a word). 'half' (a /2) is deliberately deferred — divide-by-literal is a separate admissibility path. - setup-oracle: relation_signature now canonicalizes times_as_many. Setup-oracle R1 result: 2 setup_correct (r1-01 twice; r1-05 the multi-step chain ivy/jon=3*ivy/kim=jon+2), 0 setup_WRONG, 8 setup_refused. Every hard negative stays a safe refusal: missing-base (Rosa ungrounded), ambiguous referent, distractor, inverse, partition, 'altogether'/'in total' phrasings, and 'half' (divide). wrong=0 held through the first capability addition. Gates green: setup-oracle R1 setup_wrong=0; 15-case setup gate 15/15 setup_wrong=0; relational_metric answer lane 15/15 wrong=0; binding-graph admissibility + realize + architectural invariants + chat-runtime + pipeline (122+). No serving path touched (this reader feeds the relational_metric / setup-oracle lanes, not the candidate-graph serving). --- evals/setup_oracle/signature.py | 2 + generate/quantitative_comprehension.py | 61 +++++++++++++++++++++++- generate/quantitative_expr.py | 21 +++++++- tests/test_quantitative_comprehension.py | 27 +++++++++++ tests/test_quantitative_expr.py | 17 +++++++ tests/test_setup_oracle.py | 22 +++++---- 6 files changed, 139 insertions(+), 11 deletions(-) diff --git a/evals/setup_oracle/signature.py b/evals/setup_oracle/signature.py index cd7dfe5e..061fc2ca 100644 --- a/evals/setup_oracle/signature.py +++ b/evals/setup_oracle/signature.py @@ -43,6 +43,8 @@ def relation_signature(relations: list[dict[str, Any]]) -> tuple[tuple, ...]: out.append(("fact", r["entity"], int(r["value"]))) elif kind in ("more_than", "fewer_than"): out.append((kind, r["entity"], r["ref"], int(r["delta"]))) + elif kind == "times_as_many": + out.append(("times_as_many", r["entity"], r["ref"], r["factor"])) elif kind == "sum_of": out.append(("sum_of", r["entity"], tuple(sorted(r["parts"])))) else: # an unknown relation kind is itself a structural difference, not a crash diff --git a/generate/quantitative_comprehension.py b/generate/quantitative_comprehension.py index cd026e92..ba56be3f 100644 --- a/generate/quantitative_comprehension.py +++ b/generate/quantitative_comprehension.py @@ -45,6 +45,7 @@ from generate.quantitative_expr import ( Add, Expr, Literal, + Mul, Sub, SumOf, Symbol, @@ -143,8 +144,51 @@ class _Eq: unit: str +@dataclass(frozen=True, slots=True) +class _Mul: + """Multiplicative comparative: entity = factor * ref (R1).""" + + entity: str + ref: str + factor: int + unit: str + + +#: Word factors for "twice/double/triple ... as many". 'half' (a /2, the divide path) +#: is deliberately ABSENT — divide-by-literal is a separate admissibility path, deferred. +_FACTOR_WORDS: dict[str, int] = {"twice": 2, "double": 2, "triple": 3, "quadruple": 4} + + +def _try_multiplicative(entity: str, toks: list[str], detail: str) -> "_Mul | None": + """Match "Y has as many as X" or "Y has times as many + as X" → ``_Mul``. Returns None if the clause is not multiplicative (the + caller then tries the digit-led fact/additive templates).""" + # [Y, has, FACTORWORD, as, many, UNIT, as, X] + if ( + len(toks) == 8 + and toks[2] in _FACTOR_WORDS + and toks[3] == "as" + and toks[4] == "many" + and toks[6] == "as" + ): + return _Mul(entity, _ident(toks[7], detail), _FACTOR_WORDS[toks[2]], + _resolve_unit(_ident(toks[5], detail))) + # [Y, has, N, times, as, many, UNIT, as, X] + if ( + len(toks) == 9 + and toks[2].isdigit() + and toks[3] == "times" + and toks[4] == "as" + and toks[5] == "many" + and toks[7] == "as" + ): + return _Mul(entity, _ident(toks[8], detail), int(toks[2]), + _resolve_unit(_ident(toks[6], detail))) + return None + + def _parse_sentence(body: str, detail: str): - """Return a (_Fact | _Eq | ('query', entity, unit) | ('sumquery', parts, unit)) + """Return a (_Fact | _Eq | _Mul | ('query', entity, unit) | ('sumquery', parts, unit)) spec, or None if the sentence matches no arithmetic template.""" toks = body.strip().lower().rstrip("?.!").split() if not toks: @@ -163,6 +207,11 @@ def _parse_sentence(body: str, detail: str): if len(toks) >= 4 and toks[1] == "has": entity = _ident(toks[0], detail) + # Multiplicative comparative is checked BEFORE the digit gate (its factor may be + # a word like "twice", which is not a digit). + mul = _try_multiplicative(entity, toks, detail) + if mul is not None: + return mul value = _int(toks[2], detail) if len(toks) == 4: return _Fact(entity, value, _resolve_unit(_ident(toks[3], detail))) @@ -190,6 +239,7 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh facts: list[_Fact] = [] eqs: list[_Eq] = [] + muls: list[_Mul] = [] queries: list[tuple] = [] try: @@ -201,6 +251,8 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh facts.append(spec) elif isinstance(spec, _Eq): eqs.append(spec) + elif isinstance(spec, _Mul): + muls.append(spec) else: queries.append(spec) except _QReject as rej: @@ -215,6 +267,8 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh unit_of[f.entity], role_of[f.entity] = f.unit, "count" for e in eqs: unit_of[e.entity], role_of[e.entity] = e.unit, "count" + for m in muls: + unit_of[m.entity], role_of[m.entity] = m.unit, "count" query = queries[0] sum_eq: tuple[str, tuple[str, ...]] | None = None @@ -232,6 +286,8 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh referenced.add(f.entity) for e in eqs: referenced.update((e.entity, e.ref)) + for m in muls: + referenced.update((m.entity, m.ref)) if sum_eq is not None: referenced.add(sum_eq[0]) referenced.update(sum_eq[1]) @@ -262,6 +318,9 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh (e.entity, (Add if e.op == "add" else Sub)(Symbol(e.ref), Literal(e.delta))) for e in eqs ] + expr_specs.extend( + (m.entity, Mul(Symbol(m.ref), Literal(m.factor))) for m in muls + ) if sum_eq is not None: lhs, parts = sum_eq expr_specs.append((lhs, SumOf(tuple(Symbol(p) for p in parts)))) diff --git a/generate/quantitative_expr.py b/generate/quantitative_expr.py index 04965000..e56a5ed3 100644 --- a/generate/quantitative_expr.py +++ b/generate/quantitative_expr.py @@ -43,6 +43,16 @@ class Sub: right: "Expr" +@dataclass(frozen=True, slots=True) +class Mul: + """A scalar multiple of a symbol — the multiplicative comparative ("twice/N times + as many"). ``left`` is the referenced symbol, ``right`` a dimensionless literal + factor; the product keeps the symbol's unit (``count × scalar = count``).""" + + left: "Expr" + right: "Expr" + + @dataclass(frozen=True, slots=True) class SumOf: """An aggregate over ≥2 symbols (the part-whole total).""" @@ -50,7 +60,7 @@ class SumOf: parts: tuple[Symbol, ...] -Expr = Union[Literal, Symbol, Add, Sub, SumOf] +Expr = Union[Literal, Symbol, Add, Sub, Mul, SumOf] def to_canonical_string(expr: Expr) -> str: @@ -64,6 +74,8 @@ def to_canonical_string(expr: Expr) -> str: return f"{to_canonical_string(left)} + {to_canonical_string(right)}" case Sub(left, right): return f"{to_canonical_string(left)} - {to_canonical_string(right)}" + case Mul(left, right): + return f"{to_canonical_string(left)} * {to_canonical_string(right)}" case SumOf(parts): return " + ".join(to_canonical_string(p) for p in parts) raise TypeError(f"not an Expr: {expr!r}") # pragma: no cover - exhaustive above @@ -76,7 +88,7 @@ def dependencies(expr: Expr) -> frozenset[str]: return frozenset() case Symbol(symbol_id): return frozenset({symbol_id}) - case Add(left, right) | Sub(left, right): + case Add(left, right) | Sub(left, right) | Mul(left, right): return dependencies(left) | dependencies(right) case SumOf(parts): out: frozenset[str] = frozenset() @@ -93,6 +105,8 @@ def operation_kind(expr: Expr) -> str: return "add" case Sub(_, _): return "subtract" + case Mul(_, _): + return "multiply" case _: raise TypeError(f"expression has no operation_kind: {expr!r}") @@ -108,6 +122,8 @@ def to_relation(lhs: str, expr: Expr) -> dict[str, Any] | None: return {"kind": "more_than", "entity": lhs, "ref": ref, "delta": delta} case Sub(Symbol(ref), Literal(delta)): return {"kind": "fewer_than", "entity": lhs, "ref": ref, "delta": delta} + case Mul(Symbol(ref), Literal(factor)): + return {"kind": "times_as_many", "entity": lhs, "ref": ref, "factor": factor} case SumOf(parts): return {"kind": "sum_of", "entity": lhs, "parts": [p.symbol_id for p in parts]} case _: @@ -118,6 +134,7 @@ __all__ = [ "Add", "Expr", "Literal", + "Mul", "Sub", "SumOf", "Symbol", diff --git a/tests/test_quantitative_comprehension.py b/tests/test_quantitative_comprehension.py index 862c9cb4..ea41ada6 100644 --- a/tests/test_quantitative_comprehension.py +++ b/tests/test_quantitative_comprehension.py @@ -169,3 +169,30 @@ def test_to_relational_metric_refuses_malformed_target() -> None: for n in (0, 2): comp = QuantComprehension(binding_graph=_graph_with_n_unknowns(n)) assert to_relational_metric(comp) is None # refuse rather than emit a guessed query + + +# --------------------------------------------------------------------------- # +# PR-5c — the multiplicative comparative frame ("twice / N times as many") +# --------------------------------------------------------------------------- # + + +def test_twice_as_many_builds_multiply_equation() -> None: + comp = _comp("Anna has 6 apples. Bella has twice as many apples as Anna. How many apples does Bella have?") + eq = next(e for e in comp.binding_graph.equations if e.lhs_symbol_id == "bella") + assert eq.operation_kind == "multiply" + assert eq.rhs_canonical == "anna * 2" + assert eq.admissibility_status == "admitted" # count * scalar = count, REAL check + assert single_unknown(comp.binding_graph).symbol_id == "bella" + + +def test_n_times_as_many_builds_multiply_equation() -> None: + comp = _comp("Ivy has 4 pens. Jon has 3 times as many pens as Ivy. How many pens does Jon have?") + eq = next(e for e in comp.binding_graph.equations if e.lhs_symbol_id == "jon") + assert eq.operation_kind == "multiply" and eq.rhs_canonical == "ivy * 3" + + +def test_multiplicative_missing_base_refuses() -> None: + # "twice as many as Rosa" with no value for Rosa -> Rosa is ungrounded -> REFUSE, + # never fabricate a base quantity. + comp = comprehend_quantitative("Quinn has twice as many toys as Rosa. How many toys does Quinn have?") + assert isinstance(comp, Refusal) diff --git a/tests/test_quantitative_expr.py b/tests/test_quantitative_expr.py index cf2541c1..cc9a6de5 100644 --- a/tests/test_quantitative_expr.py +++ b/tests/test_quantitative_expr.py @@ -69,3 +69,20 @@ def test_reader_carries_ir_consistent_with_rhs_canonical() -> None: for eq in comp.binding_graph.equations: assert to_canonical_string(by_lhs[eq.lhs_symbol_id]) == eq.rhs_canonical assert dependencies(by_lhs[eq.lhs_symbol_id]) == eq.dependencies + + +# --------------------------------------------------------------------------- # +# PR-5c — the multiplicative comparative (Mul) +# --------------------------------------------------------------------------- # + + +def test_mul_serialization_and_derivations() -> None: + from generate.quantitative_expr import Mul + + m = Mul(Symbol("anna"), Literal(2)) + assert to_canonical_string(m) == "anna * 2" + assert dependencies(m) == frozenset({"anna"}) + assert operation_kind(m) == "multiply" + assert to_relation("bella", m) == { + "kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2, + } diff --git a/tests/test_setup_oracle.py b/tests/test_setup_oracle.py index c375d9ca..99cc2ce6 100644 --- a/tests/test_setup_oracle.py +++ b/tests/test_setup_oracle.py @@ -147,17 +147,23 @@ def test_reader_units_read_from_the_binding_graph() -> None: # --------------------------------------------------------------------------- # -def test_r1_reader_refuses_never_misreads() -> None: +def test_r1_multiplicative_supported_rest_refused_wrong_zero() -> None: from evals.setup_oracle import run_r1 r = run_r1() assert r["total"] == 10 - # THE success criterion: the reader misreads NO unsupported R1 case. A wrong setup - # here would be a pre-existing hazard to fix before the R1 frame (PR-5c). + # THE invariant through the first capability slice: NO R1 case is misread. Adding the + # multiplicative frame turned refusals into correct readings without any setup_wrong. assert r["setup_wrong"] == 0 - # The reader cannot read these shapes yet -> every one is a safe (typed) refusal. - assert r["setup_refused"] == 10 - assert r["setup_correct"] == 0 + # The multiplicative frame (PR-5c) reads "twice as many" (r1-01) and the multi-step + # chain whose middle step is "N times as many" (r1-05); the rest stay safe refusals. + by_id = {d["id"]: d["outcome"] for d in r["details"]} + assert by_id["r1-01-twice"] == "correct" + assert by_id["r1-05-chain"] == "correct" + assert r["setup_correct"] == 2 + assert r["setup_refused"] == 8 + # No detail is ever WRONG, and every non-correct one is a typed refusal. for d in r["details"]: - assert d["outcome"] == "refused" - assert d.get("reason") # a typed refusal reason, never a silent drop + assert d["outcome"] in ("correct", "refused") + if d["outcome"] == "refused": + assert d.get("reason")