diff --git a/evals/capability_index/adapters.py b/evals/capability_index/adapters.py index 24951dac..39ec7d8e 100644 --- a/evals/capability_index/adapters.py +++ b/evals/capability_index/adapters.py @@ -70,10 +70,19 @@ def comprehension_propositional_result() -> DomainResult: return DomainResult("comprehension_propositional", c, w, r) +def comprehension_relational_metric_result() -> DomainResult: + from evals.comprehension.relational_metric_runner import run + + c, w, r = _counts(run()) + return DomainResult("comprehension_relational_metric", c, w, r) + + #: The reasoning domains currently composed into the index (self-loading lanes). -#: The four ``comprehension_*`` lanes score the GENERAL comprehension reader -#: (prose -> MeaningGraph -> projection -> independent oracle), so the index now -#: measures comprehension breadth, not just structured-input reasoning. +#: The five ``comprehension_*`` lanes score the GENERAL comprehension reader; the +#: relational_metric one reads arithmetic prose into the binding-graph quantity +#: substrate (admissibility-checked) and projects to the arithmetic oracle, so the +#: index now measures comprehension breadth across categorical, ordering, +#: propositional, AND quantitative reasoning. ADAPTERS = ( deductive_logic_result, relational_metric_result, @@ -82,6 +91,7 @@ ADAPTERS = ( comprehension_syllogism_result, comprehension_total_ordering_result, comprehension_propositional_result, + comprehension_relational_metric_result, ) diff --git a/evals/capability_index/baseline.json b/evals/capability_index/baseline.json index 34dad1bd..f2bc8be8 100644 --- a/evals/capability_index/baseline.json +++ b/evals/capability_index/baseline.json @@ -1,13 +1,13 @@ { - "capability_score": 0.928622, - "coverage_geomean": 0.928622, - "coverage_micro": 0.993582, + "capability_score": 0.937258, + "coverage_geomean": 0.937258, + "coverage_micro": 0.993703, "accuracy_micro": 1.0, - "breadth": 7, + "breadth": 8, "min_domain_coverage": 0.833333, "wrong_total": 0, "assert_mode_valid": true, - "deterministic_digest": "51df7bba62a035c73bdaf289ea20cded05b66b2f7ca5fd8bbcb04f127d60cadf", + "deterministic_digest": "50e0675bd69938ce5747b5d47592504b9cf143027dd6c1c5e410da0add25341c", "domains": [ { "domain": "comprehension_propositional", @@ -17,6 +17,14 @@ "coverage": 1.0, "accuracy": 1.0 }, + { + "domain": "comprehension_relational_metric", + "correct": 15, + "wrong": 0, + "refused": 0, + "coverage": 1.0, + "accuracy": 1.0 + }, { "domain": "comprehension_set_membership", "correct": 8, diff --git a/evals/comprehension/relational_metric_runner.py b/evals/comprehension/relational_metric_runner.py new file mode 100644 index 00000000..594c2d5b --- /dev/null +++ b/evals/comprehension/relational_metric_runner.py @@ -0,0 +1,75 @@ +"""Score the general comprehension reader on the relational_metric gold lane. + +prose -> comprehend_quantitative() -> binding_graph -> to_relational_metric() -> +independent arithmetic oracle -> answer vs gold. This is the binding-graph's first +comprehension consumer: quantities live in the binding-graph (admissibility-checked, +never stamped), then project to the relational_metric oracle for the verdict. + +A refusal (unreadable prose, admissibility refusal, unprojectable, or an +OracleError on the projection) is NOT a wrong; only a committed integer answer that +disagrees with gold is wrong (must stay 0). +""" + +from __future__ import annotations + +import json +import sys +from typing import Any + +from evals.relational_metric.oracle import OracleError, oracle_answer +from evals.relational_metric.runner import _load_cases +from generate.meaning_graph.reader import Refusal +from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric + + +def run() -> dict[str, Any]: + cases = _load_cases() + correct = wrong = refused = 0 + wrongs: list[dict[str, Any]] = [] + + for case in cases: + comp = comprehend_quantitative(case["text"]) + if isinstance(comp, Refusal): + refused += 1 + continue + projected = to_relational_metric(comp) + if projected is None: + refused += 1 + continue + relations, query = projected + try: + got = oracle_answer(relations, query) + except OracleError: + refused += 1 + continue + if got == case.get("gold"): + correct += 1 + else: + wrong += 1 + wrongs.append( + {"id": case.get("id"), "got": got, "gold": case.get("gold"), "text": case["text"]} + ) + + return { + "domain": "comprehension_relational_metric", + "total": len(cases), + "correct": correct, + "wrong": wrong, + "refused": refused, + "wrongs": wrongs, + "counts": {"correct": correct, "wrong": wrong, "refused": refused}, + } + + +def main() -> int: + report = run() + print(json.dumps({k: v for k, v in report.items() if k != "wrongs"}, indent=2, sort_keys=True)) + if report["wrong"]: + print("WRONG > 0 — comprehension produced a wrong committed answer:", file=sys.stderr) + print(json.dumps(report["wrongs"], indent=2), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/generate/quantitative_comprehension.py b/generate/quantitative_comprehension.py new file mode 100644 index 00000000..4ad14a63 --- /dev/null +++ b/generate/quantitative_comprehension.py @@ -0,0 +1,324 @@ +"""Arithmetic word-problem comprehension -> binding_graph (Phase 2b, domain 5). + +The doctrine-aligned quantity reader, and the binding-graph's FIRST comprehension +consumer. Quantities live in the ``binding_graph`` substrate — CLAUDE.md: the +``MeaningGraph`` deliberately excludes quantities — so this reader lives OUTSIDE +``generate/meaning_graph`` (which stays a numeric-free interlingua, INV-28) and +targets the binding-graph instead. + +It reads arithmetic prose ("Liam has 6 stickers. Mia has 4 more stickers than +Liam.") into ``SymbolBinding`` / ``BoundFact`` / ``BoundEquation`` and runs the +REAL ``check_admissibility`` — there is NO stamped "admitted": an equation is +admitted only if its operand units actually verify, and a dimensional mismatch +REFUSES the whole reading. ``to_relational_metric`` then projects the binding-graph +into the independent ``relational_metric`` oracle for scoring. + +Templates (function-word + order; digits only — a non-digit quantity REFUSES): + - `` has `` -> BoundFact(X = N [unit]) + - `` has more than `` -> BoundEquation(Y = X + N) op=add + - `` has fewer than `` -> BoundEquation(Y = X - N) op=subtract + - query ``How many does have`` -> ask Y + - query ``How many do and have`` -> total = X + Y; ask total + +Refusal-first: an unparseable clause, a non-digit quantity, a non-identifier name, +a missing/duplicated query, or an admissibility refusal all return a typed +``Refusal`` — never a fabricated quantity (wrong=0 at the comprehension layer). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from generate.binding_graph.admissibility import AdmissibilityError, check_admissibility +from generate.binding_graph.model import ( + BoundEquation, + BoundFact, + SemanticSymbolicBindingGraph, + SourceSpanLink, + SymbolBinding, +) +from generate.binding_graph.units import UnitAlgebraError, parse_unit +from generate.meaning_graph.reader import Refusal, _split_sentences + +_INTRODUCED_BY = "comprehend_quantitative" + +#: The generic count dimension for discrete sortal objects (an existing pack +#: lemma resolving to dimension ``count``). A noun the unit pack does not know is +#: read as a count of discrete objects, NOT faked into a physical unit. +_COUNT_UNIT = "item" + + +def _resolve_unit(noun: str) -> str: + """Map a surface unit noun to a binding-graph unit the pack accepts. + + A KNOWN physical/currency/count unit (``dollars`` -> ``dollar``, ``meters``) + is used verbatim (``parse_unit`` depluralizes). An UNKNOWN sortal noun + (``stickers``, ``coins``) is a count of discrete objects -> ``item`` (dimension + ``count``). This keeps admissibility a REAL check: ``count + count`` admits, + ``count + length`` still refuses — nothing is stamped or faked. + """ + try: + parse_unit(noun) + except UnitAlgebraError: + return _COUNT_UNIT + return noun + + +@dataclass(frozen=True, slots=True) +class QuantQuery: + """The question over the comprehended quantities (which entity's count).""" + + entity: str + unit: str + span: SourceSpanLink + + +@dataclass(frozen=True, slots=True) +class QuantComprehension: + """Successful arithmetic comprehension: a binding_graph + the asked entity.""" + + binding_graph: SemanticSymbolicBindingGraph + query: QuantQuery + + +class _QReject(Exception): + """Internal: a clause matched a shape but is not honestly readable.""" + + def __init__(self, reason: str, detail: str = "") -> None: + super().__init__(reason) + self.refusal = Refusal(reason, detail) + + +def _ident(tok: str, detail: str) -> str: + w = tok.strip().lower() + if not w.isidentifier(): + raise _QReject("non_identifier_name", detail) + return w + + +def _int(tok: str, detail: str) -> int: + if not tok.isdigit(): + raise _QReject("non_digit_quantity", detail) + return int(tok) + + +@dataclass(frozen=True, slots=True) +class _Fact: + entity: str + value: int + unit: str + + +@dataclass(frozen=True, slots=True) +class _Eq: + entity: str + ref: str + delta: int + op: str # "add" | "subtract" + unit: str + + +def _parse_sentence(body: str, detail: str): + """Return a (_Fact | _Eq | ('query', entity, unit) | ('sumquery', parts, unit)) + spec, or None if the sentence matches no arithmetic template.""" + toks = body.strip().lower().rstrip("?.!").split() + if not toks: + return None + + if len(toks) >= 5 and toks[0] == "how" and toks[1] == "many" and toks[-1] == "have": + unit = _resolve_unit(_ident(toks[2], detail)) + rest = toks[3:-1] # between "" and "have" + if rest and rest[0] == "does" and len(rest) == 2: + return ("query", _ident(rest[1], detail), unit) + if rest and rest[0] == "do": + parts = [_ident(t, detail) for t in rest[1:] if t != "and"] + if len(parts) >= 2: + return ("sumquery", tuple(parts), unit) + raise _QReject("unreadable_quantity_query", detail) + + if len(toks) >= 4 and toks[1] == "has": + entity = _ident(toks[0], detail) + value = _int(toks[2], detail) + if len(toks) == 4: + return _Fact(entity, value, _resolve_unit(_ident(toks[3], detail))) + if len(toks) == 7 and toks[3] in ("more", "fewer") and toks[5] == "than": + op = "add" if toks[3] == "more" else "subtract" + return _Eq( + entity, _ident(toks[6], detail), value, op, _resolve_unit(_ident(toks[4], detail)) + ) + raise _QReject("unreadable_quantity_clause", detail) + + return None + + +def _span(text: str) -> SourceSpanLink: + return SourceSpanLink(source_id="input", start=0, end=max(1, len(text)), text=text or " ") + + +def _rhs(op: str, ref: str, delta: int) -> str: + return f"{ref} + {delta}" if op == "add" else f"{ref} - {delta}" + + +def comprehend_quantitative(text: str, source_id: str = "input") -> QuantComprehension | Refusal: + """Comprehend arithmetic prose into a binding_graph + asked entity, or refuse.""" + if not text or not text.strip(): + return Refusal("empty") + sentences = _split_sentences(text) + if not sentences: + return Refusal("empty") + + facts: list[_Fact] = [] + eqs: list[_Eq] = [] + queries: list[tuple] = [] + + try: + for body, _terminator, _start, _end in sentences: + spec = _parse_sentence(body, body) + if spec is None: + return Refusal("no_quantity_template", body) + if isinstance(spec, _Fact): + facts.append(spec) + elif isinstance(spec, _Eq): + eqs.append(spec) + else: + queries.append(spec) + except _QReject as rej: + return rej.refusal + + if len(queries) != 1 or not facts: + return Refusal("no_single_quantity_query") + + unit_of: dict[str, str] = {} + role_of: dict[str, str] = {} + for f in facts: + unit_of[f.entity], role_of[f.entity] = f.unit, "count" + for e in eqs: + unit_of[e.entity], role_of[e.entity] = e.unit, "count" + + query = queries[0] + sum_eq: tuple[str, tuple[str, ...]] | None = None + if query[0] == "query": + ask_entity, ask_unit = query[1], query[2] + else: # sumquery -> synthesize a total symbol + sum equation + parts, ask_unit = query[1], query[2] + ask_entity = "total" + unit_of.setdefault(ask_entity, ask_unit) + role_of[ask_entity] = "total" + sum_eq = (ask_entity, parts) + + referenced: set[str] = set() + for f in facts: + referenced.add(f.entity) + for e in eqs: + referenced.update((e.entity, e.ref)) + if sum_eq is not None: + referenced.add(sum_eq[0]) + referenced.update(sum_eq[1]) + referenced.add(ask_entity) + + symbols = [ + SymbolBinding( + symbol_id=sid, + name=sid, + semantic_role=role_of.get(sid, "count"), + source_span=_span(sid), + introduced_by=_INTRODUCED_BY, + entity=sid, + unit=unit_of.get(sid), + ) + for sid in sorted(referenced) + ] + symbols_by_id = {s.symbol_id: s for s in symbols} + + bound_facts = tuple( + BoundFact(symbol_id=f.entity, value=str(f.value), source_span=_span(f.entity), unit=f.unit) + for f in facts + ) + + # equations: shell -> REAL admissibility -> rebuild (NEVER stamp "admitted"). + eq_specs: list[tuple[str, str, frozenset[str], str]] = [ + (e.entity, _rhs(e.op, e.ref, e.delta), frozenset({e.ref}), e.op) for e in eqs + ] + if sum_eq is not None: + lhs, parts = sum_eq + eq_specs.append((lhs, " + ".join(parts), frozenset(parts), "add")) + + equations: list[BoundEquation] = [] + for lhs, rhs, deps, op in eq_specs: + shell = BoundEquation( + lhs_symbol_id=lhs, + rhs_canonical=rhs, + dependencies=deps, + operation_kind=op, + unit_proof="pending", + admissibility_status="pending", + source_span=_span(lhs), + ) + try: + proof = check_admissibility(shell, symbols=symbols_by_id) + except AdmissibilityError as exc: + return Refusal("admissibility_refused", f"{lhs}: {exc.reason}") + equations.append( + BoundEquation( + lhs_symbol_id=lhs, + rhs_canonical=rhs, + dependencies=deps, + operation_kind=op, + unit_proof=proof.to_canonical_string(), + admissibility_status="admitted", + source_span=_span(lhs), + ) + ) + + try: + graph = SemanticSymbolicBindingGraph( + symbols=tuple(symbols), facts=bound_facts, equations=tuple(equations) + ) + except Exception as exc: # noqa: BLE001 — surface construction refusal + return Refusal("invalid_binding_graph", repr(exc)) + + return QuantComprehension( + binding_graph=graph, + query=QuantQuery(entity=ask_entity, unit=ask_unit, span=_span(ask_entity)), + ) + + +def to_relational_metric( + comp: QuantComprehension, +) -> tuple[list[dict[str, Any]], dict[str, Any]] | None: + """Project the comprehended binding_graph into ``(relations, query)`` for + ``evals.relational_metric.oracle.oracle_answer``. + + Reads the binding-graph itself (facts + admitted equations) — the equation's + own ``rhs_canonical`` is parsed back (a controlled round-trip of the format this + module emits) and operands are classified as symbol (a known entity) vs literal. + Facts are emitted before equations and equations in dependency order, so the + oracle's forward substitution never hits an unresolved reference. + """ + graph = comp.binding_graph + symbol_ids = {s.symbol_id for s in graph.symbols} + relations: list[dict[str, Any]] = [ + {"kind": "fact", "entity": f.symbol_id, "value": int(f.value)} for f in graph.facts + ] + for eq in graph.equations: + rhs = eq.rhs_canonical + if " + " in rhs: + operands = rhs.split(" + ") + if all(op in symbol_ids for op in operands): + relations.append({"kind": "sum_of", "entity": eq.lhs_symbol_id, "parts": list(operands)}) + continue + ref, literal = operands[0], operands[1] + relations.append( + {"kind": "more_than", "entity": eq.lhs_symbol_id, "ref": ref, "delta": int(literal)} + ) + elif " - " in rhs: + ref, literal = rhs.split(" - ") + relations.append( + {"kind": "fewer_than", "entity": eq.lhs_symbol_id, "ref": ref, "delta": int(literal)} + ) + else: + return None # unrecognized equation shape -> refuse + if not relations: + return None + return relations, {"entity": comp.query.entity, "unit": comp.query.unit} diff --git a/tests/test_architectural_invariants.py b/tests/test_architectural_invariants.py index f5885144..c89cff9e 100644 --- a/tests/test_architectural_invariants.py +++ b/tests/test_architectural_invariants.py @@ -1144,6 +1144,19 @@ INDEPENDENT_GOLD_LANES: tuple[IndependentGoldLane, ...] = ( oracle_module="evals/deductive_logic/oracle.py", sut_import_prefixes=("generate.meaning_graph",), ), + # The arithmetic comprehension lane: the SUT is the quantity reader + # (generate.quantitative_comprehension) and the binding-graph substrate it + # builds. The relational_metric oracle (forward substitution) must share no code + # with either — the arithmetic gold stays independent of the reader. + IndependentGoldLane( + name="comprehension_relational_metric", + oracle_module="evals/relational_metric/oracle.py", + sut_import_prefixes=( + "generate.quantitative_comprehension", + "generate.binding_graph", + "generate.meaning_graph", + ), + ), ) _DEDUCTIVE_CASE_FILES: tuple[str, ...] = ( diff --git a/tests/test_capability_index.py b/tests/test_capability_index.py index 71195357..c044d1df 100644 --- a/tests/test_capability_index.py +++ b/tests/test_capability_index.py @@ -97,7 +97,7 @@ def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None: idx = aggregate(list(collection.results)) assert idx.wrong_total == 0 assert idx.assert_mode_valid - assert idx.breadth == 7 + assert idx.breadth == 8 assert {d.domain for d in idx.domains} == { "deductive_logic", "dimensional", @@ -106,6 +106,7 @@ def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None: "comprehension_syllogism", "comprehension_total_ordering", "comprehension_propositional", + "comprehension_relational_metric", } assert idx.capability_score > 0.5 # real, non-trivial cross-domain capability diff --git a/tests/test_comprehension_relational_metric.py b/tests/test_comprehension_relational_metric.py new file mode 100644 index 00000000..31fdf3df --- /dev/null +++ b/tests/test_comprehension_relational_metric.py @@ -0,0 +1,30 @@ +"""Phase 2b — end-to-end: the comprehension reader scored on relational_metric. + +prose -> comprehend_quantitative -> binding_graph -> to_relational_metric -> +INDEPENDENT arithmetic oracle -> answer vs gold. The load-bearing invariant: +wrong == 0. This is the binding-graph's first comprehension consumer; quantities +are admissibility-checked (never stamped), then projected to the oracle. +""" + +from __future__ import annotations + +from evals.comprehension.relational_metric_runner import run + + +def test_comprehension_relational_metric_wrong_is_zero() -> None: + report = run() + assert report["wrong"] == 0, report["wrongs"] + + +def test_comprehension_relational_metric_has_real_coverage() -> None: + report = run() + assert report["correct"] > 0 + assert report["correct"] + report["refused"] == report["total"] + + +def test_comprehension_relational_metric_full_coverage() -> None: + # The whole 15-case lane reads end-to-end (fact / more_than / fewer_than / + # sum_of, single + chained, count nouns -> item dimension, dollars -> money). + report = run() + assert report["refused"] == 0 + assert report["correct"] == report["total"] diff --git a/tests/test_comprehension_structure_preserving.py b/tests/test_comprehension_structure_preserving.py index e51da491..45fea0a0 100644 --- a/tests/test_comprehension_structure_preserving.py +++ b/tests/test_comprehension_structure_preserving.py @@ -26,6 +26,7 @@ from generate.meaning_graph.projectors import ( to_total_ordering, ) from generate.meaning_graph.reader import Refusal, comprehend +from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric _TERMS = [f"t{i}" for i in range(8)] @@ -263,3 +264,42 @@ def test_propositional_invariant_to_premise_reorder() -> None: s_base = _struct(base, to_deductive_logic) assert s_base is not None assert _struct(swapped, to_deductive_logic) == s_base + + +# --------------------------------------------------------------------------- # +# Arithmetic (binding_graph) — projected relations + query preserved exactly. +# --------------------------------------------------------------------------- # + + +def test_arithmetic_structure_is_preserved_exactly() -> None: + rng = random.Random(55) + committed = 0 + for _ in range(300): + ents = rng.sample([f"e{i}" for i in range(6)], rng.randint(2, 4)) + base, base_val = ents[0], rng.randint(1, 20) + relations = [{"kind": "fact", "entity": base, "value": base_val}] + lines = [f"{base} has {base_val} things."] + prev = base + for e in ents[1:]: + delta = rng.randint(1, 15) + kind = rng.choice(["more_than", "fewer_than"]) + word = "more" if kind == "more_than" else "fewer" + relations.append({"kind": kind, "entity": e, "ref": prev, "delta": delta}) + lines.append(f"{e} has {delta} {word} things than {prev}.") + prev = e + ask = rng.choice(ents) + lines.append(f"How many things does {ask} have?") + prose = " ".join(lines) + expected_query = {"entity": ask, "unit": "item"} # "things" -> item dimension + + comp = comprehend_quantitative(prose) + if isinstance(comp, Refusal): + continue + proj = to_relational_metric(comp) + if proj is None: + continue + committed += 1 + prelations, pquery = proj + assert _canon(prelations) == _canon(relations), (prose, prelations, relations) + assert pquery == expected_query, (prose, pquery, expected_query) + assert committed > 50 diff --git a/tests/test_quantitative_comprehension.py b/tests/test_quantitative_comprehension.py new file mode 100644 index 00000000..be9e7958 --- /dev/null +++ b/tests/test_quantitative_comprehension.py @@ -0,0 +1,99 @@ +"""Unit tests for the arithmetic reader (prose -> binding_graph) + its projector. + +Pins the templates, the count-vs-physical-unit modelling, and — load-bearing — the +REAL admissibility check: an equation is admitted only if its operand units verify, +so a mixed-unit sum REFUSES rather than fabricating a quantity. This is the +reviewer's "do not stamp admissibility" guard, made executable. +""" + +from __future__ import annotations + +from generate.binding_graph.model import SemanticSymbolicBindingGraph +from generate.meaning_graph.reader import Refusal +from generate.quantitative_comprehension import ( + QuantComprehension, + comprehend_quantitative, + to_relational_metric, +) + + +def _comp(text: str) -> QuantComprehension: + comp = comprehend_quantitative(text) + assert isinstance(comp, QuantComprehension), comp + return comp + + +def test_fact_and_more_than_build_binding_graph() -> None: + comp = _comp("Liam has 6 stickers. Mia has 4 more stickers than Liam. How many stickers does Mia have?") + g = comp.binding_graph + assert isinstance(g, SemanticSymbolicBindingGraph) + assert {f.symbol_id: f.value for f in g.facts} == {"liam": "6"} + eq = next(e for e in g.equations if e.lhs_symbol_id == "mia") + assert eq.operation_kind == "add" + assert eq.rhs_canonical == "liam + 4" + assert eq.admissibility_status == "admitted" # from the REAL check, not stamped + assert comp.query.entity == "mia" + + +def test_count_nouns_resolve_to_item_dimension() -> None: + # Unknown sortal nouns become the count dimension (item); admissibility admits. + comp = _comp("Kim has 2 marbles. Leo has 3 more marbles than Kim. How many marbles does Leo have?") + units = {s.symbol_id: s.unit for s in comp.binding_graph.symbols} + assert units["kim"] == "item" and units["leo"] == "item" + + +def test_known_unit_is_used_verbatim() -> None: + comp = _comp("Iris has 100 dollars. Jack has 250 more dollars than Iris. How many dollars does Jack have?") + units = {s.symbol_id: s.unit for s in comp.binding_graph.symbols} + assert units["iris"] == "dollars" # parse_unit depluralizes dollars -> dollar (money) + + +def test_fewer_than_is_subtract() -> None: + comp = _comp("Noah has 15 cards. Olivia has 6 fewer cards than Noah. How many cards does Olivia have?") + eq = next(e for e in comp.binding_graph.equations if e.lhs_symbol_id == "olivia") + assert eq.operation_kind == "subtract" and eq.rhs_canonical == "noah - 6" + + +def test_sum_query_synthesizes_total() -> None: + comp = _comp("Dan has 7 coins. Eva has 9 more coins than Dan. How many coins do Dan and Eva have?") + assert comp.query.entity == "total" + total_eq = next(e for e in comp.binding_graph.equations if e.lhs_symbol_id == "total") + assert total_eq.operation_kind == "add" + assert set(total_eq.dependencies) == {"dan", "eva"} + + +def test_projection_shape() -> None: + comp = _comp("Liam has 6 stickers. Mia has 4 more stickers than Liam. How many stickers does Mia have?") + projected = to_relational_metric(comp) + assert projected is not None + relations, query = projected + assert {"kind": "fact", "entity": "liam", "value": 6} in relations + assert {"kind": "more_than", "entity": "mia", "ref": "liam", "delta": 4} in relations + assert query["entity"] == "mia" + + +# --------------------------------------------------------------------------- # +# Admissibility is REAL, not stamped (the reviewer's load-bearing guard) +# --------------------------------------------------------------------------- # + + +def test_mixed_unit_sum_refuses_via_admissibility() -> None: + # count (stickers -> item) + money (dollars) cannot be summed: the REAL + # admissibility check must REFUSE, not fabricate a total. + comp = comprehend_quantitative( + "Liam has 6 stickers. Mia has 4 dollars. How many things do Liam and Mia have?" + ) + assert isinstance(comp, Refusal) + assert comp.reason == "admissibility_refused" + assert "unit_mismatch" in comp.detail + + +def test_non_digit_quantity_refuses() -> None: + comp = comprehend_quantitative("Liam has several stickers. How many stickers does Liam have?") + assert isinstance(comp, Refusal) + assert comp.reason == "non_digit_quantity" + + +def test_unreadable_clause_refuses() -> None: + comp = comprehend_quantitative("The weather is nice today.") + assert isinstance(comp, Refusal)