feat(comprehend): arithmetic word-problems via binding_graph (5th domain, real admissibility)
The binding-graph's FIRST comprehension consumer (doctrine-aligned: quantities live in binding_graph, NOT the MeaningGraph). generate/quantitative_comprehension.py reads arithmetic prose into SymbolBinding/BoundFact/BoundEquation and runs the REAL check_admissibility (shell -> verify -> rebuild with the actual UnitProof) — there is NO stamped "admitted": an equation is admitted only if its operand units verify. Then to_relational_metric projects the binding-graph to the independent relational_metric oracle for the verdict. Templates (digits only; non-digit quantity REFUSES): "<X> has <N> <unit>" -> BoundFact(X = N) "<Y> has <N> more <unit> than <X>" -> BoundEquation(Y = X + N) op=add "<Y> has <N> fewer <unit> than <X>" -> BoundEquation(Y = X - N) op=subtract "How many <unit> does <Y> have" -> ask Y "How many <unit> do <X> and <Y> have"-> total = X + Y; ask total Unit modelling (honest, not faked): a noun the closed en_units_v1 pack knows is used verbatim (dollars -> dollar/money); an UNKNOWN sortal noun (stickers, coins) is a count of discrete objects -> the existing 'item' lemma (dimension count). So admissibility stays a REAL check: count+count admits, count+money (a mixed-unit sum) REFUSES with unit_mismatch — verified to bite. comprehension_relational_metric: 15/15 wrong=0 (full coverage). Located OUTSIDE generate/meaning_graph (it targets binding_graph, not the MeaningGraph) so INV-28 neutrality stays intact; oracle imports none of the SUT (new INV-25 lane). Capability index breadth 7->8, score 0.928622 -> 0.937258, wrong_total 0, digest 50e0675b… Tests: reader templates + count/known-unit modelling + admissibility-bite (mixed unit refuses) + non-digit refusal; end-to-end full-coverage wrong=0; arithmetic added to the structure-preservation generative panel (projected relations+query == ground truth); capability breadth 7->8; INV-25 arithmetic lane. 93 targeted + 90 smoke green; lane SHAs 8/9 (sole miss = public_demo env flake; deductive_logic + math_teaching unchanged -> no GSM8K coupling).
This commit is contained in:
parent
ed4c8bec2b
commit
a005a92fed
9 changed files with 609 additions and 9 deletions
|
|
@ -70,10 +70,19 @@ def comprehension_propositional_result() -> DomainResult:
|
||||||
return DomainResult("comprehension_propositional", c, w, r)
|
return DomainResult("comprehension_propositional", c, w, r)
|
||||||
|
|
||||||
|
|
||||||
|
def comprehension_relational_metric_result() -> DomainResult:
|
||||||
|
from evals.comprehension.relational_metric_runner import run
|
||||||
|
|
||||||
|
c, w, r = _counts(run())
|
||||||
|
return DomainResult("comprehension_relational_metric", c, w, r)
|
||||||
|
|
||||||
|
|
||||||
#: The reasoning domains currently composed into the index (self-loading lanes).
|
#: The reasoning domains currently composed into the index (self-loading lanes).
|
||||||
#: The four ``comprehension_*`` lanes score the GENERAL comprehension reader
|
#: The five ``comprehension_*`` lanes score the GENERAL comprehension reader; the
|
||||||
#: (prose -> MeaningGraph -> projection -> independent oracle), so the index now
|
#: relational_metric one reads arithmetic prose into the binding-graph quantity
|
||||||
#: measures comprehension breadth, not just structured-input reasoning.
|
#: substrate (admissibility-checked) and projects to the arithmetic oracle, so the
|
||||||
|
#: index now measures comprehension breadth across categorical, ordering,
|
||||||
|
#: propositional, AND quantitative reasoning.
|
||||||
ADAPTERS = (
|
ADAPTERS = (
|
||||||
deductive_logic_result,
|
deductive_logic_result,
|
||||||
relational_metric_result,
|
relational_metric_result,
|
||||||
|
|
@ -82,6 +91,7 @@ ADAPTERS = (
|
||||||
comprehension_syllogism_result,
|
comprehension_syllogism_result,
|
||||||
comprehension_total_ordering_result,
|
comprehension_total_ordering_result,
|
||||||
comprehension_propositional_result,
|
comprehension_propositional_result,
|
||||||
|
comprehension_relational_metric_result,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,13 @@
|
||||||
{
|
{
|
||||||
"capability_score": 0.928622,
|
"capability_score": 0.937258,
|
||||||
"coverage_geomean": 0.928622,
|
"coverage_geomean": 0.937258,
|
||||||
"coverage_micro": 0.993582,
|
"coverage_micro": 0.993703,
|
||||||
"accuracy_micro": 1.0,
|
"accuracy_micro": 1.0,
|
||||||
"breadth": 7,
|
"breadth": 8,
|
||||||
"min_domain_coverage": 0.833333,
|
"min_domain_coverage": 0.833333,
|
||||||
"wrong_total": 0,
|
"wrong_total": 0,
|
||||||
"assert_mode_valid": true,
|
"assert_mode_valid": true,
|
||||||
"deterministic_digest": "51df7bba62a035c73bdaf289ea20cded05b66b2f7ca5fd8bbcb04f127d60cadf",
|
"deterministic_digest": "50e0675bd69938ce5747b5d47592504b9cf143027dd6c1c5e410da0add25341c",
|
||||||
"domains": [
|
"domains": [
|
||||||
{
|
{
|
||||||
"domain": "comprehension_propositional",
|
"domain": "comprehension_propositional",
|
||||||
|
|
@ -17,6 +17,14 @@
|
||||||
"coverage": 1.0,
|
"coverage": 1.0,
|
||||||
"accuracy": 1.0
|
"accuracy": 1.0
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"domain": "comprehension_relational_metric",
|
||||||
|
"correct": 15,
|
||||||
|
"wrong": 0,
|
||||||
|
"refused": 0,
|
||||||
|
"coverage": 1.0,
|
||||||
|
"accuracy": 1.0
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"domain": "comprehension_set_membership",
|
"domain": "comprehension_set_membership",
|
||||||
"correct": 8,
|
"correct": 8,
|
||||||
|
|
|
||||||
75
evals/comprehension/relational_metric_runner.py
Normal file
75
evals/comprehension/relational_metric_runner.py
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
"""Score the general comprehension reader on the relational_metric gold lane.
|
||||||
|
|
||||||
|
prose -> comprehend_quantitative() -> binding_graph -> to_relational_metric() ->
|
||||||
|
independent arithmetic oracle -> answer vs gold. This is the binding-graph's first
|
||||||
|
comprehension consumer: quantities live in the binding-graph (admissibility-checked,
|
||||||
|
never stamped), then project to the relational_metric oracle for the verdict.
|
||||||
|
|
||||||
|
A refusal (unreadable prose, admissibility refusal, unprojectable, or an
|
||||||
|
OracleError on the projection) is NOT a wrong; only a committed integer answer that
|
||||||
|
disagrees with gold is wrong (must stay 0).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from evals.relational_metric.oracle import OracleError, oracle_answer
|
||||||
|
from evals.relational_metric.runner import _load_cases
|
||||||
|
from generate.meaning_graph.reader import Refusal
|
||||||
|
from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric
|
||||||
|
|
||||||
|
|
||||||
|
def run() -> dict[str, Any]:
|
||||||
|
cases = _load_cases()
|
||||||
|
correct = wrong = refused = 0
|
||||||
|
wrongs: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
for case in cases:
|
||||||
|
comp = comprehend_quantitative(case["text"])
|
||||||
|
if isinstance(comp, Refusal):
|
||||||
|
refused += 1
|
||||||
|
continue
|
||||||
|
projected = to_relational_metric(comp)
|
||||||
|
if projected is None:
|
||||||
|
refused += 1
|
||||||
|
continue
|
||||||
|
relations, query = projected
|
||||||
|
try:
|
||||||
|
got = oracle_answer(relations, query)
|
||||||
|
except OracleError:
|
||||||
|
refused += 1
|
||||||
|
continue
|
||||||
|
if got == case.get("gold"):
|
||||||
|
correct += 1
|
||||||
|
else:
|
||||||
|
wrong += 1
|
||||||
|
wrongs.append(
|
||||||
|
{"id": case.get("id"), "got": got, "gold": case.get("gold"), "text": case["text"]}
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"domain": "comprehension_relational_metric",
|
||||||
|
"total": len(cases),
|
||||||
|
"correct": correct,
|
||||||
|
"wrong": wrong,
|
||||||
|
"refused": refused,
|
||||||
|
"wrongs": wrongs,
|
||||||
|
"counts": {"correct": correct, "wrong": wrong, "refused": refused},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
report = run()
|
||||||
|
print(json.dumps({k: v for k, v in report.items() if k != "wrongs"}, indent=2, sort_keys=True))
|
||||||
|
if report["wrong"]:
|
||||||
|
print("WRONG > 0 — comprehension produced a wrong committed answer:", file=sys.stderr)
|
||||||
|
print(json.dumps(report["wrongs"], indent=2), file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
324
generate/quantitative_comprehension.py
Normal file
324
generate/quantitative_comprehension.py
Normal file
|
|
@ -0,0 +1,324 @@
|
||||||
|
"""Arithmetic word-problem comprehension -> binding_graph (Phase 2b, domain 5).
|
||||||
|
|
||||||
|
The doctrine-aligned quantity reader, and the binding-graph's FIRST comprehension
|
||||||
|
consumer. Quantities live in the ``binding_graph`` substrate — CLAUDE.md: the
|
||||||
|
``MeaningGraph`` deliberately excludes quantities — so this reader lives OUTSIDE
|
||||||
|
``generate/meaning_graph`` (which stays a numeric-free interlingua, INV-28) and
|
||||||
|
targets the binding-graph instead.
|
||||||
|
|
||||||
|
It reads arithmetic prose ("Liam has 6 stickers. Mia has 4 more stickers than
|
||||||
|
Liam.") into ``SymbolBinding`` / ``BoundFact`` / ``BoundEquation`` and runs the
|
||||||
|
REAL ``check_admissibility`` — there is NO stamped "admitted": an equation is
|
||||||
|
admitted only if its operand units actually verify, and a dimensional mismatch
|
||||||
|
REFUSES the whole reading. ``to_relational_metric`` then projects the binding-graph
|
||||||
|
into the independent ``relational_metric`` oracle for scoring.
|
||||||
|
|
||||||
|
Templates (function-word + order; digits only — a non-digit quantity REFUSES):
|
||||||
|
- ``<X> has <N> <unit>`` -> BoundFact(X = N [unit])
|
||||||
|
- ``<Y> has <N> more <unit> than <X>`` -> BoundEquation(Y = X + N) op=add
|
||||||
|
- ``<Y> has <N> fewer <unit> than <X>`` -> BoundEquation(Y = X - N) op=subtract
|
||||||
|
- query ``How many <unit> does <Y> have`` -> ask Y
|
||||||
|
- query ``How many <unit> do <X> and <Y> have`` -> total = X + Y; ask total
|
||||||
|
|
||||||
|
Refusal-first: an unparseable clause, a non-digit quantity, a non-identifier name,
|
||||||
|
a missing/duplicated query, or an admissibility refusal all return a typed
|
||||||
|
``Refusal`` — never a fabricated quantity (wrong=0 at the comprehension layer).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from generate.binding_graph.admissibility import AdmissibilityError, check_admissibility
|
||||||
|
from generate.binding_graph.model import (
|
||||||
|
BoundEquation,
|
||||||
|
BoundFact,
|
||||||
|
SemanticSymbolicBindingGraph,
|
||||||
|
SourceSpanLink,
|
||||||
|
SymbolBinding,
|
||||||
|
)
|
||||||
|
from generate.binding_graph.units import UnitAlgebraError, parse_unit
|
||||||
|
from generate.meaning_graph.reader import Refusal, _split_sentences
|
||||||
|
|
||||||
|
_INTRODUCED_BY = "comprehend_quantitative"
|
||||||
|
|
||||||
|
#: The generic count dimension for discrete sortal objects (an existing pack
|
||||||
|
#: lemma resolving to dimension ``count``). A noun the unit pack does not know is
|
||||||
|
#: read as a count of discrete objects, NOT faked into a physical unit.
|
||||||
|
_COUNT_UNIT = "item"
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_unit(noun: str) -> str:
|
||||||
|
"""Map a surface unit noun to a binding-graph unit the pack accepts.
|
||||||
|
|
||||||
|
A KNOWN physical/currency/count unit (``dollars`` -> ``dollar``, ``meters``)
|
||||||
|
is used verbatim (``parse_unit`` depluralizes). An UNKNOWN sortal noun
|
||||||
|
(``stickers``, ``coins``) is a count of discrete objects -> ``item`` (dimension
|
||||||
|
``count``). This keeps admissibility a REAL check: ``count + count`` admits,
|
||||||
|
``count + length`` still refuses — nothing is stamped or faked.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
parse_unit(noun)
|
||||||
|
except UnitAlgebraError:
|
||||||
|
return _COUNT_UNIT
|
||||||
|
return noun
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class QuantQuery:
|
||||||
|
"""The question over the comprehended quantities (which entity's count)."""
|
||||||
|
|
||||||
|
entity: str
|
||||||
|
unit: str
|
||||||
|
span: SourceSpanLink
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class QuantComprehension:
|
||||||
|
"""Successful arithmetic comprehension: a binding_graph + the asked entity."""
|
||||||
|
|
||||||
|
binding_graph: SemanticSymbolicBindingGraph
|
||||||
|
query: QuantQuery
|
||||||
|
|
||||||
|
|
||||||
|
class _QReject(Exception):
|
||||||
|
"""Internal: a clause matched a shape but is not honestly readable."""
|
||||||
|
|
||||||
|
def __init__(self, reason: str, detail: str = "") -> None:
|
||||||
|
super().__init__(reason)
|
||||||
|
self.refusal = Refusal(reason, detail)
|
||||||
|
|
||||||
|
|
||||||
|
def _ident(tok: str, detail: str) -> str:
|
||||||
|
w = tok.strip().lower()
|
||||||
|
if not w.isidentifier():
|
||||||
|
raise _QReject("non_identifier_name", detail)
|
||||||
|
return w
|
||||||
|
|
||||||
|
|
||||||
|
def _int(tok: str, detail: str) -> int:
|
||||||
|
if not tok.isdigit():
|
||||||
|
raise _QReject("non_digit_quantity", detail)
|
||||||
|
return int(tok)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _Fact:
|
||||||
|
entity: str
|
||||||
|
value: int
|
||||||
|
unit: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _Eq:
|
||||||
|
entity: str
|
||||||
|
ref: str
|
||||||
|
delta: int
|
||||||
|
op: str # "add" | "subtract"
|
||||||
|
unit: str
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_sentence(body: str, detail: str):
|
||||||
|
"""Return a (_Fact | _Eq | ('query', entity, unit) | ('sumquery', parts, unit))
|
||||||
|
spec, or None if the sentence matches no arithmetic template."""
|
||||||
|
toks = body.strip().lower().rstrip("?.!").split()
|
||||||
|
if not toks:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if len(toks) >= 5 and toks[0] == "how" and toks[1] == "many" and toks[-1] == "have":
|
||||||
|
unit = _resolve_unit(_ident(toks[2], detail))
|
||||||
|
rest = toks[3:-1] # between "<unit>" and "have"
|
||||||
|
if rest and rest[0] == "does" and len(rest) == 2:
|
||||||
|
return ("query", _ident(rest[1], detail), unit)
|
||||||
|
if rest and rest[0] == "do":
|
||||||
|
parts = [_ident(t, detail) for t in rest[1:] if t != "and"]
|
||||||
|
if len(parts) >= 2:
|
||||||
|
return ("sumquery", tuple(parts), unit)
|
||||||
|
raise _QReject("unreadable_quantity_query", detail)
|
||||||
|
|
||||||
|
if len(toks) >= 4 and toks[1] == "has":
|
||||||
|
entity = _ident(toks[0], detail)
|
||||||
|
value = _int(toks[2], detail)
|
||||||
|
if len(toks) == 4:
|
||||||
|
return _Fact(entity, value, _resolve_unit(_ident(toks[3], detail)))
|
||||||
|
if len(toks) == 7 and toks[3] in ("more", "fewer") and toks[5] == "than":
|
||||||
|
op = "add" if toks[3] == "more" else "subtract"
|
||||||
|
return _Eq(
|
||||||
|
entity, _ident(toks[6], detail), value, op, _resolve_unit(_ident(toks[4], detail))
|
||||||
|
)
|
||||||
|
raise _QReject("unreadable_quantity_clause", detail)
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _span(text: str) -> SourceSpanLink:
|
||||||
|
return SourceSpanLink(source_id="input", start=0, end=max(1, len(text)), text=text or " ")
|
||||||
|
|
||||||
|
|
||||||
|
def _rhs(op: str, ref: str, delta: int) -> str:
|
||||||
|
return f"{ref} + {delta}" if op == "add" else f"{ref} - {delta}"
|
||||||
|
|
||||||
|
|
||||||
|
def comprehend_quantitative(text: str, source_id: str = "input") -> QuantComprehension | Refusal:
|
||||||
|
"""Comprehend arithmetic prose into a binding_graph + asked entity, or refuse."""
|
||||||
|
if not text or not text.strip():
|
||||||
|
return Refusal("empty")
|
||||||
|
sentences = _split_sentences(text)
|
||||||
|
if not sentences:
|
||||||
|
return Refusal("empty")
|
||||||
|
|
||||||
|
facts: list[_Fact] = []
|
||||||
|
eqs: list[_Eq] = []
|
||||||
|
queries: list[tuple] = []
|
||||||
|
|
||||||
|
try:
|
||||||
|
for body, _terminator, _start, _end in sentences:
|
||||||
|
spec = _parse_sentence(body, body)
|
||||||
|
if spec is None:
|
||||||
|
return Refusal("no_quantity_template", body)
|
||||||
|
if isinstance(spec, _Fact):
|
||||||
|
facts.append(spec)
|
||||||
|
elif isinstance(spec, _Eq):
|
||||||
|
eqs.append(spec)
|
||||||
|
else:
|
||||||
|
queries.append(spec)
|
||||||
|
except _QReject as rej:
|
||||||
|
return rej.refusal
|
||||||
|
|
||||||
|
if len(queries) != 1 or not facts:
|
||||||
|
return Refusal("no_single_quantity_query")
|
||||||
|
|
||||||
|
unit_of: dict[str, str] = {}
|
||||||
|
role_of: dict[str, str] = {}
|
||||||
|
for f in facts:
|
||||||
|
unit_of[f.entity], role_of[f.entity] = f.unit, "count"
|
||||||
|
for e in eqs:
|
||||||
|
unit_of[e.entity], role_of[e.entity] = e.unit, "count"
|
||||||
|
|
||||||
|
query = queries[0]
|
||||||
|
sum_eq: tuple[str, tuple[str, ...]] | None = None
|
||||||
|
if query[0] == "query":
|
||||||
|
ask_entity, ask_unit = query[1], query[2]
|
||||||
|
else: # sumquery -> synthesize a total symbol + sum equation
|
||||||
|
parts, ask_unit = query[1], query[2]
|
||||||
|
ask_entity = "total"
|
||||||
|
unit_of.setdefault(ask_entity, ask_unit)
|
||||||
|
role_of[ask_entity] = "total"
|
||||||
|
sum_eq = (ask_entity, parts)
|
||||||
|
|
||||||
|
referenced: set[str] = set()
|
||||||
|
for f in facts:
|
||||||
|
referenced.add(f.entity)
|
||||||
|
for e in eqs:
|
||||||
|
referenced.update((e.entity, e.ref))
|
||||||
|
if sum_eq is not None:
|
||||||
|
referenced.add(sum_eq[0])
|
||||||
|
referenced.update(sum_eq[1])
|
||||||
|
referenced.add(ask_entity)
|
||||||
|
|
||||||
|
symbols = [
|
||||||
|
SymbolBinding(
|
||||||
|
symbol_id=sid,
|
||||||
|
name=sid,
|
||||||
|
semantic_role=role_of.get(sid, "count"),
|
||||||
|
source_span=_span(sid),
|
||||||
|
introduced_by=_INTRODUCED_BY,
|
||||||
|
entity=sid,
|
||||||
|
unit=unit_of.get(sid),
|
||||||
|
)
|
||||||
|
for sid in sorted(referenced)
|
||||||
|
]
|
||||||
|
symbols_by_id = {s.symbol_id: s for s in symbols}
|
||||||
|
|
||||||
|
bound_facts = tuple(
|
||||||
|
BoundFact(symbol_id=f.entity, value=str(f.value), source_span=_span(f.entity), unit=f.unit)
|
||||||
|
for f in facts
|
||||||
|
)
|
||||||
|
|
||||||
|
# equations: shell -> REAL admissibility -> rebuild (NEVER stamp "admitted").
|
||||||
|
eq_specs: list[tuple[str, str, frozenset[str], str]] = [
|
||||||
|
(e.entity, _rhs(e.op, e.ref, e.delta), frozenset({e.ref}), e.op) for e in eqs
|
||||||
|
]
|
||||||
|
if sum_eq is not None:
|
||||||
|
lhs, parts = sum_eq
|
||||||
|
eq_specs.append((lhs, " + ".join(parts), frozenset(parts), "add"))
|
||||||
|
|
||||||
|
equations: list[BoundEquation] = []
|
||||||
|
for lhs, rhs, deps, op in eq_specs:
|
||||||
|
shell = BoundEquation(
|
||||||
|
lhs_symbol_id=lhs,
|
||||||
|
rhs_canonical=rhs,
|
||||||
|
dependencies=deps,
|
||||||
|
operation_kind=op,
|
||||||
|
unit_proof="pending",
|
||||||
|
admissibility_status="pending",
|
||||||
|
source_span=_span(lhs),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
proof = check_admissibility(shell, symbols=symbols_by_id)
|
||||||
|
except AdmissibilityError as exc:
|
||||||
|
return Refusal("admissibility_refused", f"{lhs}: {exc.reason}")
|
||||||
|
equations.append(
|
||||||
|
BoundEquation(
|
||||||
|
lhs_symbol_id=lhs,
|
||||||
|
rhs_canonical=rhs,
|
||||||
|
dependencies=deps,
|
||||||
|
operation_kind=op,
|
||||||
|
unit_proof=proof.to_canonical_string(),
|
||||||
|
admissibility_status="admitted",
|
||||||
|
source_span=_span(lhs),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
graph = SemanticSymbolicBindingGraph(
|
||||||
|
symbols=tuple(symbols), facts=bound_facts, equations=tuple(equations)
|
||||||
|
)
|
||||||
|
except Exception as exc: # noqa: BLE001 — surface construction refusal
|
||||||
|
return Refusal("invalid_binding_graph", repr(exc))
|
||||||
|
|
||||||
|
return QuantComprehension(
|
||||||
|
binding_graph=graph,
|
||||||
|
query=QuantQuery(entity=ask_entity, unit=ask_unit, span=_span(ask_entity)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def to_relational_metric(
|
||||||
|
comp: QuantComprehension,
|
||||||
|
) -> tuple[list[dict[str, Any]], dict[str, Any]] | None:
|
||||||
|
"""Project the comprehended binding_graph into ``(relations, query)`` for
|
||||||
|
``evals.relational_metric.oracle.oracle_answer``.
|
||||||
|
|
||||||
|
Reads the binding-graph itself (facts + admitted equations) — the equation's
|
||||||
|
own ``rhs_canonical`` is parsed back (a controlled round-trip of the format this
|
||||||
|
module emits) and operands are classified as symbol (a known entity) vs literal.
|
||||||
|
Facts are emitted before equations and equations in dependency order, so the
|
||||||
|
oracle's forward substitution never hits an unresolved reference.
|
||||||
|
"""
|
||||||
|
graph = comp.binding_graph
|
||||||
|
symbol_ids = {s.symbol_id for s in graph.symbols}
|
||||||
|
relations: list[dict[str, Any]] = [
|
||||||
|
{"kind": "fact", "entity": f.symbol_id, "value": int(f.value)} for f in graph.facts
|
||||||
|
]
|
||||||
|
for eq in graph.equations:
|
||||||
|
rhs = eq.rhs_canonical
|
||||||
|
if " + " in rhs:
|
||||||
|
operands = rhs.split(" + ")
|
||||||
|
if all(op in symbol_ids for op in operands):
|
||||||
|
relations.append({"kind": "sum_of", "entity": eq.lhs_symbol_id, "parts": list(operands)})
|
||||||
|
continue
|
||||||
|
ref, literal = operands[0], operands[1]
|
||||||
|
relations.append(
|
||||||
|
{"kind": "more_than", "entity": eq.lhs_symbol_id, "ref": ref, "delta": int(literal)}
|
||||||
|
)
|
||||||
|
elif " - " in rhs:
|
||||||
|
ref, literal = rhs.split(" - ")
|
||||||
|
relations.append(
|
||||||
|
{"kind": "fewer_than", "entity": eq.lhs_symbol_id, "ref": ref, "delta": int(literal)}
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return None # unrecognized equation shape -> refuse
|
||||||
|
if not relations:
|
||||||
|
return None
|
||||||
|
return relations, {"entity": comp.query.entity, "unit": comp.query.unit}
|
||||||
|
|
@ -1144,6 +1144,19 @@ INDEPENDENT_GOLD_LANES: tuple[IndependentGoldLane, ...] = (
|
||||||
oracle_module="evals/deductive_logic/oracle.py",
|
oracle_module="evals/deductive_logic/oracle.py",
|
||||||
sut_import_prefixes=("generate.meaning_graph",),
|
sut_import_prefixes=("generate.meaning_graph",),
|
||||||
),
|
),
|
||||||
|
# The arithmetic comprehension lane: the SUT is the quantity reader
|
||||||
|
# (generate.quantitative_comprehension) and the binding-graph substrate it
|
||||||
|
# builds. The relational_metric oracle (forward substitution) must share no code
|
||||||
|
# with either — the arithmetic gold stays independent of the reader.
|
||||||
|
IndependentGoldLane(
|
||||||
|
name="comprehension_relational_metric",
|
||||||
|
oracle_module="evals/relational_metric/oracle.py",
|
||||||
|
sut_import_prefixes=(
|
||||||
|
"generate.quantitative_comprehension",
|
||||||
|
"generate.binding_graph",
|
||||||
|
"generate.meaning_graph",
|
||||||
|
),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
_DEDUCTIVE_CASE_FILES: tuple[str, ...] = (
|
_DEDUCTIVE_CASE_FILES: tuple[str, ...] = (
|
||||||
|
|
|
||||||
|
|
@ -97,7 +97,7 @@ def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None:
|
||||||
idx = aggregate(list(collection.results))
|
idx = aggregate(list(collection.results))
|
||||||
assert idx.wrong_total == 0
|
assert idx.wrong_total == 0
|
||||||
assert idx.assert_mode_valid
|
assert idx.assert_mode_valid
|
||||||
assert idx.breadth == 7
|
assert idx.breadth == 8
|
||||||
assert {d.domain for d in idx.domains} == {
|
assert {d.domain for d in idx.domains} == {
|
||||||
"deductive_logic",
|
"deductive_logic",
|
||||||
"dimensional",
|
"dimensional",
|
||||||
|
|
@ -106,6 +106,7 @@ def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None:
|
||||||
"comprehension_syllogism",
|
"comprehension_syllogism",
|
||||||
"comprehension_total_ordering",
|
"comprehension_total_ordering",
|
||||||
"comprehension_propositional",
|
"comprehension_propositional",
|
||||||
|
"comprehension_relational_metric",
|
||||||
}
|
}
|
||||||
assert idx.capability_score > 0.5 # real, non-trivial cross-domain capability
|
assert idx.capability_score > 0.5 # real, non-trivial cross-domain capability
|
||||||
|
|
||||||
|
|
|
||||||
30
tests/test_comprehension_relational_metric.py
Normal file
30
tests/test_comprehension_relational_metric.py
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
"""Phase 2b — end-to-end: the comprehension reader scored on relational_metric.
|
||||||
|
|
||||||
|
prose -> comprehend_quantitative -> binding_graph -> to_relational_metric ->
|
||||||
|
INDEPENDENT arithmetic oracle -> answer vs gold. The load-bearing invariant:
|
||||||
|
wrong == 0. This is the binding-graph's first comprehension consumer; quantities
|
||||||
|
are admissibility-checked (never stamped), then projected to the oracle.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from evals.comprehension.relational_metric_runner import run
|
||||||
|
|
||||||
|
|
||||||
|
def test_comprehension_relational_metric_wrong_is_zero() -> None:
|
||||||
|
report = run()
|
||||||
|
assert report["wrong"] == 0, report["wrongs"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_comprehension_relational_metric_has_real_coverage() -> None:
|
||||||
|
report = run()
|
||||||
|
assert report["correct"] > 0
|
||||||
|
assert report["correct"] + report["refused"] == report["total"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_comprehension_relational_metric_full_coverage() -> None:
|
||||||
|
# The whole 15-case lane reads end-to-end (fact / more_than / fewer_than /
|
||||||
|
# sum_of, single + chained, count nouns -> item dimension, dollars -> money).
|
||||||
|
report = run()
|
||||||
|
assert report["refused"] == 0
|
||||||
|
assert report["correct"] == report["total"]
|
||||||
|
|
@ -26,6 +26,7 @@ from generate.meaning_graph.projectors import (
|
||||||
to_total_ordering,
|
to_total_ordering,
|
||||||
)
|
)
|
||||||
from generate.meaning_graph.reader import Refusal, comprehend
|
from generate.meaning_graph.reader import Refusal, comprehend
|
||||||
|
from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric
|
||||||
|
|
||||||
_TERMS = [f"t{i}" for i in range(8)]
|
_TERMS = [f"t{i}" for i in range(8)]
|
||||||
|
|
||||||
|
|
@ -263,3 +264,42 @@ def test_propositional_invariant_to_premise_reorder() -> None:
|
||||||
s_base = _struct(base, to_deductive_logic)
|
s_base = _struct(base, to_deductive_logic)
|
||||||
assert s_base is not None
|
assert s_base is not None
|
||||||
assert _struct(swapped, to_deductive_logic) == s_base
|
assert _struct(swapped, to_deductive_logic) == s_base
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# Arithmetic (binding_graph) — projected relations + query preserved exactly.
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
|
||||||
|
def test_arithmetic_structure_is_preserved_exactly() -> None:
|
||||||
|
rng = random.Random(55)
|
||||||
|
committed = 0
|
||||||
|
for _ in range(300):
|
||||||
|
ents = rng.sample([f"e{i}" for i in range(6)], rng.randint(2, 4))
|
||||||
|
base, base_val = ents[0], rng.randint(1, 20)
|
||||||
|
relations = [{"kind": "fact", "entity": base, "value": base_val}]
|
||||||
|
lines = [f"{base} has {base_val} things."]
|
||||||
|
prev = base
|
||||||
|
for e in ents[1:]:
|
||||||
|
delta = rng.randint(1, 15)
|
||||||
|
kind = rng.choice(["more_than", "fewer_than"])
|
||||||
|
word = "more" if kind == "more_than" else "fewer"
|
||||||
|
relations.append({"kind": kind, "entity": e, "ref": prev, "delta": delta})
|
||||||
|
lines.append(f"{e} has {delta} {word} things than {prev}.")
|
||||||
|
prev = e
|
||||||
|
ask = rng.choice(ents)
|
||||||
|
lines.append(f"How many things does {ask} have?")
|
||||||
|
prose = " ".join(lines)
|
||||||
|
expected_query = {"entity": ask, "unit": "item"} # "things" -> item dimension
|
||||||
|
|
||||||
|
comp = comprehend_quantitative(prose)
|
||||||
|
if isinstance(comp, Refusal):
|
||||||
|
continue
|
||||||
|
proj = to_relational_metric(comp)
|
||||||
|
if proj is None:
|
||||||
|
continue
|
||||||
|
committed += 1
|
||||||
|
prelations, pquery = proj
|
||||||
|
assert _canon(prelations) == _canon(relations), (prose, prelations, relations)
|
||||||
|
assert pquery == expected_query, (prose, pquery, expected_query)
|
||||||
|
assert committed > 50
|
||||||
|
|
|
||||||
99
tests/test_quantitative_comprehension.py
Normal file
99
tests/test_quantitative_comprehension.py
Normal file
|
|
@ -0,0 +1,99 @@
|
||||||
|
"""Unit tests for the arithmetic reader (prose -> binding_graph) + its projector.
|
||||||
|
|
||||||
|
Pins the templates, the count-vs-physical-unit modelling, and — load-bearing — the
|
||||||
|
REAL admissibility check: an equation is admitted only if its operand units verify,
|
||||||
|
so a mixed-unit sum REFUSES rather than fabricating a quantity. This is the
|
||||||
|
reviewer's "do not stamp admissibility" guard, made executable.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from generate.binding_graph.model import SemanticSymbolicBindingGraph
|
||||||
|
from generate.meaning_graph.reader import Refusal
|
||||||
|
from generate.quantitative_comprehension import (
|
||||||
|
QuantComprehension,
|
||||||
|
comprehend_quantitative,
|
||||||
|
to_relational_metric,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _comp(text: str) -> QuantComprehension:
|
||||||
|
comp = comprehend_quantitative(text)
|
||||||
|
assert isinstance(comp, QuantComprehension), comp
|
||||||
|
return comp
|
||||||
|
|
||||||
|
|
||||||
|
def test_fact_and_more_than_build_binding_graph() -> None:
|
||||||
|
comp = _comp("Liam has 6 stickers. Mia has 4 more stickers than Liam. How many stickers does Mia have?")
|
||||||
|
g = comp.binding_graph
|
||||||
|
assert isinstance(g, SemanticSymbolicBindingGraph)
|
||||||
|
assert {f.symbol_id: f.value for f in g.facts} == {"liam": "6"}
|
||||||
|
eq = next(e for e in g.equations if e.lhs_symbol_id == "mia")
|
||||||
|
assert eq.operation_kind == "add"
|
||||||
|
assert eq.rhs_canonical == "liam + 4"
|
||||||
|
assert eq.admissibility_status == "admitted" # from the REAL check, not stamped
|
||||||
|
assert comp.query.entity == "mia"
|
||||||
|
|
||||||
|
|
||||||
|
def test_count_nouns_resolve_to_item_dimension() -> None:
|
||||||
|
# Unknown sortal nouns become the count dimension (item); admissibility admits.
|
||||||
|
comp = _comp("Kim has 2 marbles. Leo has 3 more marbles than Kim. How many marbles does Leo have?")
|
||||||
|
units = {s.symbol_id: s.unit for s in comp.binding_graph.symbols}
|
||||||
|
assert units["kim"] == "item" and units["leo"] == "item"
|
||||||
|
|
||||||
|
|
||||||
|
def test_known_unit_is_used_verbatim() -> None:
|
||||||
|
comp = _comp("Iris has 100 dollars. Jack has 250 more dollars than Iris. How many dollars does Jack have?")
|
||||||
|
units = {s.symbol_id: s.unit for s in comp.binding_graph.symbols}
|
||||||
|
assert units["iris"] == "dollars" # parse_unit depluralizes dollars -> dollar (money)
|
||||||
|
|
||||||
|
|
||||||
|
def test_fewer_than_is_subtract() -> None:
|
||||||
|
comp = _comp("Noah has 15 cards. Olivia has 6 fewer cards than Noah. How many cards does Olivia have?")
|
||||||
|
eq = next(e for e in comp.binding_graph.equations if e.lhs_symbol_id == "olivia")
|
||||||
|
assert eq.operation_kind == "subtract" and eq.rhs_canonical == "noah - 6"
|
||||||
|
|
||||||
|
|
||||||
|
def test_sum_query_synthesizes_total() -> None:
|
||||||
|
comp = _comp("Dan has 7 coins. Eva has 9 more coins than Dan. How many coins do Dan and Eva have?")
|
||||||
|
assert comp.query.entity == "total"
|
||||||
|
total_eq = next(e for e in comp.binding_graph.equations if e.lhs_symbol_id == "total")
|
||||||
|
assert total_eq.operation_kind == "add"
|
||||||
|
assert set(total_eq.dependencies) == {"dan", "eva"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_projection_shape() -> None:
|
||||||
|
comp = _comp("Liam has 6 stickers. Mia has 4 more stickers than Liam. How many stickers does Mia have?")
|
||||||
|
projected = to_relational_metric(comp)
|
||||||
|
assert projected is not None
|
||||||
|
relations, query = projected
|
||||||
|
assert {"kind": "fact", "entity": "liam", "value": 6} in relations
|
||||||
|
assert {"kind": "more_than", "entity": "mia", "ref": "liam", "delta": 4} in relations
|
||||||
|
assert query["entity"] == "mia"
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# Admissibility is REAL, not stamped (the reviewer's load-bearing guard)
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
|
||||||
|
def test_mixed_unit_sum_refuses_via_admissibility() -> None:
|
||||||
|
# count (stickers -> item) + money (dollars) cannot be summed: the REAL
|
||||||
|
# admissibility check must REFUSE, not fabricate a total.
|
||||||
|
comp = comprehend_quantitative(
|
||||||
|
"Liam has 6 stickers. Mia has 4 dollars. How many things do Liam and Mia have?"
|
||||||
|
)
|
||||||
|
assert isinstance(comp, Refusal)
|
||||||
|
assert comp.reason == "admissibility_refused"
|
||||||
|
assert "unit_mismatch" in comp.detail
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_digit_quantity_refuses() -> None:
|
||||||
|
comp = comprehend_quantitative("Liam has several stickers. How many stickers does Liam have?")
|
||||||
|
assert isinstance(comp, Refusal)
|
||||||
|
assert comp.reason == "non_digit_quantity"
|
||||||
|
|
||||||
|
|
||||||
|
def test_unreadable_clause_refuses() -> None:
|
||||||
|
comp = comprehend_quantitative("The weather is nice today.")
|
||||||
|
assert isinstance(comp, Refusal)
|
||||||
Loading…
Reference in a new issue