core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge pull request #612 from AssetOverflow/feat/r1-setup-gold

Browse source 
feat(comprehension): R1 gold (PR-5b) + the multiplicative comparative frame (PR-5c)
This commit is contained in:
Shay 2026-06-06 17:38:35 -07:00 committed by GitHub
commit 9a285dfbdb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 250 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
evals/setup_oracle/__init__.py
View file

@ -13,7 +13,7 @@ the problem. v1 grades structure (facts + equations + question target/state/form
modelling stays covered by the admissibility tests (a documented signature extension).
"""
from evals.setup_oracle.runner import run
from evals.setup_oracle.runner import run, run_r1
from evals.setup_oracle.signature import (
gold_unknown_signature,
reader_symbol_units,
@ -28,5 +28,6 @@ __all__ = [
"reader_unknown_signature",
"relation_signature",
"run",
"run_r1",
"symbol_unit_signature",
]

12
evals/setup_oracle/__main__.py
View file

@ -1,20 +1,24 @@
"""CLI: print the setup-oracle report.
python -m evals.setup_oracle
python -m evals.setup_oracle # the 15-case relational-metric setup gold
python -m evals.setup_oracle r1 # the independent R1 gold (PR-5b)
Exit 0 iff ``setup_wrong == 0`` — the gate the milestone rests on (a wrong reading must
never pass, and serving must not move while setup_wrong > 0).
never pass, and serving must not move while setup_wrong > 0). For ``r1`` the reader is
expected to REFUSE the unsupported shapes (setup_refused), never misread them.
"""
from __future__ import annotations
import json
import sys
from evals.setup_oracle.runner import run
from evals.setup_oracle.runner import run, run_r1
def main() -> int:
report = run()
lane = sys.argv[1] if len(sys.argv) > 1 else ""
report = run_r1() if lane == "r1" else run()
print(json.dumps(report, indent=2, default=str))
return 0 if report["setup_wrong"] == 0 else 1

10
evals/setup_oracle/r1_gold.jsonl Normal file
View file

@ -0,0 +1,10 @@
{"id": "r1-01-twice", "text": "Anna has 6 apples. Bella has twice as many apples as Anna. How many apples does Bella have?", "relations": [{"kind": "fact", "entity": "anna", "value": 6}, {"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2}], "expected_units": {"anna": "item", "bella": "item"}, "query": {"entity": "bella", "unit": "item"}, "notes": "Multiplicative: bella = 2 * anna. The reader has no multiplicative template AND 'twice' is non-digit -> must REFUSE, never misread as a fact/additive."}
{"id": "r1-02-half", "text": "Carl has 8 coins. Dora has half as many coins as Carl. How many coins does Dora have?", "relations": [{"kind": "fact", "entity": "carl", "value": 8}, {"kind": "times_as_many", "entity": "dora", "ref": "carl", "factor": 0.5}], "expected_units": {"carl": "item", "dora": "item"}, "query": {"entity": "dora", "unit": "item"}, "notes": "Multiplicative (factor 0.5). 'half' is non-digit -> must REFUSE."}
{"id": "r1-03-more-total", "text": "Finn has 10 books. Evan has 5 more books than Finn. How many books do Evan and Finn have altogether?", "relations": [{"kind": "fact", "entity": "finn", "value": 10}, {"kind": "more_than", "entity": "evan", "ref": "finn", "delta": 5}, {"kind": "sum_of", "entity": "total", "parts": ["evan", "finn"]}], "expected_units": {"finn": "item", "evan": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive + aggregate. 'altogether' (not the bare 'have') may fall outside the strict query template -> likely REFUSE; if read, structure must match exactly."}
{"id": "r1-04-fewer-total", "text": "Gail has 20 cards. Hank has 6 fewer cards than Gail. How many cards do Gail and Hank have in total?", "relations": [{"kind": "fact", "entity": "gail", "value": 20}, {"kind": "fewer_than", "entity": "hank", "ref": "gail", "delta": 6}, {"kind": "sum_of", "entity": "total", "parts": ["gail", "hank"]}], "expected_units": {"gail": "item", "hank": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive(fewer) + aggregate. 'in total' qualifier likely outside the query template -> REFUSE expected."}
{"id": "r1-05-chain", "text": "Ivy has 4 pens. Jon has 3 times as many pens as Ivy. Kim has 2 more pens than Jon. How many pens does Kim have?", "relations": [{"kind": "fact", "entity": "ivy", "value": 4}, {"kind": "times_as_many", "entity": "jon", "ref": "ivy", "factor": 3}, {"kind": "more_than", "entity": "kim", "ref": "jon", "delta": 2}], "expected_units": {"ivy": "item", "jon": "item", "kim": "item"}, "query": {"entity": "kim", "unit": "item"}, "notes": "Multi-step derived chain: jon=3*ivy (intermediate), kim=jon+2. The multiplicative middle step has no template -> must REFUSE the whole reading, never read a partial/wrong chain."}
{"id": "r1-06-subtotal-reused", "text": "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them equally into 3 boxes. How many hats are in each box?", "relations": [{"kind": "fact", "entity": "lee", "value": 5}, {"kind": "fact", "entity": "mae", "value": 7}, {"kind": "sum_of", "entity": "total", "parts": ["lee", "mae"]}, {"kind": "divide_by", "entity": "per_box", "ref": "total", "divisor": 3}], "expected_units": {"lee": "item", "mae": "item", "total": "item", "per_box": "item"}, "query": {"entity": "per_box", "unit": "item"}, "notes": "Derived subtotal reused downstream: total = lee + mae, per_box = total / 3. No partition template -> must REFUSE."}
{"id": "r1-07-inverse", "text": "Nia has 9 more beads than Omar. Nia has 15 beads. How many beads does Omar have?", "relations": [{"kind": "fact", "entity": "nia", "value": 15}, {"kind": "more_than", "entity": "nia", "ref": "omar", "delta": 9}, {"kind": "ask_base", "entity": "omar"}], "expected_units": {"nia": "item", "omar": "item"}, "query": {"entity": "omar", "unit": "item"}, "notes": "Inverse target: omar = nia - 9 = 6, asked via the base of a more_than. The reader may read the STRUCTURE (nia=15, nia=omar+9, ask omar) — that setup is correct; only the solver can't invert. So setup_correct OR refuse, but NEVER setup_wrong."}
{"id": "r1-08-ambiguous-referent", "text": "Pat has 5 marbles. He has 3 more than her. How many marbles does she have?", "relations": [], "expected_units": {}, "query": {"entity": "she", "unit": "item"}, "notes": "Ambiguous pronoun referents (he/her/she) with no grounded base for 'her'/'she' -> must REFUSE (no honest reading), never bind a guessed referent."}
{"id": "r1-09-missing-base", "text": "Quinn has twice as many toys as Rosa. How many toys does Quinn have?", "relations": [{"kind": "times_as_many", "entity": "quinn", "ref": "rosa", "factor": 2}], "expected_units": {"quinn": "item", "rosa": "item"}, "query": {"entity": "quinn", "unit": "item"}, "notes": "Missing base quantity: Rosa's count is never given, so Quinn is underdetermined -> must REFUSE."}
{"id": "r1-10-distractor", "text": "Sam has 7 pencils and 3 erasers. Tom has 4 more pencils than Sam. How many pencils does Tom have?", "relations": [{"kind": "fact", "entity": "sam", "value": 7}, {"kind": "more_than", "entity": "tom", "ref": "sam", "delta": 4}], "expected_units": {"sam": "item", "tom": "item"}, "query": {"entity": "tom", "unit": "item"}, "notes": "Distractor quantity (3 erasers) in a compound clause the 'X has N unit' template can't parse -> must REFUSE rather than mis-bind the distractor."}

79
evals/setup_oracle/runner.py
View file

@ -28,6 +28,7 @@ from generate.meaning_graph.reader import Refusal
from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric
_EXPECTED_UNITS_PATH = Path(__file__).resolve().parent / "expected_units.json"
_R1_GOLD_PATH = Path(__file__).resolve().parent / "r1_gold.jsonl"
def _load_expected_units() -> dict[str, dict[str, str]]:
@ -35,6 +36,14 @@ def _load_expected_units() -> dict[str, dict[str, str]]:
return {k: v for k, v in raw.items() if not k.startswith("_")}
def _load_r1_gold() -> list[dict[str, Any]]:
return [
json.loads(line)
for line in _R1_GOLD_PATH.read_text(encoding="utf-8").splitlines()
if line.strip()
]
def run() -> dict[str, Any]:
"""Score the reader's setup (structure + units + target) against the independent gold."""
cases = _load_cases()
@ -94,4 +103,72 @@ def run() -> dict[str, Any]:
}
__all__ = ["run"]
def run_r1() -> dict[str, Any]:
"""Score the CURRENT reader against the independent, self-contained R1 gold.
The reader cannot read most R1 shapes (multiplicative, multi-step, partition) yet.
The SUCCESS CRITERION is **``setup_wrong == 0``**: it must REFUSE every unsupported
shape, NEVER misread it as a simpler form. ``setup_refused`` is the unsupported count;
``setup_correct`` is any shape already faithfully supported. A ``setup_wrong`` is a
pre-existing wrong-reading hazard to fix BEFORE building the R1 frame (PR-5c).
"""
fixtures = _load_r1_gold()
setup_correct = setup_wrong = setup_refused = 0
details: list[dict[str, Any]] = []
for fx in fixtures:
comp = comprehend_quantitative(fx["text"])
if isinstance(comp, Refusal):
setup_refused += 1
details.append({"id": fx["id"], "outcome": "refused", "reason": comp.reason})
continue
projected = to_relational_metric(comp)
if projected is None:
setup_refused += 1
details.append({"id": fx["id"], "outcome": "refused", "reason": "unprojectable"})
continue
reader_relations, _ = projected
units = fx["expected_units"]
reader_rel = relation_signature(reader_relations)
gold_rel = relation_signature(fx["relations"])
reader_units = reader_symbol_units(comp.binding_graph)
gold_units = symbol_unit_signature(units)
reader_unk = reader_unknown_signature(comp.binding_graph)
gold_unk = gold_unknown_signature(fx["relations"], fx["query"], units)
if reader_rel == gold_rel and reader_units == gold_units and reader_unk == gold_unk:
setup_correct += 1
details.append({"id": fx["id"], "outcome": "correct"})
else:
setup_wrong += 1
details.append(
{
"id": fx["id"],
"outcome": "WRONG",
"relations_match": reader_rel == gold_rel,
"units_match": reader_units == gold_units,
"target_match": reader_unk == gold_unk,
"reader_relations": reader_rel,
"gold_relations": gold_rel,
"reader_target": reader_unk,
"gold_target": gold_unk,
}
)
return {
"lane": "setup_oracle_r1",
"total": len(fixtures),
"setup_correct": setup_correct,
"setup_wrong": setup_wrong,
"setup_refused": setup_refused,
"details": details,
"counts": {
"setup_correct": setup_correct,
"setup_wrong": setup_wrong,
"setup_refused": setup_refused,
},
}
__all__ = ["run", "run_r1"]

2
evals/setup_oracle/signature.py
View file

@ -43,6 +43,8 @@ def relation_signature(relations: list[dict[str, Any]]) -> tuple[tuple, ...]:
out.append(("fact", r["entity"], int(r["value"])))
elif kind in ("more_than", "fewer_than"):
out.append((kind, r["entity"], r["ref"], int(r["delta"])))
elif kind == "times_as_many":
out.append(("times_as_many", r["entity"], r["ref"], r["factor"]))
elif kind == "sum_of":
out.append(("sum_of", r["entity"], tuple(sorted(r["parts"]))))
else: # an unknown relation kind is itself a structural difference, not a crash

61
generate/quantitative_comprehension.py
View file

@ -45,6 +45,7 @@ from generate.quantitative_expr import (
Add,
Expr,
Literal,
Mul,
Sub,
SumOf,
Symbol,
@ -143,8 +144,51 @@ class _Eq:
unit: str
@dataclass(frozen=True, slots=True)
class _Mul:
"""Multiplicative comparative: entity = factor * ref (R1)."""
entity: str
ref: str
factor: int
unit: str
#: Word factors for "twice/double/triple ... as many". 'half' (a /2, the divide path)
#: is deliberately ABSENT — divide-by-literal is a separate admissibility path, deferred.
_FACTOR_WORDS: dict[str, int] = {"twice": 2, "double": 2, "triple": 3, "quadruple": 4}
def _try_multiplicative(entity: str, toks: list[str], detail: str) -> "_Mul | None":
"""Match "Y has <factor-word> as many <unit> as X" or "Y has <N> times as many
<unit> as X" → ``_Mul``. Returns None if the clause is not multiplicative (the
caller then tries the digit-led fact/additive templates)."""
# [Y, has, FACTORWORD, as, many, UNIT, as, X]
if (
len(toks) == 8
and toks[2] in _FACTOR_WORDS
and toks[3] == "as"
and toks[4] == "many"
and toks[6] == "as"
):
return _Mul(entity, _ident(toks[7], detail), _FACTOR_WORDS[toks[2]],
_resolve_unit(_ident(toks[5], detail)))
# [Y, has, N, times, as, many, UNIT, as, X]
if (
len(toks) == 9
and toks[2].isdigit()
and toks[3] == "times"
and toks[4] == "as"
and toks[5] == "many"
and toks[7] == "as"
):
return _Mul(entity, _ident(toks[8], detail), int(toks[2]),
_resolve_unit(_ident(toks[6], detail)))
return None
def _parse_sentence(body: str, detail: str):
"""Return a (_Fact | _Eq | ('query', entity, unit) | ('sumquery', parts, unit))
"""Return a (_Fact | _Eq | _Mul | ('query', entity, unit) | ('sumquery', parts, unit))
spec, or None if the sentence matches no arithmetic template."""
toks = body.strip().lower().rstrip("?.!").split()
if not toks:
@ -163,6 +207,11 @@ def _parse_sentence(body: str, detail: str):
if len(toks) >= 4 and toks[1] == "has":
entity = _ident(toks[0], detail)
# Multiplicative comparative is checked BEFORE the digit gate (its factor may be
# a word like "twice", which is not a digit).
mul = _try_multiplicative(entity, toks, detail)
if mul is not None:
return mul
value = _int(toks[2], detail)
if len(toks) == 4:
return _Fact(entity, value, _resolve_unit(_ident(toks[3], detail)))
@ -190,6 +239,7 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh
facts: list[_Fact] = []
eqs: list[_Eq] = []
muls: list[_Mul] = []
queries: list[tuple] = []
try:
@ -201,6 +251,8 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh
facts.append(spec)
elif isinstance(spec, _Eq):
eqs.append(spec)
elif isinstance(spec, _Mul):
muls.append(spec)
else:
queries.append(spec)
except _QReject as rej:
@ -215,6 +267,8 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh
unit_of[f.entity], role_of[f.entity] = f.unit, "count"
for e in eqs:
unit_of[e.entity], role_of[e.entity] = e.unit, "count"
for m in muls:
unit_of[m.entity], role_of[m.entity] = m.unit, "count"
query = queries[0]
sum_eq: tuple[str, tuple[str, ...]] | None = None
@ -232,6 +286,8 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh
referenced.add(f.entity)
for e in eqs:
referenced.update((e.entity, e.ref))
for m in muls:
referenced.update((m.entity, m.ref))
if sum_eq is not None:
referenced.add(sum_eq[0])
referenced.update(sum_eq[1])
@ -262,6 +318,9 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh
(e.entity, (Add if e.op == "add" else Sub)(Symbol(e.ref), Literal(e.delta)))
for e in eqs
]
expr_specs.extend(
(m.entity, Mul(Symbol(m.ref), Literal(m.factor))) for m in muls
)
if sum_eq is not None:
lhs, parts = sum_eq
expr_specs.append((lhs, SumOf(tuple(Symbol(p) for p in parts))))

21
generate/quantitative_expr.py
View file

@ -43,6 +43,16 @@ class Sub:
right: "Expr"
@dataclass(frozen=True, slots=True)
class Mul:
"""A scalar multiple of a symbol — the multiplicative comparative ("twice/N times
as many"). ``left`` is the referenced symbol, ``right`` a dimensionless literal
factor; the product keeps the symbol's unit (``count × scalar = count``)."""
left: "Expr"
right: "Expr"
@dataclass(frozen=True, slots=True)
class SumOf:
"""An aggregate over ≥2 symbols (the part-whole total)."""
@ -50,7 +60,7 @@ class SumOf:
parts: tuple[Symbol, ...]
Expr = Union[Literal, Symbol, Add, Sub, SumOf]
Expr = Union[Literal, Symbol, Add, Sub, Mul, SumOf]
def to_canonical_string(expr: Expr) -> str:
@ -64,6 +74,8 @@ def to_canonical_string(expr: Expr) -> str:
return f"{to_canonical_string(left)} + {to_canonical_string(right)}"
case Sub(left, right):
return f"{to_canonical_string(left)} - {to_canonical_string(right)}"
case Mul(left, right):
return f"{to_canonical_string(left)} * {to_canonical_string(right)}"
case SumOf(parts):
return " + ".join(to_canonical_string(p) for p in parts)
raise TypeError(f"not an Expr: {expr!r}") # pragma: no cover - exhaustive above
@ -76,7 +88,7 @@ def dependencies(expr: Expr) -> frozenset[str]:
return frozenset()
case Symbol(symbol_id):
return frozenset({symbol_id})
case Add(left, right) | Sub(left, right):
case Add(left, right) | Sub(left, right) | Mul(left, right):
return dependencies(left) | dependencies(right)
case SumOf(parts):
out: frozenset[str] = frozenset()
@ -93,6 +105,8 @@ def operation_kind(expr: Expr) -> str:
return "add"
case Sub(_, _):
return "subtract"
case Mul(_, _):
return "multiply"
case _:
raise TypeError(f"expression has no operation_kind: {expr!r}")
@ -108,6 +122,8 @@ def to_relation(lhs: str, expr: Expr) -> dict[str, Any] | None:
return {"kind": "more_than", "entity": lhs, "ref": ref, "delta": delta}
case Sub(Symbol(ref), Literal(delta)):
return {"kind": "fewer_than", "entity": lhs, "ref": ref, "delta": delta}
case Mul(Symbol(ref), Literal(factor)):
return {"kind": "times_as_many", "entity": lhs, "ref": ref, "factor": factor}
case SumOf(parts):
return {"kind": "sum_of", "entity": lhs, "parts": [p.symbol_id for p in parts]}
case _:
@ -118,6 +134,7 @@ __all__ = [
"Add",
"Expr",
"Literal",
"Mul",
"Sub",
"SumOf",
"Symbol",

27
tests/test_quantitative_comprehension.py
View file

@ -169,3 +169,30 @@ def test_to_relational_metric_refuses_malformed_target() -> None:
for n in (0, 2):
comp = QuantComprehension(binding_graph=_graph_with_n_unknowns(n))
assert to_relational_metric(comp) is None # refuse rather than emit a guessed query
# --------------------------------------------------------------------------- #
# PR-5c — the multiplicative comparative frame ("twice / N times as many")
# --------------------------------------------------------------------------- #
def test_twice_as_many_builds_multiply_equation() -> None:
comp = _comp("Anna has 6 apples. Bella has twice as many apples as Anna. How many apples does Bella have?")
eq = next(e for e in comp.binding_graph.equations if e.lhs_symbol_id == "bella")
assert eq.operation_kind == "multiply"
assert eq.rhs_canonical == "anna * 2"
assert eq.admissibility_status == "admitted" # count * scalar = count, REAL check
assert single_unknown(comp.binding_graph).symbol_id == "bella"
def test_n_times_as_many_builds_multiply_equation() -> None:
comp = _comp("Ivy has 4 pens. Jon has 3 times as many pens as Ivy. How many pens does Jon have?")
eq = next(e for e in comp.binding_graph.equations if e.lhs_symbol_id == "jon")
assert eq.operation_kind == "multiply" and eq.rhs_canonical == "ivy * 3"
def test_multiplicative_missing_base_refuses() -> None:
# "twice as many as Rosa" with no value for Rosa -> Rosa is ungrounded -> REFUSE,
# never fabricate a base quantity.
comp = comprehend_quantitative("Quinn has twice as many toys as Rosa. How many toys does Quinn have?")
assert isinstance(comp, Refusal)

17
tests/test_quantitative_expr.py
View file

@ -69,3 +69,20 @@ def test_reader_carries_ir_consistent_with_rhs_canonical() -> None:
for eq in comp.binding_graph.equations:
assert to_canonical_string(by_lhs[eq.lhs_symbol_id]) == eq.rhs_canonical
assert dependencies(by_lhs[eq.lhs_symbol_id]) == eq.dependencies
# --------------------------------------------------------------------------- #
# PR-5c — the multiplicative comparative (Mul)
# --------------------------------------------------------------------------- #
def test_mul_serialization_and_derivations() -> None:
from generate.quantitative_expr import Mul
m = Mul(Symbol("anna"), Literal(2))
assert to_canonical_string(m) == "anna * 2"
assert dependencies(m) == frozenset({"anna"})
assert operation_kind(m) == "multiply"
assert to_relation("bella", m) == {
"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2,
}

27
tests/test_setup_oracle.py
View file

@ -140,3 +140,30 @@ def test_reader_units_read_from_the_binding_graph() -> None:
)
assert reader_symbol_units(graph) == (("iris", "dollars"), ("jack", "dollars"))
assert reader_unknown_signature(graph) == ("jack", "terminal", "count", "dollars")
# --------------------------------------------------------------------------- #
# PR-5b — independent R1 gold: the reader must REFUSE, never MISREAD
# --------------------------------------------------------------------------- #
def test_r1_multiplicative_supported_rest_refused_wrong_zero() -> None:
from evals.setup_oracle import run_r1
r = run_r1()
assert r["total"] == 10
# THE invariant through the first capability slice: NO R1 case is misread. Adding the
# multiplicative frame turned refusals into correct readings without any setup_wrong.
assert r["setup_wrong"] == 0
# The multiplicative frame (PR-5c) reads "twice as many" (r1-01) and the multi-step
# chain whose middle step is "N times as many" (r1-05); the rest stay safe refusals.
by_id = {d["id"]: d["outcome"] for d in r["details"]}
assert by_id["r1-01-twice"] == "correct"
assert by_id["r1-05-chain"] == "correct"
assert r["setup_correct"] == 2
assert r["setup_refused"] == 8
# No detail is ever WRONG, and every non-correct one is a typed refusal.
for d in r["details"]:
assert d["outcome"] in ("correct", "refused")
if d["outcome"] == "refused":
assert d.get("reason")