feat: dimensional-reasoning lane — 3rd diversity-panel domain

The first non-GSM8K consumer of the binding-graph interlingua's unit algebra as a
load-bearing reasoner: given two units and an operation, decide the result's
dimension. SUT = generate.binding_graph.units; gold = evals/dimensional/oracle.py,
a genuinely INDEPENDENT dimensional reasoner (own unit->base-exponent table, own
exponent arithmetic, own canonical-string renderer; shares no code with the SUT).

12 cases (area / speed / wage / mass-density / dimensionless / 2 refused) gated by
SUT == oracle == gold (wrong=0). Registered in INV-25's INDEPENDENT_GOLD_LANES,
proving the independent-gold discipline generalizes to a SECOND oracle.

This is the 3rd structurally-distinct golded domain (logic / grounding / dimensional)
— the anti-overfit >=2-domain panel is now real, and the interlingua is load-bearing
beyond GSM8K.
This commit is contained in:
Shay 2026-06-04 16:38:56 -07:00
parent 4d964020a9
commit 96c1d4bcee
6 changed files with 279 additions and 0 deletions

View file

@ -0,0 +1,7 @@
"""Dimensional-reasoning lane — the third diversity-panel domain.
The interlingua's unit algebra (``generate.binding_graph.units``) is the system
under test; an independent dimensional oracle (``evals.dimensional.oracle``) is the
gold. Structurally distinct from logic/grounding (dimensional exponent arithmetic),
it exercises the binding-graph interlingua as a load-bearing reasoner beyond GSM8K.
"""

View file

@ -0,0 +1,99 @@
"""Independent dimensional oracle — the gold for the dimensional-reasoning lane.
This is **deliberately a second, independent decision procedure** for "what is the
dimension of ``left <op> right``?". It carries its **own** unit→base-exponent table
and its **own** exponent arithmetic and canonical-string renderer, and shares
**no code** with :mod:`generate.binding_graph.units` (the system under test). Two
independent dimensional reasoners agreeing on the same cases is real evidence the
interlingua's unit algebra is correct; a shared-code "oracle" would only prove the
algebra agrees with itself (INV-25).
Base-dimension order matches the SUT's so the rendered strings are directly
comparable (the comparison is about the *decision*, not the spelling).
"""
from __future__ import annotations
from typing import Final
# Same axis order as the SUT's BASE_DIMENSIONS so canonical strings line up.
_BASE: Final[tuple[str, ...]] = ("length", "time", "mass", "money", "count", "temperature")
# An INDEPENDENT unit→base-exponent table (hand-authored; NOT the en_units_v1 pack
# the SUT reads). Only concrete base units; composites are resolved structurally.
_UNIT_DIMS: Final[dict[str, dict[str, int]]] = {
"meter": {"length": 1}, "mile": {"length": 1}, "foot": {"length": 1},
"second": {"time": 1}, "hour": {"time": 1}, "minute": {"time": 1}, "day": {"time": 1},
"kilogram": {"mass": 1}, "pound": {"mass": 1},
"dollar": {"money": 1},
"degree": {"temperature": 1},
}
class OracleError(ValueError):
"""Unit outside the oracle's independent vocabulary — refuse, never guess."""
def _depluralize(unit: str) -> str:
"""Independent conservative plural strip (mirrors the SUT's contract, own code)."""
if unit in _UNIT_DIMS:
return unit
for cand in (
unit[:-3] + "y" if unit.endswith("ies") and len(unit) > 3 else "",
unit[:-2] if unit.endswith("es") and len(unit) > 2 else "",
unit[:-1] if unit.endswith("s") and len(unit) > 1 else "",
):
if cand and cand in _UNIT_DIMS:
return cand
return unit
def _vector(unit: str) -> tuple[int, ...]:
"""Resolve a unit id (incl. plural and ``X_per_Y`` composites) to an exponent
tuple over ``_BASE``. Raises :class:`OracleError` on anything else."""
if not isinstance(unit, str) or not unit:
raise OracleError(f"empty unit: {unit!r}")
canon = _depluralize(unit)
if canon in _UNIT_DIMS:
dims = _UNIT_DIMS[canon]
return tuple(dims.get(b, 0) for b in _BASE)
if "_per_" in unit:
num, _, denom = unit.partition("_per_")
nv = _vector(num)
dv = _vector(denom)
return tuple(a - b for a, b in zip(nv, dv))
raise OracleError(f"unknown_unit: {unit!r}")
def _render(exps: tuple[int, ...]) -> str:
"""Canonical dimension string, format-compatible with the SUT's renderer."""
nums: list[str] = []
dens: list[str] = []
for dim, e in zip(_BASE, exps):
if e > 0:
nums.append(dim if e == 1 else f"{dim}^{e}")
elif e < 0:
dens.append(dim if e == -1 else f"{dim}^{-e}")
if not nums and not dens:
return "dimensionless"
num_part = "*".join(nums) if nums else "1"
return num_part if not dens else f"{num_part}/{'*'.join(dens)}"
def dimensional_result(op: str, left: str, right: str) -> str:
"""The oracle verdict: the canonical dimension of ``left <op> right``.
``op`` is ``"product"`` or ``"quotient"``. Returns ``"refused"`` if either
unit is outside the oracle's vocabulary or ``op`` is unknown — the same
refusal posture as the SUT.
"""
try:
lv = _vector(left)
rv = _vector(right)
except OracleError:
return "refused"
if op == "product":
return _render(tuple(a + b for a, b in zip(lv, rv)))
if op == "quotient":
return _render(tuple(a - b for a, b in zip(lv, rv)))
return "refused"

View file

@ -0,0 +1,80 @@
"""Dimensional-reasoning lane runner — the interlingua's unit algebra vs. the
independent dimensional oracle.
The system under test is the binding-graph interlingua (``generate.binding_graph.
units``): given two unit ids and an operation, what is the dimension of the result?
Each committed case is scored as ``correct`` (SUT == gold), ``wrong`` (SUT
committed a different dimension than gold MUST stay 0), or ``refused`` (SUT
declined a unit outside the closed vocabulary). The gold is the independent oracle
verdict, frozen into ``cases.jsonl`` at generation (INV-25).
"""
from __future__ import annotations
import json
from collections import Counter
from pathlib import Path
from generate.binding_graph.units import (
UnitAlgebraError,
parse_unit,
unit_product,
unit_quotient,
)
_ROOT = Path(__file__).resolve().parent
_REFUSED = "refused"
def decide(op: str, left: str, right: str) -> str:
"""The SUT verdict: the canonical dimension of ``left <op> right`` via the
interlingua's unit algebra, or ``"refused"`` for an unknown unit / op."""
try:
lv = parse_unit(left)
rv = parse_unit(right)
except UnitAlgebraError:
return _REFUSED
if op == "product":
return unit_product(lv, rv).to_canonical_string()
if op == "quotient":
return unit_quotient(lv, rv).to_canonical_string()
return _REFUSED
def _load(path: Path) -> list[dict]:
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
def build_report(cases: list[dict]) -> dict:
counts: Counter[str] = Counter({"correct": 0, "wrong": 0, "refused": 0})
wrong_examples: list[dict] = []
for case in cases:
gold = case["gold"]
got = decide(case["op"], case["left"], case["right"])
if got == gold:
counts["refused" if gold == _REFUSED else "correct"] += 1
else:
counts["wrong"] += 1
if len(wrong_examples) < 10:
wrong_examples.append({"id": case["id"], "gold": gold, "got": got})
return {
"n": len(cases),
"counts": dict(counts),
"all_correct": counts["wrong"] == 0,
"wrong_examples": wrong_examples,
}
def main() -> int:
report = build_report(_load(_ROOT / "v1" / "cases.jsonl"))
c = report["counts"]
print(f"[dimensional] n={report['n']} correct={c['correct']} wrong={c['wrong']} refused={c['refused']}")
for w in report["wrong_examples"]:
print(f" WRONG {w['id']}: gold={w['gold']} got={w['got']}")
return 0 if report["all_correct"] else 1
if __name__ == "__main__":
import sys
sys.exit(main())

View file

@ -0,0 +1,12 @@
{"gold": "length^2", "id": "dim-v1-0001", "left": "meter", "op": "product", "right": "meter"}
{"gold": "length/time", "id": "dim-v1-0002", "left": "meter", "op": "quotient", "right": "second"}
{"gold": "length", "id": "dim-v1-0003", "left": "meter_per_second", "op": "product", "right": "second"}
{"gold": "money/time", "id": "dim-v1-0004", "left": "dollar", "op": "quotient", "right": "hour"}
{"gold": "time*money", "id": "dim-v1-0005", "left": "dollar", "op": "product", "right": "hour"}
{"gold": "mass/length", "id": "dim-v1-0006", "left": "kilogram", "op": "quotient", "right": "meter"}
{"gold": "length*time", "id": "dim-v1-0007", "left": "meter", "op": "product", "right": "second"}
{"gold": "length/time", "id": "dim-v1-0008", "left": "mile", "op": "quotient", "right": "hour"}
{"gold": "dimensionless", "id": "dim-v1-0009", "left": "foot", "op": "quotient", "right": "foot"}
{"gold": "length^2", "id": "dim-v1-0010", "left": "foot", "op": "product", "right": "foot"}
{"gold": "refused", "id": "dim-v1-0011", "left": "dollar", "op": "quotient", "right": "apple"}
{"gold": "refused", "id": "dim-v1-0012", "left": "gravitons", "op": "product", "right": "second"}

View file

@ -1095,6 +1095,14 @@ INDEPENDENT_GOLD_LANES: tuple[IndependentGoldLane, ...] = (
"generate.logic_equivalence",
),
),
# The dimensional-reasoning lane: the interlingua's own unit algebra
# (generate.binding_graph.units) is the SUT, so its gold oracle must share no
# code with the binding graph.
IndependentGoldLane(
name="dimensional",
oracle_module="evals/dimensional/oracle.py",
sut_import_prefixes=("generate.binding_graph",),
),
)
_DEDUCTIVE_CASE_FILES: tuple[str, ...] = (

View file

@ -0,0 +1,73 @@
"""Dimensional-reasoning lane — the third diversity-panel domain.
Proves the binding-graph interlingua's unit algebra decides the dimension of a
unit operation identically to an independent dimensional oracle (sharing no code
with it), on every committed case the same independent-gold discipline as the
deductive and finite-entity lanes, applied to a structurally distinct domain.
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from evals.dimensional.oracle import dimensional_result
from evals.dimensional.runner import decide
_FIXTURE = Path(__file__).resolve().parents[1] / "evals" / "dimensional" / "v1" / "cases.jsonl"
def _load() -> list[dict]:
return [json.loads(line) for line in _FIXTURE.read_text(encoding="utf-8").splitlines() if line.strip()]
@pytest.mark.parametrize("case", _load(), ids=lambda c: c["id"])
def test_interlingua_agrees_with_independent_oracle_and_gold(case: dict) -> None:
"""The lane gate: the interlingua's unit algebra (SUT) == the independent
oracle == the committed gold, with wrong == 0 by construction."""
oracle = dimensional_result(case["op"], case["left"], case["right"])
sut = decide(case["op"], case["left"], case["right"])
assert oracle == case["gold"], f"{case['id']}: independent oracle != committed gold"
assert sut == case["gold"], f"{case['id']}: interlingua committed a wrong dimension (wrong=0 breach)"
def test_fixture_has_nontrivial_and_refusal_signal() -> None:
"""Guard against a vacuous fixture: it must carry composite dimensions AND a
refusal, or the lane proves nothing."""
golds = [c["gold"] for c in _load()]
assert any("/" in g or "*" in g or "^" in g for g in golds), "no composite dimensions"
assert "refused" in golds, "no refusal-boundary case"
def test_oracle_is_independent_of_the_sut() -> None:
"""The oracle must not IMPORT the interlingua it checks (AST, not substring —
the docstring may name the SUT). Also enforced structurally by INV-25's
registry; this is a fast local sanity check."""
import ast
import evals.dimensional.oracle as oracle_mod
path = oracle_mod.__file__
assert path is not None
tree = ast.parse(Path(path).read_text(encoding="utf-8"))
imported: set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.Import):
imported |= {a.name for a in node.names}
elif isinstance(node, ast.ImportFrom) and node.module:
imported.add(node.module)
assert not any(m.startswith("generate.binding_graph") for m in imported), (
"oracle must share no code with the SUT (generate.binding_graph)"
)
def test_dimensionless_and_distinct_units_same_dimension() -> None:
# mile/hour and meter/second are different units, same dimension.
assert decide("quotient", "mile", "hour") == decide("quotient", "meter", "second")
assert decide("quotient", "foot", "foot") == "dimensionless"
def test_unknown_unit_refuses() -> None:
assert decide("product", "gravitons", "second") == "refused"
assert dimensional_result("product", "gravitons", "second") == "refused"