feat: dimensional-reasoning lane — 3rd diversity-panel domain
The first non-GSM8K consumer of the binding-graph interlingua's unit algebra as a load-bearing reasoner: given two units and an operation, decide the result's dimension. SUT = generate.binding_graph.units; gold = evals/dimensional/oracle.py, a genuinely INDEPENDENT dimensional reasoner (own unit->base-exponent table, own exponent arithmetic, own canonical-string renderer; shares no code with the SUT). 12 cases (area / speed / wage / mass-density / dimensionless / 2 refused) gated by SUT == oracle == gold (wrong=0). Registered in INV-25's INDEPENDENT_GOLD_LANES, proving the independent-gold discipline generalizes to a SECOND oracle. This is the 3rd structurally-distinct golded domain (logic / grounding / dimensional) — the anti-overfit >=2-domain panel is now real, and the interlingua is load-bearing beyond GSM8K.
This commit is contained in:
parent
4d964020a9
commit
96c1d4bcee
6 changed files with 279 additions and 0 deletions
7
evals/dimensional/__init__.py
Normal file
7
evals/dimensional/__init__.py
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
"""Dimensional-reasoning lane — the third diversity-panel domain.
|
||||
|
||||
The interlingua's unit algebra (``generate.binding_graph.units``) is the system
|
||||
under test; an independent dimensional oracle (``evals.dimensional.oracle``) is the
|
||||
gold. Structurally distinct from logic/grounding (dimensional exponent arithmetic),
|
||||
it exercises the binding-graph interlingua as a load-bearing reasoner beyond GSM8K.
|
||||
"""
|
||||
99
evals/dimensional/oracle.py
Normal file
99
evals/dimensional/oracle.py
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
"""Independent dimensional oracle — the gold for the dimensional-reasoning lane.
|
||||
|
||||
This is **deliberately a second, independent decision procedure** for "what is the
|
||||
dimension of ``left <op> right``?". It carries its **own** unit→base-exponent table
|
||||
and its **own** exponent arithmetic and canonical-string renderer, and shares
|
||||
**no code** with :mod:`generate.binding_graph.units` (the system under test). Two
|
||||
independent dimensional reasoners agreeing on the same cases is real evidence the
|
||||
interlingua's unit algebra is correct; a shared-code "oracle" would only prove the
|
||||
algebra agrees with itself (INV-25).
|
||||
|
||||
Base-dimension order matches the SUT's so the rendered strings are directly
|
||||
comparable (the comparison is about the *decision*, not the spelling).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
# Same axis order as the SUT's BASE_DIMENSIONS so canonical strings line up.
|
||||
_BASE: Final[tuple[str, ...]] = ("length", "time", "mass", "money", "count", "temperature")
|
||||
|
||||
# An INDEPENDENT unit→base-exponent table (hand-authored; NOT the en_units_v1 pack
|
||||
# the SUT reads). Only concrete base units; composites are resolved structurally.
|
||||
_UNIT_DIMS: Final[dict[str, dict[str, int]]] = {
|
||||
"meter": {"length": 1}, "mile": {"length": 1}, "foot": {"length": 1},
|
||||
"second": {"time": 1}, "hour": {"time": 1}, "minute": {"time": 1}, "day": {"time": 1},
|
||||
"kilogram": {"mass": 1}, "pound": {"mass": 1},
|
||||
"dollar": {"money": 1},
|
||||
"degree": {"temperature": 1},
|
||||
}
|
||||
|
||||
|
||||
class OracleError(ValueError):
|
||||
"""Unit outside the oracle's independent vocabulary — refuse, never guess."""
|
||||
|
||||
|
||||
def _depluralize(unit: str) -> str:
|
||||
"""Independent conservative plural strip (mirrors the SUT's contract, own code)."""
|
||||
if unit in _UNIT_DIMS:
|
||||
return unit
|
||||
for cand in (
|
||||
unit[:-3] + "y" if unit.endswith("ies") and len(unit) > 3 else "",
|
||||
unit[:-2] if unit.endswith("es") and len(unit) > 2 else "",
|
||||
unit[:-1] if unit.endswith("s") and len(unit) > 1 else "",
|
||||
):
|
||||
if cand and cand in _UNIT_DIMS:
|
||||
return cand
|
||||
return unit
|
||||
|
||||
|
||||
def _vector(unit: str) -> tuple[int, ...]:
|
||||
"""Resolve a unit id (incl. plural and ``X_per_Y`` composites) to an exponent
|
||||
tuple over ``_BASE``. Raises :class:`OracleError` on anything else."""
|
||||
if not isinstance(unit, str) or not unit:
|
||||
raise OracleError(f"empty unit: {unit!r}")
|
||||
canon = _depluralize(unit)
|
||||
if canon in _UNIT_DIMS:
|
||||
dims = _UNIT_DIMS[canon]
|
||||
return tuple(dims.get(b, 0) for b in _BASE)
|
||||
if "_per_" in unit:
|
||||
num, _, denom = unit.partition("_per_")
|
||||
nv = _vector(num)
|
||||
dv = _vector(denom)
|
||||
return tuple(a - b for a, b in zip(nv, dv))
|
||||
raise OracleError(f"unknown_unit: {unit!r}")
|
||||
|
||||
|
||||
def _render(exps: tuple[int, ...]) -> str:
|
||||
"""Canonical dimension string, format-compatible with the SUT's renderer."""
|
||||
nums: list[str] = []
|
||||
dens: list[str] = []
|
||||
for dim, e in zip(_BASE, exps):
|
||||
if e > 0:
|
||||
nums.append(dim if e == 1 else f"{dim}^{e}")
|
||||
elif e < 0:
|
||||
dens.append(dim if e == -1 else f"{dim}^{-e}")
|
||||
if not nums and not dens:
|
||||
return "dimensionless"
|
||||
num_part = "*".join(nums) if nums else "1"
|
||||
return num_part if not dens else f"{num_part}/{'*'.join(dens)}"
|
||||
|
||||
|
||||
def dimensional_result(op: str, left: str, right: str) -> str:
|
||||
"""The oracle verdict: the canonical dimension of ``left <op> right``.
|
||||
|
||||
``op`` is ``"product"`` or ``"quotient"``. Returns ``"refused"`` if either
|
||||
unit is outside the oracle's vocabulary or ``op`` is unknown — the same
|
||||
refusal posture as the SUT.
|
||||
"""
|
||||
try:
|
||||
lv = _vector(left)
|
||||
rv = _vector(right)
|
||||
except OracleError:
|
||||
return "refused"
|
||||
if op == "product":
|
||||
return _render(tuple(a + b for a, b in zip(lv, rv)))
|
||||
if op == "quotient":
|
||||
return _render(tuple(a - b for a, b in zip(lv, rv)))
|
||||
return "refused"
|
||||
80
evals/dimensional/runner.py
Normal file
80
evals/dimensional/runner.py
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
"""Dimensional-reasoning lane runner — the interlingua's unit algebra vs. the
|
||||
independent dimensional oracle.
|
||||
|
||||
The system under test is the binding-graph interlingua (``generate.binding_graph.
|
||||
units``): given two unit ids and an operation, what is the dimension of the result?
|
||||
Each committed case is scored as ``correct`` (SUT == gold), ``wrong`` (SUT
|
||||
committed a different dimension than gold — MUST stay 0), or ``refused`` (SUT
|
||||
declined a unit outside the closed vocabulary). The gold is the independent oracle
|
||||
verdict, frozen into ``cases.jsonl`` at generation (INV-25).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from generate.binding_graph.units import (
|
||||
UnitAlgebraError,
|
||||
parse_unit,
|
||||
unit_product,
|
||||
unit_quotient,
|
||||
)
|
||||
|
||||
_ROOT = Path(__file__).resolve().parent
|
||||
_REFUSED = "refused"
|
||||
|
||||
|
||||
def decide(op: str, left: str, right: str) -> str:
|
||||
"""The SUT verdict: the canonical dimension of ``left <op> right`` via the
|
||||
interlingua's unit algebra, or ``"refused"`` for an unknown unit / op."""
|
||||
try:
|
||||
lv = parse_unit(left)
|
||||
rv = parse_unit(right)
|
||||
except UnitAlgebraError:
|
||||
return _REFUSED
|
||||
if op == "product":
|
||||
return unit_product(lv, rv).to_canonical_string()
|
||||
if op == "quotient":
|
||||
return unit_quotient(lv, rv).to_canonical_string()
|
||||
return _REFUSED
|
||||
|
||||
|
||||
def _load(path: Path) -> list[dict]:
|
||||
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
|
||||
|
||||
def build_report(cases: list[dict]) -> dict:
|
||||
counts: Counter[str] = Counter({"correct": 0, "wrong": 0, "refused": 0})
|
||||
wrong_examples: list[dict] = []
|
||||
for case in cases:
|
||||
gold = case["gold"]
|
||||
got = decide(case["op"], case["left"], case["right"])
|
||||
if got == gold:
|
||||
counts["refused" if gold == _REFUSED else "correct"] += 1
|
||||
else:
|
||||
counts["wrong"] += 1
|
||||
if len(wrong_examples) < 10:
|
||||
wrong_examples.append({"id": case["id"], "gold": gold, "got": got})
|
||||
return {
|
||||
"n": len(cases),
|
||||
"counts": dict(counts),
|
||||
"all_correct": counts["wrong"] == 0,
|
||||
"wrong_examples": wrong_examples,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
report = build_report(_load(_ROOT / "v1" / "cases.jsonl"))
|
||||
c = report["counts"]
|
||||
print(f"[dimensional] n={report['n']} correct={c['correct']} wrong={c['wrong']} refused={c['refused']}")
|
||||
for w in report["wrong_examples"]:
|
||||
print(f" WRONG {w['id']}: gold={w['gold']} got={w['got']}")
|
||||
return 0 if report["all_correct"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
sys.exit(main())
|
||||
12
evals/dimensional/v1/cases.jsonl
Normal file
12
evals/dimensional/v1/cases.jsonl
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
{"gold": "length^2", "id": "dim-v1-0001", "left": "meter", "op": "product", "right": "meter"}
|
||||
{"gold": "length/time", "id": "dim-v1-0002", "left": "meter", "op": "quotient", "right": "second"}
|
||||
{"gold": "length", "id": "dim-v1-0003", "left": "meter_per_second", "op": "product", "right": "second"}
|
||||
{"gold": "money/time", "id": "dim-v1-0004", "left": "dollar", "op": "quotient", "right": "hour"}
|
||||
{"gold": "time*money", "id": "dim-v1-0005", "left": "dollar", "op": "product", "right": "hour"}
|
||||
{"gold": "mass/length", "id": "dim-v1-0006", "left": "kilogram", "op": "quotient", "right": "meter"}
|
||||
{"gold": "length*time", "id": "dim-v1-0007", "left": "meter", "op": "product", "right": "second"}
|
||||
{"gold": "length/time", "id": "dim-v1-0008", "left": "mile", "op": "quotient", "right": "hour"}
|
||||
{"gold": "dimensionless", "id": "dim-v1-0009", "left": "foot", "op": "quotient", "right": "foot"}
|
||||
{"gold": "length^2", "id": "dim-v1-0010", "left": "foot", "op": "product", "right": "foot"}
|
||||
{"gold": "refused", "id": "dim-v1-0011", "left": "dollar", "op": "quotient", "right": "apple"}
|
||||
{"gold": "refused", "id": "dim-v1-0012", "left": "gravitons", "op": "product", "right": "second"}
|
||||
|
|
@ -1095,6 +1095,14 @@ INDEPENDENT_GOLD_LANES: tuple[IndependentGoldLane, ...] = (
|
|||
"generate.logic_equivalence",
|
||||
),
|
||||
),
|
||||
# The dimensional-reasoning lane: the interlingua's own unit algebra
|
||||
# (generate.binding_graph.units) is the SUT, so its gold oracle must share no
|
||||
# code with the binding graph.
|
||||
IndependentGoldLane(
|
||||
name="dimensional",
|
||||
oracle_module="evals/dimensional/oracle.py",
|
||||
sut_import_prefixes=("generate.binding_graph",),
|
||||
),
|
||||
)
|
||||
|
||||
_DEDUCTIVE_CASE_FILES: tuple[str, ...] = (
|
||||
|
|
|
|||
73
tests/test_dimensional_reasoning_lane.py
Normal file
73
tests/test_dimensional_reasoning_lane.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
"""Dimensional-reasoning lane — the third diversity-panel domain.
|
||||
|
||||
Proves the binding-graph interlingua's unit algebra decides the dimension of a
|
||||
unit operation identically to an independent dimensional oracle (sharing no code
|
||||
with it), on every committed case — the same independent-gold discipline as the
|
||||
deductive and finite-entity lanes, applied to a structurally distinct domain.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from evals.dimensional.oracle import dimensional_result
|
||||
from evals.dimensional.runner import decide
|
||||
|
||||
_FIXTURE = Path(__file__).resolve().parents[1] / "evals" / "dimensional" / "v1" / "cases.jsonl"
|
||||
|
||||
|
||||
def _load() -> list[dict]:
|
||||
return [json.loads(line) for line in _FIXTURE.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", _load(), ids=lambda c: c["id"])
|
||||
def test_interlingua_agrees_with_independent_oracle_and_gold(case: dict) -> None:
|
||||
"""The lane gate: the interlingua's unit algebra (SUT) == the independent
|
||||
oracle == the committed gold, with wrong == 0 by construction."""
|
||||
oracle = dimensional_result(case["op"], case["left"], case["right"])
|
||||
sut = decide(case["op"], case["left"], case["right"])
|
||||
assert oracle == case["gold"], f"{case['id']}: independent oracle != committed gold"
|
||||
assert sut == case["gold"], f"{case['id']}: interlingua committed a wrong dimension (wrong=0 breach)"
|
||||
|
||||
|
||||
def test_fixture_has_nontrivial_and_refusal_signal() -> None:
|
||||
"""Guard against a vacuous fixture: it must carry composite dimensions AND a
|
||||
refusal, or the lane proves nothing."""
|
||||
golds = [c["gold"] for c in _load()]
|
||||
assert any("/" in g or "*" in g or "^" in g for g in golds), "no composite dimensions"
|
||||
assert "refused" in golds, "no refusal-boundary case"
|
||||
|
||||
|
||||
def test_oracle_is_independent_of_the_sut() -> None:
|
||||
"""The oracle must not IMPORT the interlingua it checks (AST, not substring —
|
||||
the docstring may name the SUT). Also enforced structurally by INV-25's
|
||||
registry; this is a fast local sanity check."""
|
||||
import ast
|
||||
import evals.dimensional.oracle as oracle_mod
|
||||
|
||||
path = oracle_mod.__file__
|
||||
assert path is not None
|
||||
tree = ast.parse(Path(path).read_text(encoding="utf-8"))
|
||||
imported: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
imported |= {a.name for a in node.names}
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
imported.add(node.module)
|
||||
assert not any(m.startswith("generate.binding_graph") for m in imported), (
|
||||
"oracle must share no code with the SUT (generate.binding_graph)"
|
||||
)
|
||||
|
||||
|
||||
def test_dimensionless_and_distinct_units_same_dimension() -> None:
|
||||
# mile/hour and meter/second are different units, same dimension.
|
||||
assert decide("quotient", "mile", "hour") == decide("quotient", "meter", "second")
|
||||
assert decide("quotient", "foot", "foot") == "dimensionless"
|
||||
|
||||
|
||||
def test_unknown_unit_refuses() -> None:
|
||||
assert decide("product", "gravitons", "second") == "refused"
|
||||
assert dimensional_result("product", "gravitons", "second") == "refused"
|
||||
Loading…
Reference in a new issue