diff --git a/evals/dimensional/__init__.py b/evals/dimensional/__init__.py new file mode 100644 index 00000000..e8985f53 --- /dev/null +++ b/evals/dimensional/__init__.py @@ -0,0 +1,7 @@ +"""Dimensional-reasoning lane — the third diversity-panel domain. + +The interlingua's unit algebra (``generate.binding_graph.units``) is the system +under test; an independent dimensional oracle (``evals.dimensional.oracle``) is the +gold. Structurally distinct from logic/grounding (dimensional exponent arithmetic), +it exercises the binding-graph interlingua as a load-bearing reasoner beyond GSM8K. +""" diff --git a/evals/dimensional/oracle.py b/evals/dimensional/oracle.py new file mode 100644 index 00000000..df04f216 --- /dev/null +++ b/evals/dimensional/oracle.py @@ -0,0 +1,99 @@ +"""Independent dimensional oracle — the gold for the dimensional-reasoning lane. + +This is **deliberately a second, independent decision procedure** for "what is the +dimension of ``left right``?". It carries its **own** unit→base-exponent table +and its **own** exponent arithmetic and canonical-string renderer, and shares +**no code** with :mod:`generate.binding_graph.units` (the system under test). Two +independent dimensional reasoners agreeing on the same cases is real evidence the +interlingua's unit algebra is correct; a shared-code "oracle" would only prove the +algebra agrees with itself (INV-25). + +Base-dimension order matches the SUT's so the rendered strings are directly +comparable (the comparison is about the *decision*, not the spelling). +""" + +from __future__ import annotations + +from typing import Final + +# Same axis order as the SUT's BASE_DIMENSIONS so canonical strings line up. +_BASE: Final[tuple[str, ...]] = ("length", "time", "mass", "money", "count", "temperature") + +# An INDEPENDENT unit→base-exponent table (hand-authored; NOT the en_units_v1 pack +# the SUT reads). Only concrete base units; composites are resolved structurally. +_UNIT_DIMS: Final[dict[str, dict[str, int]]] = { + "meter": {"length": 1}, "mile": {"length": 1}, "foot": {"length": 1}, + "second": {"time": 1}, "hour": {"time": 1}, "minute": {"time": 1}, "day": {"time": 1}, + "kilogram": {"mass": 1}, "pound": {"mass": 1}, + "dollar": {"money": 1}, + "degree": {"temperature": 1}, +} + + +class OracleError(ValueError): + """Unit outside the oracle's independent vocabulary — refuse, never guess.""" + + +def _depluralize(unit: str) -> str: + """Independent conservative plural strip (mirrors the SUT's contract, own code).""" + if unit in _UNIT_DIMS: + return unit + for cand in ( + unit[:-3] + "y" if unit.endswith("ies") and len(unit) > 3 else "", + unit[:-2] if unit.endswith("es") and len(unit) > 2 else "", + unit[:-1] if unit.endswith("s") and len(unit) > 1 else "", + ): + if cand and cand in _UNIT_DIMS: + return cand + return unit + + +def _vector(unit: str) -> tuple[int, ...]: + """Resolve a unit id (incl. plural and ``X_per_Y`` composites) to an exponent + tuple over ``_BASE``. Raises :class:`OracleError` on anything else.""" + if not isinstance(unit, str) or not unit: + raise OracleError(f"empty unit: {unit!r}") + canon = _depluralize(unit) + if canon in _UNIT_DIMS: + dims = _UNIT_DIMS[canon] + return tuple(dims.get(b, 0) for b in _BASE) + if "_per_" in unit: + num, _, denom = unit.partition("_per_") + nv = _vector(num) + dv = _vector(denom) + return tuple(a - b for a, b in zip(nv, dv)) + raise OracleError(f"unknown_unit: {unit!r}") + + +def _render(exps: tuple[int, ...]) -> str: + """Canonical dimension string, format-compatible with the SUT's renderer.""" + nums: list[str] = [] + dens: list[str] = [] + for dim, e in zip(_BASE, exps): + if e > 0: + nums.append(dim if e == 1 else f"{dim}^{e}") + elif e < 0: + dens.append(dim if e == -1 else f"{dim}^{-e}") + if not nums and not dens: + return "dimensionless" + num_part = "*".join(nums) if nums else "1" + return num_part if not dens else f"{num_part}/{'*'.join(dens)}" + + +def dimensional_result(op: str, left: str, right: str) -> str: + """The oracle verdict: the canonical dimension of ``left right``. + + ``op`` is ``"product"`` or ``"quotient"``. Returns ``"refused"`` if either + unit is outside the oracle's vocabulary or ``op`` is unknown — the same + refusal posture as the SUT. + """ + try: + lv = _vector(left) + rv = _vector(right) + except OracleError: + return "refused" + if op == "product": + return _render(tuple(a + b for a, b in zip(lv, rv))) + if op == "quotient": + return _render(tuple(a - b for a, b in zip(lv, rv))) + return "refused" diff --git a/evals/dimensional/runner.py b/evals/dimensional/runner.py new file mode 100644 index 00000000..c62ff3ad --- /dev/null +++ b/evals/dimensional/runner.py @@ -0,0 +1,80 @@ +"""Dimensional-reasoning lane runner — the interlingua's unit algebra vs. the +independent dimensional oracle. + +The system under test is the binding-graph interlingua (``generate.binding_graph. +units``): given two unit ids and an operation, what is the dimension of the result? +Each committed case is scored as ``correct`` (SUT == gold), ``wrong`` (SUT +committed a different dimension than gold — MUST stay 0), or ``refused`` (SUT +declined a unit outside the closed vocabulary). The gold is the independent oracle +verdict, frozen into ``cases.jsonl`` at generation (INV-25). +""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +from generate.binding_graph.units import ( + UnitAlgebraError, + parse_unit, + unit_product, + unit_quotient, +) + +_ROOT = Path(__file__).resolve().parent +_REFUSED = "refused" + + +def decide(op: str, left: str, right: str) -> str: + """The SUT verdict: the canonical dimension of ``left right`` via the + interlingua's unit algebra, or ``"refused"`` for an unknown unit / op.""" + try: + lv = parse_unit(left) + rv = parse_unit(right) + except UnitAlgebraError: + return _REFUSED + if op == "product": + return unit_product(lv, rv).to_canonical_string() + if op == "quotient": + return unit_quotient(lv, rv).to_canonical_string() + return _REFUSED + + +def _load(path: Path) -> list[dict]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def build_report(cases: list[dict]) -> dict: + counts: Counter[str] = Counter({"correct": 0, "wrong": 0, "refused": 0}) + wrong_examples: list[dict] = [] + for case in cases: + gold = case["gold"] + got = decide(case["op"], case["left"], case["right"]) + if got == gold: + counts["refused" if gold == _REFUSED else "correct"] += 1 + else: + counts["wrong"] += 1 + if len(wrong_examples) < 10: + wrong_examples.append({"id": case["id"], "gold": gold, "got": got}) + return { + "n": len(cases), + "counts": dict(counts), + "all_correct": counts["wrong"] == 0, + "wrong_examples": wrong_examples, + } + + +def main() -> int: + report = build_report(_load(_ROOT / "v1" / "cases.jsonl")) + c = report["counts"] + print(f"[dimensional] n={report['n']} correct={c['correct']} wrong={c['wrong']} refused={c['refused']}") + for w in report["wrong_examples"]: + print(f" WRONG {w['id']}: gold={w['gold']} got={w['got']}") + return 0 if report["all_correct"] else 1 + + +if __name__ == "__main__": + import sys + + sys.exit(main()) diff --git a/evals/dimensional/v1/cases.jsonl b/evals/dimensional/v1/cases.jsonl new file mode 100644 index 00000000..538bb77f --- /dev/null +++ b/evals/dimensional/v1/cases.jsonl @@ -0,0 +1,12 @@ +{"gold": "length^2", "id": "dim-v1-0001", "left": "meter", "op": "product", "right": "meter"} +{"gold": "length/time", "id": "dim-v1-0002", "left": "meter", "op": "quotient", "right": "second"} +{"gold": "length", "id": "dim-v1-0003", "left": "meter_per_second", "op": "product", "right": "second"} +{"gold": "money/time", "id": "dim-v1-0004", "left": "dollar", "op": "quotient", "right": "hour"} +{"gold": "time*money", "id": "dim-v1-0005", "left": "dollar", "op": "product", "right": "hour"} +{"gold": "mass/length", "id": "dim-v1-0006", "left": "kilogram", "op": "quotient", "right": "meter"} +{"gold": "length*time", "id": "dim-v1-0007", "left": "meter", "op": "product", "right": "second"} +{"gold": "length/time", "id": "dim-v1-0008", "left": "mile", "op": "quotient", "right": "hour"} +{"gold": "dimensionless", "id": "dim-v1-0009", "left": "foot", "op": "quotient", "right": "foot"} +{"gold": "length^2", "id": "dim-v1-0010", "left": "foot", "op": "product", "right": "foot"} +{"gold": "refused", "id": "dim-v1-0011", "left": "dollar", "op": "quotient", "right": "apple"} +{"gold": "refused", "id": "dim-v1-0012", "left": "gravitons", "op": "product", "right": "second"} diff --git a/tests/test_architectural_invariants.py b/tests/test_architectural_invariants.py index 5ccfad00..26cd7bdc 100644 --- a/tests/test_architectural_invariants.py +++ b/tests/test_architectural_invariants.py @@ -1095,6 +1095,14 @@ INDEPENDENT_GOLD_LANES: tuple[IndependentGoldLane, ...] = ( "generate.logic_equivalence", ), ), + # The dimensional-reasoning lane: the interlingua's own unit algebra + # (generate.binding_graph.units) is the SUT, so its gold oracle must share no + # code with the binding graph. + IndependentGoldLane( + name="dimensional", + oracle_module="evals/dimensional/oracle.py", + sut_import_prefixes=("generate.binding_graph",), + ), ) _DEDUCTIVE_CASE_FILES: tuple[str, ...] = ( diff --git a/tests/test_dimensional_reasoning_lane.py b/tests/test_dimensional_reasoning_lane.py new file mode 100644 index 00000000..458c8d12 --- /dev/null +++ b/tests/test_dimensional_reasoning_lane.py @@ -0,0 +1,73 @@ +"""Dimensional-reasoning lane — the third diversity-panel domain. + +Proves the binding-graph interlingua's unit algebra decides the dimension of a +unit operation identically to an independent dimensional oracle (sharing no code +with it), on every committed case — the same independent-gold discipline as the +deductive and finite-entity lanes, applied to a structurally distinct domain. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from evals.dimensional.oracle import dimensional_result +from evals.dimensional.runner import decide + +_FIXTURE = Path(__file__).resolve().parents[1] / "evals" / "dimensional" / "v1" / "cases.jsonl" + + +def _load() -> list[dict]: + return [json.loads(line) for line in _FIXTURE.read_text(encoding="utf-8").splitlines() if line.strip()] + + +@pytest.mark.parametrize("case", _load(), ids=lambda c: c["id"]) +def test_interlingua_agrees_with_independent_oracle_and_gold(case: dict) -> None: + """The lane gate: the interlingua's unit algebra (SUT) == the independent + oracle == the committed gold, with wrong == 0 by construction.""" + oracle = dimensional_result(case["op"], case["left"], case["right"]) + sut = decide(case["op"], case["left"], case["right"]) + assert oracle == case["gold"], f"{case['id']}: independent oracle != committed gold" + assert sut == case["gold"], f"{case['id']}: interlingua committed a wrong dimension (wrong=0 breach)" + + +def test_fixture_has_nontrivial_and_refusal_signal() -> None: + """Guard against a vacuous fixture: it must carry composite dimensions AND a + refusal, or the lane proves nothing.""" + golds = [c["gold"] for c in _load()] + assert any("/" in g or "*" in g or "^" in g for g in golds), "no composite dimensions" + assert "refused" in golds, "no refusal-boundary case" + + +def test_oracle_is_independent_of_the_sut() -> None: + """The oracle must not IMPORT the interlingua it checks (AST, not substring — + the docstring may name the SUT). Also enforced structurally by INV-25's + registry; this is a fast local sanity check.""" + import ast + import evals.dimensional.oracle as oracle_mod + + path = oracle_mod.__file__ + assert path is not None + tree = ast.parse(Path(path).read_text(encoding="utf-8")) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported |= {a.name for a in node.names} + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + assert not any(m.startswith("generate.binding_graph") for m in imported), ( + "oracle must share no code with the SUT (generate.binding_graph)" + ) + + +def test_dimensionless_and_distinct_units_same_dimension() -> None: + # mile/hour and meter/second are different units, same dimension. + assert decide("quotient", "mile", "hour") == decide("quotient", "meter", "second") + assert decide("quotient", "foot", "foot") == "dimensionless" + + +def test_unknown_unit_refuses() -> None: + assert decide("product", "gravitons", "second") == "refused" + assert dimensional_result("product", "gravitons", "second") == "refused"