Merge pull request #579 from AssetOverflow/evals/staged-gold-lanes

evals: staged independent-gold lanes (set-membership/ordering/syllogism) for Phase 2 cross-domain proof

evals/set_membership/__init__.py

@@ -0,0 +1,2 @@
+"""Staged independent-gold lane for set membership/subsumption."""
+

evals/set_membership/oracle.py

@@ -0,0 +1,109 @@
+"""Independent set-membership oracle for the staged Phase 2 gold lane.
+
+The oracle reads STRUCTURE only: finite elements, finite sets, direct membership
+facts, and subset relations. It performs a plain transitive-closure decision
+procedure and refuses malformed/out-of-grammar inputs. It imports no engine,
+reader, algebra, field, or generation code; independence will live in a future
+text reader, while this lane fixes the gold side.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class OracleError(ValueError):
+    """Malformed or out-of-grammar case; the oracle refuses, never guesses."""
+
+
+def _require_str(value: object, label: str) -> str:
+    if not isinstance(value, str) or not value:
+        raise OracleError(f"{label} must be a non-empty string: {value!r}")
+    return value
+
+
+def _require_str_list(value: object, label: str) -> list[str]:
+    if not isinstance(value, list):
+        raise OracleError(f"{label} must be a list")
+    out = [_require_str(v, label) for v in value]
+    if len(set(out)) != len(out):
+        raise OracleError(f"{label} contains duplicates")
+    return out
+
+
+def _closure(edges: dict[str, set[str]], sets: set[str]) -> dict[str, set[str]]:
+    closed = {s: {s, *edges.get(s, set())} for s in sets}
+    changed = True
+    while changed:
+        changed = False
+        for left in sorted(sets):
+            expanded = set(closed[left])
+            for mid in tuple(closed[left]):
+                expanded.update(closed.get(mid, {mid}))
+            if expanded != closed[left]:
+                closed[left] = expanded
+                changed = True
+    return closed
+
+
+def oracle_answer(structure: dict[str, Any], query: dict[str, Any]) -> bool:
+    """Return the closed-world membership/subsumption verdict.
+
+    Supported query kinds:
+    - ``member``: whether ``element`` is in ``set`` after subset propagation.
+    - ``subset``: whether ``subset`` is included in ``superset`` transitively.
+    """
+    elements = set(_require_str_list(structure.get("elements"), "elements"))
+
+    raw_sets = structure.get("sets")
+    if not isinstance(raw_sets, list) or not raw_sets:
+        raise OracleError("sets must be a non-empty list")
+
+    members: dict[str, set[str]] = {}
+    for raw in raw_sets:
+        if not isinstance(raw, dict):
+            raise OracleError(f"set entry must be an object: {raw!r}")
+        sid = _require_str(raw.get("id"), "set.id")
+        if sid in members:
+            raise OracleError(f"duplicate set id: {sid!r}")
+        direct = set(_require_str_list(raw.get("members", []), f"{sid}.members"))
+        unknown = direct - elements
+        if unknown:
+            raise OracleError(f"set {sid!r} names unknown elements: {sorted(unknown)}")
+        members[sid] = direct
+
+    set_ids = set(members)
+    edges: dict[str, set[str]] = {sid: set() for sid in set_ids}
+    raw_subsets = structure.get("subsets", [])
+    if not isinstance(raw_subsets, list):
+        raise OracleError("subsets must be a list")
+    for raw in raw_subsets:
+        if not isinstance(raw, dict):
+            raise OracleError(f"subset entry must be an object: {raw!r}")
+        subset = _require_str(raw.get("subset"), "subset")
+        superset = _require_str(raw.get("superset"), "superset")
+        if subset not in set_ids or superset not in set_ids:
+            raise OracleError(f"subset relation names unknown set: {raw!r}")
+        edges[subset].add(superset)
+
+    closed = _closure(edges, set_ids)
+    kind = query.get("kind")
+    if kind == "subset":
+        subset = _require_str(query.get("subset"), "query.subset")
+        superset = _require_str(query.get("superset"), "query.superset")
+        if subset not in set_ids or superset not in set_ids:
+            raise OracleError(f"query names unknown set: {query!r}")
+        return superset in closed[subset]
+
+    if kind == "member":
+        element = _require_str(query.get("element"), "query.element")
+        target = _require_str(query.get("set"), "query.set")
+        if element not in elements:
+            raise OracleError(f"query names unknown element: {element!r}")
+        if target not in set_ids:
+            raise OracleError(f"query names unknown set: {target!r}")
+        element_sets = {sid for sid, direct in members.items() if element in direct}
+        return any(target in closed[sid] for sid in element_sets)
+
+    raise OracleError(f"unsupported query kind: {kind!r}")
+

evals/set_membership/runner.py

@@ -0,0 +1,62 @@
+"""Gold-only runner for the staged set-membership lane.
+
+No engine reader is invoked here. The runner only checks that committed gold is
+reproducible from structured cases by the independent oracle.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from evals.set_membership.oracle import OracleError, oracle_answer
+
+_CASES = Path(__file__).resolve().parent / "v1" / "cases.jsonl"
+
+
+def _load_cases(path: Path = _CASES) -> list[dict[str, Any]]:
+    return [
+        json.loads(line)
+        for line in path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+
+def run(path: Path = _CASES) -> dict[str, Any]:
+    cases = _load_cases(path)
+    failures: list[str] = []
+    for case in cases:
+        try:
+            got = oracle_answer(case["structure"], case["query"])
+        except OracleError as exc:
+            failures.append(f"{case.get('id', '<missing>')}: refused gold case: {exc}")
+            continue
+        if got != case.get("gold"):
+            failures.append(f"{case['id']}: oracle={got!r} != gold={case.get('gold')!r}")
+
+    correct = len(cases) - len(failures)
+    return {
+        "domain": "set_membership",
+        "total": len(cases),
+        "correct": correct,
+        "wrong": 0,
+        "refused": 0,
+        "gold_integrity_failures": failures,
+        "counts": {"correct": correct, "wrong": 0, "refused": 0},
+    }
+
+
+def main() -> int:
+    report = run()
+    print(json.dumps(report, indent=2, sort_keys=True))
+    if report["gold_integrity_failures"]:
+        print("GOLD INTEGRITY FAILURE", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+

evals/set_membership/v1/cases.jsonl

@@ -0,0 +1,8 @@
+{"id":"sm-v1-0001","seed":20260605,"text":"Rhea is a raven. All ravens are birds. Is Rhea a bird?","structure":{"elements":["rhea","mira"],"sets":[{"id":"raven","members":["rhea"]},{"id":"bird","members":[]},{"id":"mammal","members":["mira"]}],"subsets":[{"subset":"raven","superset":"bird"}]},"query":{"kind":"member","element":"rhea","set":"bird"},"gold":true,"class":"membership_via_subsumption"}
+{"id":"sm-v1-0002","seed":20260605,"text":"Ada is a botanist. All botanists are scientists. All scientists are people. Is Ada a person?","structure":{"elements":["ada","ben"],"sets":[{"id":"botanist","members":["ada"]},{"id":"scientist","members":[]},{"id":"person","members":[]},{"id":"engineer","members":["ben"]}],"subsets":[{"subset":"botanist","superset":"scientist"},{"subset":"scientist","superset":"person"},{"subset":"engineer","superset":"person"}]},"query":{"kind":"member","element":"ada","set":"person"},"gold":true,"class":"membership_transitive"}
+{"id":"sm-v1-0003","seed":20260605,"text":"Fido is a dog. All dogs are mammals. All cats are mammals. Is Fido a cat?","structure":{"elements":["fido","cleo"],"sets":[{"id":"dog","members":["fido"]},{"id":"cat","members":["cleo"]},{"id":"mammal","members":[]}],"subsets":[{"subset":"dog","superset":"mammal"},{"subset":"cat","superset":"mammal"}]},"query":{"kind":"member","element":"fido","set":"cat"},"gold":false,"class":"membership_negative"}
+{"id":"sm-v1-0004","seed":20260605,"text":"All squares are rectangles. All rectangles are polygons. Are all squares polygons?","structure":{"elements":["tile"],"sets":[{"id":"square","members":["tile"]},{"id":"rectangle","members":[]},{"id":"polygon","members":[]}],"subsets":[{"subset":"square","superset":"rectangle"},{"subset":"rectangle","superset":"polygon"}]},"query":{"kind":"subset","subset":"square","superset":"polygon"},"gold":true,"class":"subsumption_transitive"}
+{"id":"sm-v1-0005","seed":20260605,"text":"All sparrows are birds. All birds are animals. Are all animals sparrows?","structure":{"elements":["pip"],"sets":[{"id":"sparrow","members":["pip"]},{"id":"bird","members":[]},{"id":"animal","members":[]}],"subsets":[{"subset":"sparrow","superset":"bird"},{"subset":"bird","superset":"animal"}]},"query":{"kind":"subset","subset":"animal","superset":"sparrow"},"gold":false,"class":"subsumption_direction_negative"}
+{"id":"sm-v1-0006","seed":20260605,"text":"Carol is a cyclist. All cyclists are athletes. Is Carol an athlete?","structure":{"elements":["carol","dion"],"sets":[{"id":"cyclist","members":["carol"]},{"id":"athlete","members":[]},{"id":"musician","members":["dion"]}],"subsets":[{"subset":"cyclist","superset":"athlete"}]},"query":{"kind":"member","element":"carol","set":"athlete"},"gold":true,"class":"membership_via_subsumption"}
+{"id":"sm-v1-0007","seed":20260605,"text":"The mug is a cup. The hammer is a tool. Is the mug a tool?","structure":{"elements":["mug","hammer"],"sets":[{"id":"cup","members":["mug"]},{"id":"tool","members":["hammer"]},{"id":"object","members":[]}],"subsets":[{"subset":"cup","superset":"object"},{"subset":"tool","superset":"object"}]},"query":{"kind":"member","element":"mug","set":"tool"},"gold":false,"class":"disjoint_membership_negative"}
+{"id":"sm-v1-0008","seed":20260605,"text":"All violins are instruments. All instruments are artifacts. Are all violins artifacts?","structure":{"elements":["strad"],"sets":[{"id":"violin","members":["strad"]},{"id":"instrument","members":[]},{"id":"artifact","members":[]}],"subsets":[{"subset":"violin","superset":"instrument"},{"subset":"instrument","superset":"artifact"}]},"query":{"kind":"subset","subset":"violin","superset":"artifact"},"gold":true,"class":"subsumption_transitive"}

evals/syllogism/__init__.py

@@ -0,0 +1,2 @@
+"""Staged independent-gold lane for categorical syllogisms."""
+

evals/syllogism/oracle.py

@@ -0,0 +1,105 @@
+"""Independent categorical-syllogism oracle for the staged Phase 2 gold lane.
+
+The oracle reads structured categorical propositions, not text. It evaluates
+validity by brute-force finite-model enumeration over a small domain. This is
+intentionally simple and independent of any future comprehension reader.
+"""
+
+from __future__ import annotations
+
+from itertools import product
+from typing import Any, Final
+
+
+class OracleError(ValueError):
+    """Malformed or out-of-grammar case; the oracle refuses, never guesses."""
+
+
+_FORMS: Final[frozenset[str]] = frozenset({"A", "E", "I", "O"})
+
+
+def _require_str(value: object, label: str) -> str:
+    if not isinstance(value, str) or not value:
+        raise OracleError(f"{label} must be a non-empty string: {value!r}")
+    return value
+
+
+def _proposition(raw: object, terms: set[str]) -> tuple[str, str, str]:
+    if not isinstance(raw, dict):
+        raise OracleError(f"proposition must be an object: {raw!r}")
+    form = _require_str(raw.get("form"), "form")
+    subject = _require_str(raw.get("subject"), "subject")
+    predicate = _require_str(raw.get("predicate"), "predicate")
+    if form not in _FORMS:
+        raise OracleError(f"unsupported categorical form: {form!r}")
+    if subject not in terms or predicate not in terms:
+        raise OracleError(f"proposition names unknown term: {raw!r}")
+    if subject == predicate:
+        raise OracleError("subject and predicate must be distinct")
+    return form, subject, predicate
+
+
+def _holds(prop: tuple[str, str, str], env: dict[str, frozenset[int]]) -> bool:
+    form, subject, predicate = prop
+    s = env[subject]
+    p = env[predicate]
+    if form == "A":
+        return s <= p
+    if form == "E":
+        return not (s & p)
+    if form == "I":
+        return bool(s & p)
+    if form == "O":
+        return bool(s - p)
+    raise OracleError(f"unknown form: {form!r}")  # pragma: no cover
+
+
+def oracle_answer(structure: dict[str, Any], query: dict[str, Any]) -> dict[str, Any]:
+    """Return ``{"valid": bool, "conclusion": conclusion-or-None}``.
+
+    Premises with no satisfying model are refused as inconsistent; otherwise a
+    candidate conclusion is valid iff it holds in every model of the premises.
+    """
+    raw_terms = structure.get("terms")
+    if not isinstance(raw_terms, list) or len(raw_terms) < 2:
+        raise OracleError("terms must be a list of at least two term names")
+    terms = [_require_str(t, "term") for t in raw_terms]
+    if len(set(terms)) != len(terms):
+        raise OracleError("terms contains duplicates")
+    term_set = set(terms)
+
+    raw_premises = structure.get("premises")
+    if not isinstance(raw_premises, list) or not raw_premises:
+        raise OracleError("premises must be a non-empty list")
+    premises = [_proposition(p, term_set) for p in raw_premises]
+
+    if query.get("kind") != "validity":
+        raise OracleError(f"unsupported query kind: {query.get('kind')!r}")
+    conclusion_payload = query.get("conclusion")
+    conclusion = _proposition(conclusion_payload, term_set)
+
+    domain = tuple(range(int(structure.get("domain_size", 3))))
+    if not domain:
+        raise OracleError("domain_size must be positive")
+    subsets = [
+        frozenset(i for i, include in enumerate(bits) if include)
+        for bits in product((False, True), repeat=len(domain))
+    ]
+
+    models = 0
+    conclusion_true = 0
+    for assignment in product(subsets, repeat=len(terms)):
+        env = dict(zip(terms, assignment))
+        if all(_holds(premise, env) for premise in premises):
+            models += 1
+            if _holds(conclusion, env):
+                conclusion_true += 1
+
+    if models == 0:
+        raise OracleError("premises have no satisfying model")
+    valid = conclusion_true == models
+    return {
+        "valid": valid,
+        "conclusion": dict(conclusion_payload) if valid else None,
+    }
+

evals/syllogism/runner.py

@@ -0,0 +1,58 @@
+"""Gold-only runner for the staged syllogism lane."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from evals.syllogism.oracle import OracleError, oracle_answer
+
+_CASES = Path(__file__).resolve().parent / "v1" / "cases.jsonl"
+
+
+def _load_cases(path: Path = _CASES) -> list[dict[str, Any]]:
+    return [
+        json.loads(line)
+        for line in path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+
+def run(path: Path = _CASES) -> dict[str, Any]:
+    cases = _load_cases(path)
+    failures: list[str] = []
+    for case in cases:
+        try:
+            got = oracle_answer(case["structure"], case["query"])
+        except OracleError as exc:
+            failures.append(f"{case.get('id', '<missing>')}: refused gold case: {exc}")
+            continue
+        if got != case.get("gold"):
+            failures.append(f"{case['id']}: oracle={got!r} != gold={case.get('gold')!r}")
+
+    correct = len(cases) - len(failures)
+    return {
+        "domain": "syllogism",
+        "total": len(cases),
+        "correct": correct,
+        "wrong": 0,
+        "refused": 0,
+        "gold_integrity_failures": failures,
+        "counts": {"correct": correct, "wrong": 0, "refused": 0},
+    }
+
+
+def main() -> int:
+    report = run()
+    print(json.dumps(report, indent=2, sort_keys=True))
+    if report["gold_integrity_failures"]:
+        print("GOLD INTEGRITY FAILURE", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+

evals/syllogism/v1/cases.jsonl

@@ -0,0 +1,8 @@
+{"id":"sy-v1-0001","seed":20260605,"text":"All mammals are animals. All whales are mammals. Therefore all whales are animals.","structure":{"terms":["whale","mammal","animal"],"domain_size":3,"premises":[{"form":"A","subject":"mammal","predicate":"animal"},{"form":"A","subject":"whale","predicate":"mammal"}]},"query":{"kind":"validity","conclusion":{"form":"A","subject":"whale","predicate":"animal"}},"gold":{"valid":true,"conclusion":{"form":"A","subject":"whale","predicate":"animal"}},"class":"barbara_valid"}
+{"id":"sy-v1-0002","seed":20260605,"text":"No reptiles are mammals. All snakes are reptiles. Therefore no snakes are mammals.","structure":{"terms":["snake","reptile","mammal"],"domain_size":3,"premises":[{"form":"E","subject":"reptile","predicate":"mammal"},{"form":"A","subject":"snake","predicate":"reptile"}]},"query":{"kind":"validity","conclusion":{"form":"E","subject":"snake","predicate":"mammal"}},"gold":{"valid":true,"conclusion":{"form":"E","subject":"snake","predicate":"mammal"}},"class":"celarent_valid"}
+{"id":"sy-v1-0003","seed":20260605,"text":"All poets are artists. Some students are poets. Therefore some students are artists.","structure":{"terms":["student","poet","artist"],"domain_size":3,"premises":[{"form":"A","subject":"poet","predicate":"artist"},{"form":"I","subject":"student","predicate":"poet"}]},"query":{"kind":"validity","conclusion":{"form":"I","subject":"student","predicate":"artist"}},"gold":{"valid":true,"conclusion":{"form":"I","subject":"student","predicate":"artist"}},"class":"darii_valid"}
+{"id":"sy-v1-0004","seed":20260605,"text":"No mammals are reptiles. Some pets are mammals. Therefore some pets are not reptiles.","structure":{"terms":["pet","mammal","reptile"],"domain_size":3,"premises":[{"form":"E","subject":"mammal","predicate":"reptile"},{"form":"I","subject":"pet","predicate":"mammal"}]},"query":{"kind":"validity","conclusion":{"form":"O","subject":"pet","predicate":"reptile"}},"gold":{"valid":true,"conclusion":{"form":"O","subject":"pet","predicate":"reptile"}},"class":"ferio_valid"}
+{"id":"sy-v1-0005","seed":20260605,"text":"All cats are mammals. All dogs are mammals. Therefore all dogs are cats.","structure":{"terms":["dog","cat","mammal"],"domain_size":3,"premises":[{"form":"A","subject":"cat","predicate":"mammal"},{"form":"A","subject":"dog","predicate":"mammal"}]},"query":{"kind":"validity","conclusion":{"form":"A","subject":"dog","predicate":"cat"}},"gold":{"valid":false,"conclusion":null},"class":"invalid_undistributed_middle"}
+{"id":"sy-v1-0006","seed":20260605,"text":"All pilots are people. All pilots are trained. Therefore some trained people exist.","structure":{"terms":["pilot","person","trained"],"domain_size":3,"premises":[{"form":"A","subject":"pilot","predicate":"person"},{"form":"A","subject":"pilot","predicate":"trained"}]},"query":{"kind":"validity","conclusion":{"form":"I","subject":"trained","predicate":"person"}},"gold":{"valid":false,"conclusion":null},"class":"invalid_existential_import"}
+{"id":"sy-v1-0007","seed":20260605,"text":"Some musicians are teachers. All teachers are workers. Therefore some musicians are workers.","structure":{"terms":["musician","teacher","worker"],"domain_size":3,"premises":[{"form":"I","subject":"musician","predicate":"teacher"},{"form":"A","subject":"teacher","predicate":"worker"}]},"query":{"kind":"validity","conclusion":{"form":"I","subject":"musician","predicate":"worker"}},"gold":{"valid":true,"conclusion":{"form":"I","subject":"musician","predicate":"worker"}},"class":"datisi_valid"}
+{"id":"sy-v1-0008","seed":20260605,"text":"No metal objects are soft objects. Some tools are metal objects. Therefore some tools are not soft objects.","structure":{"terms":["tool","metal","soft"],"domain_size":3,"premises":[{"form":"E","subject":"metal","predicate":"soft"},{"form":"I","subject":"tool","predicate":"metal"}]},"query":{"kind":"validity","conclusion":{"form":"O","subject":"tool","predicate":"soft"}},"gold":{"valid":true,"conclusion":{"form":"O","subject":"tool","predicate":"soft"}},"class":"ferio_valid"}

evals/total_ordering/__init__.py

@@ -0,0 +1,2 @@
+"""Staged independent-gold lane for finite total ordering."""
+

evals/total_ordering/oracle.py

@@ -0,0 +1,98 @@
+"""Independent finite-order oracle for the staged Phase 2 gold lane.
+
+The oracle reads only structured pairwise ``less < greater`` facts over a finite
+item set. It refuses cycles, incomparability for sort queries, unknown items, and
+malformed relations. It imports no engine, reader, algebra, field, or generation
+code.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class OracleError(ValueError):
+    """Malformed or out-of-grammar case; the oracle refuses, never guesses."""
+
+
+def _require_str(value: object, label: str) -> str:
+    if not isinstance(value, str) or not value:
+        raise OracleError(f"{label} must be a non-empty string: {value!r}")
+    return value
+
+
+def _items(raw: object) -> list[str]:
+    if not isinstance(raw, list) or not raw:
+        raise OracleError("items must be a non-empty list")
+    items = [_require_str(v, "item") for v in raw]
+    if len(set(items)) != len(items):
+        raise OracleError("items contains duplicates")
+    return items
+
+
+def _closure(items: list[str], relations: object) -> dict[str, set[str]]:
+    item_set = set(items)
+    if not isinstance(relations, list):
+        raise OracleError("relations must be a list")
+    less_than: dict[str, set[str]] = {item: set() for item in items}
+    for rel in relations:
+        if not isinstance(rel, dict):
+            raise OracleError(f"relation must be an object: {rel!r}")
+        less = _require_str(rel.get("less"), "relation.less")
+        greater = _require_str(rel.get("greater"), "relation.greater")
+        if less not in item_set or greater not in item_set:
+            raise OracleError(f"relation names unknown item: {rel!r}")
+        if less == greater:
+            raise OracleError("irreflexive order cannot relate an item to itself")
+        less_than[less].add(greater)
+
+    changed = True
+    while changed:
+        changed = False
+        for item in items:
+            expanded = set(less_than[item])
+            for mid in tuple(less_than[item]):
+                expanded.update(less_than[mid])
+            if item in expanded:
+                raise OracleError("cycle in ordering relations")
+            if expanded != less_than[item]:
+                less_than[item] = expanded
+                changed = True
+    return less_than
+
+
+def oracle_answer(structure: dict[str, Any], query: dict[str, Any]) -> list[str] | str:
+    """Return a sort or comparison result from a finite strict total order."""
+    items = _items(structure.get("items"))
+    less_than = _closure(items, structure.get("relations", []))
+    item_set = set(items)
+
+    def compare(left: str, right: str) -> str:
+        if left not in item_set or right not in item_set:
+            raise OracleError(f"query names unknown item: {left!r}, {right!r}")
+        if left == right:
+            return "equal"
+        if right in less_than[left]:
+            return "less"
+        if left in less_than[right]:
+            return "greater"
+        raise OracleError(f"items are incomparable: {left!r}, {right!r}")
+
+    kind = query.get("kind")
+    if kind == "compare":
+        return compare(
+            _require_str(query.get("left"), "query.left"),
+            _require_str(query.get("right"), "query.right"),
+        )
+    if kind == "sort":
+        order = query.get("order", "ascending")
+        if order not in ("ascending", "descending"):
+            raise OracleError(f"unsupported sort order: {order!r}")
+        for i, left in enumerate(items):
+            for right in items[i + 1:]:
+                compare(left, right)
+        sorted_items = sorted(items, key=lambda item: (-len(less_than[item]), item))
+        if order == "descending":
+            sorted_items = list(reversed(sorted_items))
+        return sorted_items
+    raise OracleError(f"unsupported query kind: {kind!r}")

evals/total_ordering/runner.py

@@ -0,0 +1,58 @@
+"""Gold-only runner for the staged total-ordering lane."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from evals.total_ordering.oracle import OracleError, oracle_answer
+
+_CASES = Path(__file__).resolve().parent / "v1" / "cases.jsonl"
+
+
+def _load_cases(path: Path = _CASES) -> list[dict[str, Any]]:
+    return [
+        json.loads(line)
+        for line in path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+
+def run(path: Path = _CASES) -> dict[str, Any]:
+    cases = _load_cases(path)
+    failures: list[str] = []
+    for case in cases:
+        try:
+            got = oracle_answer(case["structure"], case["query"])
+        except OracleError as exc:
+            failures.append(f"{case.get('id', '<missing>')}: refused gold case: {exc}")
+            continue
+        if got != case.get("gold"):
+            failures.append(f"{case['id']}: oracle={got!r} != gold={case.get('gold')!r}")
+
+    correct = len(cases) - len(failures)
+    return {
+        "domain": "total_ordering",
+        "total": len(cases),
+        "correct": correct,
+        "wrong": 0,
+        "refused": 0,
+        "gold_integrity_failures": failures,
+        "counts": {"correct": correct, "wrong": 0, "refused": 0},
+    }
+
+
+def main() -> int:
+    report = run()
+    print(json.dumps(report, indent=2, sort_keys=True))
+    if report["gold_integrity_failures"]:
+        print("GOLD INTEGRITY FAILURE", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+

evals/total_ordering/v1/cases.jsonl

@@ -0,0 +1,8 @@
+{"id":"to-v1-0001","seed":20260605,"text":"Bronze is below silver, and silver is below gold. Sort them from lowest to highest.","structure":{"items":["gold","bronze","silver"],"relations":[{"less":"bronze","greater":"silver"},{"less":"silver","greater":"gold"}]},"query":{"kind":"sort","order":"ascending"},"gold":["bronze","silver","gold"],"class":"sort_chain_three"}
+{"id":"to-v1-0002","seed":20260605,"text":"Oak is taller than birch. Pine is taller than oak. Which is the height order from shortest to tallest?","structure":{"items":["pine","birch","oak"],"relations":[{"less":"birch","greater":"oak"},{"less":"oak","greater":"pine"}]},"query":{"kind":"sort","order":"ascending"},"gold":["birch","oak","pine"],"class":"sort_chain_three"}
+{"id":"to-v1-0003","seed":20260605,"text":"A is earlier than B, B is earlier than C, and C is earlier than D. Compare A with D.","structure":{"items":["d","b","a","c"],"relations":[{"less":"a","greater":"b"},{"less":"b","greater":"c"},{"less":"c","greater":"d"}]},"query":{"kind":"compare","left":"a","right":"d"},"gold":"less","class":"compare_transitive_less"}
+{"id":"to-v1-0004","seed":20260605,"text":"North station comes after central, and central comes after south. Compare north with south.","structure":{"items":["north","south","central"],"relations":[{"less":"south","greater":"central"},{"less":"central","greater":"north"}]},"query":{"kind":"compare","left":"north","right":"south"},"gold":"greater","class":"compare_transitive_greater"}
+{"id":"to-v1-0005","seed":20260605,"text":"Red rank is below orange, orange below yellow, yellow below green. Sort descending.","structure":{"items":["orange","green","red","yellow"],"relations":[{"less":"red","greater":"orange"},{"less":"orange","greater":"yellow"},{"less":"yellow","greater":"green"}]},"query":{"kind":"sort","order":"descending"},"gold":["green","yellow","orange","red"],"class":"sort_descending_four"}
+{"id":"to-v1-0006","seed":20260605,"text":"Compare beta with beta in the same order.","structure":{"items":["alpha","beta","gamma"],"relations":[{"less":"alpha","greater":"beta"},{"less":"beta","greater":"gamma"}]},"query":{"kind":"compare","left":"beta","right":"beta"},"gold":"equal","class":"compare_equal"}
+{"id":"to-v1-0007","seed":20260605,"text":"Level one is below level two, level two below level three, level three below level four. Sort ascending.","structure":{"items":["level_four","level_one","level_three","level_two"],"relations":[{"less":"level_one","greater":"level_two"},{"less":"level_two","greater":"level_three"},{"less":"level_three","greater":"level_four"}]},"query":{"kind":"sort","order":"ascending"},"gold":["level_one","level_two","level_three","level_four"],"class":"sort_chain_four"}
+{"id":"to-v1-0008","seed":20260605,"text":"Mercury is closer than Venus, Venus closer than Earth, and Earth closer than Mars. Compare Mars with Venus.","structure":{"items":["earth","mars","mercury","venus"],"relations":[{"less":"mercury","greater":"venus"},{"less":"venus","greater":"earth"},{"less":"earth","greater":"mars"}]},"query":{"kind":"compare","left":"mars","right":"venus"},"gold":"greater","class":"compare_transitive_greater"}

tests/test_staged_gold_lanes.py

@@ -0,0 +1,155 @@
+"""Staged independent-gold lanes for future comprehension domains.
+
+These tests validate the GOLD side only: fixed structured cases, independent
+oracles, refusal-first malformed handling, and no capability-index wiring before
+readers exist.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from evals.capability_index import adapters
+from evals.set_membership import runner as set_runner
+from evals.set_membership.oracle import (
+    OracleError as SetOracleError,
+    oracle_answer as set_answer,
+)
+from evals.syllogism import runner as syllogism_runner
+from evals.syllogism.oracle import (
+    OracleError as SyllogismOracleError,
+    oracle_answer as syllogism_answer,
+)
+from evals.total_ordering import runner as order_runner
+from evals.total_ordering.oracle import (
+    OracleError as OrderOracleError,
+    oracle_answer as order_answer,
+)
+
+
+_ROOT = Path(__file__).resolve().parent.parent
+_LANES = ("set_membership", "total_ordering", "syllogism")
+
+
+def _cases(domain: str) -> list[dict]:
+    path = _ROOT / "evals" / domain / "v1" / "cases.jsonl"
+    return [
+        json.loads(line)
+        for line in path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+
+@pytest.mark.parametrize(
+    ("domain", "runner"),
+    [
+        ("set_membership", set_runner),
+        ("total_ordering", order_runner),
+        ("syllogism", syllogism_runner),
+    ],
+)
+def test_staged_gold_lane_integrity(domain, runner) -> None:
+    report = runner.run()
+    assert report["domain"] == domain
+    assert report["gold_integrity_failures"] == []
+    assert report["wrong"] == 0
+    assert report["refused"] == 0
+    assert report["total"] == report["correct"]
+    assert report["counts"] == {
+        "correct": report["correct"],
+        "wrong": 0,
+        "refused": 0,
+    }
+
+
+@pytest.mark.parametrize("domain", _LANES)
+def test_cases_are_structured_not_text_only(domain: str) -> None:
+    cases = _cases(domain)
+    assert len(cases) >= 8
+    assert [case["id"] for case in cases] == sorted(case["id"] for case in cases)
+    assert {case["seed"] for case in cases} == {20260605}
+    for case in cases:
+        assert isinstance(case["text"], str) and case["text"]
+        assert isinstance(case["structure"], dict) and case["structure"]
+        assert isinstance(case["query"], dict) and case["query"]
+        assert "gold" in case
+
+
+def test_set_membership_oracle_is_deterministic_and_refusal_first() -> None:
+    case = _cases("set_membership")[0]
+    assert set_answer(case["structure"], case["query"]) is True
+    assert set_answer(case["structure"], case["query"]) is True
+    with pytest.raises(SetOracleError):
+        set_answer(case["structure"], {"kind": "member", "element": "ghost", "set": "bird"})
+    with pytest.raises(SetOracleError):
+        set_answer(case["structure"], {"kind": "union", "left": "raven", "right": "bird"})
+
+
+def test_total_ordering_oracle_is_deterministic_and_refusal_first() -> None:
+    case = _cases("total_ordering")[0]
+    assert order_answer(case["structure"], case["query"]) == ["bronze", "silver", "gold"]
+    assert order_answer(case["structure"], case["query"]) == ["bronze", "silver", "gold"]
+    with pytest.raises(OrderOracleError):
+        order_answer(
+            {"items": ["a", "b", "c"], "relations": [{"less": "a", "greater": "b"}]},
+            {"kind": "sort", "order": "ascending"},
+        )
+    with pytest.raises(OrderOracleError):
+        order_answer(
+            {
+                "items": ["a", "b"],
+                "relations": [
+                    {"less": "a", "greater": "b"},
+                    {"less": "b", "greater": "a"},
+                ],
+            },
+            {"kind": "compare", "left": "a", "right": "b"},
+        )
+
+
+def test_syllogism_oracle_is_deterministic_and_refusal_first() -> None:
+    case = _cases("syllogism")[0]
+    expected = {
+        "valid": True,
+        "conclusion": {"form": "A", "subject": "whale", "predicate": "animal"},
+    }
+    assert syllogism_answer(case["structure"], case["query"]) == expected
+    assert syllogism_answer(case["structure"], case["query"]) == expected
+    with pytest.raises(SyllogismOracleError):
+        syllogism_answer(
+            {
+                "terms": ["a", "b"],
+                "domain_size": 3,
+                "premises": [{"form": "X", "subject": "a", "predicate": "b"}],
+            },
+            {"kind": "validity", "conclusion": {"form": "A", "subject": "a", "predicate": "b"}},
+        )
+    with pytest.raises(SyllogismOracleError):
+        syllogism_answer(
+            {
+                "terms": ["a", "b"],
+                "domain_size": 3,
+                "premises": [
+                    {"form": "A", "subject": "a", "predicate": "b"},
+                    {"form": "O", "subject": "a", "predicate": "b"},
+                ],
+            },
+            {"kind": "validity", "conclusion": {"form": "A", "subject": "a", "predicate": "b"}},
+        )
+
+
+def test_staged_lanes_are_not_capability_index_adapters() -> None:
+    adapter_names = {fn.__name__ for fn in adapters.ADAPTERS}
+    assert "set_membership_result" not in adapter_names
+    assert "total_ordering_result" not in adapter_names
+    assert "syllogism_result" not in adapter_names
+
+
+def test_new_oracles_do_not_import_engine_or_geometry_modules() -> None:
+    forbidden = ("import algebra", "import field", "import generate", "from algebra", "from field", "from generate")
+    for domain in _LANES:
+        source = (_ROOT / "evals" / domain / "oracle.py").read_text(encoding="utf-8")
+        assert not any(token in source for token in forbidden)