diff --git a/evals/set_membership/__init__.py b/evals/set_membership/__init__.py new file mode 100644 index 00000000..fb6a93c3 --- /dev/null +++ b/evals/set_membership/__init__.py @@ -0,0 +1,2 @@ +"""Staged independent-gold lane for set membership/subsumption.""" + diff --git a/evals/set_membership/oracle.py b/evals/set_membership/oracle.py new file mode 100644 index 00000000..322606df --- /dev/null +++ b/evals/set_membership/oracle.py @@ -0,0 +1,109 @@ +"""Independent set-membership oracle for the staged Phase 2 gold lane. + +The oracle reads STRUCTURE only: finite elements, finite sets, direct membership +facts, and subset relations. It performs a plain transitive-closure decision +procedure and refuses malformed/out-of-grammar inputs. It imports no engine, +reader, algebra, field, or generation code; independence will live in a future +text reader, while this lane fixes the gold side. +""" + +from __future__ import annotations + +from typing import Any + + +class OracleError(ValueError): + """Malformed or out-of-grammar case; the oracle refuses, never guesses.""" + + +def _require_str(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise OracleError(f"{label} must be a non-empty string: {value!r}") + return value + + +def _require_str_list(value: object, label: str) -> list[str]: + if not isinstance(value, list): + raise OracleError(f"{label} must be a list") + out = [_require_str(v, label) for v in value] + if len(set(out)) != len(out): + raise OracleError(f"{label} contains duplicates") + return out + + +def _closure(edges: dict[str, set[str]], sets: set[str]) -> dict[str, set[str]]: + closed = {s: {s, *edges.get(s, set())} for s in sets} + changed = True + while changed: + changed = False + for left in sorted(sets): + expanded = set(closed[left]) + for mid in tuple(closed[left]): + expanded.update(closed.get(mid, {mid})) + if expanded != closed[left]: + closed[left] = expanded + changed = True + return closed + + +def oracle_answer(structure: dict[str, Any], query: dict[str, Any]) -> bool: + """Return the closed-world membership/subsumption verdict. + + Supported query kinds: + - ``member``: whether ``element`` is in ``set`` after subset propagation. + - ``subset``: whether ``subset`` is included in ``superset`` transitively. + """ + elements = set(_require_str_list(structure.get("elements"), "elements")) + + raw_sets = structure.get("sets") + if not isinstance(raw_sets, list) or not raw_sets: + raise OracleError("sets must be a non-empty list") + + members: dict[str, set[str]] = {} + for raw in raw_sets: + if not isinstance(raw, dict): + raise OracleError(f"set entry must be an object: {raw!r}") + sid = _require_str(raw.get("id"), "set.id") + if sid in members: + raise OracleError(f"duplicate set id: {sid!r}") + direct = set(_require_str_list(raw.get("members", []), f"{sid}.members")) + unknown = direct - elements + if unknown: + raise OracleError(f"set {sid!r} names unknown elements: {sorted(unknown)}") + members[sid] = direct + + set_ids = set(members) + edges: dict[str, set[str]] = {sid: set() for sid in set_ids} + raw_subsets = structure.get("subsets", []) + if not isinstance(raw_subsets, list): + raise OracleError("subsets must be a list") + for raw in raw_subsets: + if not isinstance(raw, dict): + raise OracleError(f"subset entry must be an object: {raw!r}") + subset = _require_str(raw.get("subset"), "subset") + superset = _require_str(raw.get("superset"), "superset") + if subset not in set_ids or superset not in set_ids: + raise OracleError(f"subset relation names unknown set: {raw!r}") + edges[subset].add(superset) + + closed = _closure(edges, set_ids) + kind = query.get("kind") + if kind == "subset": + subset = _require_str(query.get("subset"), "query.subset") + superset = _require_str(query.get("superset"), "query.superset") + if subset not in set_ids or superset not in set_ids: + raise OracleError(f"query names unknown set: {query!r}") + return superset in closed[subset] + + if kind == "member": + element = _require_str(query.get("element"), "query.element") + target = _require_str(query.get("set"), "query.set") + if element not in elements: + raise OracleError(f"query names unknown element: {element!r}") + if target not in set_ids: + raise OracleError(f"query names unknown set: {target!r}") + element_sets = {sid for sid, direct in members.items() if element in direct} + return any(target in closed[sid] for sid in element_sets) + + raise OracleError(f"unsupported query kind: {kind!r}") + diff --git a/evals/set_membership/runner.py b/evals/set_membership/runner.py new file mode 100644 index 00000000..c7ed6d35 --- /dev/null +++ b/evals/set_membership/runner.py @@ -0,0 +1,62 @@ +"""Gold-only runner for the staged set-membership lane. + +No engine reader is invoked here. The runner only checks that committed gold is +reproducible from structured cases by the independent oracle. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +from evals.set_membership.oracle import OracleError, oracle_answer + +_CASES = Path(__file__).resolve().parent / "v1" / "cases.jsonl" + + +def _load_cases(path: Path = _CASES) -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def run(path: Path = _CASES) -> dict[str, Any]: + cases = _load_cases(path) + failures: list[str] = [] + for case in cases: + try: + got = oracle_answer(case["structure"], case["query"]) + except OracleError as exc: + failures.append(f"{case.get('id', '')}: refused gold case: {exc}") + continue + if got != case.get("gold"): + failures.append(f"{case['id']}: oracle={got!r} != gold={case.get('gold')!r}") + + correct = len(cases) - len(failures) + return { + "domain": "set_membership", + "total": len(cases), + "correct": correct, + "wrong": 0, + "refused": 0, + "gold_integrity_failures": failures, + "counts": {"correct": correct, "wrong": 0, "refused": 0}, + } + + +def main() -> int: + report = run() + print(json.dumps(report, indent=2, sort_keys=True)) + if report["gold_integrity_failures"]: + print("GOLD INTEGRITY FAILURE", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/evals/set_membership/v1/cases.jsonl b/evals/set_membership/v1/cases.jsonl new file mode 100644 index 00000000..10a765e5 --- /dev/null +++ b/evals/set_membership/v1/cases.jsonl @@ -0,0 +1,8 @@ +{"id":"sm-v1-0001","seed":20260605,"text":"Rhea is a raven. All ravens are birds. Is Rhea a bird?","structure":{"elements":["rhea","mira"],"sets":[{"id":"raven","members":["rhea"]},{"id":"bird","members":[]},{"id":"mammal","members":["mira"]}],"subsets":[{"subset":"raven","superset":"bird"}]},"query":{"kind":"member","element":"rhea","set":"bird"},"gold":true,"class":"membership_via_subsumption"} +{"id":"sm-v1-0002","seed":20260605,"text":"Ada is a botanist. All botanists are scientists. All scientists are people. Is Ada a person?","structure":{"elements":["ada","ben"],"sets":[{"id":"botanist","members":["ada"]},{"id":"scientist","members":[]},{"id":"person","members":[]},{"id":"engineer","members":["ben"]}],"subsets":[{"subset":"botanist","superset":"scientist"},{"subset":"scientist","superset":"person"},{"subset":"engineer","superset":"person"}]},"query":{"kind":"member","element":"ada","set":"person"},"gold":true,"class":"membership_transitive"} +{"id":"sm-v1-0003","seed":20260605,"text":"Fido is a dog. All dogs are mammals. All cats are mammals. Is Fido a cat?","structure":{"elements":["fido","cleo"],"sets":[{"id":"dog","members":["fido"]},{"id":"cat","members":["cleo"]},{"id":"mammal","members":[]}],"subsets":[{"subset":"dog","superset":"mammal"},{"subset":"cat","superset":"mammal"}]},"query":{"kind":"member","element":"fido","set":"cat"},"gold":false,"class":"membership_negative"} +{"id":"sm-v1-0004","seed":20260605,"text":"All squares are rectangles. All rectangles are polygons. Are all squares polygons?","structure":{"elements":["tile"],"sets":[{"id":"square","members":["tile"]},{"id":"rectangle","members":[]},{"id":"polygon","members":[]}],"subsets":[{"subset":"square","superset":"rectangle"},{"subset":"rectangle","superset":"polygon"}]},"query":{"kind":"subset","subset":"square","superset":"polygon"},"gold":true,"class":"subsumption_transitive"} +{"id":"sm-v1-0005","seed":20260605,"text":"All sparrows are birds. All birds are animals. Are all animals sparrows?","structure":{"elements":["pip"],"sets":[{"id":"sparrow","members":["pip"]},{"id":"bird","members":[]},{"id":"animal","members":[]}],"subsets":[{"subset":"sparrow","superset":"bird"},{"subset":"bird","superset":"animal"}]},"query":{"kind":"subset","subset":"animal","superset":"sparrow"},"gold":false,"class":"subsumption_direction_negative"} +{"id":"sm-v1-0006","seed":20260605,"text":"Carol is a cyclist. All cyclists are athletes. Is Carol an athlete?","structure":{"elements":["carol","dion"],"sets":[{"id":"cyclist","members":["carol"]},{"id":"athlete","members":[]},{"id":"musician","members":["dion"]}],"subsets":[{"subset":"cyclist","superset":"athlete"}]},"query":{"kind":"member","element":"carol","set":"athlete"},"gold":true,"class":"membership_via_subsumption"} +{"id":"sm-v1-0007","seed":20260605,"text":"The mug is a cup. The hammer is a tool. Is the mug a tool?","structure":{"elements":["mug","hammer"],"sets":[{"id":"cup","members":["mug"]},{"id":"tool","members":["hammer"]},{"id":"object","members":[]}],"subsets":[{"subset":"cup","superset":"object"},{"subset":"tool","superset":"object"}]},"query":{"kind":"member","element":"mug","set":"tool"},"gold":false,"class":"disjoint_membership_negative"} +{"id":"sm-v1-0008","seed":20260605,"text":"All violins are instruments. All instruments are artifacts. Are all violins artifacts?","structure":{"elements":["strad"],"sets":[{"id":"violin","members":["strad"]},{"id":"instrument","members":[]},{"id":"artifact","members":[]}],"subsets":[{"subset":"violin","superset":"instrument"},{"subset":"instrument","superset":"artifact"}]},"query":{"kind":"subset","subset":"violin","superset":"artifact"},"gold":true,"class":"subsumption_transitive"} diff --git a/evals/syllogism/__init__.py b/evals/syllogism/__init__.py new file mode 100644 index 00000000..3fb7a431 --- /dev/null +++ b/evals/syllogism/__init__.py @@ -0,0 +1,2 @@ +"""Staged independent-gold lane for categorical syllogisms.""" + diff --git a/evals/syllogism/oracle.py b/evals/syllogism/oracle.py new file mode 100644 index 00000000..34e7225b --- /dev/null +++ b/evals/syllogism/oracle.py @@ -0,0 +1,105 @@ +"""Independent categorical-syllogism oracle for the staged Phase 2 gold lane. + +The oracle reads structured categorical propositions, not text. It evaluates +validity by brute-force finite-model enumeration over a small domain. This is +intentionally simple and independent of any future comprehension reader. +""" + +from __future__ import annotations + +from itertools import product +from typing import Any, Final + + +class OracleError(ValueError): + """Malformed or out-of-grammar case; the oracle refuses, never guesses.""" + + +_FORMS: Final[frozenset[str]] = frozenset({"A", "E", "I", "O"}) + + +def _require_str(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise OracleError(f"{label} must be a non-empty string: {value!r}") + return value + + +def _proposition(raw: object, terms: set[str]) -> tuple[str, str, str]: + if not isinstance(raw, dict): + raise OracleError(f"proposition must be an object: {raw!r}") + form = _require_str(raw.get("form"), "form") + subject = _require_str(raw.get("subject"), "subject") + predicate = _require_str(raw.get("predicate"), "predicate") + if form not in _FORMS: + raise OracleError(f"unsupported categorical form: {form!r}") + if subject not in terms or predicate not in terms: + raise OracleError(f"proposition names unknown term: {raw!r}") + if subject == predicate: + raise OracleError("subject and predicate must be distinct") + return form, subject, predicate + + +def _holds(prop: tuple[str, str, str], env: dict[str, frozenset[int]]) -> bool: + form, subject, predicate = prop + s = env[subject] + p = env[predicate] + if form == "A": + return s <= p + if form == "E": + return not (s & p) + if form == "I": + return bool(s & p) + if form == "O": + return bool(s - p) + raise OracleError(f"unknown form: {form!r}") # pragma: no cover + + +def oracle_answer(structure: dict[str, Any], query: dict[str, Any]) -> dict[str, Any]: + """Return ``{"valid": bool, "conclusion": conclusion-or-None}``. + + Premises with no satisfying model are refused as inconsistent; otherwise a + candidate conclusion is valid iff it holds in every model of the premises. + """ + raw_terms = structure.get("terms") + if not isinstance(raw_terms, list) or len(raw_terms) < 2: + raise OracleError("terms must be a list of at least two term names") + terms = [_require_str(t, "term") for t in raw_terms] + if len(set(terms)) != len(terms): + raise OracleError("terms contains duplicates") + term_set = set(terms) + + raw_premises = structure.get("premises") + if not isinstance(raw_premises, list) or not raw_premises: + raise OracleError("premises must be a non-empty list") + premises = [_proposition(p, term_set) for p in raw_premises] + + if query.get("kind") != "validity": + raise OracleError(f"unsupported query kind: {query.get('kind')!r}") + conclusion_payload = query.get("conclusion") + conclusion = _proposition(conclusion_payload, term_set) + + domain = tuple(range(int(structure.get("domain_size", 3)))) + if not domain: + raise OracleError("domain_size must be positive") + subsets = [ + frozenset(i for i, include in enumerate(bits) if include) + for bits in product((False, True), repeat=len(domain)) + ] + + models = 0 + conclusion_true = 0 + for assignment in product(subsets, repeat=len(terms)): + env = dict(zip(terms, assignment)) + if all(_holds(premise, env) for premise in premises): + models += 1 + if _holds(conclusion, env): + conclusion_true += 1 + + if models == 0: + raise OracleError("premises have no satisfying model") + valid = conclusion_true == models + return { + "valid": valid, + "conclusion": dict(conclusion_payload) if valid else None, + } + diff --git a/evals/syllogism/runner.py b/evals/syllogism/runner.py new file mode 100644 index 00000000..7f646d7a --- /dev/null +++ b/evals/syllogism/runner.py @@ -0,0 +1,58 @@ +"""Gold-only runner for the staged syllogism lane.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +from evals.syllogism.oracle import OracleError, oracle_answer + +_CASES = Path(__file__).resolve().parent / "v1" / "cases.jsonl" + + +def _load_cases(path: Path = _CASES) -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def run(path: Path = _CASES) -> dict[str, Any]: + cases = _load_cases(path) + failures: list[str] = [] + for case in cases: + try: + got = oracle_answer(case["structure"], case["query"]) + except OracleError as exc: + failures.append(f"{case.get('id', '')}: refused gold case: {exc}") + continue + if got != case.get("gold"): + failures.append(f"{case['id']}: oracle={got!r} != gold={case.get('gold')!r}") + + correct = len(cases) - len(failures) + return { + "domain": "syllogism", + "total": len(cases), + "correct": correct, + "wrong": 0, + "refused": 0, + "gold_integrity_failures": failures, + "counts": {"correct": correct, "wrong": 0, "refused": 0}, + } + + +def main() -> int: + report = run() + print(json.dumps(report, indent=2, sort_keys=True)) + if report["gold_integrity_failures"]: + print("GOLD INTEGRITY FAILURE", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/evals/syllogism/v1/cases.jsonl b/evals/syllogism/v1/cases.jsonl new file mode 100644 index 00000000..f914f23d --- /dev/null +++ b/evals/syllogism/v1/cases.jsonl @@ -0,0 +1,8 @@ +{"id":"sy-v1-0001","seed":20260605,"text":"All mammals are animals. All whales are mammals. Therefore all whales are animals.","structure":{"terms":["whale","mammal","animal"],"domain_size":3,"premises":[{"form":"A","subject":"mammal","predicate":"animal"},{"form":"A","subject":"whale","predicate":"mammal"}]},"query":{"kind":"validity","conclusion":{"form":"A","subject":"whale","predicate":"animal"}},"gold":{"valid":true,"conclusion":{"form":"A","subject":"whale","predicate":"animal"}},"class":"barbara_valid"} +{"id":"sy-v1-0002","seed":20260605,"text":"No reptiles are mammals. All snakes are reptiles. Therefore no snakes are mammals.","structure":{"terms":["snake","reptile","mammal"],"domain_size":3,"premises":[{"form":"E","subject":"reptile","predicate":"mammal"},{"form":"A","subject":"snake","predicate":"reptile"}]},"query":{"kind":"validity","conclusion":{"form":"E","subject":"snake","predicate":"mammal"}},"gold":{"valid":true,"conclusion":{"form":"E","subject":"snake","predicate":"mammal"}},"class":"celarent_valid"} +{"id":"sy-v1-0003","seed":20260605,"text":"All poets are artists. Some students are poets. Therefore some students are artists.","structure":{"terms":["student","poet","artist"],"domain_size":3,"premises":[{"form":"A","subject":"poet","predicate":"artist"},{"form":"I","subject":"student","predicate":"poet"}]},"query":{"kind":"validity","conclusion":{"form":"I","subject":"student","predicate":"artist"}},"gold":{"valid":true,"conclusion":{"form":"I","subject":"student","predicate":"artist"}},"class":"darii_valid"} +{"id":"sy-v1-0004","seed":20260605,"text":"No mammals are reptiles. Some pets are mammals. Therefore some pets are not reptiles.","structure":{"terms":["pet","mammal","reptile"],"domain_size":3,"premises":[{"form":"E","subject":"mammal","predicate":"reptile"},{"form":"I","subject":"pet","predicate":"mammal"}]},"query":{"kind":"validity","conclusion":{"form":"O","subject":"pet","predicate":"reptile"}},"gold":{"valid":true,"conclusion":{"form":"O","subject":"pet","predicate":"reptile"}},"class":"ferio_valid"} +{"id":"sy-v1-0005","seed":20260605,"text":"All cats are mammals. All dogs are mammals. Therefore all dogs are cats.","structure":{"terms":["dog","cat","mammal"],"domain_size":3,"premises":[{"form":"A","subject":"cat","predicate":"mammal"},{"form":"A","subject":"dog","predicate":"mammal"}]},"query":{"kind":"validity","conclusion":{"form":"A","subject":"dog","predicate":"cat"}},"gold":{"valid":false,"conclusion":null},"class":"invalid_undistributed_middle"} +{"id":"sy-v1-0006","seed":20260605,"text":"All pilots are people. All pilots are trained. Therefore some trained people exist.","structure":{"terms":["pilot","person","trained"],"domain_size":3,"premises":[{"form":"A","subject":"pilot","predicate":"person"},{"form":"A","subject":"pilot","predicate":"trained"}]},"query":{"kind":"validity","conclusion":{"form":"I","subject":"trained","predicate":"person"}},"gold":{"valid":false,"conclusion":null},"class":"invalid_existential_import"} +{"id":"sy-v1-0007","seed":20260605,"text":"Some musicians are teachers. All teachers are workers. Therefore some musicians are workers.","structure":{"terms":["musician","teacher","worker"],"domain_size":3,"premises":[{"form":"I","subject":"musician","predicate":"teacher"},{"form":"A","subject":"teacher","predicate":"worker"}]},"query":{"kind":"validity","conclusion":{"form":"I","subject":"musician","predicate":"worker"}},"gold":{"valid":true,"conclusion":{"form":"I","subject":"musician","predicate":"worker"}},"class":"datisi_valid"} +{"id":"sy-v1-0008","seed":20260605,"text":"No metal objects are soft objects. Some tools are metal objects. Therefore some tools are not soft objects.","structure":{"terms":["tool","metal","soft"],"domain_size":3,"premises":[{"form":"E","subject":"metal","predicate":"soft"},{"form":"I","subject":"tool","predicate":"metal"}]},"query":{"kind":"validity","conclusion":{"form":"O","subject":"tool","predicate":"soft"}},"gold":{"valid":true,"conclusion":{"form":"O","subject":"tool","predicate":"soft"}},"class":"ferio_valid"} diff --git a/evals/total_ordering/__init__.py b/evals/total_ordering/__init__.py new file mode 100644 index 00000000..e227352e --- /dev/null +++ b/evals/total_ordering/__init__.py @@ -0,0 +1,2 @@ +"""Staged independent-gold lane for finite total ordering.""" + diff --git a/evals/total_ordering/oracle.py b/evals/total_ordering/oracle.py new file mode 100644 index 00000000..a08838d0 --- /dev/null +++ b/evals/total_ordering/oracle.py @@ -0,0 +1,98 @@ +"""Independent finite-order oracle for the staged Phase 2 gold lane. + +The oracle reads only structured pairwise ``less < greater`` facts over a finite +item set. It refuses cycles, incomparability for sort queries, unknown items, and +malformed relations. It imports no engine, reader, algebra, field, or generation +code. +""" + +from __future__ import annotations + +from typing import Any + + +class OracleError(ValueError): + """Malformed or out-of-grammar case; the oracle refuses, never guesses.""" + + +def _require_str(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise OracleError(f"{label} must be a non-empty string: {value!r}") + return value + + +def _items(raw: object) -> list[str]: + if not isinstance(raw, list) or not raw: + raise OracleError("items must be a non-empty list") + items = [_require_str(v, "item") for v in raw] + if len(set(items)) != len(items): + raise OracleError("items contains duplicates") + return items + + +def _closure(items: list[str], relations: object) -> dict[str, set[str]]: + item_set = set(items) + if not isinstance(relations, list): + raise OracleError("relations must be a list") + less_than: dict[str, set[str]] = {item: set() for item in items} + for rel in relations: + if not isinstance(rel, dict): + raise OracleError(f"relation must be an object: {rel!r}") + less = _require_str(rel.get("less"), "relation.less") + greater = _require_str(rel.get("greater"), "relation.greater") + if less not in item_set or greater not in item_set: + raise OracleError(f"relation names unknown item: {rel!r}") + if less == greater: + raise OracleError("irreflexive order cannot relate an item to itself") + less_than[less].add(greater) + + changed = True + while changed: + changed = False + for item in items: + expanded = set(less_than[item]) + for mid in tuple(less_than[item]): + expanded.update(less_than[mid]) + if item in expanded: + raise OracleError("cycle in ordering relations") + if expanded != less_than[item]: + less_than[item] = expanded + changed = True + return less_than + + +def oracle_answer(structure: dict[str, Any], query: dict[str, Any]) -> list[str] | str: + """Return a sort or comparison result from a finite strict total order.""" + items = _items(structure.get("items")) + less_than = _closure(items, structure.get("relations", [])) + item_set = set(items) + + def compare(left: str, right: str) -> str: + if left not in item_set or right not in item_set: + raise OracleError(f"query names unknown item: {left!r}, {right!r}") + if left == right: + return "equal" + if right in less_than[left]: + return "less" + if left in less_than[right]: + return "greater" + raise OracleError(f"items are incomparable: {left!r}, {right!r}") + + kind = query.get("kind") + if kind == "compare": + return compare( + _require_str(query.get("left"), "query.left"), + _require_str(query.get("right"), "query.right"), + ) + if kind == "sort": + order = query.get("order", "ascending") + if order not in ("ascending", "descending"): + raise OracleError(f"unsupported sort order: {order!r}") + for i, left in enumerate(items): + for right in items[i + 1:]: + compare(left, right) + sorted_items = sorted(items, key=lambda item: (-len(less_than[item]), item)) + if order == "descending": + sorted_items = list(reversed(sorted_items)) + return sorted_items + raise OracleError(f"unsupported query kind: {kind!r}") diff --git a/evals/total_ordering/runner.py b/evals/total_ordering/runner.py new file mode 100644 index 00000000..66a3e907 --- /dev/null +++ b/evals/total_ordering/runner.py @@ -0,0 +1,58 @@ +"""Gold-only runner for the staged total-ordering lane.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +from evals.total_ordering.oracle import OracleError, oracle_answer + +_CASES = Path(__file__).resolve().parent / "v1" / "cases.jsonl" + + +def _load_cases(path: Path = _CASES) -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def run(path: Path = _CASES) -> dict[str, Any]: + cases = _load_cases(path) + failures: list[str] = [] + for case in cases: + try: + got = oracle_answer(case["structure"], case["query"]) + except OracleError as exc: + failures.append(f"{case.get('id', '')}: refused gold case: {exc}") + continue + if got != case.get("gold"): + failures.append(f"{case['id']}: oracle={got!r} != gold={case.get('gold')!r}") + + correct = len(cases) - len(failures) + return { + "domain": "total_ordering", + "total": len(cases), + "correct": correct, + "wrong": 0, + "refused": 0, + "gold_integrity_failures": failures, + "counts": {"correct": correct, "wrong": 0, "refused": 0}, + } + + +def main() -> int: + report = run() + print(json.dumps(report, indent=2, sort_keys=True)) + if report["gold_integrity_failures"]: + print("GOLD INTEGRITY FAILURE", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/evals/total_ordering/v1/cases.jsonl b/evals/total_ordering/v1/cases.jsonl new file mode 100644 index 00000000..6d21695f --- /dev/null +++ b/evals/total_ordering/v1/cases.jsonl @@ -0,0 +1,8 @@ +{"id":"to-v1-0001","seed":20260605,"text":"Bronze is below silver, and silver is below gold. Sort them from lowest to highest.","structure":{"items":["gold","bronze","silver"],"relations":[{"less":"bronze","greater":"silver"},{"less":"silver","greater":"gold"}]},"query":{"kind":"sort","order":"ascending"},"gold":["bronze","silver","gold"],"class":"sort_chain_three"} +{"id":"to-v1-0002","seed":20260605,"text":"Oak is taller than birch. Pine is taller than oak. Which is the height order from shortest to tallest?","structure":{"items":["pine","birch","oak"],"relations":[{"less":"birch","greater":"oak"},{"less":"oak","greater":"pine"}]},"query":{"kind":"sort","order":"ascending"},"gold":["birch","oak","pine"],"class":"sort_chain_three"} +{"id":"to-v1-0003","seed":20260605,"text":"A is earlier than B, B is earlier than C, and C is earlier than D. Compare A with D.","structure":{"items":["d","b","a","c"],"relations":[{"less":"a","greater":"b"},{"less":"b","greater":"c"},{"less":"c","greater":"d"}]},"query":{"kind":"compare","left":"a","right":"d"},"gold":"less","class":"compare_transitive_less"} +{"id":"to-v1-0004","seed":20260605,"text":"North station comes after central, and central comes after south. Compare north with south.","structure":{"items":["north","south","central"],"relations":[{"less":"south","greater":"central"},{"less":"central","greater":"north"}]},"query":{"kind":"compare","left":"north","right":"south"},"gold":"greater","class":"compare_transitive_greater"} +{"id":"to-v1-0005","seed":20260605,"text":"Red rank is below orange, orange below yellow, yellow below green. Sort descending.","structure":{"items":["orange","green","red","yellow"],"relations":[{"less":"red","greater":"orange"},{"less":"orange","greater":"yellow"},{"less":"yellow","greater":"green"}]},"query":{"kind":"sort","order":"descending"},"gold":["green","yellow","orange","red"],"class":"sort_descending_four"} +{"id":"to-v1-0006","seed":20260605,"text":"Compare beta with beta in the same order.","structure":{"items":["alpha","beta","gamma"],"relations":[{"less":"alpha","greater":"beta"},{"less":"beta","greater":"gamma"}]},"query":{"kind":"compare","left":"beta","right":"beta"},"gold":"equal","class":"compare_equal"} +{"id":"to-v1-0007","seed":20260605,"text":"Level one is below level two, level two below level three, level three below level four. Sort ascending.","structure":{"items":["level_four","level_one","level_three","level_two"],"relations":[{"less":"level_one","greater":"level_two"},{"less":"level_two","greater":"level_three"},{"less":"level_three","greater":"level_four"}]},"query":{"kind":"sort","order":"ascending"},"gold":["level_one","level_two","level_three","level_four"],"class":"sort_chain_four"} +{"id":"to-v1-0008","seed":20260605,"text":"Mercury is closer than Venus, Venus closer than Earth, and Earth closer than Mars. Compare Mars with Venus.","structure":{"items":["earth","mars","mercury","venus"],"relations":[{"less":"mercury","greater":"venus"},{"less":"venus","greater":"earth"},{"less":"earth","greater":"mars"}]},"query":{"kind":"compare","left":"mars","right":"venus"},"gold":"greater","class":"compare_transitive_greater"} diff --git a/tests/test_staged_gold_lanes.py b/tests/test_staged_gold_lanes.py new file mode 100644 index 00000000..10d101d6 --- /dev/null +++ b/tests/test_staged_gold_lanes.py @@ -0,0 +1,155 @@ +"""Staged independent-gold lanes for future comprehension domains. + +These tests validate the GOLD side only: fixed structured cases, independent +oracles, refusal-first malformed handling, and no capability-index wiring before +readers exist. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from evals.capability_index import adapters +from evals.set_membership import runner as set_runner +from evals.set_membership.oracle import ( + OracleError as SetOracleError, + oracle_answer as set_answer, +) +from evals.syllogism import runner as syllogism_runner +from evals.syllogism.oracle import ( + OracleError as SyllogismOracleError, + oracle_answer as syllogism_answer, +) +from evals.total_ordering import runner as order_runner +from evals.total_ordering.oracle import ( + OracleError as OrderOracleError, + oracle_answer as order_answer, +) + + +_ROOT = Path(__file__).resolve().parent.parent +_LANES = ("set_membership", "total_ordering", "syllogism") + + +def _cases(domain: str) -> list[dict]: + path = _ROOT / "evals" / domain / "v1" / "cases.jsonl" + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +@pytest.mark.parametrize( + ("domain", "runner"), + [ + ("set_membership", set_runner), + ("total_ordering", order_runner), + ("syllogism", syllogism_runner), + ], +) +def test_staged_gold_lane_integrity(domain, runner) -> None: + report = runner.run() + assert report["domain"] == domain + assert report["gold_integrity_failures"] == [] + assert report["wrong"] == 0 + assert report["refused"] == 0 + assert report["total"] == report["correct"] + assert report["counts"] == { + "correct": report["correct"], + "wrong": 0, + "refused": 0, + } + + +@pytest.mark.parametrize("domain", _LANES) +def test_cases_are_structured_not_text_only(domain: str) -> None: + cases = _cases(domain) + assert len(cases) >= 8 + assert [case["id"] for case in cases] == sorted(case["id"] for case in cases) + assert {case["seed"] for case in cases} == {20260605} + for case in cases: + assert isinstance(case["text"], str) and case["text"] + assert isinstance(case["structure"], dict) and case["structure"] + assert isinstance(case["query"], dict) and case["query"] + assert "gold" in case + + +def test_set_membership_oracle_is_deterministic_and_refusal_first() -> None: + case = _cases("set_membership")[0] + assert set_answer(case["structure"], case["query"]) is True + assert set_answer(case["structure"], case["query"]) is True + with pytest.raises(SetOracleError): + set_answer(case["structure"], {"kind": "member", "element": "ghost", "set": "bird"}) + with pytest.raises(SetOracleError): + set_answer(case["structure"], {"kind": "union", "left": "raven", "right": "bird"}) + + +def test_total_ordering_oracle_is_deterministic_and_refusal_first() -> None: + case = _cases("total_ordering")[0] + assert order_answer(case["structure"], case["query"]) == ["bronze", "silver", "gold"] + assert order_answer(case["structure"], case["query"]) == ["bronze", "silver", "gold"] + with pytest.raises(OrderOracleError): + order_answer( + {"items": ["a", "b", "c"], "relations": [{"less": "a", "greater": "b"}]}, + {"kind": "sort", "order": "ascending"}, + ) + with pytest.raises(OrderOracleError): + order_answer( + { + "items": ["a", "b"], + "relations": [ + {"less": "a", "greater": "b"}, + {"less": "b", "greater": "a"}, + ], + }, + {"kind": "compare", "left": "a", "right": "b"}, + ) + + +def test_syllogism_oracle_is_deterministic_and_refusal_first() -> None: + case = _cases("syllogism")[0] + expected = { + "valid": True, + "conclusion": {"form": "A", "subject": "whale", "predicate": "animal"}, + } + assert syllogism_answer(case["structure"], case["query"]) == expected + assert syllogism_answer(case["structure"], case["query"]) == expected + with pytest.raises(SyllogismOracleError): + syllogism_answer( + { + "terms": ["a", "b"], + "domain_size": 3, + "premises": [{"form": "X", "subject": "a", "predicate": "b"}], + }, + {"kind": "validity", "conclusion": {"form": "A", "subject": "a", "predicate": "b"}}, + ) + with pytest.raises(SyllogismOracleError): + syllogism_answer( + { + "terms": ["a", "b"], + "domain_size": 3, + "premises": [ + {"form": "A", "subject": "a", "predicate": "b"}, + {"form": "O", "subject": "a", "predicate": "b"}, + ], + }, + {"kind": "validity", "conclusion": {"form": "A", "subject": "a", "predicate": "b"}}, + ) + + +def test_staged_lanes_are_not_capability_index_adapters() -> None: + adapter_names = {fn.__name__ for fn in adapters.ADAPTERS} + assert "set_membership_result" not in adapter_names + assert "total_ordering_result" not in adapter_names + assert "syllogism_result" not in adapter_names + + +def test_new_oracles_do_not_import_engine_or_geometry_modules() -> None: + forbidden = ("import algebra", "import field", "import generate", "from algebra", "from field", "from generate") + for domain in _LANES: + source = (_ROOT / "evals" / domain / "oracle.py").read_text(encoding="utf-8") + assert not any(token in source for token in forbidden)