diff --git a/evals/capability_index/adapters.py b/evals/capability_index/adapters.py index 0e17f68b..f11c29dc 100644 --- a/evals/capability_index/adapters.py +++ b/evals/capability_index/adapters.py @@ -84,13 +84,22 @@ def comprehension_relational_predicate_result() -> DomainResult: return DomainResult("comprehension_relational_predicate", c, w, r) +def comprehension_relational_inference_result() -> DomainResult: + from evals.comprehension.relational_inference_runner import run + + c, w, r = _counts(run()) + return DomainResult("comprehension_relational_inference", c, w, r) + + #: The reasoning domains currently composed into the index (self-loading lanes). #: The six ``comprehension_*`` lanes score the GENERAL comprehension reader; the #: relational_metric one reads arithmetic prose into the binding-graph quantity #: substrate (admissibility-checked) and projects to the arithmetic oracle, and the #: relational_predicate one (#596) reads binary-relation prose into pack-named -#: predicates — so the index now measures comprehension breadth across categorical, -#: ordering, propositional, quantitative, AND relational reasoning. +#: predicates, and the relational_inference one DETERMINES one-hop inverse/symmetric +#: entailments (mastery-v2 Step 3) — so the index measures comprehension breadth across +#: categorical, ordering, propositional, quantitative, relational-reading, AND +#: relational-inference reasoning. ADAPTERS = ( deductive_logic_result, relational_metric_result, @@ -101,6 +110,7 @@ ADAPTERS = ( comprehension_propositional_result, comprehension_relational_metric_result, comprehension_relational_predicate_result, + comprehension_relational_inference_result, ) diff --git a/evals/capability_index/baseline.json b/evals/capability_index/baseline.json index 3fa13e6c..0695c837 100644 --- a/evals/capability_index/baseline.json +++ b/evals/capability_index/baseline.json @@ -1,13 +1,13 @@ { - "capability_score": 0.94403, - "coverage_geomean": 0.94403, - "coverage_micro": 0.993835, + "capability_score": 0.949483, + "coverage_geomean": 0.949483, + "coverage_micro": 0.993932, "accuracy_micro": 1.0, - "breadth": 9, + "breadth": 10, "min_domain_coverage": 0.833333, "wrong_total": 0, "assert_mode_valid": true, - "deterministic_digest": "1ea91c1eefd6a0c4739e6fc97a40d8522d9b0758b4531bce448ec98e25789ccf", + "deterministic_digest": "35dea2b23084de9d56bd97225ccee6d9c07e650551559dd26c4fed7324806ffc", "domains": [ { "domain": "comprehension_propositional", @@ -17,6 +17,14 @@ "coverage": 1.0, "accuracy": 1.0 }, + { + "domain": "comprehension_relational_inference", + "correct": 13, + "wrong": 0, + "refused": 0, + "coverage": 1.0, + "accuracy": 1.0 + }, { "domain": "comprehension_relational_metric", "correct": 15, diff --git a/evals/comprehension/relational_inference_runner.py b/evals/comprehension/relational_inference_runner.py new file mode 100644 index 00000000..ea5ab470 --- /dev/null +++ b/evals/comprehension/relational_inference_runner.py @@ -0,0 +1,106 @@ +"""Score the one-hop relational-inference DETERMINE capability on independent gold. + +told fact(s) -> ``determine(query)`` vs hand-authored gold ``(predicate, subject, +object)``. The gold is authored by reading the inverse/symmetric relational algebra, +INDEPENDENTLY of ``generate.determine`` / ``generate.meaning_graph.relational`` +(INV-25 / INV-27) — the engine never produced it. + +A refusal is a COVERAGE miss (``refused``), never a wrong; only a ``Determined`` that +disagrees with gold — wrong predicate/subject/object, or ``answer`` not True — is +``wrong``, and wrong must stay 0. The confuser cases that MUST refuse live in +``refusals.jsonl`` + the dedicated lane test. This is the positive-coverage lane that +puts the one-hop relational-inference capability ON the capability index (breadth 9->10). +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +from chat.runtime import ChatRuntime +from generate.determine import Determined, determine +from generate.meaning_graph.relational import ( + comprehend_relational, + load_relational_pack_lemmas, +) +from generate.realize import realize_comprehension +from session.context import SessionContext + +_CASES = ( + Path(__file__).resolve().parent.parent + / "relational_inference" + / "v1" + / "cases.jsonl" +) +_HIGH = 10**9 + + +def _load_cases(path: Path = _CASES) -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def run(path: Path = _CASES) -> dict[str, Any]: + cases = _load_cases(path) + pack = load_relational_pack_lemmas() + rt = ChatRuntime(no_load_state=True) + vocab, persona = rt._context.vocab, rt._context.persona + + correct = wrong = refused = 0 + wrongs: list[dict[str, Any]] = [] + + for case in cases: + ctx = SessionContext( + vocab=vocab, persona=persona, vault_reproject_interval=_HIGH + ) + for fact in case["facts"]: + realize_comprehension(comprehend_relational(fact, pack), ctx) + res = determine(comprehend_relational(case["query"], pack), ctx) + if not isinstance(res, Determined): + refused += 1 # coverage miss, not a wrong + continue + got = [res.answer, res.predicate, res.subject, res.object, res.rule] + gold = [True, case["predicate"], case["subject"], case["object"], case["rule"]] + if got == gold: + correct += 1 + else: + wrong += 1 + wrongs.append({"id": case.get("id"), "got": got, "gold": gold}) + + return { + "domain": "comprehension_relational_inference", + "total": len(cases), + "correct": correct, + "wrong": wrong, + "refused": refused, + "wrongs": wrongs, + "counts": {"correct": correct, "wrong": wrong, "refused": refused}, + } + + +def main() -> int: + report = run() + print( + json.dumps( + {k: v for k, v in report.items() if k != "wrongs"}, + indent=2, + sort_keys=True, + ) + ) + if report["wrong"]: + print( + "WRONG > 0 — relational inference produced a wrong determination:", + file=sys.stderr, + ) + print(json.dumps(report["wrongs"], indent=2), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/relational_inference/v1/cases.jsonl b/evals/relational_inference/v1/cases.jsonl new file mode 100644 index 00000000..a72b26af --- /dev/null +++ b/evals/relational_inference/v1/cases.jsonl @@ -0,0 +1,13 @@ +{"id": "rinf-001", "facts": ["Bob is less than Alice."], "query": "Is Alice greater than Bob?", "predicate": "greater_than", "subject": "alice", "object": "bob", "rule": "inverse"} +{"id": "rinf-002", "facts": ["Alice is greater than Bob."], "query": "Is Bob less than Alice?", "predicate": "less_than", "subject": "bob", "object": "alice", "rule": "inverse"} +{"id": "rinf-003", "facts": ["Alice is the parent of Bob."], "query": "Is Bob the child of Alice?", "predicate": "child_of", "subject": "bob", "object": "alice", "rule": "inverse"} +{"id": "rinf-004", "facts": ["Bob is the child of Alice."], "query": "Is Alice the parent of Bob?", "predicate": "parent_of", "subject": "alice", "object": "bob", "rule": "inverse"} +{"id": "rinf-005", "facts": ["The box is left of the cup."], "query": "Is the cup right of the box?", "predicate": "right_of", "subject": "cup", "object": "box", "rule": "inverse"} +{"id": "rinf-006", "facts": ["The cup is right of the box."], "query": "Is the box left of the cup?", "predicate": "left_of", "subject": "box", "object": "cup", "rule": "inverse"} +{"id": "rinf-007", "facts": ["Dawn is before noon."], "query": "Is noon after dawn?", "predicate": "after_event", "subject": "noon", "object": "dawn", "rule": "inverse"} +{"id": "rinf-008", "facts": ["Noon is after dawn."], "query": "Is dawn before noon?", "predicate": "before_event", "subject": "dawn", "object": "noon", "rule": "inverse"} +{"id": "rinf-009", "facts": ["Alice is the sibling of Bob."], "query": "Is Bob the sibling of Alice?", "predicate": "sibling_of", "subject": "bob", "object": "alice", "rule": "symmetric"} +{"id": "rinf-010", "facts": ["Alice is the spouse of Bob."], "query": "Is Bob the spouse of Alice?", "predicate": "spouse_of", "subject": "bob", "object": "alice", "rule": "symmetric"} +{"id": "rinf-011", "facts": ["Gold is equal to silver."], "query": "Is silver equal to gold?", "predicate": "equal_to", "subject": "silver", "object": "gold", "rule": "symmetric"} +{"id": "rinf-012", "facts": ["Gold is distinct from silver."], "query": "Is silver distinct from gold?", "predicate": "distinct_from", "subject": "silver", "object": "gold", "rule": "symmetric"} +{"id": "rinf-013", "facts": ["The room is adjacent to the hall."], "query": "Is the hall adjacent to the room?", "predicate": "adjacent_to", "subject": "hall", "object": "room", "rule": "symmetric"} diff --git a/evals/relational_inference/v1/provenance.md b/evals/relational_inference/v1/provenance.md new file mode 100644 index 00000000..e0024c89 --- /dev/null +++ b/evals/relational_inference/v1/provenance.md @@ -0,0 +1,35 @@ +# evals/relational_inference/v1 — provenance + +Independent gold for the one-hop relational-inference DETERMINE capability +(mastery-v2 Step 3). The gold `(predicate, subject, object)` triples are authored by +reading the relational predicate algebra (inverse/converse + symmetric) +**independently** of `generate/determine/determine.py` and +`generate/meaning_graph/relational.py` (INV-25 / INV-27) — the reader/determiner never +produced them. + +## cases.jsonl — positive coverage (13) + +Each case: told fact(s) → `determine(query)` should assert the entailed relation. + +- **8 INVERSE/converse** — both directions of `less_than`/`greater_than`, + `parent_of`/`child_of`, `left_of`/`right_of`, `before_event`/`after_event`. +- **5 SYMMETRIC** — `sibling_of`, `spouse_of`, `equal_to`, `distinct_from`, + `adjacent_to`, each told one direction and queried the other. + +A refusal here is a COVERAGE miss (counted `refused`), never a wrong. + +## refusals.jsonl — confusers that MUST refuse (8) + +Transitive (direct + through-inverse), asymmetric self-converse (`less_than`, +`parent_of`), cross-predicate (`sibling_of` ≠ `parent_of`, `greater_than` ≠ +`equal_to`), object mismatch, and ungrounded. A `Determined` on any of these is a +wrong=0 breach — the lane test fails loudly. + +## Known coverage gap (honest) + +`overlaps_event` (symmetric, pack-declared `graph.edge.symmetric`) is **excluded** from +the positive gold: the relational reader requires a copula ("X is _connective_ Y"), but +`overlaps` is a finite verb — "The meeting overlaps the lunch" refuses +(`no_relational_template`) and "is overlaps" is ungrammatical. This is a **reader +surface limitation, not a determination gap** (the symmetric rule would fire if a fact +were stored). A finite-verb relational surface is brief-pack follow-up work. diff --git a/evals/relational_inference/v1/refusals.jsonl b/evals/relational_inference/v1/refusals.jsonl new file mode 100644 index 00000000..9d38df67 --- /dev/null +++ b/evals/relational_inference/v1/refusals.jsonl @@ -0,0 +1,8 @@ +{"id": "rinf-ref-001", "facts": ["Alice is less than Bob.", "Bob is less than Carol."], "query": "Is Alice less than Carol?", "why": "transitive chain — out of scope (one hop only); must refuse"} +{"id": "rinf-ref-002", "facts": ["Bob is less than Alice.", "Carol is less than Bob."], "query": "Is Alice greater than Carol?", "why": "transitive through the inverse rule — must refuse (one hop)"} +{"id": "rinf-ref-003", "facts": ["Alice is less than Bob."], "query": "Is Bob less than Alice?", "why": "less_than is asymmetric — a RealizedRecord | None: + """A realized ``predicate(a, b)`` fact (exact structural recall, R1a), or ``None``.""" + facts = recall_realized(ctx, subject=a, predicate=predicate) + return next((f for f in facts if f.relation_arguments == (a, b)), None) + + +def _relational_one_hop( + ctx: SessionContext, predicate: str, subject: str, target: str +) -> Determined | None: + """Answer ``predicate(subject, target)`` by ONE sound relational-algebra rule, or + ``None`` (the caller then refuses, open-world). Two rules, each reading a stored edge + in its OTHER lawful direction: + + INVERSE/converse ``p(subject, target)`` <= stored ``inverse(p)(target, subject)`` + SYMMETRIC ``p(subject, target)`` <= stored ``p(target, subject)`` + + NEVER transitive (one hop only), NEVER False (open-world), NEVER an undeclared rule: + inverse fires only for a declared converse pair and symmetric only for a pack-declared + symmetric predicate — so ``less_than`` is not self-inverse and ``parent_of`` is not + symmetric. + """ + inverse = INVERSE_OF.get(predicate) + if inverse is not None: + edge = _find_relational_edge(ctx, inverse, target, subject) + if edge is not None: + return _relational_determined(predicate, subject, target, edge, "inverse") + if predicate in SYMMETRIC_PREDICATES: + edge = _find_relational_edge(ctx, predicate, target, subject) + if edge is not None: + return _relational_determined(predicate, subject, target, edge, "symmetric") + return None + + +def _relational_determined( + predicate: str, subject: str, target: str, ground: RealizedRecord, rule: str +) -> Determined: + """A one-hop relational ``Determined`` — answer True, basis from the ground's + standing (as_told today), the single stored edge as grounds, the rule recorded.""" + return Determined( + answer=True, + basis=_basis((ground,)), + predicate=predicate, + subject=subject, + object=target, + grounds=(ground,), + rule=rule, + ) + + def _subset_path( start: str, target: str, supers: dict[str, list[tuple[str, RealizedRecord]]] ) -> tuple[RealizedRecord, ...] | None: @@ -272,4 +345,5 @@ def _verify_subsumption( subject=subject, object=target, grounds=grounds, + rule="subsumption", ) diff --git a/generate/meaning_graph/relational.py b/generate/meaning_graph/relational.py index e9db976e..c395d2e0 100644 --- a/generate/meaning_graph/relational.py +++ b/generate/meaning_graph/relational.py @@ -23,10 +23,12 @@ Fail-closed (wrong=0 at the comprehension layer): ("north station") canonicalizes per the join contract; an argument that still holds relational structure refuses rather than fabricate a compound entity. -Scope — ground binary relations, DIRECT reading only. NO transitive/symmetric/rule -inference: a symmetric lemma (``sibling_of``, ``equal_to``, ``adjacent_to`` …) reads -ONLY the stated direction; the converse is a sound-but-incomplete refusal at DETERMINE, -never a fabricated assertion. Negation is out of grammar (a negated surface refuses). +Scope — this reader grounds stated binary relations in the STATED DIRECTION only; it +performs no inference itself. DETERMINE then applies declared ONE-HOP relational algebra +over the realized facts — inverse/converse pairs and pack-declared symmetric predicates +(``INVERSE_OF`` / ``SYMMETRIC_PREDICATES``) — so a symmetric lemma's converse and an +inverse pair NOW determine soundly. Transitive relational closure, negation (out of +grammar — a negated surface refuses), and closed-world falsehood remain out of scope. This reader is invoked EXPLICITLY (the pack is loaded by the caller, not default-mounted); it does not perturb ``comprehend``'s templates or their wrong=0 tests. @@ -86,6 +88,56 @@ _CONNECTIVE_TO_LEMMA: dict[tuple[str, ...], str] = { #: for direct relational entailment. (The realized fact's predicate is one of these.) RELATIONAL_PREDICATES = frozenset(_CONNECTIVE_TO_LEMMA.values()) +# --------------------------------------------------------------------------- # +# Relational algebra — the SOUND one-hop rules DETERMINE may apply. Each rule +# reads a stored fact in its OTHER lawful direction; open-world (asserts only +# True), ONE hop (no transitive chaining), structurally sound by construction. +# --------------------------------------------------------------------------- # + +#: INVERSE/converse pairs: ``p(a, b)`` holds iff ``q(b, a)`` holds. The pack carries +#: no inverse metadata (only the ``graph.edge.symmetric`` tag), so the converse edges +#: are declared here as a closed table; a test pins every member to +#: ``RELATIONAL_PREDICATES`` so a typo cannot mint an unknown predicate. +_INVERSE_PAIRS: frozenset[frozenset[str]] = frozenset({ + frozenset({"parent_of", "child_of"}), + frozenset({"less_than", "greater_than"}), + frozenset({"left_of", "right_of"}), + frozenset({"before_event", "after_event"}), +}) + +#: ``lemma -> its converse lemma`` (both directions), derived from ``_INVERSE_PAIRS``. +#: A predicate NOT in this map has no converse — so ``less_than`` is never self-inverse. +INVERSE_OF: dict[str, str] = { + lemma: other + for pair in _INVERSE_PAIRS + for lemma, other in (tuple(pair), tuple(pair)[::-1]) +} + +#: The semantic-domain tag the relational pack uses to mark a symmetric predicate. +_SYMMETRIC_DOMAIN_TAG = "graph.edge.symmetric" + +#: SYMMETRIC predicates: ``p(a, b)`` holds iff ``p(b, a)`` holds. This MIRRORS the pack +#: ontology (the ``graph.edge.symmetric`` tag, see ``load_relational_pack_symmetric``); +#: a test pins the two equal so the table can never silently diverge from the pack. A +#: predicate NOT here is asymmetric — so ``parent_of`` is never read in both directions. +SYMMETRIC_PREDICATES: frozenset[str] = frozenset({ + "sibling_of", "spouse_of", "equal_to", + "distinct_from", "adjacent_to", "overlaps_event", +}) + + +def load_relational_pack_symmetric() -> frozenset[str]: + """The symmetric-predicate lemmas as DECLARED by the loaded pack ontology (the + ``graph.edge.symmetric`` semantic-domain tag). ``SYMMETRIC_PREDICATES`` is pinned + equal to this by a test — the pack is the source of truth, the constant is the + runtime-cheap mirror (no per-determination pack load).""" + return frozenset( + entry.lemma + for entry in load_pack_entries(RELATIONAL_PACK_ID) + if _SYMMETRIC_DOMAIN_TAG in entry.semantic_domains + ) + + #: The article stripped from an argument slot's edges ("the box" -> "box"). The #: copula ("is") is NOT stripped here — it is enforced STRUCTURALLY (adjacent to the #: connective) so a dangling copula ("Monday before Friday is.") cannot read as a fact. diff --git a/tests/test_architectural_invariants.py b/tests/test_architectural_invariants.py index e43a05ec..13f01c4d 100644 --- a/tests/test_architectural_invariants.py +++ b/tests/test_architectural_invariants.py @@ -2137,18 +2137,20 @@ class TestINV30OpenWorldDetermineNeverAssertsFalse: ) def test_determine_construction_sites_are_visible(self): - """30c — the scan actually sees the open-world gear's two `Determined` - sites (direct entailment + transitive subsumption), and both assert - True. If the count drops the scan went blind, not clean; if `ok` drops - the firewall was breached at its own source.""" + """30c — the scan actually sees the open-world gear's three `Determined` + sites (direct entailment, one-hop relational inverse/symmetric, and + transitive subsumption), and all assert True. If the count drops the scan + went blind, not clean; if `ok` drops the firewall was breached at its own + source. (Grew 2 -> 3 when one-hop relational inference landed; both new + rules share the single `_relational_determined` constructor.)""" determine_path = ( PROJECT_ROOT_FOR_INV21 / "generate" / "determine" / "determine.py" ) constructions = _determined_constructions( ast.parse(determine_path.read_text()) ) - assert len(constructions) == 2, ( - "Expected exactly the two open-world `Determined(answer=True)` " + assert len(constructions) == 3, ( + "Expected exactly the three open-world `Determined(answer=True)` " f"construction sites in determine.py; saw {len(constructions)}. " "If the gear legitimately changed shape, update this count — but " "confirm every site still asserts only `answer=True`." diff --git a/tests/test_determine_relational_inference.py b/tests/test_determine_relational_inference.py new file mode 100644 index 00000000..1580e8ff --- /dev/null +++ b/tests/test_determine_relational_inference.py @@ -0,0 +1,183 @@ +"""DETERMINE — one-hop sound relational entailment (mastery-v2 Step 3 lead). + +A relational query may hold by ONE sound predicate-algebra rule that reads a stored +edge in its OTHER lawful direction: + + INVERSE/converse greater_than(a, b) <= told less_than(b, a) + SYMMETRIC sibling_of(b, a) <= told sibling_of(a, b) + +Scope is deliberately narrow: OPEN-WORLD (asserts only True, never False), ONE hop +(NO transitive chaining), DECLARED rules only (less_than is NOT self-inverse; +parent_of is NOT symmetric). Closed-world / assert-False / FrameVerdict are out of +scope (a later slice). The wrong=0 discipline lives in the confuser block below. +""" + +from __future__ import annotations + +import pytest + +from chat.runtime import ChatRuntime +from generate.determine import Determined, Undetermined, determine +from generate.meaning_graph.relational import ( + INVERSE_OF, + RELATIONAL_PREDICATES, + SYMMETRIC_PREDICATES, + comprehend_relational, + load_relational_pack_lemmas, + load_relational_pack_symmetric, +) +from generate.realize import realize_comprehension +from session.context import SessionContext + +_HIGH = 10**9 + + +@pytest.fixture(scope="module") +def vocab_persona(): + rt = ChatRuntime(no_load_state=True) + return rt._context.vocab, rt._context.persona + + +@pytest.fixture(scope="module") +def pack(): + return load_relational_pack_lemmas() + + +def _ctx(vocab_persona) -> SessionContext: + vocab, persona = vocab_persona + return SessionContext(vocab=vocab, persona=persona, vault_reproject_interval=_HIGH) + + +def _tell(text: str, ctx: SessionContext, pack): + return realize_comprehension(comprehend_relational(text, pack), ctx) + + +def _ask(text: str, ctx: SessionContext, pack): + return determine(comprehend_relational(text, pack), ctx) + + +# --------------------------------------------------------------------------- # +# Positive — the engine now derives the simplest entailed relational facts +# --------------------------------------------------------------------------- # + + +def test_inverse_converse_admits_true(vocab_persona, pack) -> None: + ctx = _ctx(vocab_persona) + _tell("Bob is less than Alice.", ctx, pack) # less_than(bob, alice) + res = _ask("Is Alice greater than Bob?", ctx, pack) # greater_than(alice, bob) + assert isinstance(res, Determined) + assert res.answer is True and res.basis == "as_told" and res.rule == "inverse" + assert res.predicate == "greater_than" + assert res.subject == "alice" and res.object == "bob" + # grounds = the single stored converse edge + assert len(res.grounds) == 1 and res.grounds[0].relation_predicate == "less_than" + + +def test_symmetric_admits_true(vocab_persona, pack) -> None: + ctx = _ctx(vocab_persona) + _tell("Alice is the sibling of Bob.", ctx, pack) # sibling_of(alice, bob) + res = _ask("Is Bob the sibling of Alice?", ctx, pack) # sibling_of(bob, alice) + assert isinstance(res, Determined) and res.answer is True + assert res.rule == "symmetric" and res.predicate == "sibling_of" + assert res.subject == "bob" and res.object == "alice" + + +def test_direct_stored_direction_still_determines(vocab_persona, pack) -> None: + ctx = _ctx(vocab_persona) + _tell("Alice is the sibling of Bob.", ctx, pack) + res = _ask("Is Alice the sibling of Bob?", ctx, pack) # the stored direction + assert isinstance(res, Determined) and res.answer is True and res.rule == "direct" + + +# --------------------------------------------------------------------------- # +# wrong=0 confuser block — the rule must not over-fire +# --------------------------------------------------------------------------- # + + +def test_less_than_is_not_self_inverse(vocab_persona, pack) -> None: + """less_than is asymmetric: a None: + ctx = _ctx(vocab_persona) + _tell("Alice is the sibling of Bob.", ctx, pack) + res = _ask("Is Alice the parent of Bob?", ctx, pack) + assert isinstance(res, Undetermined) + + +def test_greater_than_does_not_imply_equal(vocab_persona, pack) -> None: + ctx = _ctx(vocab_persona) + _tell("Alice is greater than Bob.", ctx, pack) + res = _ask("Is Alice equal to Bob?", ctx, pack) + assert isinstance(res, Undetermined) + + +def test_no_transitive_chain_direct(vocab_persona, pack) -> None: + """One hop only: a None: + """The inverse rule is also one hop: it must not become a transitive bridge.""" + ctx = _ctx(vocab_persona) + _tell("Bob is less than Alice.", ctx, pack) # => greater_than(alice, bob) by inverse + _tell("Carol is less than Bob.", ctx, pack) # => greater_than(bob, carol) by inverse + res = _ask("Is Alice greater than Carol?", ctx, pack) # needs transitive — refuse + assert isinstance(res, Undetermined) + + +def test_unsupported_predicate_refuses(vocab_persona, pack) -> None: + ctx = _ctx(vocab_persona) + # No told fact at all → ungrounded refusal, never a guess. + res = _ask("Is Alice the sibling of Bob?", ctx, pack) + assert isinstance(res, Undetermined) + + +def test_never_asserts_false(vocab_persona, pack) -> None: + """Open-world: every determination is True-or-refuse; answer=False is unreachable + (INV-30 is the structural guarantee — this is the behavioral echo on this path).""" + ctx = _ctx(vocab_persona) + _tell("Alice is less than Bob.", ctx, pack) + for q in ( + "Is Bob less than Alice?", + "Is Alice greater than Bob?", + "Is Alice equal to Bob?", + "Is Alice the parent of Bob?", + "Is Alice less than Bob?", + ): + res = _ask(q, ctx, pack) + if isinstance(res, Determined): + assert res.answer is True + + +# --------------------------------------------------------------------------- # +# Ontology pins — the algebra cannot silently diverge from the pack / vocab +# --------------------------------------------------------------------------- # + + +def test_symmetric_table_matches_pack_ontology(pack) -> None: + """SYMMETRIC_PREDICATES MUST equal the pack's graph.edge.symmetric declarations — + the pack is the source of truth; the constant is the runtime-cheap mirror.""" + assert SYMMETRIC_PREDICATES == load_relational_pack_symmetric() + + +def test_algebra_members_are_relational_predicates() -> None: + """Every inverse/symmetric lemma is a real reader predicate — a typo cannot mint + an unknown predicate into the determination path.""" + for lemma in set(INVERSE_OF) | SYMMETRIC_PREDICATES: + assert lemma in RELATIONAL_PREDICATES + + +def test_inverse_is_an_involution() -> None: + """inverse(inverse(p)) == p, and no predicate is its own inverse (asymmetry).""" + for lemma, other in INVERSE_OF.items(): + assert lemma != other + assert INVERSE_OF[other] == lemma diff --git a/tests/test_relational_inference_lane.py b/tests/test_relational_inference_lane.py new file mode 100644 index 00000000..44023b60 --- /dev/null +++ b/tests/test_relational_inference_lane.py @@ -0,0 +1,79 @@ +"""Lane test — one-hop relational inference (capability-index lane + wrong=0 bite). + +The positive lane (cases.jsonl) is the capability-index coverage; the confuser lane +(refusals.jsonl) is the wrong=0 bite — every confuser MUST refuse, or the inverse/ +symmetric rule over-fired. Fixture SHAs are pinned so the gold cannot drift silently. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest + +from chat.runtime import ChatRuntime +from evals.comprehension.relational_inference_runner import _load_cases, run +from generate.determine import Determined, determine +from generate.meaning_graph.relational import ( + comprehend_relational, + load_relational_pack_lemmas, +) +from generate.realize import realize_comprehension +from session.context import SessionContext + +_V1 = Path(__file__).resolve().parent.parent / "evals" / "relational_inference" / "v1" +_CASES_SHA = "03310ecc3ab7b7bf26f0a1779709bf468e888bbca1e53a71cba1d269b5c1dd71" +_REFUSALS_SHA = "f137bbe03b91301d9aa802ab9274c7d43ff9c3966207a8ba5e5f8fc19f0f7243" +_HIGH = 10**9 + + +def _sha(p: Path) -> str: + return hashlib.sha256(p.read_bytes()).hexdigest() + + +@pytest.fixture(scope="module") +def vocab_persona(): + rt = ChatRuntime(no_load_state=True) + return rt._context.vocab, rt._context.persona + + +@pytest.fixture(scope="module") +def pack(): + return load_relational_pack_lemmas() + + +def test_fixture_shas_pinned() -> None: + assert _sha(_V1 / "cases.jsonl") == _CASES_SHA + assert _sha(_V1 / "refusals.jsonl") == _REFUSALS_SHA + + +def test_positive_lane_wrong_zero_and_covers() -> None: + r = run() + assert r["wrong"] == 0, r["wrongs"] + assert r["correct"] > 0 # coverage > 0 — else the index geomean zeroes the score + assert r["correct"] + r["wrong"] + r["refused"] == r["total"] + + +def test_confusers_all_refuse(vocab_persona, pack) -> None: + # wrong=0 BITE: a Determined on ANY confuser means the one-hop rule over-fired + # (a transitive chain, an asymmetric self-converse, a cross-predicate, an object + # mismatch, or an ungrounded guess). All must refuse. + vocab, persona = vocab_persona + asserted = [] + for case in _load_cases(_V1 / "refusals.jsonl"): + ctx = SessionContext( + vocab=vocab, persona=persona, vault_reproject_interval=_HIGH + ) + for fact in case["facts"]: + realize_comprehension(comprehend_relational(fact, pack), ctx) + res = determine(comprehend_relational(case["query"], pack), ctx) + if isinstance(res, Determined): + asserted.append( + { + "id": case["id"], + "why": case["why"], + "got": [res.answer, res.predicate, res.subject, res.object], + } + ) + assert not asserted, f"confuser(s) asserted (rule over-fired): {asserted}" diff --git a/tests/test_relational_reader.py b/tests/test_relational_reader.py index 572c3f17..83c03d8a 100644 --- a/tests/test_relational_reader.py +++ b/tests/test_relational_reader.py @@ -225,7 +225,7 @@ def test_distinct_relations_about_one_subject_stay_distinct(vocab_persona) -> No # --------------------------------------------------------------------------- # -# wrong=0 BITE — direct entailment only; no fabricated inference +# wrong=0 BITE — direct + DECLARED one-hop relational rules only; no fabrication # --------------------------------------------------------------------------- # @@ -237,19 +237,28 @@ def test_present_but_non_entailing_refuses(vocab_persona) -> None: assert isinstance(res, Undetermined) and res.reason == "not_entailed" -def test_symmetric_converse_is_not_faked(vocab_persona) -> None: - # sibling_of is symmetric in the world, but D0 does DIRECT reading only: the stored - # direction determines; the converse is a sound-but-incomplete refusal, NOT a faked - # assertion. (If this flips to Determined, symmetric inference was smuggled in.) +def test_symmetric_converse_now_soundly_determines(vocab_persona) -> None: + # sibling_of IS symmetric (pack-declared ``graph.edge.symmetric``): the converse now + # SOUNDLY determines via the one-hop symmetric rule (mastery-v2 Step 3) — a DECLARED + # sound rule, not a fake. The stored direction still determines directly. (Was + # test_symmetric_converse_is_not_faked, which pinned the pre-rule direct-only floor.) ctx = _ctx(vocab_persona) _tell("Alice is the sibling of Bob.", ctx) - assert isinstance(_ask("Is Alice the sibling of Bob?", ctx), Determined) # stored dir + direct = _ask("Is Alice the sibling of Bob?", ctx) + assert isinstance(direct, Determined) and direct.rule == "direct" # stored dir converse = _ask("Is Bob the sibling of Alice?", ctx) - # The bite is that the converse does NOT assert. Its reason is "ungrounded" (no - # sibling_of fact has "bob" as subject) rather than "not_entailed" — either way it - # is a refusal, proving symmetric inference was not smuggled in. + assert isinstance(converse, Determined) and converse.answer is True + assert converse.rule == "symmetric" + + +def test_asymmetric_converse_is_not_faked(vocab_persona) -> None: + # The wrong=0 bite, preserved: an ASYMMETRIC predicate's same-predicate converse is + # NEVER faked. parent_of is not symmetric — parent_of(alice, bob) does NOT entail + # parent_of(bob, alice). (Its sound converse is the DIFFERENT predicate child_of.) + ctx = _ctx(vocab_persona) + _tell("Alice is the parent of Bob.", ctx) + converse = _ask("Is Bob the parent of Alice?", ctx) assert isinstance(converse, Undetermined) - assert converse.reason in {"ungrounded", "not_entailed"} def test_ungrounded_relation_refuses(vocab_persona) -> None: