feat(measure): put the relational reader (#596) on the capability index

A1 of the refined sequencing — the binary-relation reader was inert w.r.t. the
yardstick (contributing 0). This adds a comprehension_relational_predicate domain:
binary-relation prose scored against hand-authored independent gold (predicate,
subject, object) triples — INV-25 independent / INV-27 reader-disjoint (the reader
never produced the gold). Index breadth 8->9, capability_score 0.937258->0.944030,
wrong_total still 0; baseline.json re-frozen to digest 1ea91c1e.

Rigor split: the index lane is POSITIVE-ONLY (clean coverage, consistent with the
other 8 lanes — mixing adversarial refuse-cases into the coverage denominator would
make 'added capability' read as a score drop). The #596 fabrication-catch lives in a
dedicated falsification test (evals/relational/v1/refusals.jsonl): the trailing-
qualifier / dangling-copula / negation / verb-form cases MUST refuse — bites if the
reader ever fabricates. Honest coverage gap recorded: overlaps_event has no copular
surface form (verb-form 'A overlaps B' refuses), so 17 positives cover 15/16 predicates.

evals/capability_index/adapters.py

@@ -77,12 +77,20 @@ def comprehension_relational_metric_result() -> DomainResult:
     return DomainResult("comprehension_relational_metric", c, w, r)
 
 
+def comprehension_relational_predicate_result() -> DomainResult:
+    from evals.comprehension.relational_predicate_runner import run
+
+    c, w, r = _counts(run())
+    return DomainResult("comprehension_relational_predicate", c, w, r)
+
+
 #: The reasoning domains currently composed into the index (self-loading lanes).
-#: The five ``comprehension_*`` lanes score the GENERAL comprehension reader; the
+#: The six ``comprehension_*`` lanes score the GENERAL comprehension reader; the
 #: relational_metric one reads arithmetic prose into the binding-graph quantity
-#: substrate (admissibility-checked) and projects to the arithmetic oracle, so the
-#: index now measures comprehension breadth across categorical, ordering,
-#: propositional, AND quantitative reasoning.
+#: substrate (admissibility-checked) and projects to the arithmetic oracle, and the
+#: relational_predicate one (#596) reads binary-relation prose into pack-named
+#: predicates — so the index now measures comprehension breadth across categorical,
+#: ordering, propositional, quantitative, AND relational reasoning.
 ADAPTERS = (
     deductive_logic_result,
     relational_metric_result,
@@ -92,6 +100,7 @@ ADAPTERS = (
     comprehension_total_ordering_result,
     comprehension_propositional_result,
     comprehension_relational_metric_result,
+    comprehension_relational_predicate_result,
 )
 
 

evals/capability_index/baseline.json

@@ -1,13 +1,13 @@
 {
-  "capability_score": 0.937258,
-  "coverage_geomean": 0.937258,
-  "coverage_micro": 0.993703,
+  "capability_score": 0.94403,
+  "coverage_geomean": 0.94403,
+  "coverage_micro": 0.993835,
   "accuracy_micro": 1.0,
-  "breadth": 8,
+  "breadth": 9,
   "min_domain_coverage": 0.833333,
   "wrong_total": 0,
   "assert_mode_valid": true,
-  "deterministic_digest": "50e0675bd69938ce5747b5d47592504b9cf143027dd6c1c5e410da0add25341c",
+  "deterministic_digest": "1ea91c1eefd6a0c4739e6fc97a40d8522d9b0758b4531bce448ec98e25789ccf",
   "domains": [
     {
       "domain": "comprehension_propositional",
@@ -25,6 +25,14 @@
       "coverage": 1.0,
       "accuracy": 1.0
     },
+    {
+      "domain": "comprehension_relational_predicate",
+      "correct": 17,
+      "wrong": 0,
+      "refused": 0,
+      "coverage": 1.0,
+      "accuracy": 1.0
+    },
     {
       "domain": "comprehension_set_membership",
       "correct": 8,

evals/comprehension/relational_predicate_runner.py

@@ -0,0 +1,87 @@
+"""Score the relational reader (#596) on binary-relation prose against independent gold.
+
+prose -> comprehend_relational() -> committed Relation vs hand-authored gold triple.
+The gold ``(predicate, subject, object)`` is authored by reading English semantics,
+INDEPENDENTLY of ``relational.py`` (INV-25 / INV-27): the reader never produced it.
+
+A refusal is a COVERAGE miss, never a wrong; only a committed relation that disagrees
+with gold — or a malformed commit (≠ exactly one relation, or a query) — is wrong, and
+wrong must stay 0. This is the positive-coverage lane that puts the just-landed reader
+ON the capability index (breadth 8→9); the adversarial fabrication cases that must
+REFUSE live in ``evals/relational/v1/refusals.jsonl`` and the dedicated lane test.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from generate.meaning_graph.reader import Refusal
+from generate.meaning_graph.relational import (
+    comprehend_relational,
+    load_relational_pack_lemmas,
+)
+
+_CASES = Path(__file__).resolve().parent.parent / "relational" / "v1" / "cases.jsonl"
+
+
+def _load_cases(path: Path = _CASES) -> list[dict[str, Any]]:
+    return [
+        json.loads(line)
+        for line in path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+
+def run(path: Path = _CASES) -> dict[str, Any]:
+    cases = _load_cases(path)
+    pack = load_relational_pack_lemmas()
+    correct = wrong = refused = 0
+    wrongs: list[dict[str, Any]] = []
+
+    for case in cases:
+        comp = comprehend_relational(case["text"], pack)
+        if isinstance(comp, Refusal):
+            refused += 1  # coverage miss, not a wrong
+            continue
+        rels = comp.meaning_graph.relations
+        if comp.queries or len(rels) != 1:
+            # a declarative fact case must yield exactly one relation and no query;
+            # anything else is a malformed commit, counted as wrong (must stay 0).
+            wrong += 1
+            wrongs.append({"id": case.get("id"), "got": "malformed_commit", "text": case["text"]})
+            continue
+        rel = rels[0]
+        got = [rel.predicate, list(rel.arguments)]
+        gold = [case["predicate"], [case["subject"], case["object"]]]
+        if got == gold:
+            correct += 1
+        else:
+            wrong += 1
+            wrongs.append({"id": case.get("id"), "got": got, "gold": gold, "text": case["text"]})
+
+    return {
+        "domain": "comprehension_relational_predicate",
+        "total": len(cases),
+        "correct": correct,
+        "wrong": wrong,
+        "refused": refused,
+        "wrongs": wrongs,
+        "counts": {"correct": correct, "wrong": wrong, "refused": refused},
+    }
+
+
+def main() -> int:
+    report = run()
+    print(json.dumps({k: v for k, v in report.items() if k != "wrongs"}, indent=2, sort_keys=True))
+    if report["wrong"]:
+        print("WRONG > 0 — relational comprehension produced a wrong committed answer:", file=sys.stderr)
+        print(json.dumps(report["wrongs"], indent=2), file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

evals/relational/__init__.py

@@ -0,0 +1,8 @@
+"""Relational-predicate gold lane — binary-relation prose with independent gold.
+
+The gold ``(predicate, subject, object)`` triples in ``v1/cases.jsonl`` are authored
+by reading English semantics, INDEPENDENTLY of ``generate/meaning_graph/relational.py``
+(INV-25 independent gold / INV-27 reader-disjoint): the reader never produced them.
+``v1/refusals.jsonl`` holds adversarial inputs that must REFUSE (the #596 fabrication
+class) — consumed by the dedicated falsification test, not the coverage metric.
+"""

evals/relational/v1/cases.jsonl

@@ -0,0 +1,17 @@
+{"id": "rel-001", "text": "Alice is the parent of Bob.", "predicate": "parent_of", "subject": "alice", "object": "bob"}
+{"id": "rel-002", "text": "Bob is the child of Alice.", "predicate": "child_of", "subject": "bob", "object": "alice"}
+{"id": "rel-003", "text": "Carol is the sibling of Dan.", "predicate": "sibling_of", "subject": "carol", "object": "dan"}
+{"id": "rel-004", "text": "Eve is the spouse of Frank.", "predicate": "spouse_of", "subject": "eve", "object": "frank"}
+{"id": "rel-005", "text": "Three is less than five.", "predicate": "less_than", "subject": "three", "object": "five"}
+{"id": "rel-006", "text": "Nine is greater than two.", "predicate": "greater_than", "subject": "nine", "object": "two"}
+{"id": "rel-007", "text": "Five is equal to five.", "predicate": "equal_to", "subject": "five", "object": "five"}
+{"id": "rel-008", "text": "Seven is distinct from eight.", "predicate": "distinct_from", "subject": "seven", "object": "eight"}
+{"id": "rel-009", "text": "The box is left of the cup.", "predicate": "left_of", "subject": "box", "object": "cup"}
+{"id": "rel-010", "text": "The lamp is right of the desk.", "predicate": "right_of", "subject": "lamp", "object": "desk"}
+{"id": "rel-011", "text": "The cat is inside of the house.", "predicate": "inside_of", "subject": "cat", "object": "house"}
+{"id": "rel-012", "text": "North station is adjacent to south station.", "predicate": "adjacent_to", "subject": "north_station", "object": "south_station"}
+{"id": "rel-013", "text": "Monday is before Friday.", "predicate": "before_event", "subject": "monday", "object": "friday"}
+{"id": "rel-014", "text": "Lunch is after breakfast.", "predicate": "after_event", "subject": "lunch", "object": "breakfast"}
+{"id": "rel-015", "text": "Recess is during morning.", "predicate": "during_event", "subject": "recess", "object": "morning"}
+{"id": "rel-016", "text": "Zorptak is the parent of Quxley.", "predicate": "parent_of", "subject": "zorptak", "object": "quxley"}
+{"id": "rel-017", "text": "Glarn is less than Brimble.", "predicate": "less_than", "subject": "glarn", "object": "brimble"}

evals/relational/v1/refusals.jsonl

@@ -0,0 +1,9 @@
+{"id": "ref-001", "text": "Carol is the sibling of Dan during school.", "why": "trailing temporal qualifier — the #596 fabrication class (sibling_of(carol, dan_during_school))"}
+{"id": "ref-002", "text": "Alice is the parent of Bob during the trip.", "why": "the asymmetric twin of ref-001 — both must refuse identically now"}
+{"id": "ref-003", "text": "Five is equal to ten during testing.", "why": "trailing qualifier on a comparison"}
+{"id": "ref-004", "text": "X is left of Y inside Z.", "why": "two relational structures in one clause"}
+{"id": "ref-005", "text": "Monday before Friday is.", "why": "dangling copula — not a fact"}
+{"id": "ref-006", "text": "Alice is not the parent of Bob.", "why": "negation is out of grammar — never reads as a positive"}
+{"id": "ref-007", "text": "Alice greets Bob.", "why": "no relational connective / no copula"}
+{"id": "ref-008", "text": "The weather is nice today.", "why": "no relational template at all"}
+{"id": "ref-009", "text": "Sunrise overlaps dawn.", "why": "verb-form relation, no copula — the honest overlaps_event coverage gap"}

tests/test_capability_index.py

@@ -87,9 +87,10 @@ def test_empty_index_is_well_defined() -> None:
 
 
 def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None:
-    # The baseline: three structured-input reasoning lanes PLUS the four
+    # The baseline: three structured-input reasoning lanes PLUS the six
     # comprehension lanes (prose -> MeaningGraph -> projection -> independent
-    # oracle) compose into the cross-domain index with zero wrong commits.
+    # oracle / pack-named predicate) compose into the cross-domain index with zero
+    # wrong commits. The relational_predicate lane (#596) is the 9th domain.
     from evals.capability_index.adapters import collect_domain_results
 
     collection = collect_domain_results()
@@ -97,7 +98,7 @@ def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None:
     idx = aggregate(list(collection.results))
     assert idx.wrong_total == 0
     assert idx.assert_mode_valid
-    assert idx.breadth == 8
+    assert idx.breadth == 9
     assert {d.domain for d in idx.domains} == {
         "deductive_logic",
         "dimensional",
@@ -107,6 +108,7 @@ def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None:
         "comprehension_total_ordering",
         "comprehension_propositional",
         "comprehension_relational_metric",
+        "comprehension_relational_predicate",
     }
     assert idx.capability_score > 0.5  # real, non-trivial cross-domain capability
 

tests/test_relational_predicate_lane.py

@@ -0,0 +1,80 @@
+"""Relational-predicate capability lane (#596 on the yardstick).
+
+Two obligations, kept separate on purpose:
+  - COVERAGE: the positive gold lane (`v1/cases.jsonl`) — the reader correctly
+    comprehends clean binary-relation prose, wrong=0, and the domain is composed into
+    the capability index (breadth 8→9).
+  - FALSIFICATION: the adversarial inputs (`v1/refusals.jsonl`) — the #596 fabrication
+    class MUST refuse. This is the bite: if the reader ever commits on one of these,
+    the lane's wrong=0 promise is a lie and this test fails.
+
+The gold is hand-authored from English semantics, independent of the reader (INV-25 /
+INV-27): a passing lane cannot be the reader grading its own output.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from evals.capability_index.adapters import collect_domain_results
+from evals.comprehension.relational_predicate_runner import run
+from generate.meaning_graph.reader import Refusal
+from generate.meaning_graph.relational import (
+    comprehend_relational,
+    load_relational_pack_lemmas,
+)
+
+_REFUSALS = Path(__file__).resolve().parent.parent / "evals" / "relational" / "v1" / "refusals.jsonl"
+
+
+def _refusal_cases() -> list[dict]:
+    return [
+        json.loads(line)
+        for line in _REFUSALS.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+
+# --------------------------------------------------------------------------- #
+# Coverage: clean prose comprehends correctly, wrong=0
+# --------------------------------------------------------------------------- #
+
+
+def test_positive_lane_is_green() -> None:
+    report = run()
+    assert report["wrong"] == 0
+    assert report["refused"] == 0  # every clean case reads
+    assert report["correct"] == report["total"] == 17
+
+
+def test_lane_is_composed_into_capability_index() -> None:
+    results = {d.domain: d for d in collect_domain_results().results}
+    dom = results.get("comprehension_relational_predicate")
+    assert dom is not None, "relational-predicate domain missing from the index"
+    assert dom.wrong == 0 and dom.correct == 17 and dom.coverage == 1.0
+
+
+# --------------------------------------------------------------------------- #
+# Falsification: the #596 fabrication class MUST refuse (the bite)
+# --------------------------------------------------------------------------- #
+
+
+def test_adversarial_inputs_all_refuse() -> None:
+    pack = load_relational_pack_lemmas()
+    cases = _refusal_cases()
+    assert len(cases) >= 9  # the #596 hazard family + negation + non-template + the verb gap
+    for case in cases:
+        comp = comprehend_relational(case["text"], pack)
+        assert isinstance(comp, Refusal), (
+            f"FABRICATION: {case['text']!r} should refuse ({case['why']}) but committed {comp!r}"
+        )
+
+
+def test_no_fabricated_relation_ever_committed() -> None:
+    # Stronger than "is a Refusal": confirm not a single relation/query is produced for
+    # any adversarial input (no empty-but-committed Comprehension slips through).
+    pack = load_relational_pack_lemmas()
+    for case in _refusal_cases():
+        comp = comprehend_relational(case["text"], pack)
+        assert isinstance(comp, Refusal), case["text"]