"""Relational-predicate capability lane (#596 on the yardstick). Two obligations, kept separate on purpose: - COVERAGE: the positive gold lane (`v1/cases.jsonl`) — the reader correctly comprehends clean binary-relation prose, wrong=0, and the domain is composed into the capability index (breadth 8→9). - FALSIFICATION: the adversarial inputs (`v1/refusals.jsonl`) — the #596 fabrication class MUST refuse. This is the bite: if the reader ever commits on one of these, the lane's wrong=0 promise is a lie and this test fails. The gold is hand-authored from English semantics, independent of the reader (INV-25 / INV-27): a passing lane cannot be the reader grading its own output. """ from __future__ import annotations import json from pathlib import Path from evals.capability_index.adapters import collect_domain_results from evals.comprehension.relational_predicate_runner import run from generate.meaning_graph.reader import Refusal from generate.meaning_graph.relational import ( comprehend_relational, load_relational_pack_lemmas, ) _REFUSALS = Path(__file__).resolve().parent.parent / "evals" / "relational" / "v1" / "refusals.jsonl" def _refusal_cases() -> list[dict]: return [ json.loads(line) for line in _REFUSALS.read_text(encoding="utf-8").splitlines() if line.strip() ] # --------------------------------------------------------------------------- # # Coverage: clean prose comprehends correctly, wrong=0 # --------------------------------------------------------------------------- # def test_positive_lane_is_green() -> None: report = run() assert report["wrong"] == 0 assert report["refused"] == 0 # every clean case reads assert report["correct"] == report["total"] == 18 # +1: overlaps_event finite-verb (B3) def test_lane_is_composed_into_capability_index() -> None: results = {d.domain: d for d in collect_domain_results().results} dom = results.get("comprehension_relational_predicate") assert dom is not None, "relational-predicate domain missing from the index" assert dom.wrong == 0 and dom.correct == 18 and dom.coverage == 1.0 # --------------------------------------------------------------------------- # # Falsification: the #596 fabrication class MUST refuse (the bite) # --------------------------------------------------------------------------- # def test_adversarial_inputs_all_refuse() -> None: pack = load_relational_pack_lemmas() cases = _refusal_cases() # the #596 fabrication family + negation + non-template + the B3 finite-verb confuser # family (adverb absorption incl. non-enumerated 'almost'/'never', coordination, # second-verb, trailing qualifier, connective-without-copula). The overlaps verb-gap # (old ref-009) MIGRATED to a positive, replaced by finite-verb confusers — the refusal # floor RISES, never drops. assert len(cases) >= 24 for case in cases: comp = comprehend_relational(case["text"], pack) assert isinstance(comp, Refusal), ( f"FABRICATION: {case['text']!r} should refuse ({case['why']}) but committed {comp!r}" ) def test_no_fabricated_relation_ever_committed() -> None: # Stronger than "is a Refusal": confirm not a single relation/query is produced for # any adversarial input (no empty-but-committed Comprehension slips through). pack = load_relational_pack_lemmas() for case in _refusal_cases(): comp = comprehend_relational(case["text"], pack) assert isinstance(comp, Refusal), case["text"]