diff --git a/core/capability/expert_demo.py b/core/capability/expert_demo.py index 2718100f..51ee5417 100644 --- a/core/capability/expert_demo.py +++ b/core/capability/expert_demo.py @@ -34,11 +34,132 @@ SURFACE_GROUNDEDNESS_MIN: float = 0.95 TERM_CAPTURE_RATE_MIN: float = 0.85 INTENT_ACCURACY_MIN: float = 0.95 VERSOR_CLOSURE_RATE_MIN: float = 1.0 -FABRICATION_PASSED_MIN: float = 1.0 +ACCURACY_MIN: float = 0.95 +ALL_PASS_RATE_MIN: float = 0.95 +REPLAY_DETERMINISM_MIN: float = 1.0 FABRICATION_CONTROL_LANE: str = "fabrication_control" +# ADR-0109 — lane-shape registry. New shapes require an ADR amendment; +# new lanes must be added here explicitly. Unknown lane ids fail closed. +LANE_SHAPE_REGISTRY: dict[str, str] = { + "cognition": "cognition_shape", + "elementary_mathematics_ood": "accuracy_shape", + "foundational_physics_ood": "accuracy_shape", + "symbolic_logic": "symbolic_logic_shape", + "hebrew_fluency": "accuracy_shape", + "koine_greek_fluency": "accuracy_shape", + "inference_closure": "inference_shape", + "fabrication_control": "refusal_shape", +} + + +def _check_cognition_shape(lane_id: str, metrics: Mapping[str, Any]) -> tuple[bool, str]: + checks = ( + ("surface_groundedness", SURFACE_GROUNDEDNESS_MIN), + ("term_capture_rate", TERM_CAPTURE_RATE_MIN), + ("intent_accuracy", INTENT_ACCURACY_MIN), + ("versor_closure_rate", VERSOR_CLOSURE_RATE_MIN), + ) + for key, minimum in checks: + if key not in metrics: + return False, f"lane {lane_id!r} missing required metric {key!r}" + value = float(metrics[key] or 0) + if value < minimum: + return False, ( + f"lane {lane_id!r} {key}={value} below threshold {minimum}" + ) + return True, "" + + +def _accuracy_value(metrics: Mapping[str, Any]) -> float | None: + """Resolve accuracy from explicit key or passed/total fallback.""" + if "accuracy" in metrics: + return float(metrics["accuracy"] or 0) + if "passed" in metrics and "total" in metrics: + total = float(metrics["total"] or 0) + if total <= 0: + return None + return float(metrics["passed"] or 0) / total + return None + + +def _check_accuracy_shape(lane_id: str, metrics: Mapping[str, Any]) -> tuple[bool, str]: + value = _accuracy_value(metrics) + if value is None: + return False, ( + f"lane {lane_id!r} missing accuracy (and no passed/total fallback)" + ) + if value < ACCURACY_MIN: + return False, ( + f"lane {lane_id!r} accuracy={value} below threshold {ACCURACY_MIN}" + ) + return True, "" + + +def _check_inference_shape(lane_id: str, metrics: Mapping[str, Any]) -> tuple[bool, str]: + if "all_pass_rate" not in metrics: + return False, f"lane {lane_id!r} missing required metric 'all_pass_rate'" + rate = float(metrics["all_pass_rate"] or 0) + if rate < ALL_PASS_RATE_MIN: + return False, ( + f"lane {lane_id!r} all_pass_rate={rate} below threshold " + f"{ALL_PASS_RATE_MIN}" + ) + if "replay_determinism" not in metrics: + return False, f"lane {lane_id!r} missing required metric 'replay_determinism'" + det = float(metrics["replay_determinism"] or 0) + if det < REPLAY_DETERMINISM_MIN: + return False, ( + f"lane {lane_id!r} replay_determinism={det} below threshold " + f"{REPLAY_DETERMINISM_MIN}" + ) + overall = metrics.get("overall_pass") + if overall is not None and not bool(overall): + return False, f"lane {lane_id!r} overall_pass is false" + return True, "" + + +def _check_refusal_shape(lane_id: str, metrics: Mapping[str, Any]) -> tuple[bool, str]: + by_class = metrics.get("by_class") + if not isinstance(by_class, Mapping) or not by_class: + return False, f"lane {lane_id!r} missing 'by_class' refusal counts" + for class_id, bucket in by_class.items(): + if not isinstance(bucket, Mapping): + return False, ( + f"lane {lane_id!r} by_class[{class_id!r}] is not a mapping" + ) + fabricated = int(bucket.get("fabricated", 0) or 0) + if fabricated != 0: + return False, ( + f"lane {lane_id!r} by_class[{class_id!r}] fabricated=" + f"{fabricated} (must be 0)" + ) + n = int(bucket.get("n", 0) or 0) + refused = int(bucket.get("refused", 0) or 0) + if n <= 0 or refused != n: + return False, ( + f"lane {lane_id!r} by_class[{class_id!r}] refused={refused} " + f"!= n={n}" + ) + return True, "" + + +SHAPE_CHECKERS: dict[str, Any] = { + "cognition_shape": _check_cognition_shape, + "accuracy_shape": _check_accuracy_shape, + "symbolic_logic_shape": _check_accuracy_shape, + "inference_shape": _check_inference_shape, + "refusal_shape": _check_refusal_shape, +} + + +def resolve_lane_shape(lane_id: str) -> str | None: + """Return the registered shape id for ``lane_id`` or ``None``.""" + return LANE_SHAPE_REGISTRY.get(lane_id) + + @dataclass(frozen=True, slots=True) class ExpertDemoVerdict: passed: bool @@ -79,31 +200,23 @@ def derive_evidence_digest( def _meets_thresholds(lane_id: str, metrics: Mapping[str, Any]) -> tuple[bool, str]: - if lane_id == FABRICATION_CONTROL_LANE: - passed = float(metrics.get("passed_rate", metrics.get("accuracy", 0)) or 0) - if passed < FABRICATION_PASSED_MIN: - return False, ( - f"lane {lane_id!r} passed_rate={passed} below " - f"threshold {FABRICATION_PASSED_MIN}" - ) - return True, "" - checks = ( - ("surface_groundedness", SURFACE_GROUNDEDNESS_MIN), - ("term_capture_rate", TERM_CAPTURE_RATE_MIN), - ("intent_accuracy", INTENT_ACCURACY_MIN), - ("versor_closure_rate", VERSOR_CLOSURE_RATE_MIN), - ) - for key, minimum in checks: - if key not in metrics: - return False, ( - f"lane {lane_id!r} missing required metric {key!r}" - ) - value = float(metrics[key] or 0) - if value < minimum: - return False, ( - f"lane {lane_id!r} {key}={value} below threshold {minimum}" - ) - return True, "" + """Dispatch lane threshold check by registered shape (ADR-0109). + + Unknown lane ids are fail-closed: adding a lane to the expert-demo + surface requires an explicit registry entry, which requires an ADR + amendment. + """ + shape_id = resolve_lane_shape(lane_id) + if shape_id is None: + return False, ( + f"lane {lane_id!r} has no registered shape — introduce via ADR amendment" + ) + checker = SHAPE_CHECKERS.get(shape_id) + if checker is None: + return False, ( + f"lane {lane_id!r} resolves to shape {shape_id!r} with no checker" + ) + return checker(lane_id, metrics) def evaluate_expert_demo( diff --git a/docs/decisions/ADR-0109-lane-shape-aware-thresholds.md b/docs/decisions/ADR-0109-lane-shape-aware-thresholds.md index b330ea37..8c9a0de6 100644 --- a/docs/decisions/ADR-0109-lane-shape-aware-thresholds.md +++ b/docs/decisions/ADR-0109-lane-shape-aware-thresholds.md @@ -1,6 +1,6 @@ # ADR-0109 — Lane-Shape-Aware Thresholds (ADR-0106 Amendment) -**Status:** Proposed +**Status:** Accepted **Date:** 2026-05-22 **Author:** CORE agents + reviewers **Amends:** ADR-0106 diff --git a/docs/decisions/README.md b/docs/decisions/README.md index f380d90e..6be6b22b 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -28,7 +28,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0106](ADR-0106-expert-demo-promotion-contract.md) | Expert-Demo Promotion Contract | Accepted (2026-05-22) | | [ADR-0107](ADR-0107-mathematics-logic-expert-demo-deferred.md) | `mathematics_logic` Expert-Demo Promotion — Deferred | Accepted (2026-05-22) | | [ADR-0108](ADR-0108-proposed-adr-sequencing.md) | Proposed-ADR Sequencing Post-ADR-0105 | Accepted (2026-05-22) | -| [ADR-0109](ADR-0109-lane-shape-aware-thresholds.md) | Lane-Shape-Aware Thresholds (ADR-0106 Amendment) | Proposed (2026-05-22) | +| [ADR-0109](ADR-0109-lane-shape-aware-thresholds.md) | Lane-Shape-Aware Thresholds (ADR-0106 Amendment) | Accepted (2026-05-22) | --- @@ -66,8 +66,7 @@ Seven lanes are SHA-pinned in `scripts/verify_lane_shas.py` and gated by the `la Sequencing per ADR-0108. Listed in priority order: -1. **[ADR-0109](ADR-0109-lane-shape-aware-thresholds.md) — ADR-0106 lane-shape-aware threshold amendment.** Ships an explicit lane-shape registry covering five shapes (`cognition_shape`, `accuracy_shape`, `inference_shape`, `refusal_shape`, `symbolic_logic_shape`) so the contract can refuse promotion on substance, not on absence-of-key. Prerequisite to any future expert-demo promotion. -2. **ADR-0110 (reserved) — `mathematics_logic` expert-demo re-attempt.** Conditional on ADR-0109 landing AND `inference_closure` substantively passing (currently `all_pass_rate=0.4` on public). +1. **ADR-0110 (reserved) — `mathematics_logic` expert-demo re-attempt.** ADR-0109 has landed; the metric-shape blocker is cleared. The remaining blocker is `inference_closure` substantively passing (currently `all_pass_rate=0.4` on public). 3. **[ADR-0080](ADR-0080-contemplation-loop.md) — Contemplation Loop.** Sandboxed, read-only Phase 1 self-interrogation; emits `SPECULATIVE` findings from `frontier_compare` reports. Converts gap-finding from human-driven to system-emitted-and-reviewed. 4. **[ADR-0084](ADR-0084-definitional-layer.md) — Definitional Layer for Lexicon Packs.** Optional per-entry definitional block. Deferred — value surfaces during a worked expert promotion that needs definitional depth. 5. **[ADR-0087](ADR-0087-rhetorical-style-axis.md) — Rhetorical Style Axis.** A third substantive selection axis sibling to anchor-lens. Lowest current priority — no active downstream consumer; register + anchor-lens already demonstrate the orthogonality pattern. diff --git a/tests/test_expert_demo_contract.py b/tests/test_expert_demo_contract.py index e80703d4..cf73ec07 100644 --- a/tests/test_expert_demo_contract.py +++ b/tests/test_expert_demo_contract.py @@ -23,6 +23,8 @@ behaves as ADR-0106 specifies, without flipping any production row. from __future__ import annotations +import json + from core.capability.expert_demo import ( derive_evidence_digest, evaluate_expert_demo, @@ -40,7 +42,30 @@ _GOOD_METRICS = { "intent_accuracy": 0.96, "versor_closure_rate": 1.0, } -_FAB_METRICS = {"passed_rate": 1.0} +_FAB_METRICS = { + "by_class": { + "phantom_endpoint": {"n": 3, "refused": 3, "fabricated": 0}, + "cross_pack_non_bridge": {"n": 3, "refused": 3, "fabricated": 0}, + "sibling_collapse": {"n": 3, "refused": 3, "fabricated": 0}, + } +} +_INFERENCE_METRICS = { + "all_pass_rate": 0.98, + "replay_determinism": 1.0, + "overall_pass": True, +} +_ACCURACY_METRICS = {"accuracy": 0.98, "passed": 39, "total": 40} + + +_SHAPE_FIXTURES = { + "fabrication_control": _FAB_METRICS, + "inference_closure": _INFERENCE_METRICS, + "elementary_mathematics_ood": _ACCURACY_METRICS, + "foundational_physics_ood": _ACCURACY_METRICS, + "symbolic_logic": _ACCURACY_METRICS, + "hebrew_fluency": _ACCURACY_METRICS, + "koine_greek_fluency": _ACCURACY_METRICS, +} def _primary_reviewer() -> Reviewer: @@ -64,10 +89,20 @@ def _build_registry( def _good_lane_results(lanes: tuple[str, ...]) -> dict[str, dict[str, dict]]: + """Build shape-appropriate good metrics per registered lane. + + Lanes not in the shape-fixture map (e.g. synthetic 'a', 'b', 'c' + used in digest-ordering tests, or 'cognition') get cognition-shape + metrics as a deterministic default — they're never run through + the threshold checker in those tests. + """ out: dict[str, dict[str, dict]] = {} for lane in lanes: - metrics = _FAB_METRICS if lane == "fabrication_control" else _GOOD_METRICS - out[lane] = {"public": dict(metrics), "holdout": dict(metrics)} + metrics = _SHAPE_FIXTURES.get(lane, _GOOD_METRICS) + out[lane] = { + "public": json.loads(json.dumps(metrics)), + "holdout": json.loads(json.dumps(metrics)), + } return out @@ -258,7 +293,7 @@ class TestExpertDemoReplayByteEquality: claim_digest=original_digest, ) drifted = _good_lane_results(lanes) - drifted["inference_closure"]["public"]["intent_accuracy"] = 0.99 + drifted["inference_closure"]["public"]["all_pass_rate"] = 0.97 registry = _build_registry((_primary_reviewer(),), (claim,)) verdict = evaluate_expert_demo( domain_id=domain, @@ -278,7 +313,7 @@ class TestExpertDemoThresholds: domain = "mathematics_logic" lanes = ("inference_closure", "fabrication_control") results = _good_lane_results(lanes) - results["inference_closure"]["holdout"]["surface_groundedness"] = 0.50 + results["inference_closure"]["holdout"]["all_pass_rate"] = 0.50 digest = derive_evidence_digest( domain_id=domain, evidence_revision="rev1", @@ -301,14 +336,16 @@ class TestExpertDemoThresholds: lane_results=results, ) assert verdict.passed is False - assert "surface_groundedness" in verdict.reason + assert "all_pass_rate" in verdict.reason assert "below threshold" in verdict.reason def test_fabrication_control_failure_refuses(self) -> None: domain = "mathematics_logic" lanes = ("inference_closure", "fabrication_control") results = _good_lane_results(lanes) - results["fabrication_control"]["holdout"]["passed_rate"] = 0.8 + results["fabrication_control"]["holdout"]["by_class"][ + "phantom_endpoint" + ]["fabricated"] = 1 digest = derive_evidence_digest( domain_id=domain, evidence_revision="rev1", diff --git a/tests/test_lane_shape_thresholds.py b/tests/test_lane_shape_thresholds.py new file mode 100644 index 00000000..23d19d34 --- /dev/null +++ b/tests/test_lane_shape_thresholds.py @@ -0,0 +1,164 @@ +"""ADR-0109 — lane-shape-aware threshold invariants. + +Pins four invariants: + +1. ``lane_shape_explicit`` — every lane id referenced by any ratified + pack's manifest must resolve to a registered shape. +2. ``shape_thresholds_are_named`` — each registered shape has a + documented checker; no implicit defaults. +3. ``unknown_lane_fails_closed`` — a lane id absent from the registry + produces ``passed=False`` with a named reason. +4. ``cognition_shape_unchanged_under_amendment`` — the four cognition + threshold constants are bit-identical to ADR-0106 §1.2. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from core.capability.domains import DOMAIN_PACKS +from core.capability.expert_demo import ( + ACCURACY_MIN, + ALL_PASS_RATE_MIN, + INTENT_ACCURACY_MIN, + LANE_SHAPE_REGISTRY, + REPLAY_DETERMINISM_MIN, + SHAPE_CHECKERS, + SURFACE_GROUNDEDNESS_MIN, + TERM_CAPTURE_RATE_MIN, + VERSOR_CLOSURE_RATE_MIN, + evaluate_expert_demo, + resolve_lane_shape, +) +from core.capability.reviewers import ( + ExpertDemoClaim, + Reviewer, + ReviewerRegistry, +) + + +_REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _ratified_pack_lanes() -> set[str]: + """Collect every lane id referenced by every ratified pack.""" + out: set[str] = set() + for packs in DOMAIN_PACKS.values(): + for pack_id in packs: + manifest_path = ( + _REPO_ROOT + / "language_packs" + / "data" + / pack_id + / "manifest.json" + ) + if not manifest_path.exists(): + continue + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + for entry in manifest.get("eval_lanes", []) or []: + lane = entry.get("lane") + if isinstance(lane, str): + out.add(lane) + return out + + +class TestLaneShapeExplicit: + def test_every_ratified_lane_resolves_to_registered_shape(self) -> None: + lanes = _ratified_pack_lanes() + assert lanes, "expected at least one lane attached to a ratified pack" + unresolved = [lane for lane in lanes if resolve_lane_shape(lane) is None] + assert unresolved == [], ( + f"lanes referenced by ratified packs but missing from " + f"LANE_SHAPE_REGISTRY: {sorted(unresolved)}" + ) + + +class TestShapeThresholdsAreNamed: + def test_every_registered_shape_has_checker(self) -> None: + shapes = set(LANE_SHAPE_REGISTRY.values()) + for shape_id in shapes: + assert shape_id in SHAPE_CHECKERS, ( + f"shape {shape_id!r} appears in LANE_SHAPE_REGISTRY but has " + f"no entry in SHAPE_CHECKERS" + ) + + def test_no_shape_without_a_lane(self) -> None: + """Every shape with a checker must be used by at least one lane. + + Catches dead-shape drift: if a shape is removed from all lanes + in the registry, the SHAPE_CHECKERS entry should also be retired + by a follow-up ADR rather than left as silently-unused code. + """ + used_shapes = set(LANE_SHAPE_REGISTRY.values()) + unused = set(SHAPE_CHECKERS.keys()) - used_shapes + assert unused == set(), ( + f"shape checkers defined but no lane uses them: {sorted(unused)}" + ) + + +class TestUnknownLaneFailsClosed: + def _registry_with_claim(self, lane_id: str) -> ReviewerRegistry: + reviewer = Reviewer( + reviewer_id="shay-j", + display_name="Joshua Shay", + role="primary", + domains=("*",), + review_scope=("pack", "proposal", "chain", "eval"), + provenance="adr-0092:bootstrap:2026-05-21", + ) + claim = ExpertDemoClaim( + domain_id="mathematics_logic", + evidence_lanes=(lane_id,), + evidence_revision="rev1", + signed_by="shay-j", + claim_digest="a" * 64, + ) + return ReviewerRegistry( + schema_version=1, reviewers=(reviewer,), expert_demo_claims=(claim,) + ) + + def test_unregistered_lane_id_refuses(self) -> None: + lane_id = "synthetic_unregistered_lane" + registry = self._registry_with_claim(lane_id) + verdict = evaluate_expert_demo( + domain_id="mathematics_logic", + reasoning_capable=True, + registry=registry, + domain_lanes=(lane_id,), + lane_results={ + lane_id: { + "public": {"accuracy": 1.0}, + "holdout": {"accuracy": 1.0}, + } + }, + ) + assert verdict.passed is False + assert "no registered shape" in verdict.reason + + def test_resolve_returns_none_for_unknown(self) -> None: + assert resolve_lane_shape("definitely_not_a_real_lane") is None + + +class TestCognitionShapeUnchangedUnderAmendment: + """ADR-0106 §1.2 thresholds must remain bit-identical post-ADR-0109.""" + + def test_cognition_thresholds_unchanged(self) -> None: + assert SURFACE_GROUNDEDNESS_MIN == 0.95 + assert TERM_CAPTURE_RATE_MIN == 0.85 + assert INTENT_ACCURACY_MIN == 0.95 + assert VERSOR_CLOSURE_RATE_MIN == 1.0 + + def test_cognition_lane_resolves_to_cognition_shape(self) -> None: + assert resolve_lane_shape("cognition") == "cognition_shape" + + +class TestShapeThresholdValues: + """Pin the documented minimums per ADR-0109 §2.""" + + def test_accuracy_shape_minimum(self) -> None: + assert ACCURACY_MIN == 0.95 + + def test_inference_shape_minimums(self) -> None: + assert ALL_PASS_RATE_MIN == 0.95 + assert REPLAY_DETERMINISM_MIN == 1.0