core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feat(capability): implement ADR-0109 lane-shape-aware thresholds (#116)

Browse source 
Replaces the cognition-shape-uniform threshold dispatch in
core/capability/expert_demo.py with an explicit LANE_SHAPE_REGISTRY
mapping 8 ratified lane ids to 5 shapes:

  cognition           -> cognition_shape
  elementary_math_ood -> accuracy_shape
  foundational_physics_ood -> accuracy_shape
  symbolic_logic      -> symbolic_logic_shape
  hebrew_fluency      -> accuracy_shape
  koine_greek_fluency -> accuracy_shape
  inference_closure   -> inference_shape
  fabrication_control -> refusal_shape

Each shape has a documented threshold checker. Unknown lane ids
fail-closed with a named reason. ADR-0106 \xc2\xa71.1/\xc2\xa71.3/\xc2\xa71.4/\xc2\xa71.5
unchanged; only \xc2\xa71.2 (threshold rules) dispatches by shape.

tests/test_lane_shape_thresholds.py pins all four ADR-0109 invariants
plus dead-shape and threshold-value gates (13 new tests).
tests/test_expert_demo_contract.py fixtures updated to provide
shape-appropriate metrics (no semantic change to those tests; same
12 cases still pin the ADR-0106 contract).

ADR-0109 status: Proposed -> Accepted. README sequencing updated
(ADR-0110 now only blocked by inference_closure, not by metric-shape
amendment).

Ledger: all five domains remain reasoning-capable, expert_demo=false.
This commit is contained in:
Shay 2026-05-22 12:11:58 -07:00 committed by GitHub
parent c2dd69611b
commit 36053317be
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 350 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

165
core/capability/expert_demo.py
View file

@ -34,11 +34,132 @@ SURFACE_GROUNDEDNESS_MIN: float = 0.95
TERM_CAPTURE_RATE_MIN: float = 0.85
INTENT_ACCURACY_MIN: float = 0.95
VERSOR_CLOSURE_RATE_MIN: float = 1.0
FABRICATION_PASSED_MIN: float = 1.0
ACCURACY_MIN: float = 0.95
ALL_PASS_RATE_MIN: float = 0.95
REPLAY_DETERMINISM_MIN: float = 1.0
FABRICATION_CONTROL_LANE: str = "fabrication_control"
# ADR-0109 — lane-shape registry. New shapes require an ADR amendment;
# new lanes must be added here explicitly. Unknown lane ids fail closed.
LANE_SHAPE_REGISTRY: dict[str, str] = {
"cognition": "cognition_shape",
"elementary_mathematics_ood": "accuracy_shape",
"foundational_physics_ood": "accuracy_shape",
"symbolic_logic": "symbolic_logic_shape",
"hebrew_fluency": "accuracy_shape",
"koine_greek_fluency": "accuracy_shape",
"inference_closure": "inference_shape",
"fabrication_control": "refusal_shape",
}
def _check_cognition_shape(lane_id: str, metrics: Mapping[str, Any]) -> tuple[bool, str]:
checks = (
("surface_groundedness", SURFACE_GROUNDEDNESS_MIN),
("term_capture_rate", TERM_CAPTURE_RATE_MIN),
("intent_accuracy", INTENT_ACCURACY_MIN),
("versor_closure_rate", VERSOR_CLOSURE_RATE_MIN),
)
for key, minimum in checks:
if key not in metrics:
return False, f"lane {lane_id!r} missing required metric {key!r}"
value = float(metrics[key] or 0)
if value < minimum:
return False, (
f"lane {lane_id!r} {key}={value} below threshold {minimum}"
)
return True, ""
def _accuracy_value(metrics: Mapping[str, Any]) -> float | None:
"""Resolve accuracy from explicit key or passed/total fallback."""
if "accuracy" in metrics:
return float(metrics["accuracy"] or 0)
if "passed" in metrics and "total" in metrics:
total = float(metrics["total"] or 0)
if total <= 0:
return None
return float(metrics["passed"] or 0) / total
return None
def _check_accuracy_shape(lane_id: str, metrics: Mapping[str, Any]) -> tuple[bool, str]:
value = _accuracy_value(metrics)
if value is None:
return False, (
f"lane {lane_id!r} missing accuracy (and no passed/total fallback)"
)
if value < ACCURACY_MIN:
return False, (
f"lane {lane_id!r} accuracy={value} below threshold {ACCURACY_MIN}"
)
return True, ""
def _check_inference_shape(lane_id: str, metrics: Mapping[str, Any]) -> tuple[bool, str]:
if "all_pass_rate" not in metrics:
return False, f"lane {lane_id!r} missing required metric 'all_pass_rate'"
rate = float(metrics["all_pass_rate"] or 0)
if rate < ALL_PASS_RATE_MIN:
return False, (
f"lane {lane_id!r} all_pass_rate={rate} below threshold "
f"{ALL_PASS_RATE_MIN}"
)
if "replay_determinism" not in metrics:
return False, f"lane {lane_id!r} missing required metric 'replay_determinism'"
det = float(metrics["replay_determinism"] or 0)
if det < REPLAY_DETERMINISM_MIN:
return False, (
f"lane {lane_id!r} replay_determinism={det} below threshold "
f"{REPLAY_DETERMINISM_MIN}"
)
overall = metrics.get("overall_pass")
if overall is not None and not bool(overall):
return False, f"lane {lane_id!r} overall_pass is false"
return True, ""
def _check_refusal_shape(lane_id: str, metrics: Mapping[str, Any]) -> tuple[bool, str]:
by_class = metrics.get("by_class")
if not isinstance(by_class, Mapping) or not by_class:
return False, f"lane {lane_id!r} missing 'by_class' refusal counts"
for class_id, bucket in by_class.items():
if not isinstance(bucket, Mapping):
return False, (
f"lane {lane_id!r} by_class[{class_id!r}] is not a mapping"
)
fabricated = int(bucket.get("fabricated", 0) or 0)
if fabricated != 0:
return False, (
f"lane {lane_id!r} by_class[{class_id!r}] fabricated="
f"{fabricated} (must be 0)"
)
n = int(bucket.get("n", 0) or 0)
refused = int(bucket.get("refused", 0) or 0)
if n <= 0 or refused != n:
return False, (
f"lane {lane_id!r} by_class[{class_id!r}] refused={refused} "
f"!= n={n}"
)
return True, ""
SHAPE_CHECKERS: dict[str, Any] = {
"cognition_shape": _check_cognition_shape,
"accuracy_shape": _check_accuracy_shape,
"symbolic_logic_shape": _check_accuracy_shape,
"inference_shape": _check_inference_shape,
"refusal_shape": _check_refusal_shape,
}
def resolve_lane_shape(lane_id: str) -> str | None:
"""Return the registered shape id for ``lane_id`` or ``None``."""
return LANE_SHAPE_REGISTRY.get(lane_id)
@dataclass(frozen=True, slots=True)
class ExpertDemoVerdict:
passed: bool
@ -79,31 +200,23 @@ def derive_evidence_digest(
def _meets_thresholds(lane_id: str, metrics: Mapping[str, Any]) -> tuple[bool, str]:
if lane_id == FABRICATION_CONTROL_LANE:
passed = float(metrics.get("passed_rate", metrics.get("accuracy", 0)) or 0)
if passed < FABRICATION_PASSED_MIN:
return False, (
f"lane {lane_id!r} passed_rate={passed} below "
f"threshold {FABRICATION_PASSED_MIN}"
)
return True, ""
checks = (
("surface_groundedness", SURFACE_GROUNDEDNESS_MIN),
("term_capture_rate", TERM_CAPTURE_RATE_MIN),
("intent_accuracy", INTENT_ACCURACY_MIN),
("versor_closure_rate", VERSOR_CLOSURE_RATE_MIN),
)
for key, minimum in checks:
if key not in metrics:
return False, (
f"lane {lane_id!r} missing required metric {key!r}"
)
value = float(metrics[key] or 0)
if value < minimum:
return False, (
f"lane {lane_id!r} {key}={value} below threshold {minimum}"
)
return True, ""
"""Dispatch lane threshold check by registered shape (ADR-0109).
Unknown lane ids are fail-closed: adding a lane to the expert-demo
surface requires an explicit registry entry, which requires an ADR
amendment.
"""
shape_id = resolve_lane_shape(lane_id)
if shape_id is None:
return False, (
f"lane {lane_id!r} has no registered shape — introduce via ADR amendment"
)
checker = SHAPE_CHECKERS.get(shape_id)
if checker is None:
return False, (
f"lane {lane_id!r} resolves to shape {shape_id!r} with no checker"
)
return checker(lane_id, metrics)
def evaluate_expert_demo(

2
docs/decisions/ADR-0109-lane-shape-aware-thresholds.md
View file

@ -1,6 +1,6 @@
# ADR-0109 — Lane-Shape-Aware Thresholds (ADR-0106 Amendment)
**Status:** Proposed
**Status:** Accepted
**Date:** 2026-05-22
**Author:** CORE agents + reviewers
**Amends:** ADR-0106

5
docs/decisions/README.md
View file

@ -28,7 +28,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt
| [ADR-0106](ADR-0106-expert-demo-promotion-contract.md) | Expert-Demo Promotion Contract | Accepted (2026-05-22) |
| [ADR-0107](ADR-0107-mathematics-logic-expert-demo-deferred.md) | `mathematics_logic` Expert-Demo Promotion — Deferred | Accepted (2026-05-22) |
| [ADR-0108](ADR-0108-proposed-adr-sequencing.md) | Proposed-ADR Sequencing Post-ADR-0105 | Accepted (2026-05-22) |
| [ADR-0109](ADR-0109-lane-shape-aware-thresholds.md) | Lane-Shape-Aware Thresholds (ADR-0106 Amendment) | Proposed (2026-05-22) |
| [ADR-0109](ADR-0109-lane-shape-aware-thresholds.md) | Lane-Shape-Aware Thresholds (ADR-0106 Amendment) | Accepted (2026-05-22) |
---
@ -66,8 +66,7 @@ Seven lanes are SHA-pinned in `scripts/verify_lane_shas.py` and gated by the `la
Sequencing per ADR-0108. Listed in priority order:
1. **[ADR-0109](ADR-0109-lane-shape-aware-thresholds.md) — ADR-0106 lane-shape-aware threshold amendment.** Ships an explicit lane-shape registry covering five shapes (`cognition_shape`, `accuracy_shape`, `inference_shape`, `refusal_shape`, `symbolic_logic_shape`) so the contract can refuse promotion on substance, not on absence-of-key. Prerequisite to any future expert-demo promotion.
2. **ADR-0110 (reserved) — `mathematics_logic` expert-demo re-attempt.** Conditional on ADR-0109 landing AND `inference_closure` substantively passing (currently `all_pass_rate=0.4` on public).
1. **ADR-0110 (reserved) — `mathematics_logic` expert-demo re-attempt.** ADR-0109 has landed; the metric-shape blocker is cleared. The remaining blocker is `inference_closure` substantively passing (currently `all_pass_rate=0.4` on public).
3. **[ADR-0080](ADR-0080-contemplation-loop.md) — Contemplation Loop.** Sandboxed, read-only Phase 1 self-interrogation; emits `SPECULATIVE` findings from `frontier_compare` reports. Converts gap-finding from human-driven to system-emitted-and-reviewed.
4. **[ADR-0084](ADR-0084-definitional-layer.md) — Definitional Layer for Lexicon Packs.** Optional per-entry definitional block. Deferred — value surfaces during a worked expert promotion that needs definitional depth.
5. **[ADR-0087](ADR-0087-rhetorical-style-axis.md) — Rhetorical Style Axis.** A third substantive selection axis sibling to anchor-lens. Lowest current priority — no active downstream consumer; register + anchor-lens already demonstrate the orthogonality pattern.

51
tests/test_expert_demo_contract.py
View file

@ -23,6 +23,8 @@ behaves as ADR-0106 specifies, without flipping any production row.
from __future__ import annotations
import json
from core.capability.expert_demo import (
derive_evidence_digest,
evaluate_expert_demo,
@ -40,7 +42,30 @@ _GOOD_METRICS = {
"intent_accuracy": 0.96,
"versor_closure_rate": 1.0,
}
_FAB_METRICS = {"passed_rate": 1.0}
_FAB_METRICS = {
"by_class": {
"phantom_endpoint": {"n": 3, "refused": 3, "fabricated": 0},
"cross_pack_non_bridge": {"n": 3, "refused": 3, "fabricated": 0},
"sibling_collapse": {"n": 3, "refused": 3, "fabricated": 0},
}
}
_INFERENCE_METRICS = {
"all_pass_rate": 0.98,
"replay_determinism": 1.0,
"overall_pass": True,
}
_ACCURACY_METRICS = {"accuracy": 0.98, "passed": 39, "total": 40}
_SHAPE_FIXTURES = {
"fabrication_control": _FAB_METRICS,
"inference_closure": _INFERENCE_METRICS,
"elementary_mathematics_ood": _ACCURACY_METRICS,
"foundational_physics_ood": _ACCURACY_METRICS,
"symbolic_logic": _ACCURACY_METRICS,
"hebrew_fluency": _ACCURACY_METRICS,
"koine_greek_fluency": _ACCURACY_METRICS,
}
def _primary_reviewer() -> Reviewer:
@ -64,10 +89,20 @@ def _build_registry(
def _good_lane_results(lanes: tuple[str, ...]) -> dict[str, dict[str, dict]]:
"""Build shape-appropriate good metrics per registered lane.
Lanes not in the shape-fixture map (e.g. synthetic 'a', 'b', 'c'
used in digest-ordering tests, or 'cognition') get cognition-shape
metrics as a deterministic default — they're never run through
the threshold checker in those tests.
"""
out: dict[str, dict[str, dict]] = {}
for lane in lanes:
metrics = _FAB_METRICS if lane == "fabrication_control" else _GOOD_METRICS
out[lane] = {"public": dict(metrics), "holdout": dict(metrics)}
metrics = _SHAPE_FIXTURES.get(lane, _GOOD_METRICS)
out[lane] = {
"public": json.loads(json.dumps(metrics)),
"holdout": json.loads(json.dumps(metrics)),
}
return out
@ -258,7 +293,7 @@ class TestExpertDemoReplayByteEquality:
claim_digest=original_digest,
)
drifted = _good_lane_results(lanes)
drifted["inference_closure"]["public"]["intent_accuracy"] = 0.99
drifted["inference_closure"]["public"]["all_pass_rate"] = 0.97
registry = _build_registry((_primary_reviewer(),), (claim,))
verdict = evaluate_expert_demo(
domain_id=domain,
@ -278,7 +313,7 @@ class TestExpertDemoThresholds:
domain = "mathematics_logic"
lanes = ("inference_closure", "fabrication_control")
results = _good_lane_results(lanes)
results["inference_closure"]["holdout"]["surface_groundedness"] = 0.50
results["inference_closure"]["holdout"]["all_pass_rate"] = 0.50
digest = derive_evidence_digest(
domain_id=domain,
evidence_revision="rev1",
@ -301,14 +336,16 @@ class TestExpertDemoThresholds:
lane_results=results,
)
assert verdict.passed is False
assert "surface_groundedness" in verdict.reason
assert "all_pass_rate" in verdict.reason
assert "below threshold" in verdict.reason
def test_fabrication_control_failure_refuses(self) -> None:
domain = "mathematics_logic"
lanes = ("inference_closure", "fabrication_control")
results = _good_lane_results(lanes)
results["fabrication_control"]["holdout"]["passed_rate"] = 0.8
results["fabrication_control"]["holdout"]["by_class"][
"phantom_endpoint"
]["fabricated"] = 1
digest = derive_evidence_digest(
domain_id=domain,
evidence_revision="rev1",

164
tests/test_lane_shape_thresholds.py Normal file
View file

@ -0,0 +1,164 @@
"""ADR-0109 — lane-shape-aware threshold invariants.
Pins four invariants:
1. ``lane_shape_explicit`` — every lane id referenced by any ratified
pack's manifest must resolve to a registered shape.
2. ``shape_thresholds_are_named`` — each registered shape has a
documented checker; no implicit defaults.
3. ``unknown_lane_fails_closed`` — a lane id absent from the registry
produces ``passed=False`` with a named reason.
4. ``cognition_shape_unchanged_under_amendment`` — the four cognition
threshold constants are bit-identical to ADR-0106 §1.2.
"""
from __future__ import annotations
import json
from pathlib import Path
from core.capability.domains import DOMAIN_PACKS
from core.capability.expert_demo import (
ACCURACY_MIN,
ALL_PASS_RATE_MIN,
INTENT_ACCURACY_MIN,
LANE_SHAPE_REGISTRY,
REPLAY_DETERMINISM_MIN,
SHAPE_CHECKERS,
SURFACE_GROUNDEDNESS_MIN,
TERM_CAPTURE_RATE_MIN,
VERSOR_CLOSURE_RATE_MIN,
evaluate_expert_demo,
resolve_lane_shape,
)
from core.capability.reviewers import (
ExpertDemoClaim,
Reviewer,
ReviewerRegistry,
)
_REPO_ROOT = Path(__file__).resolve().parent.parent
def _ratified_pack_lanes() -> set[str]:
"""Collect every lane id referenced by every ratified pack."""
out: set[str] = set()
for packs in DOMAIN_PACKS.values():
for pack_id in packs:
manifest_path = (
_REPO_ROOT
/ "language_packs"
/ "data"
/ pack_id
/ "manifest.json"
)
if not manifest_path.exists():
continue
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
for entry in manifest.get("eval_lanes", []) or []:
lane = entry.get("lane")
if isinstance(lane, str):
out.add(lane)
return out
class TestLaneShapeExplicit:
def test_every_ratified_lane_resolves_to_registered_shape(self) -> None:
lanes = _ratified_pack_lanes()
assert lanes, "expected at least one lane attached to a ratified pack"
unresolved = [lane for lane in lanes if resolve_lane_shape(lane) is None]
assert unresolved == [], (
f"lanes referenced by ratified packs but missing from "
f"LANE_SHAPE_REGISTRY: {sorted(unresolved)}"
)
class TestShapeThresholdsAreNamed:
def test_every_registered_shape_has_checker(self) -> None:
shapes = set(LANE_SHAPE_REGISTRY.values())
for shape_id in shapes:
assert shape_id in SHAPE_CHECKERS, (
f"shape {shape_id!r} appears in LANE_SHAPE_REGISTRY but has "
f"no entry in SHAPE_CHECKERS"
)
def test_no_shape_without_a_lane(self) -> None:
"""Every shape with a checker must be used by at least one lane.
Catches dead-shape drift: if a shape is removed from all lanes
in the registry, the SHAPE_CHECKERS entry should also be retired
by a follow-up ADR rather than left as silently-unused code.
"""
used_shapes = set(LANE_SHAPE_REGISTRY.values())
unused = set(SHAPE_CHECKERS.keys()) - used_shapes
assert unused == set(), (
f"shape checkers defined but no lane uses them: {sorted(unused)}"
)
class TestUnknownLaneFailsClosed:
def _registry_with_claim(self, lane_id: str) -> ReviewerRegistry:
reviewer = Reviewer(
reviewer_id="shay-j",
display_name="Joshua Shay",
role="primary",
domains=("*",),
review_scope=("pack", "proposal", "chain", "eval"),
provenance="adr-0092:bootstrap:2026-05-21",
)
claim = ExpertDemoClaim(
domain_id="mathematics_logic",
evidence_lanes=(lane_id,),
evidence_revision="rev1",
signed_by="shay-j",
claim_digest="a" * 64,
)
return ReviewerRegistry(
schema_version=1, reviewers=(reviewer,), expert_demo_claims=(claim,)
)
def test_unregistered_lane_id_refuses(self) -> None:
lane_id = "synthetic_unregistered_lane"
registry = self._registry_with_claim(lane_id)
verdict = evaluate_expert_demo(
domain_id="mathematics_logic",
reasoning_capable=True,
registry=registry,
domain_lanes=(lane_id,),
lane_results={
lane_id: {
"public": {"accuracy": 1.0},
"holdout": {"accuracy": 1.0},
}
},
)
assert verdict.passed is False
assert "no registered shape" in verdict.reason
def test_resolve_returns_none_for_unknown(self) -> None:
assert resolve_lane_shape("definitely_not_a_real_lane") is None
class TestCognitionShapeUnchangedUnderAmendment:
"""ADR-0106 §1.2 thresholds must remain bit-identical post-ADR-0109."""
def test_cognition_thresholds_unchanged(self) -> None:
assert SURFACE_GROUNDEDNESS_MIN == 0.95
assert TERM_CAPTURE_RATE_MIN == 0.85
assert INTENT_ACCURACY_MIN == 0.95
assert VERSOR_CLOSURE_RATE_MIN == 1.0
def test_cognition_lane_resolves_to_cognition_shape(self) -> None:
assert resolve_lane_shape("cognition") == "cognition_shape"
class TestShapeThresholdValues:
"""Pin the documented minimums per ADR-0109 §2."""
def test_accuracy_shape_minimum(self) -> None:
assert ACCURACY_MIN == 0.95
def test_inference_shape_minimums(self) -> None:
assert ALL_PASS_RATE_MIN == 0.95
assert REPLAY_DETERMINISM_MIN == 1.0