From dec98ea0d019aea03f99afa369d9563b16d3f772 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 23 May 2026 18:55:34 -0700 Subject: [PATCH] =?UTF-8?q?feat(ADR-0120=20math,=20ledger=20flip):=20mathe?= =?UTF-8?q?matics=5Flogic=20=E2=86=92=20expert=20tier=20(first-ever)=20(#1?= =?UTF-8?q?95)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bundles the three pieces needed to consummate the promotion after the reviewer signature lands: 1. Wire the expert tier in the capability ledger 2. Path-stability fix (digest filesystem-independence) 3. Reviewer-registry allow-list extension (regression fix for #194) Result: mathematics_logic is now the first expert-tier domain in the capability ledger. $ ledger_report() -> mathematics_logic row: status: "expert" predicates: { seeded, grounded, reasoning_capable, audit_passed, expert: True } expert_reason: "ADR-0120-math composer admitted" 1. Ledger wiring (core/capability/reporting.py): - _EXPERT_DOMAIN_STATUSES extends to 6 tiers with "expert" after "audit-passed" (strict super-tier). - New _EXPERT_COMPOSERS dict — per-domain registry of composer module names. Currently only mathematics_logic -> core.capability.expert_promotion_math. - New `expert` predicate computation gated on audit_passed; calls registered composer's evaluate_math_expert_promotion() and reads promote_admitted as the verdict. Fail-closed on exception or missing composer. - status = "expert" when predicate True. - predicates dict gains "expert" key; row gains expert_reason. 2. Path-stability fix (composite_math_gate.py + expert_promotion_math.py): - New _rel(path) helpers return repo-root-relative POSIX strings instead of str(absolute_path). - claim_digest now commits to relative paths, so operator A on ~/work/core and operator B on /srv/checkouts/core compute the SAME digest for identical evidence. - Without this fix no signature would ever match across filesystems — a real bug that would have blocked every signing attempt. 3. Allow-list regression fix (core/capability/reviewers.py): - ALLOWED_TOP_LEVEL_KEYS extended with "math_expert_claims". - PR #194 added the section to docs/reviewers.yaml but didn't extend the allow-list, silently breaking the audit_passed predicate for ALL 3 prior domains (loader rejected the file). This PR's test_allowed_top_level_keys_includes_math_expert_claims regression-pins the fix. Reviewer signature (operator-only action by shay-j) carried in docs/reviewers.yaml: math_expert_claims: - domain_id: mathematics_logic signed_by: shay-j claim_digest: "94149794e8c19896851e062cf1f921cfa9ba04770b674bc3b4c33023f7c7331b" The auto-mode safeguard correctly blocked the agent from self- signing during PR construction; the signature was performed by the reviewer directly and brought into this PR. Future signatures stay human-only. Tests: 12/12 new ledger-flip tests + 174/174 across full obligation auditor / composer / composite-gate / expert-demo / reviewer-registry regression. Updated #194's awaiting-state snapshot to reflect the new promote_admitted=True state on main. GSM8K (honest disclosure, not gating): still 0/50 admission, wrong=0, safety_rail_intact=True, substrate=candidate_graph. Probe lift is future work (bounded pronoun coref is the highest-leverage item — ~28% of refusals route through it). The promotion does not depend on GSM8K per ADR-0131. --- core/capability/composite_math_gate.py | 27 ++- core/capability/expert_promotion_math.py | 55 +++-- core/capability/reporting.py | 50 ++++- core/capability/reviewers.py | 2 +- .../ADR-0120-math-expert-ledger-flip.md | 199 ++++++++++++++++++ docs/reviewers.yaml | 31 +-- .../v1/expert_claims_math_v1_signed.json | 41 ++-- tests/test_adr_0120_math_expert_promotion.py | 19 +- tests/test_mathlogic_expert_ledger_flip.py | 192 +++++++++++++++++ 9 files changed, 545 insertions(+), 71 deletions(-) create mode 100644 docs/decisions/ADR-0120-math-expert-ledger-flip.md create mode 100644 tests/test_mathlogic_expert_ledger_flip.py diff --git a/core/capability/composite_math_gate.py b/core/capability/composite_math_gate.py index f1b9b1b1..cded5b22 100644 --- a/core/capability/composite_math_gate.py +++ b/core/capability/composite_math_gate.py @@ -56,6 +56,23 @@ WRONG_MAX: int = 0 # core/capability/composite_math_gate.py -> ../../. _REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +def _rel(path: Path) -> str: + """Repo-root-relative POSIX string of ``path``. + + The composite-gate claim_digest commits to ``report_path`` / + ``probe_path`` strings; making them repo-relative keeps the digest + stable across operator filesystems (different worktree locations + would otherwise produce different digests for identical evidence). + Falls back to the absolute POSIX string if ``path`` is outside the + repo root. + """ + p = Path(path).resolve() + try: + return p.relative_to(_REPO_ROOT).as_posix() + except ValueError: + return p.as_posix() + # Default benchmark report paths. The evaluator accepts override # paths so tests can point at fixture data without touching main's # committed artifacts. @@ -199,7 +216,7 @@ def _evaluate_one(benchmark_id: str, path: Path) -> BenchmarkVerdict: except CompositeMathGateError as exc: return BenchmarkVerdict( benchmark_id=benchmark_id, - report_path=str(path), + report_path=_rel(path), correct=0, wrong=0, cases_total=0, @@ -213,7 +230,7 @@ def _evaluate_one(benchmark_id: str, path: Path) -> BenchmarkVerdict: wrong_zero = wrong <= WRONG_MAX return BenchmarkVerdict( benchmark_id=benchmark_id, - report_path=str(path), + report_path=_rel(path), correct=correct, wrong=wrong, cases_total=total, @@ -235,7 +252,7 @@ def _gsm8k_honest_disclosure(path: Path) -> dict[str, Any]: """ if not path.exists(): return { - "probe_path": str(path), + "probe_path": _rel(path), "available": False, "note": "probe report missing; honest disclosure unavailable", } @@ -243,13 +260,13 @@ def _gsm8k_honest_disclosure(path: Path) -> dict[str, Any]: report = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: return { - "probe_path": str(path), + "probe_path": _rel(path), "available": False, "note": f"probe report unparseable: {exc}", } metrics = report.get("metrics", {}) return { - "probe_path": str(path), + "probe_path": _rel(path), "available": True, "admitted_solved": metrics.get("admitted_solved", 0), "admitted_wrong": metrics.get("admitted_wrong", 0), diff --git a/core/capability/expert_promotion_math.py b/core/capability/expert_promotion_math.py index cd03d72b..f7bd63b3 100644 --- a/core/capability/expert_promotion_math.py +++ b/core/capability/expert_promotion_math.py @@ -48,6 +48,23 @@ from core.capability.perturbation_b3 import validate_perturbation_suite _REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +def _rel(path: Path) -> str: + """Return ``path`` as a POSIX string repo-root-relative. + + The claim digest commits to evidence-pointer strings; making them + repo-relative keeps the digest stable across operator filesystems + (different worktree locations would otherwise produce different + digests for identical evidence). Falls back to the absolute POSIX + string if ``path`` is outside the repo root. + """ + p = Path(path).resolve() + try: + return p.relative_to(_REPO_ROOT).as_posix() + except ValueError: + return p.as_posix() + + DEFAULT_REVIEWERS_YAML: Path = _REPO_ROOT / "docs" / "reviewers.yaml" # Evidence-bundle paths the digest commits to (in canonical order). @@ -132,7 +149,7 @@ def _evaluate_obligation_1(b1_sealed_path: Path = DEFAULT_B1_SEALED) -> Obligati if not b1_sealed_path.exists(): return ObligationVerdict( obligation_id="1", title="sealed holdout discipline", - passed=False, evidence_pointer=str(b1_sealed_path), + passed=False, evidence_pointer=_rel(b1_sealed_path), refusal_reason="sealed report missing", ) report = json.loads(b1_sealed_path.read_text(encoding="utf-8")) @@ -143,7 +160,7 @@ def _evaluate_obligation_1(b1_sealed_path: Path = DEFAULT_B1_SEALED) -> Obligati return ObligationVerdict( obligation_id="1", title="sealed holdout discipline", passed=passed, - evidence_pointer=str(b1_sealed_path), + evidence_pointer=_rel(b1_sealed_path), refusal_reason=( "" if passed else f"sealed report: wrong={wrong}, exit_passed={exit_crit.get('passed')}" @@ -156,7 +173,7 @@ def _evaluate_obligation_2() -> ObligationVerdict: return ObligationVerdict( obligation_id="2", title="OOD surface variation ratio ≥ 0.95", passed=bool(r.obligation_2_passed), - evidence_pointer=str(_REPO_ROOT / "evals" / "obligation_2_ood_ratio"), + evidence_pointer=_rel(_REPO_ROOT / "evals" / "obligation_2_ood_ratio"), refusal_reason="" if r.obligation_2_passed else r.refusal_reason, ) @@ -169,7 +186,7 @@ def _evaluate_obligation_3(b_reports: tuple[Path, ...]) -> ObligationVerdict: if not path.exists(): return ObligationVerdict( obligation_id="3", title="replay-equal trace", - passed=False, evidence_pointer=str(path), + passed=False, evidence_pointer=_rel(path), refusal_reason=f"B-lane report missing: {path}", ) report = json.loads(path.read_text(encoding="utf-8")) @@ -181,12 +198,12 @@ def _evaluate_obligation_3(b_reports: tuple[Path, ...]) -> ObligationVerdict: if missing: return ObligationVerdict( obligation_id="3", title="replay-equal trace", - passed=False, evidence_pointer=str(b_reports[0].parent), + passed=False, evidence_pointer=_rel(b_reports[0].parent), refusal_reason=f"{len(missing)} correct case(s) missing trace_hash; first: {missing[0]}", ) return ObligationVerdict( obligation_id="3", title="replay-equal trace", - passed=True, evidence_pointer=str(b_reports[0].parent), + passed=True, evidence_pointer=_rel(b_reports[0].parent), ) @@ -196,7 +213,7 @@ def _evaluate_obligation_4(b_reports: tuple[Path, ...]) -> ObligationVerdict: if not path.exists(): return ObligationVerdict( obligation_id="4", title="typed refusal + wrong == 0", - passed=False, evidence_pointer=str(path), + passed=False, evidence_pointer=_rel(path), refusal_reason=f"B-lane report missing: {path}", ) report = json.loads(path.read_text(encoding="utf-8")) @@ -209,12 +226,12 @@ def _evaluate_obligation_4(b_reports: tuple[Path, ...]) -> ObligationVerdict: if wrong is None or int(wrong) > 0: return ObligationVerdict( obligation_id="4", title="typed refusal + wrong == 0", - passed=False, evidence_pointer=str(path), + passed=False, evidence_pointer=_rel(path), refusal_reason=f"{path.name}: wrong={wrong}", ) return ObligationVerdict( obligation_id="4", title="typed refusal + wrong == 0", - passed=True, evidence_pointer=str(b_reports[0].parent), + passed=True, evidence_pointer=_rel(b_reports[0].parent), ) @@ -231,7 +248,7 @@ def _evaluate_obligation_5() -> ObligationVerdict: return ObligationVerdict( obligation_id="5", title="reasoning-isolation perturbation suite", passed=bool(passed), - evidence_pointer=str(_REPO_ROOT / "evals" / "obligation_5_perturbation"), + evidence_pointer=_rel(_REPO_ROOT / "evals" / "obligation_5_perturbation"), refusal_reason="" if passed else getattr(r, "refusal_reason", "obligation_5 evaluator returned non-pass"), ) @@ -243,7 +260,7 @@ def _evaluate_obligation_6() -> ObligationVerdict: return ObligationVerdict( obligation_id="6", title="compositional-depth curve", passed=passed, - evidence_pointer=str(_REPO_ROOT / "evals" / "obligation_6_depth_curve"), + evidence_pointer=_rel(_REPO_ROOT / "evals" / "obligation_6_depth_curve"), refusal_reason="" if passed else r.refusal_reason, ) @@ -255,7 +272,7 @@ def _evaluate_obligation_7(frontier_dir: Path = DEFAULT_FRONTIER_DIR) -> Obligat if not frontier_dir.exists(): return ObligationVerdict( obligation_id="7", title="frontier-baseline comparison", - passed=False, evidence_pointer=str(frontier_dir), + passed=False, evidence_pointer=_rel(frontier_dir), refusal_reason="frontier directory missing", ) artifacts = sorted( @@ -265,12 +282,12 @@ def _evaluate_obligation_7(frontier_dir: Path = DEFAULT_FRONTIER_DIR) -> Obligat if not artifacts: return ObligationVerdict( obligation_id="7", title="frontier-baseline comparison", - passed=False, evidence_pointer=str(frontier_dir), + passed=False, evidence_pointer=_rel(frontier_dir), refusal_reason="no frontier comparison artifacts found", ) return ObligationVerdict( obligation_id="7", title="frontier-baseline comparison", - passed=True, evidence_pointer=str(frontier_dir), + passed=True, evidence_pointer=_rel(frontier_dir), ) @@ -279,7 +296,7 @@ def _evaluate_obligation_8() -> ObligationVerdict: return ObligationVerdict( obligation_id="8", title="adversarial generation; misparse zero", passed=bool(r.obligation_8_passed), - evidence_pointer=str(_REPO_ROOT / "evals" / "obligation_8_adversarial"), + evidence_pointer=_rel(_REPO_ROOT / "evals" / "obligation_8_adversarial"), refusal_reason="" if r.obligation_8_passed else r.refusal_reason, ) @@ -294,7 +311,7 @@ def _evaluate_obligation_9(b_reports: tuple[Path, ...]) -> ObligationVerdict: if not path.exists(): return ObligationVerdict( obligation_id="9", title="determinism", - passed=False, evidence_pointer=str(path), + passed=False, evidence_pointer=_rel(path), refusal_reason=f"B-lane report missing: {path}", ) try: @@ -302,12 +319,12 @@ def _evaluate_obligation_9(b_reports: tuple[Path, ...]) -> ObligationVerdict: except json.JSONDecodeError as exc: return ObligationVerdict( obligation_id="9", title="determinism", - passed=False, evidence_pointer=str(path), + passed=False, evidence_pointer=_rel(path), refusal_reason=f"{path.name}: invalid JSON ({exc})", ) return ObligationVerdict( obligation_id="9", title="determinism", - passed=True, evidence_pointer=str(b_reports[0].parent), + passed=True, evidence_pointer=_rel(b_reports[0].parent), ) @@ -316,7 +333,7 @@ def _evaluate_obligation_10() -> ObligationVerdict: return ObligationVerdict( obligation_id="10", title="operation provenance via pack", passed=bool(r.obligation_10_passed), - evidence_pointer=str(_REPO_ROOT / "evals" / "obligation_10_pack_provenance"), + evidence_pointer=_rel(_REPO_ROOT / "evals" / "obligation_10_pack_provenance"), refusal_reason="" if r.obligation_10_passed else r.refusal_reason, ) diff --git a/core/capability/reporting.py b/core/capability/reporting.py index 9f3a084b..cd65f3f1 100644 --- a/core/capability/reporting.py +++ b/core/capability/reporting.py @@ -34,7 +34,17 @@ from language_packs.domain_contract import validate_domain_contract_pack _REPO_ROOT = Path(__file__).resolve().parent.parent.parent _CHAINS_PER_OPERATOR_DOMAIN = 8 _TARGET_INTENT_SHAPES = ("cause", "verification", "comparison", "procedure", "correction") -_EXPERT_DOMAIN_STATUSES = ("blocked", "seeded", "grounded", "reasoning-capable", "audit-passed") +_EXPERT_DOMAIN_STATUSES = ( + "blocked", "seeded", "grounded", "reasoning-capable", "audit-passed", "expert", +) + +# ADR-0120 reserves the ``expert`` tier; the predicate computation +# is per-domain because each domain ratifies its own composer +# (ADR-0131.4 → ADR-0120-math for ``mathematics_logic``). Adding a +# new expert-tier domain means wiring its own composer entry here. +_EXPERT_COMPOSERS: dict[str, str] = { + "mathematics_logic": "core.capability.expert_promotion_math", +} _DOMAIN_FOUNDATION_GAPS: dict[str, tuple[str, ...]] = { "hebrew_greek_textual_reasoning": ( "gap:grc_he_glosses_absent", @@ -449,7 +459,41 @@ def ledger_report() -> dict[str, Any]: ) audit_passed = verdict.passed audit_passed_reason = verdict.reason - if audit_passed: + # ADR-0120 expert tier: per-domain composer (mathematics_logic + # via ADR-0120-math). Only evaluated when audit-passed already + # holds — expert is a strict super-tier of audit-passed. + expert = False + expert_reason = "audit-passed predicate must hold first" + if audit_passed and domain in _EXPERT_COMPOSERS: + try: + from importlib import import_module + module = import_module(_EXPERT_COMPOSERS[domain]) + evaluator = getattr(module, "evaluate_math_expert_promotion", None) + if evaluator is not None: + expert_verdict = evaluator() + expert = bool(getattr(expert_verdict, "promote_admitted", False)) + expert_reason = ( + "ADR-0120-math composer admitted" + if expert + else getattr(expert_verdict, "refusal_reason", "composer refused") + ) + else: + expert_reason = ( + f"composer module {_EXPERT_COMPOSERS[domain]!r} " + f"missing evaluate_math_expert_promotion()" + ) + except Exception as exc: # noqa: BLE001 — fail closed + expert_reason = ( + f"composer raised {type(exc).__name__}: {exc}" + ) + elif audit_passed: + expert_reason = ( + f"no expert composer wired for domain {domain!r}" + ) + + if expert: + status = "expert" + elif audit_passed: status = "audit-passed" elif reasoning_capable: status = "reasoning-capable" @@ -491,8 +535,10 @@ def ledger_report() -> dict[str, Any]: "grounded": grounded, "reasoning_capable": reasoning_capable, "audit_passed": audit_passed, + "expert": expert, }, "audit_passed_reason": audit_passed_reason, + "expert_reason": expert_reason, "known_gaps": domain_gaps, "open_gaps": open_gaps, "blocked_lift": blocked_lift, diff --git a/core/capability/reviewers.py b/core/capability/reviewers.py index c41ca02a..6552b7cf 100644 --- a/core/capability/reviewers.py +++ b/core/capability/reviewers.py @@ -18,7 +18,7 @@ import yaml ALLOWED_TOP_LEVEL_KEYS: frozenset[str] = frozenset( - {"schema_version", "reviewers", "audit_passed_claims"} + {"schema_version", "reviewers", "audit_passed_claims", "math_expert_claims"} ) ALLOWED_REVIEWER_KEYS: frozenset[str] = frozenset( {"reviewer_id", "display_name", "role", "domains", "review_scope", "provenance"} diff --git a/docs/decisions/ADR-0120-math-expert-ledger-flip.md b/docs/decisions/ADR-0120-math-expert-ledger-flip.md new file mode 100644 index 00000000..1b04a07c --- /dev/null +++ b/docs/decisions/ADR-0120-math-expert-ledger-flip.md @@ -0,0 +1,199 @@ +# ADR-0120 (math, ledger flip) — Mathematics-Logic Domain Promoted to `expert` + +**Status:** Accepted — first `expert`-tier domain in the capability ledger +**Date:** 2026-05-23 +**Author:** CORE main agent (Opus 4.7) + reviewer (shay-j) +**Depends on:** ADR-0120 (expert tier contract), ADR-0120-math +(composer, PR #194), ADR-0131.4 (composite math gate, PR #188), +ADR-0114a.{1,2,3,4,5,6,7,8,9,10} (obligation auditors) +**Predecessor PRs:** #173, #176-#180, #182-#194 + +--- + +## Context + +After all 10 ADR-0114a obligation auditors + the ADR-0131.4 composite +gate + the ADR-0120-math composer (#194) landed, three pieces +remained before the first-ever ledger flip: + +1. The reviewer signature itself (operator-only action). +2. The capability ledger's `_EXPERT_DOMAIN_STATUSES` tuple still + topped out at `"audit-passed"`; the `expert` tier was reserved + but never wired into the row construction. +3. A digest-stability bug: PR #194's composer baked absolute + filesystem paths into the `claim_digest`, which would have made + any operator's signature fail on any other operator's checkout. + +This PR closes all three. + +## Decision + +### 1. Reviewer signature (operator action by shay-j) + +`docs/reviewers.yaml` now carries the signed +`math_expert_claims` entry: + +```yaml +math_expert_claims: + - domain_id: mathematics_logic + evidence_lanes: + - math_symbolic_equivalence/v1 (public) + - math_symbolic_equivalence/v1 (sealed) + - math_teaching_corpus/v1 + - math_bounded_grammar/v1 + evidence_revision: "adr-0120-math:reviewed:2026-05-23" + signed_by: shay-j + claim_digest: "94149794e8c19896851e062cf1f921cfa9ba04770b674bc3b4c33023f7c7331b" +``` + +### 2. Ledger row gains an `expert` tier + +`core/capability/reporting.py`: + +- `_EXPERT_DOMAIN_STATUSES` extended with `"expert"` (now 6 tiers). +- New `_EXPERT_COMPOSERS: dict[str, str]` registry — per-domain + module name of the expert composer. Currently only + `mathematics_logic → core.capability.expert_promotion_math`. +- New `expert` predicate computation in `_compute_domain_row`: + - Gated on `audit_passed=True` (strict super-tier). + - Calls the registered composer's + `evaluate_math_expert_promotion()` and reads + `promote_admitted` as the verdict. + - Fail-closed on exception or missing composer. +- New `expert_reason` field on the row mirroring `audit_passed_reason`. +- `status = "expert"` when the predicate passes. +- `predicates: { ..., "expert": }` added. + +### 3. Path-stability fix (digest filesystem-independence) + +Both `core/capability/composite_math_gate.py` and +`core/capability/expert_promotion_math.py` now use a `_rel(path)` +helper that returns the **repo-root-relative POSIX string** +instead of `str(path)`. The `claim_digest` SHA-256 commits to +these relative strings, so: + +- Operator A on `~/work/core` and Operator B on `/srv/checkouts/core` + now compute the **same digest** for identical evidence. +- Re-signing isn't needed on every checkout. +- The `evidence_revision` field tells operators when the bundle + semantically changed; the digest tells them whether their on- + disk bytes still match what the signer endorsed. + +### 4. Reviewer-registry allow-list extended (regression fix) + +`ALLOWED_TOP_LEVEL_KEYS` in `core/capability/reviewers.py` now +includes `"math_expert_claims"`. PR #194 added the section to +`docs/reviewers.yaml` but **didn't extend the loader's allow-list** +— a real regression that silently broke the `audit_passed` +predicate for all 3 prior domains (mathematics_logic, physics, +systems_software) since the loader rejected the whole file and +the catch-all `_load_registry_for_expert_demo` returned an empty +registry. This PR's `test_allowed_top_level_keys_includes_math_expert_claims` +regression-pins the fix. + +## Empirical verdict on current main (after this PR) + +``` +$ python3 -c "from core.capability import ledger_report; \ + import json; r = ledger_report(); \ + row = next(d for d in r['domains'] if d['domain'] == 'mathematics_logic'); \ + print(json.dumps({k: row[k] for k in ('domain','status','predicates','expert_reason')}, indent=2))" + +{ + "domain": "mathematics_logic", + "status": "expert", + "predicates": { + "seeded": true, + "grounded": true, + "reasoning_capable": true, + "audit_passed": true, + "expert": true + }, + "expert_reason": "ADR-0120-math composer admitted" +} +``` + +**`mathematics_logic` is now the first `expert`-tier domain in the +capability ledger.** + +## GSM8K — honest disclosure unchanged + +The same `core capability math-expert-promote` artifact carries the +GSM8K probe's `honest_disclosure` block, which today reports +`admission_rate: 0/50, wrong: 0, safety_rail_intact: true, +substrate: candidate_graph`. **GSM8K does not gate** per ADR-0131; +it's reported as honest disclosure. Probe admission lift will +accumulate naturally as the parser layer matures (bounded pronoun +coreference is the highest-leverage next item — ~28% of GSM8K +refusals would route through it). + +## What this PR does NOT do + +- Does NOT promote any other domain. The pattern transfers, but + each domain needs its own composer module + reviewer-registry + section. +- Does NOT auto-promote on subsequent evidence-bundle changes. If + any B-lane re-runs (different correct/wrong/refused counts), the + composer's digest changes and the existing signature stops + matching — the verdict flips back to "awaiting reviewer + signature" and the ledger row drops back to `audit-passed`. This + is the load-bearing safety property. +- Does NOT remove the auto-mode safeguard that blocks the + composer-agent from self-signing. The signature stays a + human-only action; this PR carries the signature only because + the reviewer (shay-j) explicitly performed it. + +## Trust boundary + +- **Reads**: full obligation auditor reports + composer evidence + + `docs/reviewers.yaml` (transitively via the existing registry + loader) +- **Writes**: artifact path only (default + `evals/math_expert_claims/v1/expert_claims_math_v1_signed.json`) +- No new dynamic imports beyond what `_EXPERT_COMPOSERS` declares + (a single allow-listed module name); the lookup is fail-closed + if a domain isn't registered. +- Pure deterministic — verified by digest reproducibility + + artifact byte-equality + filesystem-independence tests. + +## Tests + +`tests/test_mathlogic_expert_ledger_flip.py` — 12 tests: + +| Group | What it pins | +|---|---| +| Allow-list regression | `math_expert_claims` is in `ALLOWED_TOP_LEVEL_KEYS`; reviewers.yaml loads after fix | +| Tier wiring | `expert` is in `_EXPERT_DOMAIN_STATUSES`; ordered after `audit-passed`; `mathematics_logic` composer registered | +| Path-stability | `_rel()` returns repo-relative POSIX; falls back to absolute outside repo; composer + composite-gate digests use relative paths only | +| Snapshot pass | `mathematics_logic.status == "expert"`; `predicates.expert == True`; row carries `expert_reason` | +| Other domains | non-mathematics_logic domains keep `expert: False` (no composer wired) | +| Refusal mode | without a matching signature, composer reports `promote_admitted: False` with awaiting-signature message | + +Plus the prior #194 snapshot test updated to reflect the new +post-signature state (`promote_admitted: True`). + +Full regression: **174/174 tests pass** across this PR's tests + +ADR-0120 composer (#194) + all 5 obligation auditors + composite +gate + existing expert-demo / reviewer-registry / ADR-0110 / ADR-0111 +/ ADR-0121 (math/physics/deferred). + +## CLAUDE.md PR-checklist + +- **Capability added:** first ledger flip to `expert` tier; + per-domain composer registry pattern future domains can adopt; + fixed two real bugs (path-instability + allow-list regression) + surfaced during the flip. +- **Invariant proving field validity:** ledger reports `expert` + iff and only iff signed evidence + every auditor pass; + composer digest filesystem-independent; allow-list permits + schema extension without silent failure. +- **CLI/eval proving the lane:** + `python3 -m core.cli capability math-expert-promote` + + `pytest tests/test_mathlogic_expert_ledger_flip.py`. +- **Avoided hidden normalization / stochastic / approximate / + unreviewed mutation:** Yes. The signature is operator-only; + the auto-mode safeguard explicitly blocks the agent from + self-signing; the bug fixes preserve every existing invariant. +- **Trust boundary:** read-only inputs from documented paths; + single deterministic write; signature remains the only gate + switch; mismatched signature refused with explicit diagnostic. diff --git a/docs/reviewers.yaml b/docs/reviewers.yaml index 3f138d1c..b2d9c27f 100644 --- a/docs/reviewers.yaml +++ b/docs/reviewers.yaml @@ -51,23 +51,14 @@ audit_passed_claims: evidence_revision: "adr-0124:reviewed:2026-05-22" signed_by: shay-j claim_digest: "17e24436b6875b89f6d1a5c2992557413c7ef456250f549d463159f54438c407" - -# Reviewer-signed math-expert promotion claims (ADR-0120 / ADR-0131 / -# wired by the ADR-0120 math-expert wire-up PR). -# -# Each entry signs the canonical evidence-bundle SHA-256 the -# expert_promotion_math composer derives from: -# - all 10 ADR-0114a obligation auditor verdicts -# - the ADR-0131.4 composite math gate verdict (B1+B2+B3) -# -# Re-derivation must reproduce ``claim_digest`` byte-for-byte. A -# mismatch means the evidence bundle has changed since the signature -# was added (a B-lane was re-run, a pack was edited, an obligation -# auditor's report changed); the operator must re-inspect the new -# digest and re-sign explicitly. -# -# A populated entry here is what flips -# ``MathExpertPromotionVerdict.promote_admitted`` to True. Until -# then the verdict reports ``awaiting reviewer signature`` along -# with the digest the operator should sign. -math_expert_claims: [] +math_expert_claims: + - domain_id: mathematics_logic + evidence_lanes: + - math_symbolic_equivalence/v1 (public) + - math_symbolic_equivalence/v1 (sealed) + - math_teaching_corpus/v1 + - math_bounded_grammar/v1 + evidence_revision: "adr-0120-math:reviewed:2026-05-23" + signed_by: shay-j + claim_digest: "94149794e8c19896851e062cf1f921cfa9ba04770b674bc3b4c33023f7c7331b" + \ No newline at end of file diff --git a/evals/math_expert_claims/v1/expert_claims_math_v1_signed.json b/evals/math_expert_claims/v1/expert_claims_math_v1_signed.json index 41576782..1c1baab4 100644 --- a/evals/math_expert_claims/v1/expert_claims_math_v1_signed.json +++ b/evals/math_expert_claims/v1/expert_claims_math_v1_signed.json @@ -1,86 +1,97 @@ { "adr": "0120-math", "all_obligations_passed": true, - "claim_digest": "d164866975341d9b82503caf50c0404ee140eab21fd60f589536c6daf6e1d706", + "claim_digest": "94149794e8c19896851e062cf1f921cfa9ba04770b674bc3b4c33023f7c7331b", "composite_gate_passed": true, "composite_gate_refusal": "", "domain": "mathematics_logic", "obligations": [ { - "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/math_symbolic_equivalence/v1/sealed_report.json", + "evidence_pointer": "evals/math_symbolic_equivalence/v1/sealed_report.json", "obligation_id": "1", "passed": true, "refusal_reason": "", "title": "sealed holdout discipline" }, { - "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/obligation_2_ood_ratio", + "evidence_pointer": "evals/obligation_2_ood_ratio", "obligation_id": "2", "passed": true, "refusal_reason": "", "title": "OOD surface variation ratio \u2265 0.95" }, { - "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/math_symbolic_equivalence/v1", + "evidence_pointer": "evals/math_symbolic_equivalence/v1", "obligation_id": "3", "passed": true, "refusal_reason": "", "title": "replay-equal trace" }, { - "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/math_symbolic_equivalence/v1", + "evidence_pointer": "evals/math_symbolic_equivalence/v1", "obligation_id": "4", "passed": true, "refusal_reason": "", "title": "typed refusal + wrong == 0" }, { - "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/obligation_5_perturbation", + "evidence_pointer": "evals/obligation_5_perturbation", "obligation_id": "5", "passed": true, "refusal_reason": "", "title": "reasoning-isolation perturbation suite" }, { - "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/obligation_6_depth_curve", + "evidence_pointer": "evals/obligation_6_depth_curve", "obligation_id": "6", "passed": true, "refusal_reason": "", "title": "compositional-depth curve" }, { - "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/math_symbolic_equivalence/v1/frontier", + "evidence_pointer": "evals/math_symbolic_equivalence/v1/frontier", "obligation_id": "7", "passed": true, "refusal_reason": "", "title": "frontier-baseline comparison" }, { - "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/obligation_8_adversarial", + "evidence_pointer": "evals/obligation_8_adversarial", "obligation_id": "8", "passed": true, "refusal_reason": "", "title": "adversarial generation; misparse zero" }, { - "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/math_symbolic_equivalence/v1", + "evidence_pointer": "evals/math_symbolic_equivalence/v1", "obligation_id": "9", "passed": true, "refusal_reason": "", "title": "determinism" }, { - "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/obligation_10_pack_provenance", + "evidence_pointer": "evals/obligation_10_pack_provenance", "obligation_id": "10", "passed": true, "refusal_reason": "", "title": "operation provenance via pack" } ], - "promote_admitted": false, - "refusal_reason": "awaiting reviewer signature \u2014 add an entry to docs/reviewers.yaml under 'math_expert_claims' for domain 'mathematics_logic' with claim_digest=d164866975341d9b82503caf50c0404ee140eab21fd60f589536c6daf6e1d706", - "reviewer_signature": null, - "reviewer_signature_matches": false, + "promote_admitted": true, + "refusal_reason": "", + "reviewer_signature": { + "claim_digest": "94149794e8c19896851e062cf1f921cfa9ba04770b674bc3b4c33023f7c7331b", + "domain_id": "mathematics_logic", + "evidence_lanes": [ + "math_symbolic_equivalence/v1 (public)", + "math_symbolic_equivalence/v1 (sealed)", + "math_teaching_corpus/v1", + "math_bounded_grammar/v1" + ], + "evidence_revision": "adr-0120-math:reviewed:2026-05-23", + "signed_by": "shay-j" + }, + "reviewer_signature_matches": true, "schema_version": 1, "technical_pass": true } diff --git a/tests/test_adr_0120_math_expert_promotion.py b/tests/test_adr_0120_math_expert_promotion.py index 48a84850..3e521aa0 100644 --- a/tests/test_adr_0120_math_expert_promotion.py +++ b/tests/test_adr_0120_math_expert_promotion.py @@ -159,11 +159,11 @@ def test_obligation_9_refuses_when_a_lane_is_invalid_json(tmp_path: Path) -> Non def test_composer_runs_on_current_main_with_all_obligations_passing() -> None: - """The load-bearing snapshot. As of this PR every obligation - auditor reports pass + the composite gate reports pass, so - technical_pass is True. The reviewer signature is intentionally - absent — promote_admitted is False with refusal pointing the - operator at the digest to sign. + """The load-bearing snapshot. Updated by the ledger-flip PR after + the reviewer added the signed entry to ``docs/reviewers.yaml`` — + every obligation auditor passes, the composite gate passes, + technical_pass is True, the reviewer signature is present and + matches the computed digest, so promote_admitted is True. """ v = evaluate_math_expert_promotion() assert v.all_obligations_passed is True, ( @@ -172,10 +172,11 @@ def test_composer_runs_on_current_main_with_all_obligations_passing() -> None: ) assert v.composite_gate_passed is True assert v.technical_pass is True - assert v.reviewer_signature is None - assert v.reviewer_signature_matches is False - assert v.promote_admitted is False - assert "awaiting reviewer signature" in v.refusal_reason + assert v.reviewer_signature is not None + assert v.reviewer_signature.get("signed_by") == "shay-j" + assert v.reviewer_signature_matches is True + assert v.promote_admitted is True + assert v.refusal_reason == "" assert v.claim_digest # non-empty assert len(v.claim_digest) == 64 # SHA-256 hex diff --git a/tests/test_mathlogic_expert_ledger_flip.py b/tests/test_mathlogic_expert_ledger_flip.py new file mode 100644 index 00000000..a5ebb437 --- /dev/null +++ b/tests/test_mathlogic_expert_ledger_flip.py @@ -0,0 +1,192 @@ +"""ADR-0120 math-expert ledger-flip tests. + +Pins: + - reviewers.yaml ``math_expert_claims:`` top-level key is allow-listed + (regression test for the bug PR #194 introduced) + - ``_EXPERT_DOMAIN_STATUSES`` includes ``"expert"`` + - per-domain expert composer wiring (``_EXPERT_COMPOSERS``) + - claim_digest is filesystem-independent (path-stability fix) + - mathematics_logic row reports ``status: expert`` + ``predicates.expert: True`` + given the on-disk evidence + signed reviewer entry + - other domains stay at their existing tier (composer absent → expert=False) +""" + +from __future__ import annotations + +import json +import yaml +from pathlib import Path + +import pytest + +from core.capability.composite_math_gate import _rel as _gate_rel, evaluate_composite_math_gate +from core.capability.expert_promotion_math import ( + DOMAIN_ID, + EXPERT_CLAIMS_KEY, + _rel as _composer_rel, + evaluate_math_expert_promotion, +) +from core.capability.reporting import _EXPERT_COMPOSERS, _EXPERT_DOMAIN_STATUSES, ledger_report +from core.capability.reviewers import ALLOWED_TOP_LEVEL_KEYS, load_reviewer_registry + + +_REPO_ROOT = Path(__file__).resolve().parent.parent + + +# --------------------------------------------------------------------------- +# Allow-list regression (bug introduced in PR #194, fixed here) +# --------------------------------------------------------------------------- + + +def test_allowed_top_level_keys_includes_math_expert_claims() -> None: + """PR #194 added the ``math_expert_claims:`` section to + docs/reviewers.yaml but didn't extend the loader's allow-list, + silently breaking the audit_passed predicate for ALL 3 prior + domains (the loader rejected the whole file).""" + assert "math_expert_claims" in ALLOWED_TOP_LEVEL_KEYS + + +def test_reviewers_yaml_loads_after_fix() -> None: + """The real file should parse without raising — proving the + allow-list regression is fixed end-to-end.""" + registry = load_reviewer_registry(_REPO_ROOT / "docs" / "reviewers.yaml") + # Existing 3 audit-passed entries must still resolve. + assert registry.expert_demo_claim_for("mathematics_logic") is not None + assert registry.expert_demo_claim_for("physics") is not None + assert registry.expert_demo_claim_for("systems_software") is not None + + +# --------------------------------------------------------------------------- +# Expert tier wiring +# --------------------------------------------------------------------------- + + +def test_expert_status_in_status_order() -> None: + assert "expert" in _EXPERT_DOMAIN_STATUSES + # Must come after audit-passed (strict super-tier). + assert _EXPERT_DOMAIN_STATUSES.index("expert") > _EXPERT_DOMAIN_STATUSES.index("audit-passed") + + +def test_mathematics_logic_composer_wired() -> None: + assert _EXPERT_COMPOSERS["mathematics_logic"] == "core.capability.expert_promotion_math" + + +# --------------------------------------------------------------------------- +# Path-stability fix (digest filesystem-independence) +# --------------------------------------------------------------------------- + + +def test_rel_returns_repo_relative_posix() -> None: + """``_rel`` must return repo-relative POSIX, not absolute path, + so the digest is stable across operator filesystems.""" + p = _REPO_ROOT / "evals" / "math_bounded_grammar" / "v1" / "cases.jsonl" + rel = _composer_rel(p) + assert not rel.startswith("/") # not absolute + assert rel == "evals/math_bounded_grammar/v1/cases.jsonl" + # composite_math_gate's _rel must agree + assert _gate_rel(p) == rel + + +def test_rel_falls_back_to_absolute_outside_repo(tmp_path: Path) -> None: + """Paths outside the repo root should fall back to absolute + POSIX (rather than raise).""" + outside = tmp_path / "elsewhere.json" + rel = _composer_rel(outside) + assert rel == outside.as_posix() + + +def test_composer_digest_uses_relative_paths() -> None: + """Sanity: the digest's input bundle should contain repo-relative + paths (not absolute), so it's filesystem-independent.""" + v = evaluate_math_expert_promotion() + for o in v.obligations: + assert not o.evidence_pointer.startswith("/"), ( + f"obligation {o.obligation_id} has absolute path: {o.evidence_pointer}" + ) + + +def test_composite_gate_digest_uses_relative_paths() -> None: + v = evaluate_composite_math_gate() + for b in v.benchmarks: + assert not b.report_path.startswith("/"), ( + f"benchmark {b.benchmark_id} has absolute report_path: {b.report_path}" + ) + + +# --------------------------------------------------------------------------- +# Ledger flip — mathematics_logic must reach "expert" on current main +# --------------------------------------------------------------------------- + + +def _math_row(report: dict) -> dict: + for row in report["domains"]: + if row["domain"] == "mathematics_logic": + return row + raise AssertionError("mathematics_logic row missing from ledger") + + +def test_mathlogic_status_is_expert_with_signed_evidence() -> None: + """The load-bearing snapshot. Given the signed entry in + docs/reviewers.yaml AND every obligation auditor passing AND the + composite gate passing, the ledger reports + ``mathematics_logic.status == "expert"``.""" + report = ledger_report() + row = _math_row(report) + assert row["status"] == "expert", ( + f"expected status=expert; got {row['status']!r}; " + f"audit_passed_reason={row.get('audit_passed_reason')}; " + f"expert_reason={row.get('expert_reason')}" + ) + assert row["predicates"]["expert"] is True + assert row["predicates"]["audit_passed"] is True + + +def test_mathlogic_row_carries_expert_reason() -> None: + report = ledger_report() + row = _math_row(report) + assert "expert_reason" in row + assert "admitted" in row["expert_reason"].lower() + + +def test_other_domains_have_expert_false_no_composer_wired() -> None: + """Domains without an entry in _EXPERT_COMPOSERS keep expert=False + even when audit_passed=True. Currently only mathematics_logic + is wired.""" + report = ledger_report() + for row in report["domains"]: + if row["domain"] == "mathematics_logic": + continue + # Other domains' expert predicate must be False. + assert row["predicates"]["expert"] is False, ( + f"{row['domain']} unexpectedly reports expert=True" + ) + + +# --------------------------------------------------------------------------- +# Failure mode: signature missing → status stays at audit-passed +# --------------------------------------------------------------------------- + + +def test_composer_refuses_without_signature(tmp_path: Path) -> None: + """If the reviewers file omits the math_expert_claims entry, + promote_admitted stays False with the expected awaiting-signature + message.""" + fake = tmp_path / "reviewers.yaml" + fake.write_text(yaml.safe_dump({ + "schema_version": 1, + "reviewers": [ + { + "reviewer_id": "shay-j", + "display_name": "test", + "role": "primary", + "domains": ["*"], + "review_scope": ["pack", "proposal", "chain", "eval"], + "provenance": "test", + } + ], + EXPERT_CLAIMS_KEY: [], + }), encoding="utf-8") + v = evaluate_math_expert_promotion(reviewers_path=fake) + assert v.technical_pass is True # evidence still passes + assert v.promote_admitted is False + assert "awaiting reviewer signature" in v.refusal_reason