From 60da4f0cd0dcdbcd58ef9a05d37d4d5222599924 Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 21 May 2026 21:02:36 -0700 Subject: [PATCH] feat(claims): auto-generate CLAIMS.md from ledger + pinned lane SHAs CLAIMS.md is now mechanically derived from two ground-truth sources: - core.capability.ledger_report (Tier 1: ratified domains) - scripts/verify_lane_shas.PINNED_SHAS (Tier 2: pinned lane reports) The generator is deterministic and gated by tests/test_claims_md_is_current.py + the lane-shas CI workflow's new 'verify CLAIMS.md is current' step. Drift between in-tree state and the published claims fails CI before merge. Tier 1 (5 ratified domains) and Tier 2 (6 pinned lanes) cover every ADR-0092..0102 invariant currently CI-pinned. --- .github/workflows/lane-shas.yml | 6 + CLAIMS.md | 49 ++++++ scripts/generate_claims.py | 264 +++++++++++++++++++++++++++++ tests/test_claims_md_is_current.py | 38 +++++ 4 files changed, 357 insertions(+) create mode 100644 CLAIMS.md create mode 100644 scripts/generate_claims.py create mode 100644 tests/test_claims_md_is_current.py diff --git a/.github/workflows/lane-shas.yml b/.github/workflows/lane-shas.yml index 226764b4..077fce74 100644 --- a/.github/workflows/lane-shas.yml +++ b/.github/workflows/lane-shas.yml @@ -50,6 +50,12 @@ jobs: run: | python scripts/verify_lane_shas.py + - name: verify CLAIMS.md is current + env: + PYTHONPATH: ${{ github.workspace }} + run: | + python scripts/generate_claims.py --check + - name: emit machine-readable report (on failure) if: failure() env: diff --git a/CLAIMS.md b/CLAIMS.md new file mode 100644 index 00000000..d5fbee0d --- /dev/null +++ b/CLAIMS.md @@ -0,0 +1,49 @@ +# CLAIMS + + + +Every row below is mechanically derived from in-tree state and +verified by CI. Tier 1 rows come from `core.capability.ledger_report`; +Tier 2 rows come from `scripts/verify_lane_shas.py`'s pinned SHAs. + +## Tier 1 — Ratified domains + +Each row asserts: the domain's packs pass all nine ADR-0091 contract +predicates, declared operator chains meet the ≥8 / ≥3-intent floor, +the reviewer registry resolves the primary reviewer, and the ledger +status predicate evaluates to `reasoning-capable` with no open gaps. + +| Domain | Status | ADR | Packs | Open gaps | +| --- | --- | --- | --- | --- | +| `philosophy_theology` | reasoning-capable | ADR-0085 | 2 | 0 | +| `mathematics_logic` | reasoning-capable | ADR-0097 | 1 | 0 | +| `physics` | reasoning-capable | ADR-0100 | 1 | 0 | +| `systems_software` | reasoning-capable | ADR-0101 | 1 | 0 | +| `hebrew_greek_textual_reasoning` | reasoning-capable | ADR-0102 | 4 | 0 | + +## Tier 2 — Pinned lane reports + +Each row asserts: running the lane's runner produces a JSON +report whose SHA-256 matches the pinned value below. Mismatch +is a CI failure (`.github/workflows/lane-shas.yml`). + +| ADR | Lane | Purpose | Report path | Pinned SHA-256 | +| --- | --- | --- | --- | --- | +| ADR-0092 | `reviewer_registry` | Reviewer registry schema validates + bootstrap entry self-seals | `evals/reviewer_registry/results/v1_dev.json` | `681a2aab5aa4ffd58cd837ce5673c8b2a9545b570117aec3c02726a12f6876e6` | +| ADR-0093 | `domain_contract_validation` | All ratified packs satisfy the 9 ADR-0091 contract predicates | `evals/domain_contract_validation/results/v1_dev.json` | `f9c06cdeea8fb36a0d3c320007618c3afc92d67702ef31bd36ebd9ae9ced473f` | +| ADR-0095 | `miner_loop_closure` | Miner-sourced proposals route through single reviewed teaching path | `evals/miner_loop_closure/results/v1_dev.json` | `9f071733abe7dcacf759f928548ce738fb639af3fd6e4c621a651b306d7e77ce` | +| ADR-0096 | `fabrication_control_summary` | Phantom endpoints / cross-pack non-bridges / sibling collapses refuse | `evals/fabrication_control/results/v1_summary.json` | `01e1b6b711141f2b4a14551d7df3ea482d8d6dd7b364a25c509f4f8d08cda8a8` | +| ADR-0098 | `demo_composition` | Demos compose from shipped modules; no parallel mechanism | `evals/demo_composition/results/v1_dev.json` | `27d838241bf3ed9e15d0e918ec6d89a823494d7e17c2dab9777825af7188f20f` | +| ADR-0099 | `public_demo` | Public showcase runs deterministically under 30s; all claims supported | `evals/public_demo/results/v1_dev.json` | `4be6f47509435a24984713acfcebd88e61f4e1278096fa5dc88a09e8af2f87ba` | + +## Verification + +```bash +python3 scripts/verify_lane_shas.py # verify all Tier 2 SHAs +core test --suite full -q # exercises Tier 1 invariants +``` diff --git a/scripts/generate_claims.py b/scripts/generate_claims.py new file mode 100644 index 00000000..a3aec135 --- /dev/null +++ b/scripts/generate_claims.py @@ -0,0 +1,264 @@ +"""Generate CLAIMS.md from the capability ledger and pinned lane SHAs. + +CLAIMS.md is a single-page, auditable list of every CI-pinned +capability claim CORE currently makes. It is **mechanically derived** +from two ground-truth sources: + +- ``core.capability.ledger_report()`` — domain ratification rows + (Tier 1: "this domain is reasoning-capable"). +- ``scripts.verify_lane_shas.PINNED_SHAS`` + ``LANE_SPECS`` — + per-lane pinned report SHA-256 (Tier 2: "this lane's report bytes + hash to X under deterministic replay"). + +The generator is **deterministic**: same inputs → byte-identical +output. ``tests/test_claims_md_is_current.py`` regenerates the file +into a temp path and asserts byte equality with the on-disk copy. +""" + +from __future__ import annotations + +import argparse +import hashlib +import sys +from dataclasses import dataclass +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +CLAIMS_PATH = REPO_ROOT / "CLAIMS.md" + +# Import via path so this script remains runnable without installing the +# package. +sys.path.insert(0, str(REPO_ROOT)) + +from core.capability import ledger_report # noqa: E402 +from scripts.verify_lane_shas import LANE_SPECS, PINNED_SHAS # noqa: E402 # type: ignore[import-not-found] + + +# ADR → human-readable lane purpose. Keep this in lockstep with the +# ADR docs themselves; the generator fails fast if any lane lacks a +# mapping here, which surfaces drift before CI does. +_LANE_ADR: dict[str, tuple[str, str]] = { + "reviewer_registry": ( + "ADR-0092", + "Reviewer registry schema validates + bootstrap entry self-seals", + ), + "miner_loop_closure": ( + "ADR-0095", + "Miner-sourced proposals route through single reviewed teaching path", + ), + "domain_contract_validation": ( + "ADR-0093", + "All ratified packs satisfy the 9 ADR-0091 contract predicates", + ), + "fabrication_control_summary": ( + "ADR-0096", + "Phantom endpoints / cross-pack non-bridges / sibling collapses refuse", + ), + "demo_composition": ( + "ADR-0098", + "Demos compose from shipped modules; no parallel mechanism", + ), + "public_demo": ( + "ADR-0099", + "Public showcase runs deterministically under 30s; all claims supported", + ), +} + + +# Domain → ratifying ADR. The ledger row already carries provenance +# inside each contract (e.g. "adr-0101:reviewed:2026-05-21"), but we +# pin the canonical ADR id here so the table reads cleanly even if a +# pack later carries a different provenance prefix. +_DOMAIN_ADR: dict[str, str] = { + "systems_software": "ADR-0101", + "mathematics_logic": "ADR-0097", + "physics": "ADR-0100", + "hebrew_greek_textual_reasoning": "ADR-0102", + "philosophy_theology": "ADR-0085", +} + + +@dataclass(frozen=True, slots=True) +class DomainRow: + domain: str + status: str + adr: str + pack_count: int + open_gaps_count: int + + +@dataclass(frozen=True, slots=True) +class LaneRow: + lane_id: str + adr: str + purpose: str + report_path: str + pinned_sha: str + + +def _collect_domain_rows() -> list[DomainRow]: + report = ledger_report() + rows: list[DomainRow] = [] + for entry in report["domains"]: + domain = entry["domain"] + if domain not in _DOMAIN_ADR: + raise RuntimeError( + f"domain {domain!r} missing from _DOMAIN_ADR mapping in " + f"scripts/generate_claims.py — add its ratifying ADR id." + ) + rows.append( + DomainRow( + domain=domain, + status=entry["status"], + adr=_DOMAIN_ADR[domain], + pack_count=len(entry["packs"]), + open_gaps_count=len(entry["open_gaps"]), + ) + ) + # Deterministic ordering: by ADR id (lexicographic), then domain. + return sorted(rows, key=lambda r: (r.adr, r.domain)) + + +def _collect_lane_rows() -> list[LaneRow]: + rows: list[LaneRow] = [] + for spec in LANE_SPECS: + if spec.lane_id not in _LANE_ADR: + raise RuntimeError( + f"lane {spec.lane_id!r} missing from _LANE_ADR mapping in " + f"scripts/generate_claims.py — add its ADR id and purpose." + ) + adr, purpose = _LANE_ADR[spec.lane_id] + pinned = PINNED_SHAS.get(spec.lane_id) + if not pinned: + raise RuntimeError( + f"lane {spec.lane_id!r} has no entry in PINNED_SHAS — " + f"run scripts/verify_lane_shas.py --update first." + ) + rows.append( + LaneRow( + lane_id=spec.lane_id, + adr=adr, + purpose=purpose, + report_path=spec.report_relative, + pinned_sha=pinned, + ) + ) + return sorted(rows, key=lambda r: r.adr) + + +def _render( + *, domain_rows: list[DomainRow], lane_rows: list[LaneRow] +) -> str: + """Render CLAIMS.md as a deterministic markdown string. + + Two tables. No timestamps, no git revs, no wall-clock — anything + volatile would break the "regenerate and compare bytes" test. + """ + lines: list[str] = [ + "# CLAIMS", + "", + "", + "", + "Every row below is mechanically derived from in-tree state and", + "verified by CI. Tier 1 rows come from `core.capability.ledger_report`;", + "Tier 2 rows come from `scripts/verify_lane_shas.py`'s pinned SHAs.", + "", + "## Tier 1 — Ratified domains", + "", + "Each row asserts: the domain's packs pass all nine ADR-0091 contract", + "predicates, declared operator chains meet the ≥8 / ≥3-intent floor,", + "the reviewer registry resolves the primary reviewer, and the ledger", + "status predicate evaluates to `reasoning-capable` with no open gaps.", + "", + "| Domain | Status | ADR | Packs | Open gaps |", + "| --- | --- | --- | --- | --- |", + ] + for row in domain_rows: + lines.append( + f"| `{row.domain}` | {row.status} | {row.adr} | {row.pack_count} | " + f"{row.open_gaps_count} |" + ) + lines.extend( + [ + "", + "## Tier 2 — Pinned lane reports", + "", + "Each row asserts: running the lane's runner produces a JSON", + "report whose SHA-256 matches the pinned value below. Mismatch", + "is a CI failure (`.github/workflows/lane-shas.yml`).", + "", + "| ADR | Lane | Purpose | Report path | Pinned SHA-256 |", + "| --- | --- | --- | --- | --- |", + ] + ) + for row in lane_rows: + lines.append( + f"| {row.adr} | `{row.lane_id}` | {row.purpose} | " + f"`{row.report_path}` | `{row.pinned_sha}` |" + ) + lines.extend( + [ + "", + "## Verification", + "", + "```bash", + "python3 scripts/verify_lane_shas.py # verify all Tier 2 SHAs", + "core test --suite full -q # exercises Tier 1 invariants", + "```", + "", + ] + ) + return "\n".join(lines) + + +def render_claims() -> bytes: + domain_rows = _collect_domain_rows() + lane_rows = _collect_lane_rows() + text = _render(domain_rows=domain_rows, lane_rows=lane_rows) + return text.encode("utf-8") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="generate CLAIMS.md") + parser.add_argument( + "--check", + action="store_true", + help="exit non-zero if CLAIMS.md differs from regenerated bytes", + ) + parser.add_argument( + "--output", + type=Path, + default=CLAIMS_PATH, + help="write to this path (default: repo-root CLAIMS.md)", + ) + args = parser.parse_args(argv) + + payload = render_claims() + sha = hashlib.sha256(payload).hexdigest() + + if args.check: + if not CLAIMS_PATH.exists(): + print(f"FAIL: {CLAIMS_PATH} does not exist; run without --check") + return 1 + on_disk = CLAIMS_PATH.read_bytes() + if on_disk != payload: + print("FAIL: CLAIMS.md is stale — regenerate with:") + print(" python3 scripts/generate_claims.py") + print(f" on-disk sha256: {hashlib.sha256(on_disk).hexdigest()}") + print(f" computed sha256: {sha}") + return 1 + print(f"OK: CLAIMS.md is current (sha256: {sha})") + return 0 + + args.output.write_bytes(payload) + print(f"wrote {args.output} (sha256: {sha})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_claims_md_is_current.py b/tests/test_claims_md_is_current.py new file mode 100644 index 00000000..825d3123 --- /dev/null +++ b/tests/test_claims_md_is_current.py @@ -0,0 +1,38 @@ +"""CLAIMS.md must be deterministically regenerable from in-tree state. + +The generator (``scripts/generate_claims.py``) reads only the capability +ledger and ``PINNED_SHAS`` — both already gated by other tests. This +test asserts the on-disk CLAIMS.md matches the freshly-rendered bytes, +so any drift between the file and its sources fails CI before merge. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +from scripts.generate_claims import CLAIMS_PATH, render_claims + + +def test_claims_md_matches_generator_output() -> None: + rendered = render_claims() + on_disk = CLAIMS_PATH.read_bytes() + if rendered != on_disk: + raise AssertionError( + "CLAIMS.md is stale. Regenerate with:\n" + " python3 scripts/generate_claims.py\n" + f" on-disk sha256: {hashlib.sha256(on_disk).hexdigest()}\n" + f" computed sha256: {hashlib.sha256(rendered).hexdigest()}" + ) + + +def test_claims_md_render_is_deterministic() -> None: + """Two render() calls in the same process must be byte-identical.""" + a = render_claims() + b = render_claims() + assert a == b + + +def test_claims_md_path_is_at_repo_root() -> None: + assert CLAIMS_PATH.name == "CLAIMS.md" + assert CLAIMS_PATH.parent == Path(__file__).resolve().parent.parent