diff --git a/.github/workflows/lane-shas.yml b/.github/workflows/lane-shas.yml new file mode 100644 index 00000000..226764b4 --- /dev/null +++ b/.github/workflows/lane-shas.yml @@ -0,0 +1,58 @@ +name: lane-shas + +# Verify that every ADR-0092..0099 lane produces its pinned SHA-256 +# report. A failing job means a lane's deterministic output changed +# without an explicit ADR-tracked pin update via: +# +# python scripts/verify_lane_shas.py --update +# +# Single source of truth for the pinned values is scripts/verify_lane_shas.py. + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: lane-shas-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + name: verify pinned lane SHAs + runs-on: ubuntu-latest + timeout-minutes: 12 + + steps: + - name: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: set up python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . pyyaml pytest + + - name: verify lane SHAs + env: + PYTHONPATH: ${{ github.workspace }} + run: | + python scripts/verify_lane_shas.py + + - name: emit machine-readable report (on failure) + if: failure() + env: + PYTHONPATH: ${{ github.workspace }} + run: | + python scripts/verify_lane_shas.py --json || true diff --git a/evals/public_demo/results/v1_dev.json b/evals/public_demo/results/v1_dev.json index 0d94e08d..9a75e69d 100644 --- a/evals/public_demo/results/v1_dev.json +++ b/evals/public_demo/results/v1_dev.json @@ -13,7 +13,7 @@ { "case_id": "determinism_run_to_run_byte_equality", "details": { - "sha256": "141d5f919c0bf7c47dd19b913cb9082f740e5b9f10e5d4b0d8bec39982687f5a" + "sha256": "a8a24def3636ecf34e3ade88fde932ad7dce9469862a9f927303dd1677fd1115" }, "divergence": null, "passed": true diff --git a/scripts/verify_lane_shas.py b/scripts/verify_lane_shas.py new file mode 100644 index 00000000..cf7a7841 --- /dev/null +++ b/scripts/verify_lane_shas.py @@ -0,0 +1,294 @@ +"""Verify ADR-0092..0099 lane SHA-256 pins. + +Each ADR lane writes a deterministic JSON report. This script runs +every pinned lane and asserts the SHA-256 of the report bytes matches +the value pinned below. Pinned SHAs come from the commits that landed +each ADR: + +- ADR-0092 reviewer_registry afdd2ee +- ADR-0095 miner_loop_closure 7dc7e9d +- ADR-0093 domain_contract_validation 7784c39 +- ADR-0096 fabrication_control d7713b0 +- ADR-0098 demo_composition 4f640af +- ADR-0099 public_demo bfb54fb + +Update the pins with ``--update`` when an ADR-tracked change to the +lane is intentional. The diff between the in-tree pin and the freshly +computed SHA is the audit trail. + +Usage: + + python scripts/verify_lane_shas.py # verify, exit non-zero on mismatch + python scripts/verify_lane_shas.py --update # rewrite the pin block in this file + python scripts/verify_lane_shas.py --json # machine-readable report +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +# --------------------------------------------------------------------------- +# Pinned SHA-256 of each lane's canonical report bytes. +# Generated by running the lane's runner module; update with --update. +# --------------------------------------------------------------------------- +PINNED_SHAS: dict[str, str] = { + "reviewer_registry": "681a2aab5aa4ffd58cd837ce5673c8b2a9545b570117aec3c02726a12f6876e6", + "miner_loop_closure": "9f071733abe7dcacf759f928548ce738fb639af3fd6e4c621a651b306d7e77ce", + "domain_contract_validation": "f9c06cdeea8fb36a0d3c320007618c3afc92d67702ef31bd36ebd9ae9ced473f", + "fabrication_control_summary": "01e1b6b711141f2b4a14551d7df3ea482d8d6dd7b364a25c509f4f8d08cda8a8", + "demo_composition": "27d838241bf3ed9e15d0e918ec6d89a823494d7e17c2dab9777825af7188f20f", + "public_demo": "21751aaf2b60897ec7ca9b6498219e3c5eb28aec6b17eb77369ed139db093b29", +} + + +@dataclass(frozen=True, slots=True) +class LaneSpec: + lane_id: str + runner_module: str + report_relative: str + accepts_report_flag: bool = True + extra_args: tuple[str, ...] = field(default_factory=tuple) + + @property + def runner_path(self) -> Path: + return REPO_ROOT / self.runner_module + + @property + def canonical_report(self) -> Path: + return REPO_ROOT / self.report_relative + + +LANE_SPECS: tuple[LaneSpec, ...] = ( + LaneSpec( + lane_id="reviewer_registry", + runner_module="evals/reviewer_registry/runner.py", + report_relative="evals/reviewer_registry/results/v1_dev.json", + ), + LaneSpec( + lane_id="miner_loop_closure", + runner_module="evals/miner_loop_closure/runner.py", + report_relative="evals/miner_loop_closure/results/v1_dev.json", + ), + LaneSpec( + lane_id="domain_contract_validation", + runner_module="evals/domain_contract_validation/runner.py", + report_relative="evals/domain_contract_validation/results/v1_dev.json", + ), + LaneSpec( + # fabrication_control runner emits per-split + summary reports + # into its own results/ directory; ``--lane-dir`` is the + # accepted redirect, not ``--report``. We inspect the + # ``v1_summary.json`` written by that runner. + lane_id="fabrication_control_summary", + runner_module="evals/fabrication_control/runner.py", + report_relative="evals/fabrication_control/results/v1_summary.json", + accepts_report_flag=False, + ), + LaneSpec( + lane_id="demo_composition", + runner_module="evals/demo_composition/runner.py", + report_relative="evals/demo_composition/results/v1_dev.json", + ), + LaneSpec( + lane_id="public_demo", + runner_module="evals/public_demo/runner.py", + report_relative="evals/public_demo/results/v1_dev.json", + ), +) + + +def _invoke_runner(spec: LaneSpec, *, target_path: Path | None = None) -> Path: + """Execute a lane runner as a subprocess and return the report path. + + Subprocess isolation guarantees each lane runs in a clean Python + state — relevant for adapters that mutate global runtime config or + cache pack loads at module import time. + """ + import os + + env = {"PYTHONPATH": str(REPO_ROOT), **os.environ} + args: list[str] = [sys.executable, str(spec.runner_path)] + if target_path is not None and spec.accepts_report_flag: + args.extend(["--report", str(target_path)]) + args.extend(spec.extra_args) + result = subprocess.run( + args, + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=300, + ) + if result.returncode != 0: + raise RuntimeError( + f"lane runner {spec.lane_id} exited non-zero " + f"(code={result.returncode})\nSTDOUT:\n{result.stdout}\n" + f"STDERR:\n{result.stderr}" + ) + # If the runner accepts --report, the report lives at target_path; + # otherwise the runner wrote to its canonical in-tree location. + if target_path is not None and spec.accepts_report_flag: + report_path = target_path + else: + report_path = spec.canonical_report + if not report_path.exists(): + raise RuntimeError( + f"lane {spec.lane_id} runner returned 0 but report not found at {report_path}" + ) + return report_path + + +def _sha_of(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +@dataclass(frozen=True, slots=True) +class LaneVerification: + lane_id: str + pinned_sha: str + actual_sha: str + matched: bool + report_path: str + error: str | None = None + + def as_dict(self) -> dict[str, Any]: + return { + "lane_id": self.lane_id, + "pinned_sha": self.pinned_sha, + "actual_sha": self.actual_sha, + "matched": self.matched, + "report_path": self.report_path, + "error": self.error, + } + + +def verify_all(*, ephemeral: bool = True) -> list[LaneVerification]: + """Run every pinned lane and verify its SHA matches the pin. + + ``ephemeral`` writes the report to a temp directory so the in-tree + artifacts are not perturbed during verification. Disable to rewrite + the canonical results path (used by ``--update``). + """ + results: list[LaneVerification] = [] + for spec in LANE_SPECS: + pinned = PINNED_SHAS.get(spec.lane_id, "") + try: + if ephemeral: + with tempfile.TemporaryDirectory(prefix=f"lane_{spec.lane_id}_") as d: + target = Path(d) / "report.json" + report_path = _invoke_runner(spec, target_path=target) + actual = _sha_of(report_path) + else: + report_path = _invoke_runner(spec) + actual = _sha_of(report_path) + except Exception as exc: # pylint: disable=broad-except + results.append( + LaneVerification( + lane_id=spec.lane_id, + pinned_sha=pinned, + actual_sha="", + matched=False, + report_path=str(spec.canonical_report), + error=f"{type(exc).__name__}: {exc}", + ) + ) + continue + results.append( + LaneVerification( + lane_id=spec.lane_id, + pinned_sha=pinned, + actual_sha=actual, + matched=(actual == pinned), + report_path=str(report_path), + ) + ) + return results + + +_PIN_BLOCK_START = "PINNED_SHAS: dict[str, str] = {" +_PIN_BLOCK_END = "}" + + +def _rewrite_pins(new_pins: dict[str, str]) -> None: + """Rewrite the PINNED_SHAS block in this file with new values.""" + text = Path(__file__).read_text(encoding="utf-8") + start = text.index(_PIN_BLOCK_START) + # Find the *next* closing brace after start. + rel_end = text[start:].index(_PIN_BLOCK_END) + end = start + rel_end + 1 + new_block_lines = [_PIN_BLOCK_START] + for lane_id, sha in new_pins.items(): + new_block_lines.append(f' "{lane_id}": "{sha}",') + new_block_lines.append("}") + new_block = "\n".join(new_block_lines) + Path(__file__).write_text(text[:start] + new_block + text[end:], encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="verify ADR lane SHAs") + parser.add_argument( + "--update", + action="store_true", + help="rewrite PINNED_SHAS in this file with freshly-computed values", + ) + parser.add_argument( + "--json", + action="store_true", + help="emit machine-readable JSON report", + ) + args = parser.parse_args(argv) + + if args.update: + results = verify_all(ephemeral=False) + new_pins = {r.lane_id: r.actual_sha for r in results if not r.error} + _rewrite_pins(new_pins) + if args.json: + print(json.dumps({"updated": new_pins}, indent=2, sort_keys=True)) + else: + print("Updated PINNED_SHAS:") + for lane_id, sha in new_pins.items(): + print(f" {lane_id:>32}: {sha}") + return 0 + + results = verify_all() + if args.json: + payload = { + "total": len(results), + "matched": sum(1 for r in results if r.matched), + "mismatched": [r.as_dict() for r in results if not r.matched], + "results": [r.as_dict() for r in results], + } + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + width = max(len(r.lane_id) for r in results) + for r in results: + mark = "✓" if r.matched else "✗" + print(f" {mark} {r.lane_id:<{width}} {r.actual_sha[:16]}..", end="") + if not r.matched: + if r.error: + print(f" ERROR: {r.error}") + else: + print(f" expected {r.pinned_sha[:16]}..") + else: + print() + total = len(results) + matched = sum(1 for r in results if r.matched) + print(f"\nlanes: {matched}/{total} match pinned SHAs") + + return 0 if all(r.matched for r in results) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_lane_sha_verifier.py b/tests/test_lane_sha_verifier.py new file mode 100644 index 00000000..af48bade --- /dev/null +++ b/tests/test_lane_sha_verifier.py @@ -0,0 +1,103 @@ +"""Pin scripts/verify_lane_shas.py's pin-block shape. + +The full verify-all suite (which re-runs every lane) is expensive +(~30s). It is exercised in CI by the lane-shas workflow. Locally +``pytest`` covers two cheaper guarantees: + +1. Every shipped ADR lane has a pin in ``PINNED_SHAS``. +2. Every ``LaneSpec.runner_module`` actually exists on disk and is + marked as a Python file. + +These two together mean no lane silently drops out of CI coverage. +""" + +from __future__ import annotations + +import importlib.util +import re +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _load_verifier_module(): + """Load scripts/verify_lane_shas.py as a module. + + The scripts directory is not on the default import path; load + directly so the pins/specs remain authoritative without forcing a + package reshuffle. + """ + path = REPO_ROOT / "scripts" / "verify_lane_shas.py" + spec = importlib.util.spec_from_file_location("_verify_lane_shas", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["_verify_lane_shas"] = module + spec.loader.exec_module(module) + return module + + +verifier = _load_verifier_module() + + +class TestPinBlockShape: + def test_every_lane_spec_has_a_pin(self) -> None: + lane_ids = {spec.lane_id for spec in verifier.LANE_SPECS} + pinned = set(verifier.PINNED_SHAS.keys()) + missing = lane_ids - pinned + assert not missing, f"lanes without pinned SHA: {missing}" + + def test_no_orphan_pins(self) -> None: + """A pin without a matching LaneSpec is dead code.""" + lane_ids = {spec.lane_id for spec in verifier.LANE_SPECS} + pinned = set(verifier.PINNED_SHAS.keys()) + orphans = pinned - lane_ids + assert not orphans, f"orphan pins (no matching LaneSpec): {orphans}" + + def test_every_pin_is_64_hex_chars(self) -> None: + for lane_id, sha in verifier.PINNED_SHAS.items(): + assert re.fullmatch(r"[0-9a-f]{64}", sha), ( + f"pin for {lane_id!r} is not a 64-char hex SHA-256: {sha!r}" + ) + + def test_every_lane_spec_runner_exists(self) -> None: + for spec in verifier.LANE_SPECS: + assert spec.runner_path.exists(), ( + f"lane {spec.lane_id!r} runner not found at {spec.runner_path}" + ) + assert spec.runner_path.suffix == ".py" + + def test_every_lane_spec_canonical_report_path_under_repo(self) -> None: + for spec in verifier.LANE_SPECS: + assert spec.canonical_report.is_relative_to(REPO_ROOT) + + +class TestExpectedLaneCoverage: + """The verifier MUST cover all six ADR-0092..0099 lanes. + + Hard-code the canonical lane ids so silently dropping any one + fails this test. ADR-0094 and ADR-0097 are schema/ratification + only — no eval lane — and intentionally absent. + """ + + EXPECTED_LANES = frozenset( + { + "reviewer_registry", # ADR-0092 + "miner_loop_closure", # ADR-0095 + "domain_contract_validation", # ADR-0093 + "fabrication_control_summary", # ADR-0096 + "demo_composition", # ADR-0098 + "public_demo", # ADR-0099 + } + ) + + def test_all_expected_lanes_covered(self) -> None: + actual = {spec.lane_id for spec in verifier.LANE_SPECS} + missing = self.EXPECTED_LANES - actual + extra = actual - self.EXPECTED_LANES + assert not missing, f"missing expected lanes: {missing}" + assert not extra, ( + f"unexpected extra lanes: {extra} (add to EXPECTED_LANES in test " + "if intentional)" + )