diff --git a/.github/workflows/lane-shas.yml b/.github/workflows/lane-shas.yml index 2a7978bf..aa3fb8de 100644 --- a/.github/workflows/lane-shas.yml +++ b/.github/workflows/lane-shas.yml @@ -49,9 +49,9 @@ jobs: - name: verify lane SHAs env: PYTHONPATH: ${{ github.workspace }} - # Align with full-pytest.yml: wall-clock is env-sensitive on cold - # Act runners; content cases remain hard gates in public_demo. - CORE_SHOWCASE_SKIP_BUDGET: "1" + # public_demo wall-clock is soft by default (see evals/public_demo/runner.py). + # Do not set CORE_SHOWCASE_HARD_BUDGET here — cold Act runners exceed 60s. + # Content cases (claims, determinism, pure composition) remain hard gates. run: | uv run python scripts/verify_lane_shas.py diff --git a/CLAIMS.md b/CLAIMS.md index df76bdbf..a11598ad 100644 --- a/CLAIMS.md +++ b/CLAIMS.md @@ -38,8 +38,8 @@ is a CI failure (`.github/workflows/lane-shas.yml`). | ADR-0093 | `domain_contract_validation` | All ratified packs satisfy the 9 ADR-0091 contract predicates | `evals/domain_contract_validation/results/v1_dev.json` | `98ace04e3f02bbc5a8ad655bb6593c3f1ee64cb67014f1122fe6c3c85f48d22f` | | ADR-0095 | `miner_loop_closure` | Miner-sourced proposals route through single reviewed teaching path | `evals/miner_loop_closure/results/v1_dev.json` | `9f071733abe7dcacf759f928548ce738fb639af3fd6e4c621a651b306d7e77ce` | | ADR-0096 | `fabrication_control_summary` | Phantom endpoints / cross-pack non-bridges / sibling collapses refuse | `evals/fabrication_control/results/v1_summary.json` | `01e1b6b711141f2b4a14551d7df3ea482d8d6dd7b364a25c509f4f8d08cda8a8` | -| ADR-0098 | `demo_composition` | Demos compose from shipped modules; no parallel mechanism | `evals/demo_composition/results/v1_dev.json` | `c8bb37e03b85f2558eb31f6caa20d4a1d2e39c4a7c711393aa88e180b1c39a3a` | -| ADR-0099 | `public_demo` | Public showcase runs deterministically under 30s; all claims supported | `evals/public_demo/results/v1_dev.json` | `ed1668a64490f73f4d9b701e611e07841c149fd36cb90703436e3e33732fcd76` | +| ADR-0098 | `demo_composition` | Demos compose from shipped modules; no parallel mechanism | `evals/demo_composition/results/v1_dev.json` | `e2ba2314d8768459fb6a8db082a4bbcf4107b5161d869804a4b2a33c3724081a` | +| ADR-0099 | `public_demo` | Public showcase runs deterministically under 30s; all claims supported | `evals/public_demo/results/v1_dev.json` | `7d8ba0dbae9287cfe0bf15d231fa78a75abc627121c14900439293e01e1cc1d3` | | ADR-0104 | `curriculum_loop_closure` | Curriculum-sourced proposals route through single reviewed teaching path | `evals/curriculum_loop_closure/results/v1_dev.json` | `b46d56b2d209172cc3ffaf3776dc8dcfe55093f13587c5cb67372be6dfa23e8d` | | ADR-0131 | `math_teaching_corpus_v1` | Math teaching corpus replays deterministically; all chains pass exit criterion (correct_rate=1.0, wrong=0) | `evals/math_teaching_corpus/v1/report.json` | `eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4` | | ADR-0206 | `deductive_logic_v1` | Propositional entailment scored against an independent truth-table oracle; dev+holdout+external 716/716 correct, wrong=0, refused=0 | `evals/deductive_logic/report.json` | `97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f` | diff --git a/core/demos/contract.py b/core/demos/contract.py index 2d5cd258..b58de1d8 100644 --- a/core/demos/contract.py +++ b/core/demos/contract.py @@ -164,6 +164,58 @@ _TRACKED_MODULES: tuple[str, ...] = ( ) +# Env keys whose values are host/isolation paths. Mutation is still +# detected (key present/absent/changed), but absolute path *values* +# never enter divergence messages — those would make lane SHA pins +# depend on tempfile locations and break hermetic CI. +_PATH_LIKE_ENV_SUFFIXES: tuple[str, ...] = ("_DIR", "_PATH", "_HOME", "_ROOT") + + +def _is_path_like_env_key(key: str) -> bool: + if key == "CORE_ENGINE_STATE_DIR": + return True + return any(key.endswith(suffix) for suffix in _PATH_LIKE_ENV_SUFFIXES) + + +def _format_env_value(key: str, value: str) -> str: + """Stable, pin-safe rendering of an env value for divergence text.""" + if _is_path_like_env_key(key): + return "" + return value + + +def _env_subset_divergences( + before_env: tuple[tuple[str, str], ...] | tuple[()] | Any, + after_env: tuple[tuple[str, str], ...] | tuple[()] | Any, +) -> tuple[str, ...]: + """Key-level env delta — not a full before/after dump. + + Full tuple dumps embed every ambient ``CORE_*`` value (including + hermetic engine-state temp paths). That is correct for *detection* + when compared as raw snapshots, but wrong for *report text*: the + lane SHA pin must be host-independent. Only keys that actually + changed appear in the message. + """ + before_map = dict(before_env or ()) + after_map = dict(after_env or ()) + messages: list[str] = [] + for key in sorted(set(before_map) | set(after_map)): + b = before_map.get(key) + a = after_map.get(key) + if b == a: + continue + if b is None: + messages.append(f"env_subset: +{key}={_format_env_value(key, a)}") + elif a is None: + messages.append(f"env_subset: -{key}={_format_env_value(key, b)}") + else: + messages.append( + "env_subset: " + f"{key} {_format_env_value(key, b)!r} -> {_format_env_value(key, a)!r}" + ) + return tuple(messages) + + def _global_state_snapshot() -> dict[str, Any]: """Capture a load-bearing subset of process state for diff checking. @@ -206,9 +258,14 @@ def verify_no_global_state_mutation( when the adapter does its own deferred imports. Only id → id rebindings (the module object was replaced) and value-set divergences on env vars are flagged. + + Env divergences are reported as a **key-level delta** (added / + removed / changed keys). Full before/after env dumps are forbidden + in the divergence text: they embed host-volatile values such as + ``CORE_ENGINE_STATE_DIR`` temp paths and make lane SHA pins flaky. """ divergences: list[str] = [] - for key in set(before.keys()) | set(after.keys()): + for key in sorted(set(before.keys()) | set(after.keys())): b = before.get(key) a = after.get(key) if b == a: @@ -217,9 +274,10 @@ def verify_no_global_state_mutation( # Lazy import: a module that wasn't yet loaded is now # loaded. Benign and unavoidable. continue - divergences.append( - f"{key}: before={b!r} after={a!r}" - ) + if key == "env_subset": + divergences.extend(_env_subset_divergences(b, a)) + continue + divergences.append(f"{key}: before={b!r} after={a!r}") return (not divergences, tuple(divergences)) diff --git a/evals/demo_composition/results/v1_dev.json b/evals/demo_composition/results/v1_dev.json index accf90d0..dc7bec99 100644 --- a/evals/demo_composition/results/v1_dev.json +++ b/evals/demo_composition/results/v1_dev.json @@ -77,7 +77,7 @@ "case_id": "stateful_fixture_rejected", "details": { "divergences": [ - "env_subset: before=() after=(('CORE_STATEFUL_FIXTURE_FLAG', '1'),)" + "env_subset: +CORE_STATEFUL_FIXTURE_FLAG=1" ] }, "divergence": null, diff --git a/evals/demo_composition/runner.py b/evals/demo_composition/runner.py index 93f8755e..a7bc8d4a 100644 --- a/evals/demo_composition/runner.py +++ b/evals/demo_composition/runner.py @@ -170,6 +170,16 @@ def _run_stateful_fixture_rejected(tmp_root: Path) -> dict[str, Any]: def run() -> dict[str, Any]: + import os + + # Hermetic engine state (same pattern as public_demo): lived engine_state/ + # on the developer's machine or a dirty CI workspace must not leak into + # adapter JSON hashes and break the lane pin. + os.environ.setdefault( + "CORE_ENGINE_STATE_DIR", + tempfile.mkdtemp(prefix="demo_composition_engine_state_"), + ) + tmp_root = Path(tempfile.mkdtemp(prefix="demo_composition_lane_")) try: cases: list[dict[str, Any]] = [] diff --git a/evals/public_demo/results/v1_dev.json b/evals/public_demo/results/v1_dev.json index 32b96be5..40401e5e 100644 --- a/evals/public_demo/results/v1_dev.json +++ b/evals/public_demo/results/v1_dev.json @@ -13,7 +13,7 @@ { "case_id": "determinism_run_to_run_byte_equality", "details": { - "sha256": "d4e5a840a03a57dac9d12b9f00b36928271cf76f59b134ec5847318048431e06" + "sha256": "7f03086d43869979b1bebd15f4f58a8b3887c907f7fbe0e7d4b8cad33b7507f2" }, "divergence": null, "passed": true diff --git a/scripts/verify_lane_shas.py b/scripts/verify_lane_shas.py index ac4483ff..be02ce4f 100644 --- a/scripts/verify_lane_shas.py +++ b/scripts/verify_lane_shas.py @@ -38,8 +38,8 @@ PINNED_SHAS: dict[str, str] = { "curriculum_loop_closure": "b46d56b2d209172cc3ffaf3776dc8dcfe55093f13587c5cb67372be6dfa23e8d", "domain_contract_validation": "98ace04e3f02bbc5a8ad655bb6593c3f1ee64cb67014f1122fe6c3c85f48d22f", "fabrication_control_summary": "01e1b6b711141f2b4a14551d7df3ea482d8d6dd7b364a25c509f4f8d08cda8a8", - "demo_composition": "c8bb37e03b85f2558eb31f6caa20d4a1d2e39c4a7c711393aa88e180b1c39a3a", - "public_demo": "ed1668a64490f73f4d9b701e611e07841c149fd36cb90703436e3e33732fcd76", + "demo_composition": "e2ba2314d8768459fb6a8db082a4bbcf4107b5161d869804a4b2a33c3724081a", + "public_demo": "7d8ba0dbae9287cfe0bf15d231fa78a75abc627121c14900439293e01e1cc1d3", "math_teaching_corpus_v1": "eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4", "deductive_logic_v1": "97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f", } @@ -128,6 +128,12 @@ def _invoke_runner(spec: LaneSpec, *, target_path: Path | None = None) -> Path: import os env = {"PYTHONPATH": str(REPO_ROOT), **os.environ} + # Force hermetic engine_state for every lane so local lived state and CI + # workspaces cannot change demo / showcase report bytes. + env.setdefault( + "CORE_ENGINE_STATE_DIR", + tempfile.mkdtemp(prefix=f"lane_{spec.lane_id}_engine_"), + ) if spec.run_as_module: args = [sys.executable, "-m", spec.runner_dotted] else: diff --git a/tests/test_demo_composition.py b/tests/test_demo_composition.py index 9072a34d..9428b1b0 100644 --- a/tests/test_demo_composition.py +++ b/tests/test_demo_composition.py @@ -174,6 +174,47 @@ class TestGlobalStateDetector: passed, divergences = verify_no_global_state_mutation(before=before, after=after) assert passed is False assert any("env_subset" in d for d in divergences) + # Key-level delta only — never a full env dump (host paths break pins). + assert any( + d == "env_subset: +CORE_DETECTOR_TEST_FLAG=1" for d in divergences + ) + assert not any("before=" in d for d in divergences) + + def test_env_delta_stable_across_hermetic_engine_state_paths(self) -> None: + """Lane pins must not depend on CORE_ENGINE_STATE_DIR temp paths.""" + before_a = { + "env_subset": (("CORE_ENGINE_STATE_DIR", "/tmp/run_a_path"),), + } + after_a = { + "env_subset": ( + ("CORE_ENGINE_STATE_DIR", "/tmp/run_a_path"), + ("CORE_STATEFUL_FIXTURE_FLAG", "1"), + ), + } + before_b = { + "env_subset": (("CORE_ENGINE_STATE_DIR", "/var/folders/xyz/run_b"),), + } + after_b = { + "env_subset": ( + ("CORE_ENGINE_STATE_DIR", "/var/folders/xyz/run_b"), + ("CORE_STATEFUL_FIXTURE_FLAG", "1"), + ), + } + passed_a, div_a = verify_no_global_state_mutation(before=before_a, after=after_a) + passed_b, div_b = verify_no_global_state_mutation(before=before_b, after=after_b) + assert passed_a is False and passed_b is False + assert div_a == div_b + assert div_a == ("env_subset: +CORE_STATEFUL_FIXTURE_FLAG=1",) + + def test_path_like_env_change_detected_without_absolute_path(self) -> None: + before = {"env_subset": (("CORE_ENGINE_STATE_DIR", "/tmp/old"),)} + after = {"env_subset": (("CORE_ENGINE_STATE_DIR", "/tmp/new"),)} + passed, divergences = verify_no_global_state_mutation(before=before, after=after) + assert passed is False + assert divergences == ( + "env_subset: CORE_ENGINE_STATE_DIR '' -> ''", + ) + assert "/tmp/" not in "".join(divergences) def test_lazy_import_not_flagged(self) -> None: """None → module id transition is benign (lazy import)."""