fix(ci): path-stable env deltas so lane SHA pins stop thrashing
All checks were successful
smoke / smoke (-m "not quarantine") (pull_request) Successful in 14m5s
lane-shas / verify pinned lane SHAs (pull_request) Successful in 25m23s

Root cause of demo_composition pin thrash was not cancelled CI runs:
verify_no_global_state_mutation dumped full env_subset before/after
tuples into the lane report, embedding CORE_ENGINE_STATE_DIR temp
paths (and any ambient CORE_* host values). Each hermetic run then
produced a different report SHA.

Fix for good:
- Report key-level env deltas only (+/- / changed), with path-like
  CORE_* values redacted to <path> in the message text.
- Detection still uses raw snapshots (mutations are not ignored).
- Hermetic CORE_ENGINE_STATE_DIR for demo_composition and all
  verify_lane_shas invocations.
- Dual-run-stable re-pin: demo_composition e2ba2314…,
  public_demo 7d8ba0db… (matches CI-observed soft-budget SHA).
- Drop dead CORE_SHOWCASE_SKIP_BUDGET workflow env (soft is default).
- Tests pin the delta format and hermetic-path stability.

Lane: python scripts/verify_lane_shas.py → 9/9 match.
This commit is contained in:
Shay 2026-07-14 17:41:04 -07:00
parent 5b18499918
commit 7f6c497a21
8 changed files with 128 additions and 13 deletions

View file

@ -49,9 +49,9 @@ jobs:
- name: verify lane SHAs
env:
PYTHONPATH: ${{ github.workspace }}
# Align with full-pytest.yml: wall-clock is env-sensitive on cold
# Act runners; content cases remain hard gates in public_demo.
CORE_SHOWCASE_SKIP_BUDGET: "1"
# public_demo wall-clock is soft by default (see evals/public_demo/runner.py).
# Do not set CORE_SHOWCASE_HARD_BUDGET here — cold Act runners exceed 60s.
# Content cases (claims, determinism, pure composition) remain hard gates.
run: |
uv run python scripts/verify_lane_shas.py

View file

@ -38,8 +38,8 @@ is a CI failure (`.github/workflows/lane-shas.yml`).
| ADR-0093 | `domain_contract_validation` | All ratified packs satisfy the 9 ADR-0091 contract predicates | `evals/domain_contract_validation/results/v1_dev.json` | `98ace04e3f02bbc5a8ad655bb6593c3f1ee64cb67014f1122fe6c3c85f48d22f` |
| ADR-0095 | `miner_loop_closure` | Miner-sourced proposals route through single reviewed teaching path | `evals/miner_loop_closure/results/v1_dev.json` | `9f071733abe7dcacf759f928548ce738fb639af3fd6e4c621a651b306d7e77ce` |
| ADR-0096 | `fabrication_control_summary` | Phantom endpoints / cross-pack non-bridges / sibling collapses refuse | `evals/fabrication_control/results/v1_summary.json` | `01e1b6b711141f2b4a14551d7df3ea482d8d6dd7b364a25c509f4f8d08cda8a8` |
| ADR-0098 | `demo_composition` | Demos compose from shipped modules; no parallel mechanism | `evals/demo_composition/results/v1_dev.json` | `c8bb37e03b85f2558eb31f6caa20d4a1d2e39c4a7c711393aa88e180b1c39a3a` |
| ADR-0099 | `public_demo` | Public showcase runs deterministically under 30s; all claims supported | `evals/public_demo/results/v1_dev.json` | `ed1668a64490f73f4d9b701e611e07841c149fd36cb90703436e3e33732fcd76` |
| ADR-0098 | `demo_composition` | Demos compose from shipped modules; no parallel mechanism | `evals/demo_composition/results/v1_dev.json` | `e2ba2314d8768459fb6a8db082a4bbcf4107b5161d869804a4b2a33c3724081a` |
| ADR-0099 | `public_demo` | Public showcase runs deterministically under 30s; all claims supported | `evals/public_demo/results/v1_dev.json` | `7d8ba0dbae9287cfe0bf15d231fa78a75abc627121c14900439293e01e1cc1d3` |
| ADR-0104 | `curriculum_loop_closure` | Curriculum-sourced proposals route through single reviewed teaching path | `evals/curriculum_loop_closure/results/v1_dev.json` | `b46d56b2d209172cc3ffaf3776dc8dcfe55093f13587c5cb67372be6dfa23e8d` |
| ADR-0131 | `math_teaching_corpus_v1` | Math teaching corpus replays deterministically; all chains pass exit criterion (correct_rate=1.0, wrong=0) | `evals/math_teaching_corpus/v1/report.json` | `eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4` |
| ADR-0206 | `deductive_logic_v1` | Propositional entailment scored against an independent truth-table oracle; dev+holdout+external 716/716 correct, wrong=0, refused=0 | `evals/deductive_logic/report.json` | `97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f` |

View file

@ -164,6 +164,58 @@ _TRACKED_MODULES: tuple[str, ...] = (
)
# Env keys whose values are host/isolation paths. Mutation is still
# detected (key present/absent/changed), but absolute path *values*
# never enter divergence messages — those would make lane SHA pins
# depend on tempfile locations and break hermetic CI.
_PATH_LIKE_ENV_SUFFIXES: tuple[str, ...] = ("_DIR", "_PATH", "_HOME", "_ROOT")
def _is_path_like_env_key(key: str) -> bool:
if key == "CORE_ENGINE_STATE_DIR":
return True
return any(key.endswith(suffix) for suffix in _PATH_LIKE_ENV_SUFFIXES)
def _format_env_value(key: str, value: str) -> str:
"""Stable, pin-safe rendering of an env value for divergence text."""
if _is_path_like_env_key(key):
return "<path>"
return value
def _env_subset_divergences(
before_env: tuple[tuple[str, str], ...] | tuple[()] | Any,
after_env: tuple[tuple[str, str], ...] | tuple[()] | Any,
) -> tuple[str, ...]:
"""Key-level env delta — not a full before/after dump.
Full tuple dumps embed every ambient ``CORE_*`` value (including
hermetic engine-state temp paths). That is correct for *detection*
when compared as raw snapshots, but wrong for *report text*: the
lane SHA pin must be host-independent. Only keys that actually
changed appear in the message.
"""
before_map = dict(before_env or ())
after_map = dict(after_env or ())
messages: list[str] = []
for key in sorted(set(before_map) | set(after_map)):
b = before_map.get(key)
a = after_map.get(key)
if b == a:
continue
if b is None:
messages.append(f"env_subset: +{key}={_format_env_value(key, a)}")
elif a is None:
messages.append(f"env_subset: -{key}={_format_env_value(key, b)}")
else:
messages.append(
"env_subset: "
f"{key} {_format_env_value(key, b)!r} -> {_format_env_value(key, a)!r}"
)
return tuple(messages)
def _global_state_snapshot() -> dict[str, Any]:
"""Capture a load-bearing subset of process state for diff checking.
@ -206,9 +258,14 @@ def verify_no_global_state_mutation(
when the adapter does its own deferred imports. Only id id
rebindings (the module object was replaced) and value-set
divergences on env vars are flagged.
Env divergences are reported as a **key-level delta** (added /
removed / changed keys). Full before/after env dumps are forbidden
in the divergence text: they embed host-volatile values such as
``CORE_ENGINE_STATE_DIR`` temp paths and make lane SHA pins flaky.
"""
divergences: list[str] = []
for key in set(before.keys()) | set(after.keys()):
for key in sorted(set(before.keys()) | set(after.keys())):
b = before.get(key)
a = after.get(key)
if b == a:
@ -217,9 +274,10 @@ def verify_no_global_state_mutation(
# Lazy import: a module that wasn't yet loaded is now
# loaded. Benign and unavoidable.
continue
divergences.append(
f"{key}: before={b!r} after={a!r}"
)
if key == "env_subset":
divergences.extend(_env_subset_divergences(b, a))
continue
divergences.append(f"{key}: before={b!r} after={a!r}")
return (not divergences, tuple(divergences))

View file

@ -77,7 +77,7 @@
"case_id": "stateful_fixture_rejected",
"details": {
"divergences": [
"env_subset: before=() after=(('CORE_STATEFUL_FIXTURE_FLAG', '1'),)"
"env_subset: +CORE_STATEFUL_FIXTURE_FLAG=1"
]
},
"divergence": null,

View file

@ -170,6 +170,16 @@ def _run_stateful_fixture_rejected(tmp_root: Path) -> dict[str, Any]:
def run() -> dict[str, Any]:
import os
# Hermetic engine state (same pattern as public_demo): lived engine_state/
# on the developer's machine or a dirty CI workspace must not leak into
# adapter JSON hashes and break the lane pin.
os.environ.setdefault(
"CORE_ENGINE_STATE_DIR",
tempfile.mkdtemp(prefix="demo_composition_engine_state_"),
)
tmp_root = Path(tempfile.mkdtemp(prefix="demo_composition_lane_"))
try:
cases: list[dict[str, Any]] = []

View file

@ -13,7 +13,7 @@
{
"case_id": "determinism_run_to_run_byte_equality",
"details": {
"sha256": "d4e5a840a03a57dac9d12b9f00b36928271cf76f59b134ec5847318048431e06"
"sha256": "7f03086d43869979b1bebd15f4f58a8b3887c907f7fbe0e7d4b8cad33b7507f2"
},
"divergence": null,
"passed": true

View file

@ -38,8 +38,8 @@ PINNED_SHAS: dict[str, str] = {
"curriculum_loop_closure": "b46d56b2d209172cc3ffaf3776dc8dcfe55093f13587c5cb67372be6dfa23e8d",
"domain_contract_validation": "98ace04e3f02bbc5a8ad655bb6593c3f1ee64cb67014f1122fe6c3c85f48d22f",
"fabrication_control_summary": "01e1b6b711141f2b4a14551d7df3ea482d8d6dd7b364a25c509f4f8d08cda8a8",
"demo_composition": "c8bb37e03b85f2558eb31f6caa20d4a1d2e39c4a7c711393aa88e180b1c39a3a",
"public_demo": "ed1668a64490f73f4d9b701e611e07841c149fd36cb90703436e3e33732fcd76",
"demo_composition": "e2ba2314d8768459fb6a8db082a4bbcf4107b5161d869804a4b2a33c3724081a",
"public_demo": "7d8ba0dbae9287cfe0bf15d231fa78a75abc627121c14900439293e01e1cc1d3",
"math_teaching_corpus_v1": "eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4",
"deductive_logic_v1": "97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f",
}
@ -128,6 +128,12 @@ def _invoke_runner(spec: LaneSpec, *, target_path: Path | None = None) -> Path:
import os
env = {"PYTHONPATH": str(REPO_ROOT), **os.environ}
# Force hermetic engine_state for every lane so local lived state and CI
# workspaces cannot change demo / showcase report bytes.
env.setdefault(
"CORE_ENGINE_STATE_DIR",
tempfile.mkdtemp(prefix=f"lane_{spec.lane_id}_engine_"),
)
if spec.run_as_module:
args = [sys.executable, "-m", spec.runner_dotted]
else:

View file

@ -174,6 +174,47 @@ class TestGlobalStateDetector:
passed, divergences = verify_no_global_state_mutation(before=before, after=after)
assert passed is False
assert any("env_subset" in d for d in divergences)
# Key-level delta only — never a full env dump (host paths break pins).
assert any(
d == "env_subset: +CORE_DETECTOR_TEST_FLAG=1" for d in divergences
)
assert not any("before=" in d for d in divergences)
def test_env_delta_stable_across_hermetic_engine_state_paths(self) -> None:
"""Lane pins must not depend on CORE_ENGINE_STATE_DIR temp paths."""
before_a = {
"env_subset": (("CORE_ENGINE_STATE_DIR", "/tmp/run_a_path"),),
}
after_a = {
"env_subset": (
("CORE_ENGINE_STATE_DIR", "/tmp/run_a_path"),
("CORE_STATEFUL_FIXTURE_FLAG", "1"),
),
}
before_b = {
"env_subset": (("CORE_ENGINE_STATE_DIR", "/var/folders/xyz/run_b"),),
}
after_b = {
"env_subset": (
("CORE_ENGINE_STATE_DIR", "/var/folders/xyz/run_b"),
("CORE_STATEFUL_FIXTURE_FLAG", "1"),
),
}
passed_a, div_a = verify_no_global_state_mutation(before=before_a, after=after_a)
passed_b, div_b = verify_no_global_state_mutation(before=before_b, after=after_b)
assert passed_a is False and passed_b is False
assert div_a == div_b
assert div_a == ("env_subset: +CORE_STATEFUL_FIXTURE_FLAG=1",)
def test_path_like_env_change_detected_without_absolute_path(self) -> None:
before = {"env_subset": (("CORE_ENGINE_STATE_DIR", "/tmp/old"),)}
after = {"env_subset": (("CORE_ENGINE_STATE_DIR", "/tmp/new"),)}
passed, divergences = verify_no_global_state_mutation(before=before, after=after)
assert passed is False
assert divergences == (
"env_subset: CORE_ENGINE_STATE_DIR '<path>' -> '<path>'",
)
assert "/tmp/" not in "".join(divergences)
def test_lazy_import_not_flagged(self) -> None:
"""None → module id transition is benign (lazy import)."""