fix(ci): path-stable env deltas so lane SHA pins stop thrashing
Root cause of demo_composition pin thrash was not cancelled CI runs: verify_no_global_state_mutation dumped full env_subset before/after tuples into the lane report, embedding CORE_ENGINE_STATE_DIR temp paths (and any ambient CORE_* host values). Each hermetic run then produced a different report SHA. Fix for good: - Report key-level env deltas only (+/- / changed), with path-like CORE_* values redacted to <path> in the message text. - Detection still uses raw snapshots (mutations are not ignored). - Hermetic CORE_ENGINE_STATE_DIR for demo_composition and all verify_lane_shas invocations. - Dual-run-stable re-pin: demo_composition e2ba2314…, public_demo 7d8ba0db… (matches CI-observed soft-budget SHA). - Drop dead CORE_SHOWCASE_SKIP_BUDGET workflow env (soft is default). - Tests pin the delta format and hermetic-path stability. Lane: python scripts/verify_lane_shas.py → 9/9 match.
This commit is contained in:
parent
5b18499918
commit
7f6c497a21
8 changed files with 128 additions and 13 deletions
6
.github/workflows/lane-shas.yml
vendored
6
.github/workflows/lane-shas.yml
vendored
|
|
@ -49,9 +49,9 @@ jobs:
|
|||
- name: verify lane SHAs
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
# Align with full-pytest.yml: wall-clock is env-sensitive on cold
|
||||
# Act runners; content cases remain hard gates in public_demo.
|
||||
CORE_SHOWCASE_SKIP_BUDGET: "1"
|
||||
# public_demo wall-clock is soft by default (see evals/public_demo/runner.py).
|
||||
# Do not set CORE_SHOWCASE_HARD_BUDGET here — cold Act runners exceed 60s.
|
||||
# Content cases (claims, determinism, pure composition) remain hard gates.
|
||||
run: |
|
||||
uv run python scripts/verify_lane_shas.py
|
||||
|
||||
|
|
|
|||
|
|
@ -38,8 +38,8 @@ is a CI failure (`.github/workflows/lane-shas.yml`).
|
|||
| ADR-0093 | `domain_contract_validation` | All ratified packs satisfy the 9 ADR-0091 contract predicates | `evals/domain_contract_validation/results/v1_dev.json` | `98ace04e3f02bbc5a8ad655bb6593c3f1ee64cb67014f1122fe6c3c85f48d22f` |
|
||||
| ADR-0095 | `miner_loop_closure` | Miner-sourced proposals route through single reviewed teaching path | `evals/miner_loop_closure/results/v1_dev.json` | `9f071733abe7dcacf759f928548ce738fb639af3fd6e4c621a651b306d7e77ce` |
|
||||
| ADR-0096 | `fabrication_control_summary` | Phantom endpoints / cross-pack non-bridges / sibling collapses refuse | `evals/fabrication_control/results/v1_summary.json` | `01e1b6b711141f2b4a14551d7df3ea482d8d6dd7b364a25c509f4f8d08cda8a8` |
|
||||
| ADR-0098 | `demo_composition` | Demos compose from shipped modules; no parallel mechanism | `evals/demo_composition/results/v1_dev.json` | `c8bb37e03b85f2558eb31f6caa20d4a1d2e39c4a7c711393aa88e180b1c39a3a` |
|
||||
| ADR-0099 | `public_demo` | Public showcase runs deterministically under 30s; all claims supported | `evals/public_demo/results/v1_dev.json` | `ed1668a64490f73f4d9b701e611e07841c149fd36cb90703436e3e33732fcd76` |
|
||||
| ADR-0098 | `demo_composition` | Demos compose from shipped modules; no parallel mechanism | `evals/demo_composition/results/v1_dev.json` | `e2ba2314d8768459fb6a8db082a4bbcf4107b5161d869804a4b2a33c3724081a` |
|
||||
| ADR-0099 | `public_demo` | Public showcase runs deterministically under 30s; all claims supported | `evals/public_demo/results/v1_dev.json` | `7d8ba0dbae9287cfe0bf15d231fa78a75abc627121c14900439293e01e1cc1d3` |
|
||||
| ADR-0104 | `curriculum_loop_closure` | Curriculum-sourced proposals route through single reviewed teaching path | `evals/curriculum_loop_closure/results/v1_dev.json` | `b46d56b2d209172cc3ffaf3776dc8dcfe55093f13587c5cb67372be6dfa23e8d` |
|
||||
| ADR-0131 | `math_teaching_corpus_v1` | Math teaching corpus replays deterministically; all chains pass exit criterion (correct_rate=1.0, wrong=0) | `evals/math_teaching_corpus/v1/report.json` | `eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4` |
|
||||
| ADR-0206 | `deductive_logic_v1` | Propositional entailment scored against an independent truth-table oracle; dev+holdout+external 716/716 correct, wrong=0, refused=0 | `evals/deductive_logic/report.json` | `97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f` |
|
||||
|
|
|
|||
|
|
@ -164,6 +164,58 @@ _TRACKED_MODULES: tuple[str, ...] = (
|
|||
)
|
||||
|
||||
|
||||
# Env keys whose values are host/isolation paths. Mutation is still
|
||||
# detected (key present/absent/changed), but absolute path *values*
|
||||
# never enter divergence messages — those would make lane SHA pins
|
||||
# depend on tempfile locations and break hermetic CI.
|
||||
_PATH_LIKE_ENV_SUFFIXES: tuple[str, ...] = ("_DIR", "_PATH", "_HOME", "_ROOT")
|
||||
|
||||
|
||||
def _is_path_like_env_key(key: str) -> bool:
|
||||
if key == "CORE_ENGINE_STATE_DIR":
|
||||
return True
|
||||
return any(key.endswith(suffix) for suffix in _PATH_LIKE_ENV_SUFFIXES)
|
||||
|
||||
|
||||
def _format_env_value(key: str, value: str) -> str:
|
||||
"""Stable, pin-safe rendering of an env value for divergence text."""
|
||||
if _is_path_like_env_key(key):
|
||||
return "<path>"
|
||||
return value
|
||||
|
||||
|
||||
def _env_subset_divergences(
|
||||
before_env: tuple[tuple[str, str], ...] | tuple[()] | Any,
|
||||
after_env: tuple[tuple[str, str], ...] | tuple[()] | Any,
|
||||
) -> tuple[str, ...]:
|
||||
"""Key-level env delta — not a full before/after dump.
|
||||
|
||||
Full tuple dumps embed every ambient ``CORE_*`` value (including
|
||||
hermetic engine-state temp paths). That is correct for *detection*
|
||||
when compared as raw snapshots, but wrong for *report text*: the
|
||||
lane SHA pin must be host-independent. Only keys that actually
|
||||
changed appear in the message.
|
||||
"""
|
||||
before_map = dict(before_env or ())
|
||||
after_map = dict(after_env or ())
|
||||
messages: list[str] = []
|
||||
for key in sorted(set(before_map) | set(after_map)):
|
||||
b = before_map.get(key)
|
||||
a = after_map.get(key)
|
||||
if b == a:
|
||||
continue
|
||||
if b is None:
|
||||
messages.append(f"env_subset: +{key}={_format_env_value(key, a)}")
|
||||
elif a is None:
|
||||
messages.append(f"env_subset: -{key}={_format_env_value(key, b)}")
|
||||
else:
|
||||
messages.append(
|
||||
"env_subset: "
|
||||
f"{key} {_format_env_value(key, b)!r} -> {_format_env_value(key, a)!r}"
|
||||
)
|
||||
return tuple(messages)
|
||||
|
||||
|
||||
def _global_state_snapshot() -> dict[str, Any]:
|
||||
"""Capture a load-bearing subset of process state for diff checking.
|
||||
|
||||
|
|
@ -206,9 +258,14 @@ def verify_no_global_state_mutation(
|
|||
when the adapter does its own deferred imports. Only id → id
|
||||
rebindings (the module object was replaced) and value-set
|
||||
divergences on env vars are flagged.
|
||||
|
||||
Env divergences are reported as a **key-level delta** (added /
|
||||
removed / changed keys). Full before/after env dumps are forbidden
|
||||
in the divergence text: they embed host-volatile values such as
|
||||
``CORE_ENGINE_STATE_DIR`` temp paths and make lane SHA pins flaky.
|
||||
"""
|
||||
divergences: list[str] = []
|
||||
for key in set(before.keys()) | set(after.keys()):
|
||||
for key in sorted(set(before.keys()) | set(after.keys())):
|
||||
b = before.get(key)
|
||||
a = after.get(key)
|
||||
if b == a:
|
||||
|
|
@ -217,9 +274,10 @@ def verify_no_global_state_mutation(
|
|||
# Lazy import: a module that wasn't yet loaded is now
|
||||
# loaded. Benign and unavoidable.
|
||||
continue
|
||||
divergences.append(
|
||||
f"{key}: before={b!r} after={a!r}"
|
||||
)
|
||||
if key == "env_subset":
|
||||
divergences.extend(_env_subset_divergences(b, a))
|
||||
continue
|
||||
divergences.append(f"{key}: before={b!r} after={a!r}")
|
||||
return (not divergences, tuple(divergences))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@
|
|||
"case_id": "stateful_fixture_rejected",
|
||||
"details": {
|
||||
"divergences": [
|
||||
"env_subset: before=() after=(('CORE_STATEFUL_FIXTURE_FLAG', '1'),)"
|
||||
"env_subset: +CORE_STATEFUL_FIXTURE_FLAG=1"
|
||||
]
|
||||
},
|
||||
"divergence": null,
|
||||
|
|
|
|||
|
|
@ -170,6 +170,16 @@ def _run_stateful_fixture_rejected(tmp_root: Path) -> dict[str, Any]:
|
|||
|
||||
|
||||
def run() -> dict[str, Any]:
|
||||
import os
|
||||
|
||||
# Hermetic engine state (same pattern as public_demo): lived engine_state/
|
||||
# on the developer's machine or a dirty CI workspace must not leak into
|
||||
# adapter JSON hashes and break the lane pin.
|
||||
os.environ.setdefault(
|
||||
"CORE_ENGINE_STATE_DIR",
|
||||
tempfile.mkdtemp(prefix="demo_composition_engine_state_"),
|
||||
)
|
||||
|
||||
tmp_root = Path(tempfile.mkdtemp(prefix="demo_composition_lane_"))
|
||||
try:
|
||||
cases: list[dict[str, Any]] = []
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
{
|
||||
"case_id": "determinism_run_to_run_byte_equality",
|
||||
"details": {
|
||||
"sha256": "d4e5a840a03a57dac9d12b9f00b36928271cf76f59b134ec5847318048431e06"
|
||||
"sha256": "7f03086d43869979b1bebd15f4f58a8b3887c907f7fbe0e7d4b8cad33b7507f2"
|
||||
},
|
||||
"divergence": null,
|
||||
"passed": true
|
||||
|
|
|
|||
|
|
@ -38,8 +38,8 @@ PINNED_SHAS: dict[str, str] = {
|
|||
"curriculum_loop_closure": "b46d56b2d209172cc3ffaf3776dc8dcfe55093f13587c5cb67372be6dfa23e8d",
|
||||
"domain_contract_validation": "98ace04e3f02bbc5a8ad655bb6593c3f1ee64cb67014f1122fe6c3c85f48d22f",
|
||||
"fabrication_control_summary": "01e1b6b711141f2b4a14551d7df3ea482d8d6dd7b364a25c509f4f8d08cda8a8",
|
||||
"demo_composition": "c8bb37e03b85f2558eb31f6caa20d4a1d2e39c4a7c711393aa88e180b1c39a3a",
|
||||
"public_demo": "ed1668a64490f73f4d9b701e611e07841c149fd36cb90703436e3e33732fcd76",
|
||||
"demo_composition": "e2ba2314d8768459fb6a8db082a4bbcf4107b5161d869804a4b2a33c3724081a",
|
||||
"public_demo": "7d8ba0dbae9287cfe0bf15d231fa78a75abc627121c14900439293e01e1cc1d3",
|
||||
"math_teaching_corpus_v1": "eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4",
|
||||
"deductive_logic_v1": "97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f",
|
||||
}
|
||||
|
|
@ -128,6 +128,12 @@ def _invoke_runner(spec: LaneSpec, *, target_path: Path | None = None) -> Path:
|
|||
import os
|
||||
|
||||
env = {"PYTHONPATH": str(REPO_ROOT), **os.environ}
|
||||
# Force hermetic engine_state for every lane so local lived state and CI
|
||||
# workspaces cannot change demo / showcase report bytes.
|
||||
env.setdefault(
|
||||
"CORE_ENGINE_STATE_DIR",
|
||||
tempfile.mkdtemp(prefix=f"lane_{spec.lane_id}_engine_"),
|
||||
)
|
||||
if spec.run_as_module:
|
||||
args = [sys.executable, "-m", spec.runner_dotted]
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -174,6 +174,47 @@ class TestGlobalStateDetector:
|
|||
passed, divergences = verify_no_global_state_mutation(before=before, after=after)
|
||||
assert passed is False
|
||||
assert any("env_subset" in d for d in divergences)
|
||||
# Key-level delta only — never a full env dump (host paths break pins).
|
||||
assert any(
|
||||
d == "env_subset: +CORE_DETECTOR_TEST_FLAG=1" for d in divergences
|
||||
)
|
||||
assert not any("before=" in d for d in divergences)
|
||||
|
||||
def test_env_delta_stable_across_hermetic_engine_state_paths(self) -> None:
|
||||
"""Lane pins must not depend on CORE_ENGINE_STATE_DIR temp paths."""
|
||||
before_a = {
|
||||
"env_subset": (("CORE_ENGINE_STATE_DIR", "/tmp/run_a_path"),),
|
||||
}
|
||||
after_a = {
|
||||
"env_subset": (
|
||||
("CORE_ENGINE_STATE_DIR", "/tmp/run_a_path"),
|
||||
("CORE_STATEFUL_FIXTURE_FLAG", "1"),
|
||||
),
|
||||
}
|
||||
before_b = {
|
||||
"env_subset": (("CORE_ENGINE_STATE_DIR", "/var/folders/xyz/run_b"),),
|
||||
}
|
||||
after_b = {
|
||||
"env_subset": (
|
||||
("CORE_ENGINE_STATE_DIR", "/var/folders/xyz/run_b"),
|
||||
("CORE_STATEFUL_FIXTURE_FLAG", "1"),
|
||||
),
|
||||
}
|
||||
passed_a, div_a = verify_no_global_state_mutation(before=before_a, after=after_a)
|
||||
passed_b, div_b = verify_no_global_state_mutation(before=before_b, after=after_b)
|
||||
assert passed_a is False and passed_b is False
|
||||
assert div_a == div_b
|
||||
assert div_a == ("env_subset: +CORE_STATEFUL_FIXTURE_FLAG=1",)
|
||||
|
||||
def test_path_like_env_change_detected_without_absolute_path(self) -> None:
|
||||
before = {"env_subset": (("CORE_ENGINE_STATE_DIR", "/tmp/old"),)}
|
||||
after = {"env_subset": (("CORE_ENGINE_STATE_DIR", "/tmp/new"),)}
|
||||
passed, divergences = verify_no_global_state_mutation(before=before, after=after)
|
||||
assert passed is False
|
||||
assert divergences == (
|
||||
"env_subset: CORE_ENGINE_STATE_DIR '<path>' -> '<path>'",
|
||||
)
|
||||
assert "/tmp/" not in "".join(divergences)
|
||||
|
||||
def test_lazy_import_not_flagged(self) -> None:
|
||||
"""None → module id transition is benign (lazy import)."""
|
||||
|
|
|
|||
Loading…
Reference in a new issue