fix(capability): correct discourse_planner flag catalog + commit-independent public_demo pin
Two pre-existing latent issues fixed:
1. discourse_planner flag catalog drift (test_flag_report failure)
On 2026-05-21 the discourse_planner default was flipped to True
after byte-equality verification (per inline comment in
core/config.py:130-138), but the capability flag catalog at
core/capability/reporting.py was not updated — it still claimed
"flag_shipped_default_off". The test
test_flag_report_tracks_default_off_flags_without_enabling_them
correctly caught the inconsistency; it had been failing across
every commit since ADR-0092 first ran the suite.
Fix:
- New "flag_shipped_default_on" state in _FLAG_CATALOG, added
to flag_report() grouped output
- discourse_planner moved from default_off → default_on
- Test renamed to test_flag_report_classification_matches_actual_defaults,
enforces BOTH directions of the contract (catalog claim must
match DEFAULT_CONFIG value)
- New test test_flag_catalog_state_is_consistent_with_default_config
cross-checks every catalog entry against DEFAULT_CONFIG;
catches future drift before it lands
2. public_demo lane SHA shifted every commit
Each commit advances the showcase's generated_at_revision field
(git HEAD SHA). _strip_volatile in the lane runner was stripping
wall-clock and per-run paths but NOT generated_at_revision, so
the byte-equality case's details.sha256 changed with every commit
even when underlying demos produced identical content. That made
the pin a "did this run today" check rather than a "did the code
produce the right artifact" check — exactly the failure mode
the verifier was supposed to prevent.
Fix:
- Add generated_at_revision to _VOLATILE_KEYS in the public_demo
runner. Lane's invariant is "same code → same SHA," not
"same HEAD → same SHA"; HEAD belongs in the showcase output
(operators need it) but not in the lane's equality projection.
- Pin refreshed once to capture the now-commit-independent SHA;
subsequent commits won't shift it unless underlying demo content
actually changes.
After fix:
- Capability tests: 6/6 passing (was 4/5 with discourse_planner failing)
- Lane SHAs: 6/6 match pinned values; public_demo pin will now survive
routine code changes
- Smoke 67/67, cognition eval byte-identical 100/100/100/100
This is the single known pre-existing test failure cleaned up.
This commit is contained in:
Shay
parent
b9a6f2ddb5
commit
a8c12670ec
5 changed files with 70 additions and 9 deletions
|
|
@ -43,7 +43,7 @@ _FLAG_CATALOG: dict[str, dict[str, str]] = {
|
|||
"transitive_surface": {"state": "flag_shipped_default_off", "adr": "ADR-0083"},
|
||||
"gloss_aware_cause": {"state": "flag_shipped_default_off", "adr": "ADR-0085"},
|
||||
"thread_anaphora": {"state": "flag_shipped_default_off", "adr": "P3.2"},
|
||||
"discourse_planner": {"state": "flag_shipped_default_off", "adr": "ADR-0089"},
|
||||
"discourse_planner": {"state": "flag_shipped_default_on", "adr": "ADR-0089"},
|
||||
"compound_intent_dispatch": {"state": "substrate_shipped_flag_missing", "adr": "ADR-0089-C2"},
|
||||
"inference_trace": {"state": "substrate_missing", "adr": "ADR-0024"},
|
||||
}
|
||||
|
|
@ -355,6 +355,7 @@ def chain_report() -> dict[str, Any]:
|
|||
def flag_report() -> dict[str, Any]:
|
||||
grouped: dict[str, list[dict[str, str]]] = {
|
||||
"flag_shipped_default_off": [],
|
||||
"flag_shipped_default_on": [],
|
||||
"substrate_shipped_flag_missing": [],
|
||||
"substrate_missing": [],
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
{
|
||||
"case_id": "determinism_run_to_run_byte_equality",
|
||||
"details": {
|
||||
"sha256": "f35ba6c28f008f7f80eef51fdecd36c951b7f38eea1d64dc163c5f45e5c86050"
|
||||
"sha256": "b166949828921da9f45b19e3f7c2e6cf76a2073302244778dc2a4009bd4926f4"
|
||||
},
|
||||
"divergence": null,
|
||||
"passed": true
|
||||
|
|
|
|||
|
|
@ -69,6 +69,11 @@ _VOLATILE_KEYS: frozenset[str] = frozenset(
|
|||
"total_runtime_ms", # wall-clock
|
||||
"json_path", # adapter output paths (per-run temp dirs)
|
||||
"transient_corpus", # learning-loop embeds its temp corpus path
|
||||
# ``generated_at_revision`` advances every commit; the lane's
|
||||
# invariant is "same code → same SHA," not "same HEAD → same
|
||||
# SHA." Stripping this here keeps the pinned lane SHA stable
|
||||
# across commits unless the underlying demos' content changes.
|
||||
"generated_at_revision",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ PINNED_SHAS: dict[str, str] = {
|
|||
"domain_contract_validation": "f9c06cdeea8fb36a0d3c320007618c3afc92d67702ef31bd36ebd9ae9ced473f",
|
||||
"fabrication_control_summary": "01e1b6b711141f2b4a14551d7df3ea482d8d6dd7b364a25c509f4f8d08cda8a8",
|
||||
"demo_composition": "27d838241bf3ed9e15d0e918ec6d89a823494d7e17c2dab9777825af7188f20f",
|
||||
"public_demo": "71090323943b3b09bf563d3d5217426922ab027f9a20bee3af4173435dda4c8c",
|
||||
"public_demo": "4be6f47509435a24984713acfcebd88e61f4e1278096fa5dc88a09e8af2f87ba",
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -46,9 +46,19 @@ def test_chain_report_exposes_required_count_axes() -> None:
|
|||
assert len(philosophy_intents) >= 3
|
||||
|
||||
|
||||
def test_flag_report_tracks_default_off_flags_without_enabling_them() -> None:
|
||||
def test_flag_report_classification_matches_actual_defaults() -> None:
|
||||
"""Catalog classification must match DEFAULT_CONFIG's actual values.
|
||||
|
||||
A flag in ``flag_shipped_default_off`` MUST be False on DEFAULT_CONFIG;
|
||||
a flag in ``flag_shipped_default_on`` MUST be True. Drift between the
|
||||
catalog and the runtime default is itself a bug — catching it here
|
||||
prevents the kind of silent re-classification that landed when
|
||||
``discourse_planner`` was flipped to True on 2026-05-21 without
|
||||
updating the catalog.
|
||||
"""
|
||||
report = flag_report()
|
||||
shipped = {row["flag"] for row in report["flag_shipped_default_off"]}
|
||||
default_off = {row["flag"] for row in report["flag_shipped_default_off"]}
|
||||
default_on = {row["flag"] for row in report["flag_shipped_default_on"]}
|
||||
|
||||
for flag in (
|
||||
"realizer_grounded_authority",
|
||||
|
|
@ -56,17 +66,62 @@ def test_flag_report_tracks_default_off_flags_without_enabling_them() -> None:
|
|||
"transitive_surface",
|
||||
"gloss_aware_cause",
|
||||
"thread_anaphora",
|
||||
"discourse_planner",
|
||||
):
|
||||
assert flag in shipped
|
||||
assert getattr(DEFAULT_CONFIG, flag) is False
|
||||
assert "stop_tokens" in shipped
|
||||
assert flag in default_off, f"{flag!r} expected in default_off"
|
||||
assert getattr(DEFAULT_CONFIG, flag) is False, (
|
||||
f"{flag!r} catalog says default_off but DEFAULT_CONFIG has it True"
|
||||
)
|
||||
|
||||
# discourse_planner shipped flag-off and was flipped to default-on
|
||||
# on 2026-05-21 after cognition-eval byte-equality verification.
|
||||
# The catalog state must match.
|
||||
assert "discourse_planner" in default_on, (
|
||||
"discourse_planner expected in default_on after the 2026-05-21 flip"
|
||||
)
|
||||
assert DEFAULT_CONFIG.discourse_planner is True
|
||||
|
||||
# stop_tokens is None-by-default (not a bool); kept in default_off
|
||||
# because its absence is the "off" state.
|
||||
assert "stop_tokens" in default_off
|
||||
assert DEFAULT_CONFIG.stop_tokens is None
|
||||
|
||||
assert report["substrate_shipped_flag_missing"] == [
|
||||
{"flag": "compound_intent_dispatch", "adr": "ADR-0089-C2"}
|
||||
]
|
||||
|
||||
|
||||
def test_flag_catalog_state_is_consistent_with_default_config() -> None:
|
||||
"""Cross-check every catalog entry against DEFAULT_CONFIG.
|
||||
|
||||
Catches any future drift: if someone changes a default in
|
||||
``core.config`` without updating the catalog state, this test
|
||||
fails before the lane SHAs can shift.
|
||||
"""
|
||||
report = flag_report()
|
||||
for row in report["flag_shipped_default_off"]:
|
||||
flag = row["flag"]
|
||||
if not hasattr(DEFAULT_CONFIG, flag):
|
||||
continue # substrate flag without a runtime knob
|
||||
value = getattr(DEFAULT_CONFIG, flag)
|
||||
# ``stop_tokens`` is None-by-default (not a bool). All others
|
||||
# should be ``False`` to honor the ``default_off`` classification.
|
||||
if flag == "stop_tokens":
|
||||
assert value is None
|
||||
else:
|
||||
assert value is False, (
|
||||
f"catalog classifies {flag!r} as default_off but "
|
||||
f"DEFAULT_CONFIG has it {value!r}"
|
||||
)
|
||||
for row in report["flag_shipped_default_on"]:
|
||||
flag = row["flag"]
|
||||
if not hasattr(DEFAULT_CONFIG, flag):
|
||||
continue
|
||||
assert getattr(DEFAULT_CONFIG, flag) is True, (
|
||||
f"catalog classifies {flag!r} as default_on but "
|
||||
f"DEFAULT_CONFIG has it {getattr(DEFAULT_CONFIG, flag)!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_ledger_status_is_predicate_derived() -> None:
|
||||
report = ledger_report()
|
||||
rows = {row["domain"]: row for row in report["domains"]}
|
||||
|
|
|
|||
Loading…
Reference in a new issue