diff --git a/core/capability/reporting.py b/core/capability/reporting.py index 9bd30d6e..7582feb8 100644 --- a/core/capability/reporting.py +++ b/core/capability/reporting.py @@ -43,7 +43,7 @@ _FLAG_CATALOG: dict[str, dict[str, str]] = { "transitive_surface": {"state": "flag_shipped_default_off", "adr": "ADR-0083"}, "gloss_aware_cause": {"state": "flag_shipped_default_off", "adr": "ADR-0085"}, "thread_anaphora": {"state": "flag_shipped_default_off", "adr": "P3.2"}, - "discourse_planner": {"state": "flag_shipped_default_off", "adr": "ADR-0089"}, + "discourse_planner": {"state": "flag_shipped_default_on", "adr": "ADR-0089"}, "compound_intent_dispatch": {"state": "substrate_shipped_flag_missing", "adr": "ADR-0089-C2"}, "inference_trace": {"state": "substrate_missing", "adr": "ADR-0024"}, } @@ -355,6 +355,7 @@ def chain_report() -> dict[str, Any]: def flag_report() -> dict[str, Any]: grouped: dict[str, list[dict[str, str]]] = { "flag_shipped_default_off": [], + "flag_shipped_default_on": [], "substrate_shipped_flag_missing": [], "substrate_missing": [], } diff --git a/evals/public_demo/results/v1_dev.json b/evals/public_demo/results/v1_dev.json index 7b544c4d..01e42073 100644 --- a/evals/public_demo/results/v1_dev.json +++ b/evals/public_demo/results/v1_dev.json @@ -13,7 +13,7 @@ { "case_id": "determinism_run_to_run_byte_equality", "details": { - "sha256": "f35ba6c28f008f7f80eef51fdecd36c951b7f38eea1d64dc163c5f45e5c86050" + "sha256": "b166949828921da9f45b19e3f7c2e6cf76a2073302244778dc2a4009bd4926f4" }, "divergence": null, "passed": true diff --git a/evals/public_demo/runner.py b/evals/public_demo/runner.py index d60f0c17..b525e42b 100644 --- a/evals/public_demo/runner.py +++ b/evals/public_demo/runner.py @@ -69,6 +69,11 @@ _VOLATILE_KEYS: frozenset[str] = frozenset( "total_runtime_ms", # wall-clock "json_path", # adapter output paths (per-run temp dirs) "transient_corpus", # learning-loop embeds its temp corpus path + # ``generated_at_revision`` advances every commit; the lane's + # invariant is "same code → same SHA," not "same HEAD → same + # SHA." Stripping this here keeps the pinned lane SHA stable + # across commits unless the underlying demos' content changes. + "generated_at_revision", } ) diff --git a/scripts/verify_lane_shas.py b/scripts/verify_lane_shas.py index 022339cd..6d2269ce 100644 --- a/scripts/verify_lane_shas.py +++ b/scripts/verify_lane_shas.py @@ -49,7 +49,7 @@ PINNED_SHAS: dict[str, str] = { "domain_contract_validation": "f9c06cdeea8fb36a0d3c320007618c3afc92d67702ef31bd36ebd9ae9ced473f", "fabrication_control_summary": "01e1b6b711141f2b4a14551d7df3ea482d8d6dd7b364a25c509f4f8d08cda8a8", "demo_composition": "27d838241bf3ed9e15d0e918ec6d89a823494d7e17c2dab9777825af7188f20f", - "public_demo": "71090323943b3b09bf563d3d5217426922ab027f9a20bee3af4173435dda4c8c", + "public_demo": "4be6f47509435a24984713acfcebd88e61f4e1278096fa5dc88a09e8af2f87ba", } diff --git a/tests/test_capability_reports.py b/tests/test_capability_reports.py index 0d058073..d8215687 100644 --- a/tests/test_capability_reports.py +++ b/tests/test_capability_reports.py @@ -46,9 +46,19 @@ def test_chain_report_exposes_required_count_axes() -> None: assert len(philosophy_intents) >= 3 -def test_flag_report_tracks_default_off_flags_without_enabling_them() -> None: +def test_flag_report_classification_matches_actual_defaults() -> None: + """Catalog classification must match DEFAULT_CONFIG's actual values. + + A flag in ``flag_shipped_default_off`` MUST be False on DEFAULT_CONFIG; + a flag in ``flag_shipped_default_on`` MUST be True. Drift between the + catalog and the runtime default is itself a bug — catching it here + prevents the kind of silent re-classification that landed when + ``discourse_planner`` was flipped to True on 2026-05-21 without + updating the catalog. + """ report = flag_report() - shipped = {row["flag"] for row in report["flag_shipped_default_off"]} + default_off = {row["flag"] for row in report["flag_shipped_default_off"]} + default_on = {row["flag"] for row in report["flag_shipped_default_on"]} for flag in ( "realizer_grounded_authority", @@ -56,17 +66,62 @@ def test_flag_report_tracks_default_off_flags_without_enabling_them() -> None: "transitive_surface", "gloss_aware_cause", "thread_anaphora", - "discourse_planner", ): - assert flag in shipped - assert getattr(DEFAULT_CONFIG, flag) is False - assert "stop_tokens" in shipped + assert flag in default_off, f"{flag!r} expected in default_off" + assert getattr(DEFAULT_CONFIG, flag) is False, ( + f"{flag!r} catalog says default_off but DEFAULT_CONFIG has it True" + ) + + # discourse_planner shipped flag-off and was flipped to default-on + # on 2026-05-21 after cognition-eval byte-equality verification. + # The catalog state must match. + assert "discourse_planner" in default_on, ( + "discourse_planner expected in default_on after the 2026-05-21 flip" + ) + assert DEFAULT_CONFIG.discourse_planner is True + + # stop_tokens is None-by-default (not a bool); kept in default_off + # because its absence is the "off" state. + assert "stop_tokens" in default_off assert DEFAULT_CONFIG.stop_tokens is None + assert report["substrate_shipped_flag_missing"] == [ {"flag": "compound_intent_dispatch", "adr": "ADR-0089-C2"} ] +def test_flag_catalog_state_is_consistent_with_default_config() -> None: + """Cross-check every catalog entry against DEFAULT_CONFIG. + + Catches any future drift: if someone changes a default in + ``core.config`` without updating the catalog state, this test + fails before the lane SHAs can shift. + """ + report = flag_report() + for row in report["flag_shipped_default_off"]: + flag = row["flag"] + if not hasattr(DEFAULT_CONFIG, flag): + continue # substrate flag without a runtime knob + value = getattr(DEFAULT_CONFIG, flag) + # ``stop_tokens`` is None-by-default (not a bool). All others + # should be ``False`` to honor the ``default_off`` classification. + if flag == "stop_tokens": + assert value is None + else: + assert value is False, ( + f"catalog classifies {flag!r} as default_off but " + f"DEFAULT_CONFIG has it {value!r}" + ) + for row in report["flag_shipped_default_on"]: + flag = row["flag"] + if not hasattr(DEFAULT_CONFIG, flag): + continue + assert getattr(DEFAULT_CONFIG, flag) is True, ( + f"catalog classifies {flag!r} as default_on but " + f"DEFAULT_CONFIG has it {getattr(DEFAULT_CONFIG, flag)!r}" + ) + + def test_ledger_status_is_predicate_derived() -> None: report = ledger_report() rows = {row["domain"]: row for row in report["domains"]}