From a4d68682df1194f9f556c105b7d740c5b9c75671 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 24 Jul 2026 17:13:19 -0700 Subject: [PATCH] =?UTF-8?q?chore(generalization):=20Tier=20S=20housekeepin?= =?UTF-8?q?g=20=E2=80=94=20smoke=20promotion,=20capability-index=20entries?= =?UTF-8?q?,=20promotion=20sweep=20(S5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Promote tests/test_register_substantive_consumption.py into the smoke suite (local core/cli_test.py + mirrored .github/workflows/smoke.yml): the falsifiable register-axis (ADR-0069/0071/0077) contract was red on main outside every gate for the 2026-07-22..24 window, masked by an unrelated flake label. 36 tests, ~4.6s. - Found in passing: test_core_test_deductive_suite_expands_to_entailment_lane was already red on clean main (an exact-tuple pin from when the `deductive` suite had one file; it has since grown to 12). Fixed to a contains-style assertion so it can't go stale the same way again. - Two new capability-index adapters: deduction_serve_existential_result (Band v6-EX alone — the one band among six that changes the decision procedure itself) and curriculum_serve_result. Breadth 11 -> 13; baseline re-frozen (a deliberate re-freeze per its own docstring). - Promotion sweep: confirmed via the existing wrong=0 lane gate (run repeatedly this session across 166/166 deduction-serve cases) that no split beyond the already-applied ds-mem-0020 has a stale declined-gold case a current band now decides differently. --- .github/workflows/smoke.yml | 10 ++- core/cli_test.py | 10 +++ .../tier-s-housekeeping-2026-07-24.md | 71 +++++++++++++++++++ evals/capability_index/adapters.py | 42 ++++++++++- evals/capability_index/baseline.json | 26 +++++-- tests/test_capability_index.py | 10 ++- tests/test_cli_test_suites.py | 21 ++++-- 7 files changed, 171 insertions(+), 19 deletions(-) create mode 100644 docs/research/tier-s-housekeeping-2026-07-24.md diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index bb052d5a..b8ad47d7 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -3,9 +3,12 @@ name: smoke # Fast PR gate — runs the smoke suite on every PR push. # Covers: chat runtime, pipeline, architectural invariants, audio sensorium # (tests/test_audio_*.py — compiler/eval-gates/mount/CRDT-merge/teachers, ~3s), -# and identity falsifiability (tests/test_pack_measurements_phase2.py, ADR-0043 +# identity falsifiability (tests/test_pack_measurements_phase2.py, ADR-0043 # — ratified packs diverge directionally; pack-invariant refusal floor; no -# fabrication). The falsifiability lane adds ~4 min but blocks-on-regression. +# fabrication; adds ~4 min but blocks-on-regression), and the register axis +# (tests/test_register_substantive_consumption.py, ADR-0069/0071/0077 — terse/ +# convivial registers must actually differ from neutral on served surfaces; +# promoted after a 2-day silent regression, generalization-arc Tier S). # # Post-merge on main: full-pytest.yml runs the FAST lane # (-m "not quarantine and not slow"). Soak / proof / register-matrix coverage @@ -56,4 +59,5 @@ jobs: tests/test_cognitive_turn_pipeline.py \ tests/test_architectural_invariants.py \ tests/test_audio_*.py \ - tests/test_pack_measurements_phase2.py + tests/test_pack_measurements_phase2.py \ + tests/test_register_substantive_consumption.py diff --git a/core/cli_test.py b/core/cli_test.py index a804ae4f..bdb12027 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -46,6 +46,16 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_pack_measurements_phase2.py", # ADR-0253 dual-pack boundary — draft he/grc trees must not be serve imports. "tests/test_pack_draft_serve_boundary.py", + # Register axis (ADR-0069/0071/0077) — the e2e tests here are the + # falsifiable contract that terse/convivial registers actually differ + # from neutral on pipeline-served surfaces. Red on main outside every + # gate for 2026-07-20..24 (the register-axis serving regression, + # docs/research/lane-drift-investigation-2026-07-24.md) because this + # file lived only under `full`; the lane-shas job was the sole gate + # that runs the demo lanes this axis shows up in, and its failures + # were conflated with an unrelated flake. Promoted so this axis can + # never silently regress again. + "tests/test_register_substantive_consumption.py", ), "runtime": ( "tests/test_chat_runtime.py", diff --git a/docs/research/tier-s-housekeeping-2026-07-24.md b/docs/research/tier-s-housekeeping-2026-07-24.md new file mode 100644 index 00000000..c9b940ac --- /dev/null +++ b/docs/research/tier-s-housekeeping-2026-07-24.md @@ -0,0 +1,71 @@ +# Tier S Housekeeping (S5) + +**Tier S5** (generalization-arc-2026-07-24 §6). Four items; results below. + +## 1. `test_register_substantive_consumption.py` promoted into smoke + +Added to `core/cli_test.py`'s `TEST_SUITES["smoke"]` (the local pre-push +gate) and `.github/workflows/smoke.yml` (the mirrored CI gate — kept in +parity rather than diverging from the local list). 36 tests, ~4.6s. This +file is the falsifiable contract for the register axis (ADR-0069/0071/0077) +and was red on `main`, outside every gate, for the 2026-07-22→24 window +documented in `docs/research/public-demo-lane-drift-2026-07-24.md` §3.2 and +[[project-register-axis-serving-regression]] — the exact "silent red" class +the lane-drift investigation flagged as a follow-up. It can no longer +regress without blocking a push. + +**Found in passing:** `tests/test_cli_test_suites.py::test_core_test_deductive_suite_expands_to_entailment_lane` +was already red on clean `main`, unrelated to any Tier S work — it pinned +the `deductive` suite to an exact single-file tuple from when that suite +had one entry; the suite has since grown to 12 files across the +deduction-serve/curriculum-serve arcs. A leaky-gate case in the same +"pinned list goes stale" family the smoke-coverage pin exists to catch, on +a *different* suite the coverage pin doesn't watch. Fixed to a contains-style +assertion (matching the neighboring `fast`-suite test's own style) so it +cannot go stale the same way again. + +## 2. Capability-index entries for `curriculum_serve_v1` and the `v2_exist` split + +Two new adapters in `evals/capability_index/adapters.py`: + +- `deduction_serve_existential_result()` — Band v6-EX (ADR-0261) alone, not + the full six-band deduction-serve lane. The other five bands are grammar + widenings over the same propositional-entailment substrate the existing + `deductive_logic` adapter already scores; v6-EX is the one that changes + the decision procedure itself (domain widening with Skolem witnesses / an + arbitrary element), making it the genuinely new capability worth its own + index domain. +- `curriculum_serve_result()` — the curriculum-grounded serving lane + (ADR-0262) as a whole, scored against its independent oracle. + +Index breadth: 11 → 13 domains. Re-froze `evals/capability_index/baseline.json` +(a deliberate re-freeze, per the file's own docstring — adding capability +domains is exactly the "accepted improvement" case) and updated +`tests/test_capability_baseline.py`'s implicit dependents +(`tests/test_capability_index.py`'s `breadth == 13` and the explicit +13-domain name set). `not_covered: []`, `wrong_total: 0` on the live run. + +## 3. Promotion sweep — nothing beyond `ds-mem-0020` + +The mechanism that surfaced `ds-mem-0020` (a case whose committed +`gold="declined"` stopped matching `decide()`'s output the moment Band +v6-EX could read and decide it — caught as a lane `wrong=1` failure, +diagnosed as a promotion, gold corrected) is not a one-time manual check: +it is exactly what `evals/deduction_serve/runner.py`'s `wrong == 0` gate +enforces on every run, over every split, using the CURRENT full band +cascade. Any other currently-`declined`-gold case that a band now decides +differently would show up the same way. This session ran that gate +repeatedly (166/166 wrong=0 both standalone and inside the 279-test +`deductive` suite run, post-S3's vocab-trigger instrument addition) with +zero further mismatches. **The sweep's proof is the passing gate, not a +separate pass** — nothing else in the older splits is decidable that isn't +already reflected in committed gold. + +## 4. Dead code + +None found unambiguous enough to remove during this sweep beyond what S2 +(`public_demo` pin) and S4 +(`generate/determine/derived_close_proposals.py::DEFAULT_SINK` off-by-one) +already fixed as part of their own scope — see those PRs' own notes. + +Relates to [[project-generalization-arc]]. diff --git a/evals/capability_index/adapters.py b/evals/capability_index/adapters.py index 8d0a2ab0..09919af8 100644 --- a/evals/capability_index/adapters.py +++ b/evals/capability_index/adapters.py @@ -98,6 +98,41 @@ def comprehension_relational_transitive_result() -> DomainResult: return DomainResult("comprehension_relational_transitive", c, w, r) +def deduction_serve_existential_result() -> DomainResult: + """Band v6-EX (ADR-0261) alone — the existential-argument split of the + deduction-serve lane, scored against its own independently-authored + gold. Deliberately just this one split, not the full six-band lane: the + other bands (v1/v1b propositional+categorical, v2-EN/v3-MEM/v4-CM/v5-VP) + are pure grammar widenings over the SAME propositional-entailment + substrate the ``deductive_logic`` adapter above already scores; the + existential band is the one that changes the DECISION PROCEDURE itself + (domain widening with Skolem witnesses / an arbitrary element — no + existential import), so it is the one genuinely new reasoning capability + among the six worth its own index domain.""" + from evals.deduction_serve.runner import build_combined_report + + counts = build_combined_report()["splits"]["v2_exist"]["counts"] + return DomainResult( + "deduction_serve_existential", + int(counts["correct"]), + int(counts["wrong"]), + int(counts["declined"]), + ) + + +def curriculum_serve_result() -> DomainResult: + """The curriculum-grounded serving lane (ADR-0262): exam questions + answered from the ratified domain-chain curriculum alone, scored against + an independent closed-world reachability oracle (``evals.curriculum_serve.oracle``, + a different decision procedure than the serving path it checks).""" + from evals.curriculum_serve.runner import build_combined_report + + agg = build_combined_report()["aggregate"] + return DomainResult( + "curriculum_serve", int(agg["correct"]), int(agg["wrong"]), int(agg["declined"]) + ) + + #: The reasoning domains currently composed into the index (self-loading lanes). #: The six ``comprehension_*`` lanes score the GENERAL comprehension reader; the #: relational_metric one reads arithmetic prose into the binding-graph quantity @@ -108,7 +143,10 @@ def comprehension_relational_transitive_result() -> DomainResult: #: same-predicate transitive closure over the declared strict orders (mastery-v2 Step-3 #: Brief B2) — so the index measures comprehension breadth across categorical, ordering, #: propositional, quantitative, relational-reading, one-hop relational-inference, AND -#: transitive relational-inference reasoning. +#: transitive relational-inference reasoning. ``deduction_serve_existential`` (ADR-0261) +#: and ``curriculum_serve`` (ADR-0262, generalization arc Tier S) add the PRODUCTION +#: serving deciders (not bare-engine lanes) for existential-domain widening and +#: curriculum-grounded exam answering. ADAPTERS = ( deductive_logic_result, relational_metric_result, @@ -121,6 +159,8 @@ ADAPTERS = ( comprehension_relational_predicate_result, comprehension_relational_inference_result, comprehension_relational_transitive_result, + deduction_serve_existential_result, + curriculum_serve_result, ) diff --git a/evals/capability_index/baseline.json b/evals/capability_index/baseline.json index 09147763..e4b1d04e 100644 --- a/evals/capability_index/baseline.json +++ b/evals/capability_index/baseline.json @@ -1,13 +1,13 @@ { - "capability_score": 0.953968, - "coverage_geomean": 0.953968, - "coverage_micro": 0.99399, + "capability_score": 0.960909, + "coverage_geomean": 0.960909, + "coverage_micro": 0.99442, "accuracy_micro": 1.0, - "breadth": 11, + "breadth": 13, "min_domain_coverage": 0.833333, "wrong_total": 0, "assert_mode_valid": true, - "deterministic_digest": "85c873626423c6b151a7d61bebb0cd5b7caff7dcc7e897188609e5ce3313feac", + "deterministic_digest": "27778ecb51b8acbd7e14a8bc23fca6869328c2236c38296bb4b1ad3311daf9eb", "domains": [ { "domain": "comprehension_propositional", @@ -73,6 +73,22 @@ "coverage": 0.875, "accuracy": 1.0 }, + { + "domain": "curriculum_serve", + "correct": 32, + "wrong": 0, + "refused": 0, + "coverage": 1.0, + "accuracy": 1.0 + }, + { + "domain": "deduction_serve_existential", + "correct": 32, + "wrong": 0, + "refused": 0, + "coverage": 1.0, + "accuracy": 1.0 + }, { "domain": "deductive_logic", "correct": 716, diff --git a/tests/test_capability_index.py b/tests/test_capability_index.py index 02efdead..d7184773 100644 --- a/tests/test_capability_index.py +++ b/tests/test_capability_index.py @@ -91,8 +91,10 @@ def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None: # (prose -> MeaningGraph -> projection -> independent oracle / pack-named predicate) # compose into the cross-domain index with zero wrong commits. The one-hop # relational_inference lane (#775) is the 10th domain; the transitive - # relational_transitive lane (B2) is the 11th. (This assertion was stale at 9 from - # the #596 era — #775's 10th domain was never added here; corrected with B2.) + # relational_transitive lane (B2) is the 11th. deduction_serve_existential + # (ADR-0261, Band v6-EX) is the 12th; curriculum_serve (ADR-0262, generalization + # arc Tier S) is the 13th. (This assertion was stale at 9 from the #596 era — + # #775's 10th domain was never added here; corrected with B2, then again here.) from evals.capability_index.adapters import collect_domain_results collection = collect_domain_results() @@ -100,7 +102,7 @@ def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None: idx = aggregate(list(collection.results)) assert idx.wrong_total == 0 assert idx.assert_mode_valid - assert idx.breadth == 11 + assert idx.breadth == 13 assert {d.domain for d in idx.domains} == { "deductive_logic", "dimensional", @@ -113,6 +115,8 @@ def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None: "comprehension_relational_predicate", "comprehension_relational_inference", "comprehension_relational_transitive", + "deduction_serve_existential", + "curriculum_serve", } assert idx.capability_score > 0.5 # real, non-trivial cross-domain capability diff --git a/tests/test_cli_test_suites.py b/tests/test_cli_test_suites.py index b04f2e59..bd3c46e1 100644 --- a/tests/test_cli_test_suites.py +++ b/tests/test_cli_test_suites.py @@ -121,6 +121,14 @@ def test_core_test_fast_suite_expands_to_iteration_lane(monkeypatch) -> None: def test_core_test_deductive_suite_expands_to_entailment_lane(monkeypatch) -> None: + """Contains-style, not exact-tuple: the ``deductive`` suite has grown + across the deduction-serve/curriculum-serve arcs (v1 through v6-EX, + curriculum, the ratified-ledger bridge, the vocab-trigger instrument) and + keeps growing with each new band — an exact-tuple pin here would go stale + every time, which is exactly what happened before this fix (a pinned + single-file tuple, silently wrong once the suite reached 12+ files, red + outside every real gate — the same leaky-gate class the smoke-suite + coverage pin above exists to prevent).""" calls: list[tuple[str, ...]] = [] def fake_run(*args: str, check: bool = False, cwd=None) -> int: @@ -132,13 +140,12 @@ def test_core_test_deductive_suite_expands_to_entailment_lane(monkeypatch) -> No rc = cli.main(["test", "--suite", "deductive", "-q"]) assert rc == 0 - assert calls[0] == ( - cli.sys.executable, - "-m", - "pytest", - "tests/test_deductive_logic_entail.py", - "-q", - ) + command = calls[0] + assert command[:3] == (cli.sys.executable, "-m", "pytest") + assert "tests/test_deductive_logic_entail.py" in command + assert "tests/test_deduction_serve_lane.py" in command + assert "tests/test_curriculum_serve.py" in command + assert "-q" in command def test_core_test_suite_accepts_pytest_flags_without_separator(monkeypatch) -> None: