From e1fb715eec8484e2b1b845e5c4babc6461f1e4b7 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 17 Jun 2026 14:33:33 -0700 Subject: [PATCH] fix(gsm8k): soften microscope live pins and hygiene for PR #806 Remove exact live top-bucket count assertions per #804 doctrine; keep structural invariants and case 0002 Gate A2a seed pin. Fix EOF newlines and duplicate _PRIMITIVE_BY_SUBFAMILY key. --- ...olving-capability-roadmap-v2-2026-06-17.md | 2 +- .../gsm8k_post_gate_a1_frontier_microscope.py | 1 - tests/test_gsm8k_frontier_report.py | 8 ++- ..._gsm8k_post_gate_a1_frontier_microscope.py | 64 ++++++++++--------- 4 files changed, 41 insertions(+), 34 deletions(-) diff --git a/docs/analysis/core-problem-solving-capability-roadmap-v2-2026-06-17.md b/docs/analysis/core-problem-solving-capability-roadmap-v2-2026-06-17.md index 05ed1910..fa4eb2dc 100644 --- a/docs/analysis/core-problem-solving-capability-roadmap-v2-2026-06-17.md +++ b/docs/analysis/core-problem-solving-capability-roadmap-v2-2026-06-17.md @@ -80,4 +80,4 @@ Additive comparative is **not** roadmap Gate A2. Roadmap Gate A2 remains partiti 1. **Ratify** Gate A2a unit partition / chunking primitive (shared narrow primitive — see lookback evidence). Implementation follows ratification only. 2. Preserve wrong=0 and monotonic counts on every future slice. -3. Gate A1b / Comparative-A2 (additive comparative) and Inc4 denom-state remain deferred until separately ratified. \ No newline at end of file +3. Gate A1b / Comparative-A2 (additive comparative) and Inc4 denom-state remain deferred until separately ratified. diff --git a/scripts/gsm8k_post_gate_a1_frontier_microscope.py b/scripts/gsm8k_post_gate_a1_frontier_microscope.py index 01d7b423..e6b2e1a6 100644 --- a/scripts/gsm8k_post_gate_a1_frontier_microscope.py +++ b/scripts/gsm8k_post_gate_a1_frontier_microscope.py @@ -127,7 +127,6 @@ _PRIMITIVE_BY_SUBFAMILY: dict[str, str] = { "numeric_expression_duration": "duration_multiplier", "numeric_expression_recurrence": "recurrence_frame", "partition_statement_parser_gap": "unit_partition", - "affine_equation_fraction_target": "fractional_delta", "unclassified": "diagnostic_hold", "inverse_residual_more": "inverse_residual_question", "ratio_times_entity": "ratio_question_frame", diff --git a/tests/test_gsm8k_frontier_report.py b/tests/test_gsm8k_frontier_report.py index aab6faf7..0a031a5c 100644 --- a/tests/test_gsm8k_frontier_report.py +++ b/tests/test_gsm8k_frontier_report.py @@ -215,8 +215,8 @@ def test_markdown_render_is_stable_and_mentions_rate(): assert render_markdown(summary) == md -def test_frontier_report_aligns_with_post_gate_a1_microscope_top_buckets(): - """Live ephemeral top buckets match the post-Gate-A1 microscope partition.""" +def test_frontier_report_aligns_with_post_gate_a1_microscope_structure(): + """Live microscope stays structurally complete with closed injector buckets.""" from scripts.gsm8k_post_gate_a1_frontier_microscope import build_microscope_report cases_path = _REPO_ROOT / "evals/gsm8k_math/train_sample/v1/cases.jsonl" @@ -227,7 +227,9 @@ def test_frontier_report_aligns_with_post_gate_a1_microscope_top_buckets(): ] microscope = build_microscope_report(cases) assert_monotonic_serving_counts(microscope["counts"]) - assert microscope["top_buckets"]["recognized_no_injection"] == 31 + refused = microscope["counts"]["refused"] + assert len(microscope["refusal_table"]) == refused + assert sum(microscope["top_buckets"].values()) == refused assert microscope["closed_injector_buckets"]["rate_with_currency_no_injection"] == 0 assert microscope["closed_injector_buckets"]["comparative_with_unit_no_injection"] == 0 diff --git a/tests/test_gsm8k_post_gate_a1_frontier_microscope.py b/tests/test_gsm8k_post_gate_a1_frontier_microscope.py index 73cbecff..5109047a 100644 --- a/tests/test_gsm8k_post_gate_a1_frontier_microscope.py +++ b/tests/test_gsm8k_post_gate_a1_frontier_microscope.py @@ -15,6 +15,29 @@ from tests.gsm8k_train_sample_baseline import assert_monotonic_serving_counts _REPO_ROOT = Path(__file__).resolve().parents[1] _CASES_PATH = _REPO_ROOT / "evals/gsm8k_math/train_sample/v1/cases.jsonl" +_APPROVED_TOP_BUCKETS = frozenset({ + "recognized_no_injection", + "no_admissible_statement", + "no_admissible_question", + "no_solvable_branch", + "incomplete_reading", + "other_refused", + "other", +}) + +_REFUSAL_TABLE_FIELDS = frozenset({ + "case_id", + "verdict", + "reason", + "top_refusal_bucket", + "subfamily", + "matched_recognizer_category", + "first_blocking_layer", + "candidate_next_primitive", + "expected_movement", + "evidence_snippet", +}) + def _load_cases() -> list[dict]: return [ @@ -36,43 +59,26 @@ def test_live_microscope_meets_monotonic_contract_and_closed_injectors(): def test_live_microscope_refusal_partition_is_complete(): summary = build_microscope_report(_load_cases()) - assert summary["counts"]["refused"] == 44 - assert len(summary["per_case"]) == 44 - assert len(summary["refusal_table"]) == 44 - assert sum(summary["top_buckets"].values()) == 44 + refused = summary["counts"]["refused"] + assert len(summary["refusal_table"]) == refused + assert len(summary["per_case"]) == refused + assert sum(summary["top_buckets"].values()) == refused + assert set(summary["top_buckets"]) <= _APPROVED_TOP_BUCKETS + assert summary["top_buckets"].get("recognized_no_injection", 0) >= 0 - required = { - "case_id", - "verdict", - "reason", - "top_refusal_bucket", - "subfamily", - "matched_recognizer_category", - "first_blocking_layer", - "candidate_next_primitive", - "expected_movement", - "evidence_snippet", - } for row in summary["refusal_table"]: - assert required <= set(row.keys()) - - # Post-Gate-A1 stable top-level buckets (ephemeral main @ bb083004 family) - assert summary["top_buckets"]["recognized_no_injection"] == 31 - assert summary["top_buckets"]["no_admissible_statement"] == 7 - assert summary["top_buckets"]["no_admissible_question"] == 5 - assert summary["top_buckets"]["no_solvable_branch"] == 1 + assert _REFUSAL_TABLE_FIELDS <= set(row.keys()) + assert row["verdict"] == "refused" + assert row["reason"] + assert row["evidence_snippet"] -def test_live_microscope_dcs_and_slice_candidate_pins(): +def test_live_microscope_partition_seed_case_is_tagged(): summary = build_microscope_report(_load_cases()) - - assert summary["recognized_no_injection_by_category"]["discrete_count_statement"] == 19 - assert summary["dcs_no_injection_subfamilies"]["dcs_misroute_unit_partition"] == 1 assert ( "gsm8k-train-sample-v1-0002" in summary["implementation_slice_candidates"]["partition_chunking"]["case_ids"] ) - assert summary["no_solvable_branch_subfamilies"]["rate_graph_unsolvable"] == 1 def test_microscope_output_is_deterministic(): @@ -112,4 +118,4 @@ def test_case_0002_ratification_candidate_fields(): assert ( summary["recommended_next_ratification_candidate"] == "Gate A2a unit_partition / chunking primitive" - ) \ No newline at end of file + )