fix(gsm8k): soften microscope live pins and hygiene for PR #806

Remove exact live top-bucket count assertions per #804 doctrine; keep
structural invariants and case 0002 Gate A2a seed pin. Fix EOF newlines
and duplicate _PRIMITIVE_BY_SUBFAMILY key.
This commit is contained in:
Shay 2026-06-17 14:33:33 -07:00
parent 51e6196133
commit e1fb715eec
4 changed files with 41 additions and 34 deletions

View file

@ -80,4 +80,4 @@ Additive comparative is **not** roadmap Gate A2. Roadmap Gate A2 remains partiti
1. **Ratify** Gate A2a unit partition / chunking primitive (shared narrow primitive — see lookback evidence). Implementation follows ratification only.
2. Preserve wrong=0 and monotonic counts on every future slice.
3. Gate A1b / Comparative-A2 (additive comparative) and Inc4 denom-state remain deferred until separately ratified.
3. Gate A1b / Comparative-A2 (additive comparative) and Inc4 denom-state remain deferred until separately ratified.

View file

@ -127,7 +127,6 @@ _PRIMITIVE_BY_SUBFAMILY: dict[str, str] = {
"numeric_expression_duration": "duration_multiplier",
"numeric_expression_recurrence": "recurrence_frame",
"partition_statement_parser_gap": "unit_partition",
"affine_equation_fraction_target": "fractional_delta",
"unclassified": "diagnostic_hold",
"inverse_residual_more": "inverse_residual_question",
"ratio_times_entity": "ratio_question_frame",

View file

@ -215,8 +215,8 @@ def test_markdown_render_is_stable_and_mentions_rate():
assert render_markdown(summary) == md
def test_frontier_report_aligns_with_post_gate_a1_microscope_top_buckets():
"""Live ephemeral top buckets match the post-Gate-A1 microscope partition."""
def test_frontier_report_aligns_with_post_gate_a1_microscope_structure():
"""Live microscope stays structurally complete with closed injector buckets."""
from scripts.gsm8k_post_gate_a1_frontier_microscope import build_microscope_report
cases_path = _REPO_ROOT / "evals/gsm8k_math/train_sample/v1/cases.jsonl"
@ -227,7 +227,9 @@ def test_frontier_report_aligns_with_post_gate_a1_microscope_top_buckets():
]
microscope = build_microscope_report(cases)
assert_monotonic_serving_counts(microscope["counts"])
assert microscope["top_buckets"]["recognized_no_injection"] == 31
refused = microscope["counts"]["refused"]
assert len(microscope["refusal_table"]) == refused
assert sum(microscope["top_buckets"].values()) == refused
assert microscope["closed_injector_buckets"]["rate_with_currency_no_injection"] == 0
assert microscope["closed_injector_buckets"]["comparative_with_unit_no_injection"] == 0

View file

@ -15,6 +15,29 @@ from tests.gsm8k_train_sample_baseline import assert_monotonic_serving_counts
_REPO_ROOT = Path(__file__).resolve().parents[1]
_CASES_PATH = _REPO_ROOT / "evals/gsm8k_math/train_sample/v1/cases.jsonl"
_APPROVED_TOP_BUCKETS = frozenset({
"recognized_no_injection",
"no_admissible_statement",
"no_admissible_question",
"no_solvable_branch",
"incomplete_reading",
"other_refused",
"other",
})
_REFUSAL_TABLE_FIELDS = frozenset({
"case_id",
"verdict",
"reason",
"top_refusal_bucket",
"subfamily",
"matched_recognizer_category",
"first_blocking_layer",
"candidate_next_primitive",
"expected_movement",
"evidence_snippet",
})
def _load_cases() -> list[dict]:
return [
@ -36,43 +59,26 @@ def test_live_microscope_meets_monotonic_contract_and_closed_injectors():
def test_live_microscope_refusal_partition_is_complete():
summary = build_microscope_report(_load_cases())
assert summary["counts"]["refused"] == 44
assert len(summary["per_case"]) == 44
assert len(summary["refusal_table"]) == 44
assert sum(summary["top_buckets"].values()) == 44
refused = summary["counts"]["refused"]
assert len(summary["refusal_table"]) == refused
assert len(summary["per_case"]) == refused
assert sum(summary["top_buckets"].values()) == refused
assert set(summary["top_buckets"]) <= _APPROVED_TOP_BUCKETS
assert summary["top_buckets"].get("recognized_no_injection", 0) >= 0
required = {
"case_id",
"verdict",
"reason",
"top_refusal_bucket",
"subfamily",
"matched_recognizer_category",
"first_blocking_layer",
"candidate_next_primitive",
"expected_movement",
"evidence_snippet",
}
for row in summary["refusal_table"]:
assert required <= set(row.keys())
# Post-Gate-A1 stable top-level buckets (ephemeral main @ bb083004 family)
assert summary["top_buckets"]["recognized_no_injection"] == 31
assert summary["top_buckets"]["no_admissible_statement"] == 7
assert summary["top_buckets"]["no_admissible_question"] == 5
assert summary["top_buckets"]["no_solvable_branch"] == 1
assert _REFUSAL_TABLE_FIELDS <= set(row.keys())
assert row["verdict"] == "refused"
assert row["reason"]
assert row["evidence_snippet"]
def test_live_microscope_dcs_and_slice_candidate_pins():
def test_live_microscope_partition_seed_case_is_tagged():
summary = build_microscope_report(_load_cases())
assert summary["recognized_no_injection_by_category"]["discrete_count_statement"] == 19
assert summary["dcs_no_injection_subfamilies"]["dcs_misroute_unit_partition"] == 1
assert (
"gsm8k-train-sample-v1-0002"
in summary["implementation_slice_candidates"]["partition_chunking"]["case_ids"]
)
assert summary["no_solvable_branch_subfamilies"]["rate_graph_unsolvable"] == 1
def test_microscope_output_is_deterministic():
@ -112,4 +118,4 @@ def test_case_0002_ratification_candidate_fields():
assert (
summary["recommended_next_ratification_candidate"]
== "Gate A2a unit_partition / chunking primitive"
)
)