fix(gsm8k): soften microscope live pins and hygiene for PR #806
Remove exact live top-bucket count assertions per #804 doctrine; keep structural invariants and case 0002 Gate A2a seed pin. Fix EOF newlines and duplicate _PRIMITIVE_BY_SUBFAMILY key.
This commit is contained in:
parent
51e6196133
commit
e1fb715eec
4 changed files with 41 additions and 34 deletions
|
|
@ -80,4 +80,4 @@ Additive comparative is **not** roadmap Gate A2. Roadmap Gate A2 remains partiti
|
|||
|
||||
1. **Ratify** Gate A2a unit partition / chunking primitive (shared narrow primitive — see lookback evidence). Implementation follows ratification only.
|
||||
2. Preserve wrong=0 and monotonic counts on every future slice.
|
||||
3. Gate A1b / Comparative-A2 (additive comparative) and Inc4 denom-state remain deferred until separately ratified.
|
||||
3. Gate A1b / Comparative-A2 (additive comparative) and Inc4 denom-state remain deferred until separately ratified.
|
||||
|
|
|
|||
|
|
@ -127,7 +127,6 @@ _PRIMITIVE_BY_SUBFAMILY: dict[str, str] = {
|
|||
"numeric_expression_duration": "duration_multiplier",
|
||||
"numeric_expression_recurrence": "recurrence_frame",
|
||||
"partition_statement_parser_gap": "unit_partition",
|
||||
"affine_equation_fraction_target": "fractional_delta",
|
||||
"unclassified": "diagnostic_hold",
|
||||
"inverse_residual_more": "inverse_residual_question",
|
||||
"ratio_times_entity": "ratio_question_frame",
|
||||
|
|
|
|||
|
|
@ -215,8 +215,8 @@ def test_markdown_render_is_stable_and_mentions_rate():
|
|||
assert render_markdown(summary) == md
|
||||
|
||||
|
||||
def test_frontier_report_aligns_with_post_gate_a1_microscope_top_buckets():
|
||||
"""Live ephemeral top buckets match the post-Gate-A1 microscope partition."""
|
||||
def test_frontier_report_aligns_with_post_gate_a1_microscope_structure():
|
||||
"""Live microscope stays structurally complete with closed injector buckets."""
|
||||
from scripts.gsm8k_post_gate_a1_frontier_microscope import build_microscope_report
|
||||
|
||||
cases_path = _REPO_ROOT / "evals/gsm8k_math/train_sample/v1/cases.jsonl"
|
||||
|
|
@ -227,7 +227,9 @@ def test_frontier_report_aligns_with_post_gate_a1_microscope_top_buckets():
|
|||
]
|
||||
microscope = build_microscope_report(cases)
|
||||
assert_monotonic_serving_counts(microscope["counts"])
|
||||
assert microscope["top_buckets"]["recognized_no_injection"] == 31
|
||||
refused = microscope["counts"]["refused"]
|
||||
assert len(microscope["refusal_table"]) == refused
|
||||
assert sum(microscope["top_buckets"].values()) == refused
|
||||
assert microscope["closed_injector_buckets"]["rate_with_currency_no_injection"] == 0
|
||||
assert microscope["closed_injector_buckets"]["comparative_with_unit_no_injection"] == 0
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,29 @@ from tests.gsm8k_train_sample_baseline import assert_monotonic_serving_counts
|
|||
_REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
_CASES_PATH = _REPO_ROOT / "evals/gsm8k_math/train_sample/v1/cases.jsonl"
|
||||
|
||||
_APPROVED_TOP_BUCKETS = frozenset({
|
||||
"recognized_no_injection",
|
||||
"no_admissible_statement",
|
||||
"no_admissible_question",
|
||||
"no_solvable_branch",
|
||||
"incomplete_reading",
|
||||
"other_refused",
|
||||
"other",
|
||||
})
|
||||
|
||||
_REFUSAL_TABLE_FIELDS = frozenset({
|
||||
"case_id",
|
||||
"verdict",
|
||||
"reason",
|
||||
"top_refusal_bucket",
|
||||
"subfamily",
|
||||
"matched_recognizer_category",
|
||||
"first_blocking_layer",
|
||||
"candidate_next_primitive",
|
||||
"expected_movement",
|
||||
"evidence_snippet",
|
||||
})
|
||||
|
||||
|
||||
def _load_cases() -> list[dict]:
|
||||
return [
|
||||
|
|
@ -36,43 +59,26 @@ def test_live_microscope_meets_monotonic_contract_and_closed_injectors():
|
|||
def test_live_microscope_refusal_partition_is_complete():
|
||||
summary = build_microscope_report(_load_cases())
|
||||
|
||||
assert summary["counts"]["refused"] == 44
|
||||
assert len(summary["per_case"]) == 44
|
||||
assert len(summary["refusal_table"]) == 44
|
||||
assert sum(summary["top_buckets"].values()) == 44
|
||||
refused = summary["counts"]["refused"]
|
||||
assert len(summary["refusal_table"]) == refused
|
||||
assert len(summary["per_case"]) == refused
|
||||
assert sum(summary["top_buckets"].values()) == refused
|
||||
assert set(summary["top_buckets"]) <= _APPROVED_TOP_BUCKETS
|
||||
assert summary["top_buckets"].get("recognized_no_injection", 0) >= 0
|
||||
|
||||
required = {
|
||||
"case_id",
|
||||
"verdict",
|
||||
"reason",
|
||||
"top_refusal_bucket",
|
||||
"subfamily",
|
||||
"matched_recognizer_category",
|
||||
"first_blocking_layer",
|
||||
"candidate_next_primitive",
|
||||
"expected_movement",
|
||||
"evidence_snippet",
|
||||
}
|
||||
for row in summary["refusal_table"]:
|
||||
assert required <= set(row.keys())
|
||||
|
||||
# Post-Gate-A1 stable top-level buckets (ephemeral main @ bb083004 family)
|
||||
assert summary["top_buckets"]["recognized_no_injection"] == 31
|
||||
assert summary["top_buckets"]["no_admissible_statement"] == 7
|
||||
assert summary["top_buckets"]["no_admissible_question"] == 5
|
||||
assert summary["top_buckets"]["no_solvable_branch"] == 1
|
||||
assert _REFUSAL_TABLE_FIELDS <= set(row.keys())
|
||||
assert row["verdict"] == "refused"
|
||||
assert row["reason"]
|
||||
assert row["evidence_snippet"]
|
||||
|
||||
|
||||
def test_live_microscope_dcs_and_slice_candidate_pins():
|
||||
def test_live_microscope_partition_seed_case_is_tagged():
|
||||
summary = build_microscope_report(_load_cases())
|
||||
|
||||
assert summary["recognized_no_injection_by_category"]["discrete_count_statement"] == 19
|
||||
assert summary["dcs_no_injection_subfamilies"]["dcs_misroute_unit_partition"] == 1
|
||||
assert (
|
||||
"gsm8k-train-sample-v1-0002"
|
||||
in summary["implementation_slice_candidates"]["partition_chunking"]["case_ids"]
|
||||
)
|
||||
assert summary["no_solvable_branch_subfamilies"]["rate_graph_unsolvable"] == 1
|
||||
|
||||
|
||||
def test_microscope_output_is_deterministic():
|
||||
|
|
@ -112,4 +118,4 @@ def test_case_0002_ratification_candidate_fields():
|
|||
assert (
|
||||
summary["recommended_next_ratification_candidate"]
|
||||
== "Gate A2a unit_partition / chunking primitive"
|
||||
)
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue