From 9da61b96a019f57caf9c8172f530508da1e715b4 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 27 May 2026 06:44:46 -0700 Subject: [PATCH] feat(ADR-0167/W2-A): audit-to-evidence adapter (#352) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wave 2, parallel with W2-B/C/D. Implements the type-A→type-B converter from AuditRow to MathReaderRefusalEvidence per ADR-0167 W2-A brief. Deliverables: - teaching/math_contemplation.py: - audit_to_evidence(audit_rows): pure deterministic adapter, uses SUB_TYPE_FOR_OPERATOR for subtype assignment, skips rows where missing_operator is None, leaves claim_signature="" (W2-B will fill) - audit_problem_to_evidence(problem_text, case_id): convenience wrapper that runs the reader and adapts the output - tests/test_math_contemplation_adapter.py: 8 tests covering determinism, input-order preservation, sub-type mapping exhaustiveness, distinct hashes across cases, empty input handling, None-operator skip, and round-trip from problem text Invariants: - Deterministic across reruns (verified by determinism rerun) - No I/O in adapter path - Input order preserved (no internal sort) - claim_signature == "" for all W2-A records (W2-B coordination) Validation: - tests/test_math_contemplation_adapter.py: 8 passed - tests/test_math_evidence_schema.py: 11 passed (W1-A regression) - tests/test_brief_11b_audit_artifact.py + step2_lexicon + brief_11_audit: 45 passed (regression) - Determinism rerun: identical results --- teaching/math_contemplation.py | 47 +++++++ tests/test_math_contemplation_adapter.py | 148 +++++++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 teaching/math_contemplation.py create mode 100644 tests/test_math_contemplation_adapter.py diff --git a/teaching/math_contemplation.py b/teaching/math_contemplation.py new file mode 100644 index 00000000..04973a9c --- /dev/null +++ b/teaching/math_contemplation.py @@ -0,0 +1,47 @@ +"""ADR-0167 W2-A — audit-to-evidence adapter. + +Pure deterministic conversion from comprehension audit rows into typed +``MathReaderRefusalEvidence`` records for the teaching corridor. +""" + +from __future__ import annotations + +from typing import Iterable + +from generate.comprehension.audit import AuditRow, audit_problem +from teaching.math_evidence import ( + MathReaderRefusalEvidence, + SUB_TYPE_FOR_OPERATOR, + from_audit_row, +) + + +def audit_to_evidence( + audit_rows: Iterable[AuditRow], +) -> tuple[MathReaderRefusalEvidence, ...]: + """Convert audit rows into typed teaching-corridor evidence records. + + Pure function. Deterministic. No filesystem, no network, no mutation. + Sub-type assignment from ``teaching.math_evidence.SUB_TYPE_FOR_OPERATOR``. + Skips rows with ``missing_operator=None`` (no sub_type → no candidate). + """ + out: list[MathReaderRefusalEvidence] = [] + for row in audit_rows: + if row.missing_operator is None: + continue + sub_type = SUB_TYPE_FOR_OPERATOR[row.missing_operator] + out.append(from_audit_row(row, sub_type, claim_signature="")) + return tuple(out) + + +def audit_problem_to_evidence( + problem_text: str, + *, + case_id: str, +) -> tuple[MathReaderRefusalEvidence, ...]: + """Run reader audit on ``problem_text`` and return mapped evidence rows.""" + _result, rows = audit_problem(problem_text, case_id=case_id) + return audit_to_evidence(rows) + + +__all__ = ["audit_problem_to_evidence", "audit_to_evidence"] diff --git a/tests/test_math_contemplation_adapter.py b/tests/test_math_contemplation_adapter.py new file mode 100644 index 00000000..cfff2ee6 --- /dev/null +++ b/tests/test_math_contemplation_adapter.py @@ -0,0 +1,148 @@ +"""Tests for ADR-0167 W2-A — audit-to-evidence adapter.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from generate.comprehension.audit import AuditRow +from teaching.math_contemplation import audit_problem_to_evidence, audit_to_evidence +from teaching.math_evidence import SUB_TYPE_FOR_OPERATOR + + +def _row_from_artifact_case(case: dict[str, object]) -> AuditRow: + return AuditRow( + case_id=str(case["case_id"]), + sentence_index=int(case["sentence_index"]), + token_index=int(case["token_index"]), + token_text=str(case["token_text"]), + recognized_terms=tuple(str(t) for t in case["recognized_terms"]), + skipped_frame=( + None + if case["skipped_frame"] is None + else str(case["skipped_frame"]) + ), + missing_operator=( + None + if case["missing_operator"] is None + else str(case["missing_operator"]) + ), + refusal_reason=str(case["refusal_reason"]), + refusal_detail=str(case["refusal_detail"]), + ) + + +def _load_artifact_rows() -> list[AuditRow]: + artifact = json.loads( + Path("evals/gsm8k_math/train_sample/v1/audit_brief_11.json").read_text() + ) + return [_row_from_artifact_case(case) for case in artifact["per_case"]] + + +def test_adapter_round_trips_full_audit_artifact() -> None: + rows = _load_artifact_rows() + out = audit_to_evidence(rows) + expected = sum(1 for row in rows if row.missing_operator is not None) + assert len(out) == expected + + +def test_adapter_is_deterministic() -> None: + rows = _load_artifact_rows() + baseline = audit_to_evidence(rows) + for _ in range(10): + assert audit_to_evidence(rows) == baseline + + +def test_skips_none_operator_rows() -> None: + rows = [ + AuditRow( + case_id="a", + sentence_index=0, + token_index=0, + token_text="?", + recognized_terms=(), + skipped_frame=None, + missing_operator=None, + refusal_reason="no_question_target", + refusal_detail="none", + ) + ] + assert audit_to_evidence(rows) == () + + +def test_every_missing_operator_maps_to_a_sub_type() -> None: + rows = _load_artifact_rows() + for row in rows: + if row.missing_operator is not None: + assert row.missing_operator in SUB_TYPE_FOR_OPERATOR + assert len(audit_to_evidence(rows)) > 0 + + +def test_evidence_hashes_distinct_across_cases() -> None: + row1 = AuditRow( + case_id="case-1", + sentence_index=0, + token_index=0, + token_text="crayons", + recognized_terms=("She", "has"), + skipped_frame="descriptive_frame", + missing_operator="lexicon_entry", + refusal_reason="unknown_word", + refusal_detail="no primitive or lexicon match for 'crayons'", + ) + row2 = AuditRow( + case_id="case-2", + sentence_index=0, + token_index=0, + token_text="crayons", + recognized_terms=("She", "has"), + skipped_frame="descriptive_frame", + missing_operator="lexicon_entry", + refusal_reason="unknown_word", + refusal_detail="no primitive or lexicon match for 'crayons'", + ) + out1 = audit_to_evidence([row1])[0] + out2 = audit_to_evidence([row2])[0] + assert out1.evidence_hash != out2.evidence_hash + + +def test_empty_audit_returns_empty_tuple() -> None: + assert audit_to_evidence([]) == () + + +def test_adapter_preserves_input_order() -> None: + rows = [ + AuditRow( + case_id="x-1", + sentence_index=0, + token_index=1, + token_text="a", + recognized_terms=("one",), + skipped_frame="initial_state", + missing_operator="lexicon_entry", + refusal_reason="unknown_word", + refusal_detail="a", + ), + AuditRow( + case_id="x-2", + sentence_index=1, + token_index=2, + token_text="b", + recognized_terms=("two",), + skipped_frame="operation_frame", + missing_operator="question_target_slot", + refusal_reason="no_question_target", + refusal_detail="b", + ), + ] + out = audit_to_evidence(rows) + assert [ev.case_id for ev in out] == ["x-1", "x-2"] + + +def test_audit_problem_to_evidence_round_trip() -> None: + out = audit_problem_to_evidence( + "Tom opens an amusement park. How many days?", + case_id="probe", + ) + assert len(out) >= 1 + assert all(ev.case_id == "probe" for ev in out)