From a45eab1fe3a017205c4eb2301d7d8b0c69898c7b Mon Sep 17 00:00:00 2001 From: Shay Date: Sun, 24 May 2026 11:20:32 -0700 Subject: [PATCH] fix(epistemic): Phase 2 known bug repairs (#219) * fix(epistemic): make empty resonance evidence undetermined * fix(evals): classify verified realizer failures separately * fix(packs): treat absent domain manifests as valid noop * test(packs): cover missing manifests and scope boundary domains * test(epistemic): cover phase 2 known bug fixes * fix(vault): make FALSIFIED exclusion explicit in _status_admits FALSIFIED entries previously fell through to the ADMISSIBLE_AS_EVIDENCE set-check, which excluded them correctly but left the distinction between CONTRADICTED (FALSIFIED) and UNVERIFIED-POSSIBLE (SPECULATIVE) implicit. Add an early guard so FALSIFIED is explicitly rejected before the tier filter, matching the CONTRADICTED semantics from the epistemic taxonomy. --- evals/gsm8k_math/runner.py | 81 +++++++++++++++++------ language_packs/domain_contract.py | 6 +- language_packs/evidence.py | 14 +++- tests/test_domain_pack_contract.py | 14 +++- tests/test_epistemic_phase2_known_bugs.py | 78 ++++++++++++++++++++++ vault/store.py | 13 ++-- 6 files changed, 174 insertions(+), 32 deletions(-) create mode 100644 tests/test_epistemic_phase2_known_bugs.py diff --git a/evals/gsm8k_math/runner.py b/evals/gsm8k_math/runner.py index 85506254..33e805f1 100644 --- a/evals/gsm8k_math/runner.py +++ b/evals/gsm8k_math/runner.py @@ -1,7 +1,8 @@ """ADR-0119.3 — GSM8K math eval lane runner. Composes the Phases 1-4 pipeline (parser → solver → verifier → realizer) -into a per-case scoring decision: ``correct`` / ``wrong`` / ``refused``. +into a per-case scoring decision: ``correct`` / ``wrong`` / ``refused`` / +``decoded_unarticulated``. Outcome categorization (ADR-0114a Obligation #4 — the load-bearing "refusal is first-class; misparse rate zero" discipline): @@ -11,12 +12,15 @@ Outcome categorization (ADR-0114a Obligation #4 — the load-bearing | ``parse_problem(text)`` raised ``ParseError`` | refused | typed parser error | | ``solve(graph)`` raised ``SolveError`` | refused | typed solver error | | ``verify(graph, trace)`` returned ``passed=False`` | wrong | verifier reason | +| ``realize(graph.initial_state, trace)`` raised ``RealizerError`` after verifier pass | decoded_unarticulated | typed realizer error | | Everything succeeds AND ``trace.answer_value == expected_answer`` AND ``trace.answer_unit == expected_unit`` | correct | empty | | Everything succeeds BUT answer or unit differs | wrong | "answer/unit mismatch" | **`wrong == 0` is the gate** — ADR-0114a Obligation #4 requires CORE to refuse rather than confabulate. A nonzero ``wrong`` count -invalidates the lane regardless of ``correct`` rate. +invalidates the lane regardless of ``correct`` rate. A verified trace +whose surface realization fails is not a wrong answer; it is counted as +``decoded_unarticulated``. The runner is pure / deterministic: same case set → same :class:`LaneReport.canonical_bytes()`. @@ -36,13 +40,15 @@ from generate.math_realizer import RealizerError, realize from generate.math_solver import SolveError, solve from generate.math_verifier import verify +DECODED_UNARTICULATED_OUTCOME = "decoded_unarticulated" + @dataclass(frozen=True, slots=True) class CaseOutcome: """Per-case scoring decision with full audit trail.""" case_id: str - outcome: str # "correct" | "wrong" | "refused" + outcome: str # "correct" | "wrong" | "refused" | "decoded_unarticulated" reason: str expected_answer: float expected_unit: str @@ -82,6 +88,29 @@ class LaneReport: return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") +def _decoded_unarticulated_outcome( + *, + case_id: str, + reason: str, + expected_answer: float, + expected_unit: str, + actual_answer: float, + actual_unit: str, + trace_hash: str, +) -> CaseOutcome: + return CaseOutcome( + case_id=case_id, + outcome=DECODED_UNARTICULATED_OUTCOME, + reason=reason, + expected_answer=expected_answer, + expected_unit=expected_unit, + actual_answer=actual_answer, + actual_unit=actual_unit, + trace_hash=trace_hash, + realized_prose=None, + ) + + def _score_one(case: dict[str, Any]) -> CaseOutcome: """Run the full pipeline against one case and classify the outcome.""" case_id = case["id"] @@ -137,22 +166,20 @@ def _score_one(case: dict[str, Any]) -> CaseOutcome: realized_prose=None, ) - # Stage 4 — realize (failures here are treated as wrong, not refused, - # because the trace already verified) + # Stage 4 — realize. A failure here happens after replay verification, + # so the answer remains DECODED; only the articulation surface failed. try: realized = realize(graph.initial_state, trace) prose = realized.as_prose() except RealizerError as exc: - return CaseOutcome( + return _decoded_unarticulated_outcome( case_id=case_id, - outcome="wrong", reason=f"realizer: {exc}", expected_answer=expected_answer, expected_unit=expected_unit, actual_answer=trace.answer_value, actual_unit=trace.answer_unit, trace_hash=trace_hash, - realized_prose=None, ) # Stage 5 — compare against expected. @@ -281,21 +308,20 @@ def _score_one_candidate_graph(case: dict[str, Any]) -> CaseOutcome: realized_prose=None, ) - # Stage 4 — realize. + # Stage 4 — realize. A failure here happens after replay verification, + # so the answer remains DECODED; only the articulation surface failed. try: realized = realize(graph.initial_state, trace) prose = realized.as_prose() except RealizerError as exc: - return CaseOutcome( + return _decoded_unarticulated_outcome( case_id=case_id, - outcome="wrong", reason=f"realizer: {exc}", expected_answer=expected_answer, expected_unit=expected_unit, actual_answer=trace.answer_value, actual_unit=trace.answer_unit, trace_hash=trace_hash, - realized_prose=None, ) # Stage 5 — expected-answer comparison (same logic as _score_one). @@ -355,15 +381,17 @@ def run_lane( two calls with the same input list. Aggregate metrics: - cases_total int - correct int - wrong int (gate: must == 0) - refused int - correct_rate float = correct / total - wrong_rate float = wrong / total - refused_rate float = refused / total - wrong_count_is_zero bool = wrong == 0 - overall_pass bool = wrong == 0 AND correct + refused == total + cases_total int + correct int + wrong int (gate: must == 0) + refused int + decoded_unarticulated int + correct_rate float = correct / total + wrong_rate float = wrong / total + refused_rate float = refused / total + decoded_unarticulated_rate float = decoded_unarticulated / total + wrong_count_is_zero bool = wrong == 0 + overall_pass bool = wrong == 0 AND correct + refused + decoded_unarticulated == total """ outcomes = [_score_one(c) for c in cases] @@ -371,18 +399,27 @@ def run_lane( correct = sum(1 for o in outcomes if o.outcome == "correct") wrong = sum(1 for o in outcomes if o.outcome == "wrong") refused = sum(1 for o in outcomes if o.outcome == "refused") + decoded_unarticulated = sum( + 1 for o in outcomes if o.outcome == DECODED_UNARTICULATED_OUTCOME + ) wrong_count_is_zero = wrong == 0 - overall_pass = wrong_count_is_zero and (correct + refused == total) + overall_pass = wrong_count_is_zero and ( + correct + refused + decoded_unarticulated == total + ) metrics = { "cases_total": total, "correct": correct, "wrong": wrong, "refused": refused, + "decoded_unarticulated": decoded_unarticulated, "correct_rate": (correct / total) if total else 0.0, "wrong_rate": (wrong / total) if total else 0.0, "refused_rate": (refused / total) if total else 0.0, + "decoded_unarticulated_rate": ( + decoded_unarticulated / total + ) if total else 0.0, "wrong_count_is_zero": wrong_count_is_zero, "overall_pass": overall_pass, } diff --git a/language_packs/domain_contract.py b/language_packs/domain_contract.py index b681bbae..2b90f80e 100644 --- a/language_packs/domain_contract.py +++ b/language_packs/domain_contract.py @@ -20,6 +20,7 @@ _KNOWN_DOMAIN_IDS = { "hebrew_greek_textual_reasoning", "philosophy_theology", } +_SCOPE_BOUNDARY_PREFIX = "scope_boundary" @dataclass(frozen=True, slots=True) @@ -125,7 +126,7 @@ def parse_domain_contract(manifest: dict[str, Any], *, pack_id: str) -> DomainCo else: domain_id_s = domain_id.strip() if domain_id_s not in _KNOWN_DOMAIN_IDS: - errors.append("domain_id:unknown") + errors.append(f"{_SCOPE_BOUNDARY_PREFIX}:domain_id:unknown") axioms = _optional_path(manifest.get("axioms"), field_name="axioms", errors=errors) rules = _optional_path(manifest.get("rules"), field_name="rules", errors=errors) @@ -195,8 +196,7 @@ def validate_domain_contract_pack(pack_id: str, *, data_root: Path | None = None return DomainContractValidation( pack_id=pack_id, present=False, - valid=False, - errors=("manifest:not_found",), + valid=True, ) try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) diff --git a/language_packs/evidence.py b/language_packs/evidence.py index e37483c3..1fc0a91c 100644 --- a/language_packs/evidence.py +++ b/language_packs/evidence.py @@ -10,6 +10,16 @@ from algebra.cga import cga_inner from algebra.holonomy import holonomy_encode +UNDETERMINED_SCORE: float = float("nan") +"""Numeric sentinel for evidence that could not be computed. + +An empty evidence-pair set is not neutral evidence. Returning ``0.0`` +made "no evidence" indistinguishable from a real measured zero. ``NaN`` +keeps the return type stable while forcing callers to treat the score as +UNDETERMINED rather than as weak/negative evidence. +""" + + @dataclass(frozen=True, slots=True) class ResonanceEvidence: case_id: str @@ -18,6 +28,8 @@ class ResonanceEvidence: @property def passes(self) -> bool: + if not np.isfinite(self.aligned_score) or not np.isfinite(self.contrast_score): + return False return self.aligned_score > self.contrast_score @@ -27,7 +39,7 @@ def encode_clause(manifold, tokens: tuple[str, ...] | list[str]) -> np.ndarray: def mean_pair_score(manifold, pairs: tuple[tuple[str, str], ...]) -> float: if not pairs: - return 0.0 + return UNDETERMINED_SCORE return float( np.mean( [ diff --git a/tests/test_domain_pack_contract.py b/tests/test_domain_pack_contract.py index 58993a57..37ce59a5 100644 --- a/tests/test_domain_pack_contract.py +++ b/tests/test_domain_pack_contract.py @@ -13,6 +13,15 @@ def test_absent_domain_contract_is_valid_noop() -> None: assert result.contract is None +def test_missing_manifest_is_valid_absent_noop(tmp_path) -> None: + result = validate_domain_contract_pack("missing_pack", data_root=tmp_path) + + assert result.present is False + assert result.valid is True + assert result.errors == () + assert result.contract is None + + def test_valid_domain_contract_parses_optional_axioms_rules() -> None: result = parse_domain_contract( { @@ -43,7 +52,7 @@ def test_valid_domain_contract_parses_optional_axioms_rules() -> None: assert result.contract.rules is None -def test_domain_contract_rejects_unsafe_paths_and_unknown_domain() -> None: +def test_domain_contract_rejects_unsafe_paths_and_marks_unknown_domain_scope_boundary() -> None: result = parse_domain_contract( { "domain_contract_version": 1, @@ -56,7 +65,8 @@ def test_domain_contract_rejects_unsafe_paths_and_unknown_domain() -> None: ) assert result.valid is False - assert "domain_id:unknown" in result.errors + assert "scope_boundary:domain_id:unknown" in result.errors + assert "domain_id:unknown" not in result.errors assert "axioms:unsafe_path" in result.errors assert "rules:unsafe_path" in result.errors assert "provenance:required" in result.errors diff --git a/tests/test_epistemic_phase2_known_bugs.py b/tests/test_epistemic_phase2_known_bugs.py new file mode 100644 index 00000000..3802f4e6 --- /dev/null +++ b/tests/test_epistemic_phase2_known_bugs.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import math + +import numpy as np +import pytest + +from evals.gsm8k_math import runner as gsm8k_runner +from generate.math_realizer import RealizerError +from language_packs.evidence import mean_pair_score, resonance_evidence + + +class _DummyManifold: + def get_versor(self, token: str) -> np.ndarray: + return np.ones(32, dtype=np.float32) + + +def test_mean_pair_score_empty_is_undetermined_nan() -> None: + score = mean_pair_score(_DummyManifold(), ()) + + assert math.isnan(score) + + +def test_resonance_evidence_empty_pairs_do_not_pass() -> None: + evidence = resonance_evidence( + case_id="empty", + manifold=_DummyManifold(), + aligned_pairs=(), + contrast_pairs=(), + ) + + assert math.isnan(evidence.aligned_score) + assert math.isnan(evidence.contrast_score) + assert evidence.passes is False + + +def test_verified_trace_realizer_error_is_decoded_unarticulated(monkeypatch: pytest.MonkeyPatch) -> None: + case = { + "id": "realizer-breaks-after-verify", + "problem": "Sam has 2 apples. How many apples does Sam have?", + "expected_answer": 2.0, + "expected_unit": "apples", + } + + def fail_realize(*args, **kwargs): + raise RealizerError("forced articulation failure") + + monkeypatch.setattr(gsm8k_runner, "realize", fail_realize) + + outcome = gsm8k_runner._score_one(case) + + assert outcome.outcome == gsm8k_runner.DECODED_UNARTICULATED_OUTCOME + assert outcome.reason == "realizer: forced articulation failure" + assert outcome.actual_answer == 2.0 + assert outcome.actual_unit == "apples" + assert outcome.trace_hash is not None + assert outcome.realized_prose is None + + +def test_decoded_unarticulated_does_not_increment_wrong(monkeypatch: pytest.MonkeyPatch) -> None: + case = { + "id": "realizer-breaks-after-verify", + "problem": "Sam has 2 apples. How many apples does Sam have?", + "expected_answer": 2.0, + "expected_unit": "apples", + } + + def fail_realize(*args, **kwargs): + raise RealizerError("forced articulation failure") + + monkeypatch.setattr(gsm8k_runner, "realize", fail_realize) + + report = gsm8k_runner.run_lane([case]) + + assert report.metrics["wrong"] == 0 + assert report.metrics["decoded_unarticulated"] == 1 + assert report.metrics["wrong_count_is_zero"] is True + assert report.metrics["overall_pass"] is True diff --git a/vault/store.py b/vault/store.py index 874c504c..4cc0b887 100644 --- a/vault/store.py +++ b/vault/store.py @@ -27,11 +27,16 @@ def _versor_key(F: np.ndarray) -> bytes: def _status_admits(entry_status: EpistemicStatus, min_status: EpistemicStatus) -> bool: """Return True iff `entry_status` is admissible at the `min_status` tier. - Today the only meaningful tier-filter is `min_status=COHERENT`, which - means "must be in ADMISSIBLE_AS_EVIDENCE." CONTESTED, SPECULATIVE, - and FALSIFIED entries are excluded. If the admissibility set grows - in the future (it should not, per ADR-0021), only this helper changes. + FALSIFIED entries are never admissible as evidence regardless of the + requested tier — they carry CONTRADICTED semantics and are retained only + for provenance and Stage-3 inversion (ADR-0021 §3). SPECULATIVE entries + are separately excluded at the COHERENT tier (UNVERIFIED-POSSIBLE semantics + — not yet coherent, but distinct from actively falsified). If the + admissibility set grows in the future (it should not, per ADR-0021), only + this helper changes. """ + if entry_status is EpistemicStatus.FALSIFIED: + return False # CONTRADICTED — never evidence regardless of requested tier if min_status is EpistemicStatus.COHERENT: return entry_status in ADMISSIBLE_AS_EVIDENCE return entry_status is min_status