From ca3b6011d49402dec76642f1c0770e9a36be715e Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 23 May 2026 10:44:23 -0700 Subject: [PATCH] feat(ADR-0131.1.S): sealed holdout for symbolic equivalence v1 (#173) --- docs/decisions/ADR-0131.1.S-sealed-holdout.md | 61 +++++++ .../v1/sealed_holdout.age | Bin 0 -> 3014 bytes .../v1/sealed_holdout.pubkey | 1 + .../v1/sealed_report.json | 132 ++++++++++++++ .../v1/sealed_runner.py | 169 ++++++++++++++++++ tests/test_adr_0131_1_sealed_holdout.py | 139 ++++++++++++++ 6 files changed, 502 insertions(+) create mode 100644 docs/decisions/ADR-0131.1.S-sealed-holdout.md create mode 100644 evals/math_symbolic_equivalence/v1/sealed_holdout.age create mode 100644 evals/math_symbolic_equivalence/v1/sealed_holdout.pubkey create mode 100644 evals/math_symbolic_equivalence/v1/sealed_report.json create mode 100644 evals/math_symbolic_equivalence/v1/sealed_runner.py create mode 100644 tests/test_adr_0131_1_sealed_holdout.py diff --git a/docs/decisions/ADR-0131.1.S-sealed-holdout.md b/docs/decisions/ADR-0131.1.S-sealed-holdout.md new file mode 100644 index 00000000..af101ba0 --- /dev/null +++ b/docs/decisions/ADR-0131.1.S-sealed-holdout.md @@ -0,0 +1,61 @@ +# ADR-0131.1.S — Sealed Holdout for Benchmark 1 (Symbolic Equivalence v1) + +**Status:** Accepted +**Date:** 2026-05-23 +**Author:** CORE agents + reviewers +**Depends on:** ADR-0131 (composite math expert promotion), ADR-0119.7 (sealed GSM8K test set), #167 (symbolic equivalence v1) + +--- + +## Context + +Benchmark 1 of [ADR-0131](ADR-0131-math-expert-rebench.md), merged in PR #167, tests whether the engine can determine algebraic equivalence of univariate integer polynomials under exact recall. The benchmark has a 100% baseline correct rate on a curated set of 30 public cases in `cases.jsonl`. + +To make the benchmark's claims externally credible, we must prevent the development team from "peeking" at or optimizing against the evaluation cases used to measure expert promotion. Per the methodology proven in [ADR-0119.7](ADR-0119.7-sealed-gsm8k-test.md) for GSM8K, we introduce a sealed holdout split for Benchmark 1 using pyrage age-encryption. + +--- + +## Decision + +We establish the Benchmark 1 sealed holdout split: + +1. **Crypto Keypair**: Generate a dedicated X25519 key pair for the B1 sealed holdout. The public key is stored in the repo at `evals/math_symbolic_equivalence/v1/sealed_holdout.pubkey`. The private key stays off-repo per the seal discipline of ADR-0119.7. +2. **Sealed Holdout Cases**: Hand-curated 14 new evaluation cases strictly within the v1 univariate integer polynomial scope. The categories strictly match the public cases (e.g. `commutative_add`, `distributive`, `square_of_binomial`, `difference_of_squares`, `collect_like_terms`, `zero_cancellation`, `repeated_addition`, `exponent_combine`, `different_constant`, `sign_flipped`, `distributive_miss`, `out_of_scope_variable`, `out_of_scope_division`). +3. **Disjointness Invariant**: The holdout cases are guaranteed to be disjoint from the public cases — they do not duplicate any `case_id` or any `(expression_a, expression_b)` pair. +4. **Sealed File**: The plaintext cases are encrypted with the public key and written to `evals/math_symbolic_equivalence/v1/sealed_holdout.age`. No plaintext representation of the holdout cases resides in the repository. +5. **Sealed Runner**: We ship `evals/math_symbolic_equivalence/v1/sealed_runner.py` which decrypts the sealed file using the age identity file provided by the `CORE_SEALED_KEY` environment variable. If the environment variable is unset or missing, it fails cleanly with an `EnvironmentError`. +6. **Pass Criterion**: The sealed runner writes its results to `sealed_report.json` and gates the split on: + - `wrong == 0` (no confabulation) + - `correct_rate >= 0.95` (statistical threshold matching public splitter) + +--- + +## Security and Trust Boundaries + +Per **Security and Trust Boundaries** in [CLAUDE.md](file:///Users/kaizenpro/Projects/core/CLAUDE.md#L140-L167), we enforce strict trust boundaries on holdout decryption: +- **No plaintext companion files** are written to disk during the runner's execution (only memory-based decryption). +- **No logging or echos** of the decrypted plaintext cases to stdout or stderr are allowed. +- The decryption key `CORE_SEALED_KEY` is never set in standard CI runs; tests requiring decryption gracefully skip when the key is absent to avoid blocking standard pull request flows. + +--- + +## Invariants + +### `adr_0131_1_s_sealed_file_present_and_age_formatted` +`evals/math_symbolic_equivalence/v1/sealed_holdout.age` exists, is non-empty, and begins with the `age-encryption.org/` magic header. + +### `adr_0131_1_s_no_plaintext_leaks` +No plaintext companion files (`cases_plaintext.jsonl`, etc.) exist in the holdout directory. + +### `adr_0131_1_s_runner_refuses_without_key` +The sealed runner fails with `EnvironmentError` when `CORE_SEALED_KEY` is not configured, preventing silent fallback or execution. + +### `adr_0131_1_s_decrypted_disjointness` +When decrypted, all holdout cases have distinct `case_id` values and distinct expression pairs compared to the public `cases.jsonl` set. + +--- + +## Consequences + +- **Credentialed Claims**: Benchmark 1's validation claims are now externally reviewable and immune to developer-level overfitting. +- **Zero-Confabulation Gate**: `wrong == 0` is strictly enforced against the sealed holdout. Any algebraic regression that triggers a wrong verdict on held-out cases rejects the build. diff --git a/evals/math_symbolic_equivalence/v1/sealed_holdout.age b/evals/math_symbolic_equivalence/v1/sealed_holdout.age new file mode 100644 index 0000000000000000000000000000000000000000..503c6fc89f5d54f1d57fac7b47a882b3e5157e10 GIT binary patch literal 3014 zcmV;%3pw;*XJsvAZewzJaCB*JZZ2U-Nl8#NWlDE2ct>=2PDWBrNH_{DJ|KN8Avt<=Uq~%ya%Ew2Wgsa|d}~58ad8Sq zFH1pocX={Pb!{;@R8ci6crjFCazQvnQC2W@WM)}5FmO;*HbrwsXl*xfaaeCjb4+Jv zFl|~)7RZ4kHaztWcc{F1yaxhqPSV1&IO)x@IVrfA%H8^chOHN2@ zXH7*&Q%rbmWl=P2a7aODRabdpQZiX)3N=v*EiEk|M^j95cx*#!IdV5KSXpasYgJ}6 zPjf;>aaVOhc4?OnBCDvwFySSffFWF<*AD?&sy_I7H6HWZ}$Pn5YRH zwKe)gUaQDY%nTN09^@{724x%hlvh5j&{of3E{DS`ZWG6>=R79d?nmFJnPzc_5yI3* z-*P0q$PyI3KyW-CxKg~52TEtTq%kAqAp`sm zR(6uz!@!W+m79xzUh&E}Fn8p02d&AjJm9U~F^T|4-jDKjSO7W;2Wm-{c5kY5w#`{M zlvgJFo&8SCEIl~^o+2hM9u!tQt0c0L&~uGk<~VxQfM%@t8xc6@MDe|{r`0dxmeH=N zF0agr#Vr}=Ue3w}cp&E`lUgr37$Sgl*I*{A#kx4vSN#&tv}w2&1J8Sis;~8gv;-5B zkQjY~v&5mQAa8T$ECZ4%l=^ouGrLnaJrt5u(n+`jcTdawe7Vf{)n;l*=YKw!VEU|t;XqrnWR&5=jY_(BfUnhC2q9k3ZwjigCe3=Bq3mjYH1eFZvxhBALzxxk_IiorLk%`2p;k2j67>=xE=vj z9lFPx6Mm??Kx=mQMXVUfU4LHsJG-<+KxbNpS3hU06GJNhg(qqJ^7aY`J3Q{AIzqxD zm{lL>dZpUB^yXf0#!)fnj7FgTf2)&D+W5s;1p;Ze#SSIk5wN;pxJA`op$y&J3y_n~ zM#b*u^aiGe7RX;yGVf(6vjin2_V#Px`Sm&`{~hL)dYBjjq0Mr)f!v7kyiT%*MI8#b z8Z@gxG0OIx`*kZ}RwDw1tDg|dp>liHvH;p4%bGmo>^_jx+`*9&rYm4P$D=TKJDQ@W0x0ESZ6OwJw40ikl@l4mD8Y&mk0{u@7ld+ZJwK?1gVbFzc#~wIDVU*-zYA(C^$@ zZXsbrRm&3ZY2M?8J|S0nwA(=fNeESq4|i-OH_0Wgt5(bEbYUjt>a%v2BV;@tn4UC> z0lOVr^GIJL?4;=iHczU_z0%zSPIZv%aQ;a;iD*vx6}TwH5rIp}HZ*vY_UVt9%h_2y zl+!8dJc^FwFTr*&MG}M*g}T?cuc!E2KDiNg$Pl|jiU7H}at0xsvTBYV z#Kw&2KWGOX3ANaRUDm`GXcQTyCkC9Jt8fw2M)?+GRH#9+Z%+CB)PAYv5=j~JTAO(+ zTmL{vC5w&>bJr1fW;%Dsq#M*3MC?NbryS)1Q@w@z@ad`7N&ON+cym_w3vsm3453IA z71YHT!$qXMW?6}E&lq)WS6QeH-k$==mr;Fa(5`WiG`$-sgeZ-RwIS2D7a1<{;CY(p z(&A?Bj7+V5Ba)K7gqKL-`&{p-Svkq8t^BDu{FrMswseKTu{PUKS*-Hca}KmJ@Rq$W zf9u8mV(kCc;rq^>?Tzur;{+P;s~A7DNwV91O~12woTgR30Y$|X4kB#69Cnz~Eh#xm zM^0X1^T0A4tq+>CaPH2FH&B56T;F0M^QE2Bo?R*-G8l(O*j)F^S-K4!vkU0@O5haAF|z)V!=%VAHRAg|g1;)Fdw7Vf{Da*ayH1B!+7(t#D-^ zyb9G)uoHT;^wJ=K(kE^zK+UP3zuD1`IBCjlksqW4l1mP9@rnAMcA0fc zASG2~p^#|Ua09t0*Kr4(57mwK*~pY(notb4;4Fw}(pEghP_vNt8Sw}W2_!N2`@Y-c z@DqjL1UaypdD#TdALS47c%bx;2v$(4j&xclW`nE-rc7gCEq*V{#!j;`V>aU{qluSQ zUM!&WL6gs4c{GGYwH~(B)0jAZyhK+A9+c7V{t`SKZ<^>P&RQr5h*0B5Rc%U`l3yAj zQASEu2{@ZS7JcG-N-Ru)-^KW9#Dx{4&m_+NGkzC?O%wphLG3zcVH3dX6)^*(+FJPW z>5v=w{a4Aj#A|kTS49eaB8gS7TjU^J*eZv-LV0p*MAml|RlyC;$)=VF*92~G)vmIj z`+nFTJc@6mF6ZogMDjp{gC?edt=L%9glbsPTWve~6-Vret&y6I#9W=@*gmrF#0vvvB>xkGcw zXC$L>BUiCkUBzK%pVDc0ktxu{f}%;}9`lNac#R13z36Ee6PV@BsG`E$CG&de0gRs@ zxi;(HQl8bf+!Qr5bQe58QgS(A4<*0EgaTaNCz+VcR0;tK3=E9%c$@Ainj^A>7jEul zt$LI}@U8ksooaSXj7myu^XRvOJZFl(SjXO*6#*TZjlR_>Oplq}A=WgLpp81=xwz5{ zsV}0i0_3nu0Y2tdjR@Un-_rPTFlAzQYD+~IPjB=x<<9NUWq4~Y#_)Qp@KXG5=iIf` zVT*nJ*LEnf&+NqVZ%d0}L8V$@m;lR?YSaDl9DiM@=D7|ThBUSgrLqv-LPsik zfCBi9igw6q_qGH53Jxnp=mKcj6%UDujssqO<`T+}1cxnvOO~K_je4wpRPEZ0hI-%1 z;9chMYyN9c{iyzyku_rJ7`!U07K3L&*Ed}-P5i*%@0#H|$&tACx<(8K<@L~=mhiR+ zL4a{cEv*7zX}w(pLCC&@sPOFUgL=P6^PA)1c0uDhVL{?ZCCwH_61?*Nl3-t_s0)Ag zxnrMp6}MrRKu@nKhQ1M{*GcbAdT8Y|)j}qra+X9!2nxzlVs7bd`sqg8@OrE!#iOY; z8TCt=S&cMe4*ssc@JYHJJm9tisxazuqvP|ToJ_@h2t$?Kp>?YD^-+373i)PY%Y$54 zLK_^bzpZd1YZzLzJgk52tnyL9ZMCx-9!>?vU=7iHB9=KC^+PCmIU#-4ID)0dN?IHQ IM^Tk~Z8 dict[str, str]: + return { + "case_id": self.case_id, + "category": self.category, + "expected": self.expected, + "actual": self.actual, + "verdict_class": self.verdict_class, + "reason": self.reason, + } + + +def _score_one(case: dict[str, Any]) -> CaseOutcome: + expected = case["expected"] + v = check_equivalence(case["expression_a"], case["expression_b"]) + actual = v.verdict.value + + if actual == expected: + verdict_class = "correct" + reason = "" + elif actual == Verdict.REFUSED.value: + verdict_class = "refused" + reason = v.reason + else: + verdict_class = "wrong" + reason = ( + f"engine={actual!r} expected={expected!r}; " + f"canonical_a={v.canonical_a!r} canonical_b={v.canonical_b!r}" + ) + + return CaseOutcome( + case_id=case["case_id"], + category=case["category"], + expected=expected, + actual=actual, + verdict_class=verdict_class, + reason=reason, + ) + + +def decrypt_cases(sealed_path: Path = _SEALED_PATH) -> list[dict[str, Any]]: + """Decrypt the sealed file using the key from CORE_SEALED_KEY. + + Raises EnvironmentError if the env var is missing or invalid. + """ + key_path_str = os.environ.get("CORE_SEALED_KEY") + if not key_path_str: + raise EnvironmentError( + "CORE_SEALED_KEY environment variable is not set; " + "cannot decrypt sealed holdout." + ) + + key_path = Path(key_path_str) + if not key_path.exists(): + raise EnvironmentError( + f"CORE_SEALED_KEY file path does not exist: {key_path}" + ) + + try: + import pyrage + from pyrage.x25519 import Identity + except ImportError as exc: + raise RuntimeError("pyrage package is not installed.") from exc + + try: + identity = Identity.from_str(key_path.read_text(encoding="utf-8").strip()) + plaintext = pyrage.decrypt(sealed_path.read_bytes(), [identity]) + except Exception as exc: + raise ValueError(f"Decryption failed: {exc}") from exc + + records: list[dict[str, Any]] = [] + for line in plaintext.decode("utf-8").splitlines(): + line = line.strip() + if not line: + continue + records.append(json.loads(line)) + return records + + +def build_report(cases: list[dict[str, Any]]) -> dict[str, Any]: + outcomes = [_score_one(c) for c in cases] + counts = {"correct": 0, "wrong": 0, "refused": 0} + for o in outcomes: + counts[o.verdict_class] += 1 + + total = len(outcomes) + correct_rate = counts["correct"] / total if total else 0.0 + passed = (correct_rate >= _CORRECT_RATE_MIN) and (counts["wrong"] <= _WRONG_MAX) + + return { + "schema_version": 1, + "adr": "0131.1.S", + "benchmark": "symbolic_equivalence_holdout_v1", + "cases_path": str(_SEALED_PATH.relative_to(_HERE.parent.parent.parent)), + "sample_count": total, + "counts": counts, + "correct_rate": correct_rate, + "exit_criterion": { + "correct_rate_min": _CORRECT_RATE_MIN, + "wrong_max": _WRONG_MAX, + "passed": passed, + }, + "per_case": [o.as_dict() for o in outcomes], + } + + +def write_report(report: dict[str, Any], path: Path = _REPORT_PATH) -> None: + path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main() -> int: + try: + cases = decrypt_cases() + except EnvironmentError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + except Exception as exc: + print(f"error: {exc.__class__.__name__}: {exc}", file=sys.stderr) + return 1 + + report = build_report(cases) + write_report(report) + return 0 if report["exit_criterion"]["passed"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_adr_0131_1_sealed_holdout.py b/tests/test_adr_0131_1_sealed_holdout.py new file mode 100644 index 00000000..6bf285b4 --- /dev/null +++ b/tests/test_adr_0131_1_sealed_holdout.py @@ -0,0 +1,139 @@ +"""ADR-0131.1.S — Sealed holdout tests for symbolic equivalence v1. + +Pins: +1. Sealed holdout file exists and is age-formatted. +2. Decryption fails (raises EnvironmentError) if CORE_SEALED_KEY is unset or points to a non-existent file. +3. Decryption yields valid JSONL cases matching the schema. +4. The sealed runner passes its exit criterion (wrong == 0, correct_rate >= 0.95). +5. The holdout cases are strictly disjoint from the public cases (no duplicate case_id or expression pairs). +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from evals.math_symbolic_equivalence.v1.sealed_runner import ( + build_report, + decrypt_cases, +) + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_SEALED_PATH = _REPO_ROOT / "evals" / "math_symbolic_equivalence" / "v1" / "sealed_holdout.age" +_PUBLIC_CASES_PATH = _REPO_ROOT / "evals" / "math_symbolic_equivalence" / "v1" / "cases.jsonl" + + +def _decrypt_or_skip() -> list[dict]: + """Helper to decrypt cases or skip if key is missing.""" + key = os.environ.get("CORE_SEALED_KEY") + if not key: + pytest.skip("CORE_SEALED_KEY not set; skipping decryption test") + key_path = Path(key) + if not key_path.exists(): + pytest.skip(f"CORE_SEALED_KEY path {key_path} does not exist; skipping decryption test") + try: + return decrypt_cases(_SEALED_PATH) + except Exception as exc: + pytest.fail(f"Decryption failed: {exc}") + + +class TestSealedFileExists: + def test_sealed_file_present(self) -> None: + assert _SEALED_PATH.exists(), f"missing sealed file: {_SEALED_PATH}" + assert _SEALED_PATH.is_file() + assert _SEALED_PATH.stat().st_size > 0 + + def test_sealed_file_is_age_formatted(self) -> None: + head = _SEALED_PATH.read_bytes()[:64] + assert head.startswith(b"age-encryption.org/"), ( + f"sealed file does not look age-formatted; head={head!r}" + ) + + +class TestDecryptionGated: + def test_decryption_refuses_without_key(self) -> None: + old_key = os.environ.get("CORE_SEALED_KEY") + if "CORE_SEALED_KEY" in os.environ: + del os.environ["CORE_SEALED_KEY"] + + try: + with pytest.raises(EnvironmentError) as excinfo: + decrypt_cases(_SEALED_PATH) + assert "CORE_SEALED_KEY environment variable is not set" in str(excinfo.value) + finally: + if old_key is not None: + os.environ["CORE_SEALED_KEY"] = old_key + + def test_decryption_refuses_with_invalid_key_path(self) -> None: + old_key = os.environ.get("CORE_SEALED_KEY") + os.environ["CORE_SEALED_KEY"] = "/nonexistent/path/to/key.txt" + + try: + with pytest.raises(EnvironmentError) as excinfo: + decrypt_cases(_SEALED_PATH) + assert "CORE_SEALED_KEY file path does not exist" in str(excinfo.value) + finally: + if old_key is not None: + os.environ["CORE_SEALED_KEY"] = old_key + + +class TestSealedFileContent: + def test_decrypt_produces_valid_schema(self) -> None: + cases = _decrypt_or_skip() + assert len(cases) >= 10, f"expected at least 10 holdout cases, got {len(cases)}" + + for c in cases: + assert "case_id" in c + assert c["case_id"].startswith("sym-eq-v1-hld-") + assert "expression_a" in c + assert isinstance(c["expression_a"], str) and c["expression_a"] + assert "expression_b" in c + assert isinstance(c["expression_b"], str) and c["expression_b"] + assert "expected" in c + assert c["expected"] in ("equivalent", "not_equivalent", "refused") + assert "category" in c + assert isinstance(c["category"], str) and c["category"] + assert "provenance" in c + assert "adr-0131.1" in c["provenance"] + + def test_disjointness_from_public_cases(self) -> None: + sealed_cases = _decrypt_or_skip() + + # Load public cases + public_cases = [] + with _PUBLIC_CASES_PATH.open("r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line: + public_cases.append(json.loads(line)) + + public_ids = {c["case_id"] for c in public_cases} + public_pairs = { + (c["expression_a"].strip(), c["expression_b"].strip()) + for c in public_cases + } + + for c in sealed_cases: + assert c["case_id"] not in public_ids, ( + f"Duplicate case_id {c['case_id']} found in sealed holdout" + ) + pair = (c["expression_a"].strip(), c["expression_b"].strip()) + reverse_pair = (c["expression_b"].strip(), c["expression_a"].strip()) + assert pair not in public_pairs and reverse_pair not in public_pairs, ( + f"Duplicate expression pair {pair} found in sealed holdout" + ) + + +class TestSealedRunnerGate: + def test_runner_passes_exit_criterion(self) -> None: + cases = _decrypt_or_skip() + report = build_report(cases) + assert report["exit_criterion"]["passed"], ( + f"Sealed holdout runner failed: correct_rate={report['correct_rate']}, " + f"wrong={report['counts']['wrong']}" + ) + assert report["counts"]["wrong"] == 0, "Sealed holdout must have wrong == 0" + assert report["correct_rate"] >= 0.95, "Sealed holdout correct rate must be >= 0.95"