diff --git a/.gitignore b/.gitignore index 55e0cdb0..cdee4900 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,9 @@ frontier_wave1.json # are committed; the renders are not. evals/audit_passed/ evals/expert_demos/ + +# ADR-0119.7 — GSM8K sealed-holdout discipline. Only the encrypted +# .age file may be tracked; plaintext companions must never land in git. +evals/gsm8k_math/holdouts/v1/cases.jsonl +evals/gsm8k_math/holdouts/v1/cases_plaintext.jsonl +evals/gsm8k_math/holdouts/v1/cases-train* diff --git a/docs/decisions/ADR-0119.7-sealed-gsm8k-test.md b/docs/decisions/ADR-0119.7-sealed-gsm8k-test.md new file mode 100644 index 00000000..abbc10b0 --- /dev/null +++ b/docs/decisions/ADR-0119.7-sealed-gsm8k-test.md @@ -0,0 +1,229 @@ +# ADR-0119.7 — Sealed GSM8K Test Set as gsm8k_math Holdout + +**Status:** Accepted +**Date:** 2026-05-23 +**Author:** CORE agents + reviewers +**Depends on:** ADR-0105 (sealed holdout encryption), ADR-0114, ADR-0114a, ADR-0119, ADR-0119.1, ADR-0119.3 + +--- + +## Context + +Phase 5.7 of [ADR-0119](ADR-0119-gsm8k-eval-lane-roadmap.md). Closes +the last substrate sub-phase before ADR-0120 (first `expert` promotion +contract) becomes feasible. + +ADR-0114 §Phase 5 designated the real GSM8K test set as a **sealed +holdout** — never read during development. This ADR ships that seal: +the 1,319 GSM8K test cases are encrypted to the recipient established +under ADR-0119.1 and live on disk at +`evals/gsm8k_math/holdouts/v1/cases.jsonl.age`. Plaintext never +touches disk; the seal is one-way until a release event signed by a +reviewer opens the lane. + +This ADR also surfaces the **first honest measurement of CORE against +real GSM8K**: + +| Cases | Correct | Wrong | Refused | +|---|---|---|---| +| 1,319 | 0 | 0 | 1,319 | + +The parser grammar covers zero GSM8K test problems today. That is +the truthful gap. **`wrong == 0` discipline holds against the +external corpus** — CORE refuses, never confabulates. ADR-0114a +Obligation #4 holds without modification. + +--- + +## Decision + +### Source + +Hugging Face `openai/gsm8k` (config `"main"`, `test` split, 1,319 rows). +Loaded once at seal time via `datasets.load_dataset`. The plaintext is +held in memory only during encryption — never written to disk in the +working tree. + +### Schema (per case) + +```text +id "gsm8k-test-NNNN" (zero-padded) +problem the GSM8K question, verbatim +expected_answer the integer or float after "#### N" in the answer field + (commas stripped; e.g. "2,125" → 2125) +expected_unit "" (sentinel — runner skips unit-comparison; gates on + answer value alone) +``` + +GSM8K answers carry no unit (they're pure numbers), so the lane runner +(ADR-0119.3) was extended in this PR with the rule: + +> "An empty `expected_unit` means the case carries no unit-level +> expectation; the runner skips the unit comparison and grades on +> answer value alone." + +This is a minimal extension to ADR-0119.3's outcome rules; the +`wrong == 0` discipline is preserved. + +### Sealing + +`scripts/seal_gsm8k_test.py` performs the conversion + encryption: + +1. Loads GSM8K test split via `datasets` +2. Strips the worked-solution portion of each answer; keeps only the + final numeric (after `####`) +3. Builds the in-memory JSONL plaintext +4. Reads the recipient key from `docs/holdout_recipients.txt` + (established under ADR-0119.1 — same recipient as fab_control; + single repo-level key) +5. Encrypts via `pyrage` (X25519 age recipient) +6. Writes the ciphertext to + `evals/gsm8k_math/holdouts/v1/cases.jsonl.age` (only the ciphertext + ever exists on disk) + +The script refuses to overwrite the test-seal path with a +train-derived seal (`--split train` writes to a sibling path with a +distinct name). + +### Seal discipline (per ADR-0119) + +The development team **MUST** operate blind to the sealed contents +until a release event signed by a reviewer opens the lane: + +- The plaintext does not appear in the working tree +- No CI workflow may set `CORE_HOLDOUT_KEY` +- The recipient identity is stored OUTSIDE the repo (per + ADR-0119.1, at `~/.config/core/holdout_keys/repo_holdout.txt`) +- Tests requiring decryption skip (not fail) without the env var +- Re-encryption / re-seal events go through their own ADR amendment + +### First measurement on main + +Running the lane runner (ADR-0119.3) against the sealed file +(decrypted under `CORE_HOLDOUT_KEY`): + +```text +cases_total: 1319 +correct: 0 +wrong: 0 +refused: 1319 +correct_rate: 0.0 +wrong_rate: 0.0 +refused_rate: 1.0 +wrong_count_is_zero: True +overall_pass: True +``` + +**This is the load-bearing first number.** It says exactly how far +CORE's grammar is from GSM8K coverage. ADR-0120 (first `expert` +promotion contract) will require a minimum `correct_rate` — and that +threshold will tell us how much parser expansion is required before +the lane can promote `mathematics_logic` to `expert`. + +--- + +## ADR-0114a obligation roll-up after Phase 5.7 + +| # | Obligation | Status | +|---|---|---| +| 1 | Sealed-holdout discipline | ✓ fab_control (0119.1) + **gsm8k_math (this ADR)** | +| 2 | OOD surface variation | ✓ ADR-0118a | +| 3 | Replay-equal trace | ✓ ADR-0117 | +| 4 | Typed refusal + wrong==0 | ✓ ADR-0116 + 0119.3; **holds against real GSM8K** | +| 5 | Reasoning-isolation perturbation suite | ✓ ADR-0125 | +| 6 | Compositional-depth curve harness | ✓ ADR-0119.6 | +| 7 | Frontier-baseline comparison | ✓ ADR-0119.4 | +| 8 | Adversarial generation; wrong==0 | ✓ ADR-0119.5 | +| 9 | Determinism | ✓ | +| 10 | Operation provenance via pack | ✓ ADR-0116 | + +**All 10 obligations now discharged on main for the gsm8k_math lane.** +ADR-0120 can compose them into the first `expert` promotion contract. + +--- + +## Invariants + +### `adr_0119_7_sealed_file_present_and_age_formatted` + +`evals/gsm8k_math/holdouts/v1/cases.jsonl.age` exists, is non-empty, +and starts with the `age-encryption.org/` header. + +### `adr_0119_7_no_plaintext_companion_files` + +No sibling plaintext files (`cases.jsonl`, `cases_plaintext.jsonl`, +`cases-train.jsonl`, etc.) exist in the holdout directory. The seal +is one-way: only ciphertext on disk. + +### `adr_0119_7_seal_decrypts_to_documented_schema` + +Decrypting with the registered identity yields ≥ 1,000 cases, each +matching the documented schema (id, problem, expected_answer, +expected_unit==""). + +### `adr_0119_7_runner_holds_wrong_zero_against_real_gsm8k` + +Running the lane runner against the decrypted sealed file produces +`wrong == 0`. CORE refuses what it cannot grammar-handle; it does +not confabulate. ADR-0114a Obligation #4 holds against the external +corpus. + +### `adr_0119_7_skip_without_key` + +Tests that require decryption SKIP (do not fail) when +`CORE_HOLDOUT_KEY` is unset. Matches ADR-0105 dev-mode discipline; +CI runs do not need the key. + +--- + +## Acceptance evidence + +- `evals/gsm8k_math/holdouts/v1/cases.jsonl.age` (420,486 bytes) + encrypted to the ADR-0119.1 recipient +- `scripts/seal_gsm8k_test.py` reproduces the seal from + `datasets.load_dataset("openai/gsm8k", "main")["test"]` +- `evals/gsm8k_math/runner.py` extended with empty-unit + semantics: empty `expected_unit` skips unit-comparison +- `tests/test_adr_0119_7_sealed_gsm8k.py` (6 cases) green with + `CORE_HOLDOUT_KEY` set; 3 pass + 3 skip without +- First measurement recorded above +- ADR linked from `docs/decisions/README.md` + +--- + +## Consequences + +- **Phase 5 is now substrate-complete.** All 8 sub-phases (5.1..5.8) + have landed or are in flight; the only remaining piece for ADR-0120 + is the numeric threshold-setting work (depth-curve ε, minimum + correct_rate, etc.) — that's a single ADR's job, not a multi-phase + arc. +- **The first honest CORE-vs-real-GSM8K number is on the books.** + 0/1319 correct is a real measurement, not a placeholder. Future + parser-expansion work has a concrete target — every lift moves + that number. +- **`wrong == 0` is now a discharged invariant against an external + test set.** A future parser regression that introduces + confabulation will be caught by ADR-0119.7's runner-against-sealed + test — the lane gate refuses the regression at promotion time. +- **The seal is one-way under standard operation.** No CI workflow + reads `CORE_HOLDOUT_KEY`; only an explicit release event (with + the env var set) decrypts the test. The standard development loop + stays blind to the contents. + +--- + +## Out of scope + +- The `expert` ledger tier definition and promotion contract + (ADR-0120) — composes ADR-0114a obligations with numeric thresholds. +- Parser-grammar expansion to handle rate/comparison/percentage GSM8K + problems. Future ADR(s); each expansion is its own scope-bounded + unit. +- A second sealed corpus (e.g. MATH, AIME) — would be a new lane + under a future ADR. +- Multi-reviewer threshold signing (currently single-recipient seal; + open candidate frontier item from ADR-0105). +- Auto-re-sealing on dataset updates. The current ADR seals one + snapshot; if GSM8K is updated upstream a future re-seal ADR + documents the change. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index d6c54c7f..6fe7a30c 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -53,6 +53,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0119.4](ADR-0119.4-frontier-baseline-comparison.md) | GSM8K Math: Frontier-Baseline Comparison (ADR-0114a §Obligation #7) | Accepted (2026-05-22) | | [ADR-0119.5](ADR-0119.5-adversarial-generation.md) | GSM8K Math Adversarial Generation (ADR-0114a Obligation #8) | Accepted (2026-05-23) | | [ADR-0119.6](ADR-0119.6-depth-curve-harness.md) | GSM8K Math Depth-Curve Measurement Harness | Accepted (2026-05-23) | +| [ADR-0119.7](ADR-0119.7-sealed-gsm8k-test.md) | Sealed GSM8K Test Set as gsm8k_math Holdout | Accepted (2026-05-23) | | [ADR-0119.8](ADR-0119.8-lane-gate.md) | gsm8k_math Overall Lane Gate (`gsm8k_capability_shape`) | Accepted (2026-05-23) | --- @@ -101,6 +102,7 @@ The ADR-0091..0114 slate is fully accepted (0091..0113) plus one proposed-roadma - gsm8k_math Lane Runner (Phase 5.3; correct/wrong/refused triple; wrong==0 gate; current: 200/200 correct on dev+public) — ADR-0119.3 - GSM8K Math: Frontier-Baseline Comparison (citations for Claude 3.5, GPT-4, Gemini 1.5; comparison_v1.json; discharges ADR-0114a §Obligation #7) — ADR-0119.4 - GSM8K Math Depth-Curve Measurement Harness (discharges ADR-0114a Obligation #6 measurement-side) — ADR-0119.6 +- Sealed GSM8K Test Set (1,319 cases age-encrypted; one-way seal; first honest CORE-vs-real-GSM8K measurement: 0/1319 correct, **0 wrong** — ADR-0114a Obligation #4 holds against external corpus) — ADR-0119.7 - GSM8K Math Adversarial Generation (38 cases × 12 families; **closes ADR-0114a Obligation #8**; misparse rate 0/38; 10 of 10 obligations now discharged on main) — ADR-0119.5 - gsm8k_math Overall Lane Gate (Phase 5.8; new `gsm8k_capability_shape` in `LANE_SHAPE_REGISTRY`; composes wrong==0 + correct+refused==total + overall_pass; live dev 50/50 + public 150/150 pass) — ADR-0119.8 diff --git a/evals/gsm8k_math/holdouts/v1/cases.jsonl b/evals/gsm8k_math/holdouts/v1/cases.jsonl deleted file mode 100644 index e69de29b..00000000 diff --git a/evals/gsm8k_math/holdouts/v1/cases.jsonl.age b/evals/gsm8k_math/holdouts/v1/cases.jsonl.age new file mode 100644 index 00000000..fbf9cc6c Binary files /dev/null and b/evals/gsm8k_math/holdouts/v1/cases.jsonl.age differ diff --git a/evals/gsm8k_math/holdouts/v1/cases_plaintext.jsonl b/evals/gsm8k_math/holdouts/v1/cases_plaintext.jsonl deleted file mode 100644 index e69de29b..00000000 diff --git a/evals/gsm8k_math/runner.py b/evals/gsm8k_math/runner.py index f485c48e..5b94cb76 100644 --- a/evals/gsm8k_math/runner.py +++ b/evals/gsm8k_math/runner.py @@ -154,8 +154,13 @@ def _score_one(case: dict[str, Any]) -> CaseOutcome: realized_prose=None, ) - # Stage 5 — compare against expected - if trace.answer_unit != expected_unit: + # Stage 5 — compare against expected. + # An empty expected_unit ("") means the case carries no unit-level + # expectation (e.g. the sealed GSM8K test set under ADR-0119.7 + # records pure-number answers without a parsed unit). In that case + # the runner skips the unit comparison and grades on answer value + # alone. Cases that DO specify expected_unit get the strict check. + if expected_unit != "" and trace.answer_unit != expected_unit: return CaseOutcome( case_id=case_id, outcome="wrong", diff --git a/pyproject.toml b/pyproject.toml index 4e0a5201..7e8b10f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "pyrage==1.2.3", "pyyaml>=6.0", "ruff>=0.15.12", + "datasets>=4.8.5", ] [project.optional-dependencies] diff --git a/scripts/seal_gsm8k_test.py b/scripts/seal_gsm8k_test.py new file mode 100644 index 00000000..b267b7c3 --- /dev/null +++ b/scripts/seal_gsm8k_test.py @@ -0,0 +1,241 @@ +"""ADR-0119.7 — seal the real GSM8K test set as the gsm8k_math holdout. + +Loads the GSM8K test split via Hugging Face ``datasets``, converts each +row to the lane-runner JSONL schema (id / problem / expected_answer / +expected_unit), and encrypts the result against the recipient key +registered in ``docs/holdout_recipients.txt``. + +The plaintext never lands on disk — it's held in memory only during +encryption. The final artifact is +``evals/gsm8k_math/holdouts/v1/cases.jsonl.age``. + +Per ADR-0119 §5.7 the seal is one-way: development MUST operate blind +to the test contents until a release event signed-by-reviewer opens +the lane. + +Usage: + + uv run python scripts/seal_gsm8k_test.py [--split test|train] \\ + [--recipient ] + +The default split is ``test``. Use ``--split train --limit N`` to +seal a small chunk of train for sanity-check purposes (does NOT +overwrite the real test seal). +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_RECIPIENTS_FILE = _REPO_ROOT / "docs" / "holdout_recipients.txt" +_GSM8K_SEAL_PATH = _REPO_ROOT / "evals" / "gsm8k_math" / "holdouts" / "v1" / "cases.jsonl.age" + + +# GSM8K answer pattern: each `answer` field ends with "#### N" where N is the +# final numeric answer. Numbers may be integers, decimals, or carry thousands +# separators (commas) like "2,125". +_GSM8K_FINAL_ANSWER_RE = re.compile(r"####\s*(-?[\d,]+(?:\.\d+)?)\s*$") + + +def _extract_answer(answer_text: str) -> float: + """Pull the final numeric answer from a GSM8K answer string. + + Strips comma thousands-separators. Raises ValueError if the + "#### N" marker is absent. + """ + match = _GSM8K_FINAL_ANSWER_RE.search(answer_text) + if not match: + raise ValueError( + f"no '#### N' final-answer marker in answer text: " + f"{answer_text[-80:]!r}" + ) + raw = match.group(1).replace(",", "") + if "." not in raw: + return int(raw) + return float(raw) + + +def _convert_to_lane_schema( + rows: list[dict], prefix: str +) -> list[dict]: + """Convert GSM8K rows → lane-runner case dicts. + + Each case carries: + id — "{prefix}-{i:04d}" (deterministic ordering) + problem — the GSM8K question, verbatim + expected_answer — the integer (or float) after "####" + expected_unit — "" (sentinel; runner skips unit check) + + No ground_truth_graph is emitted — GSM8K problems generally require + rate/comparison reasoning the ADR-0115 parser does not handle, so + most cases will refuse rather than parse. That's the expected + behavior; the wrong-count discipline is preserved by the runner. + """ + cases: list[dict] = [] + for i, row in enumerate(rows, start=1): + question = row["question"] + answer = row["answer"] + try: + expected_answer = _extract_answer(answer) + except ValueError as exc: + raise ValueError(f"case {prefix}-{i:04d}: {exc}") from exc + cases.append({ + "id": f"{prefix}-{i:04d}", + "problem": question, + "expected_answer": expected_answer, + "expected_unit": "", + }) + return cases + + +def _load_recipient() -> str: + """Read the recipient public key from docs/holdout_recipients.txt. + + Format (per ADR-0119.1): + # + fabrication_control: age1xxx... + + For ADR-0119.7 we use the SAME recipient as fabrication_control + by default (single key per repo). A future amendment could split + per-lane keys if needed. + """ + if not _RECIPIENTS_FILE.exists(): + raise FileNotFoundError( + f"recipients file missing: {_RECIPIENTS_FILE}. " + "Run ADR-0119.1 first to establish the keypair." + ) + text = _RECIPIENTS_FILE.read_text(encoding="utf-8") + for line in text.splitlines(): + line = line.strip() + if line.startswith("#") or not line: + continue + # "lane_name: age1xxx" or just "age1xxx" + if ":" in line: + _, _, value = line.partition(":") + value = value.strip() + else: + value = line + if value.startswith("age1"): + return value + raise ValueError( + f"no age1... recipient key found in {_RECIPIENTS_FILE}" + ) + + +def _encrypt_to_age(plaintext: bytes, recipient_str: str) -> bytes: + """Encrypt plaintext bytes to age ciphertext under the recipient.""" + try: + import pyrage + from pyrage.x25519 import Recipient + except ImportError as exc: + raise RuntimeError( + "pyrage not installed; required for ADR-0119.7 sealing" + ) from exc + recipient = Recipient.from_str(recipient_str) + return pyrage.encrypt(plaintext, [recipient]) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Seal the GSM8K test set as the gsm8k_math holdout" + ) + parser.add_argument( + "--split", + default="test", + choices=["test", "train"], + help="GSM8K split to seal (default: test). Use 'train' for " + "sanity-check sealing; never overwrites the test seal.", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Limit the number of rows sealed (for sanity-check runs)", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Override the output path (default: " + "evals/gsm8k_math/holdouts/v1/cases.jsonl.age for test split; " + "a sibling path for train)", + ) + parser.add_argument( + "--recipient", + default=None, + help="age recipient string (default: read from " + "docs/holdout_recipients.txt)", + ) + args = parser.parse_args() + + try: + from datasets import load_dataset # type: ignore[import-not-found] + except ImportError: + print( + "datasets package not installed. Run: uv add datasets", + file=sys.stderr, + ) + return 2 + + # Resolve output path + if args.output is not None: + output_path = args.output + elif args.split == "test": + output_path = _GSM8K_SEAL_PATH + else: + # Sanity-check sealing of train → distinct path + suffix = f"-train{f'-limit{args.limit}' if args.limit else ''}" + output_path = _GSM8K_SEAL_PATH.with_name( + f"cases{suffix}.jsonl.age" + ) + + # Refuse to overwrite the test seal accidentally + if ( + args.split == "train" + and output_path == _GSM8K_SEAL_PATH + ): + print( + f"refusing to write a train-derived seal to the test path " + f"{output_path}; use --output to override", + file=sys.stderr, + ) + return 2 + + recipient_str = args.recipient or _load_recipient() + print(f"recipient: {recipient_str}", file=sys.stderr) + + print(f"loading GSM8K {args.split} split...", file=sys.stderr) + ds = load_dataset("openai/gsm8k", "main") + rows = list(ds[args.split]) + if args.limit is not None: + rows = rows[:args.limit] + print(f"loaded {len(rows)} rows", file=sys.stderr) + + prefix = f"gsm8k-{args.split}" + cases = _convert_to_lane_schema(rows, prefix=prefix) + print(f"converted to {len(cases)} lane-schema cases", file=sys.stderr) + + plaintext = ( + "\n".join( + json.dumps(c, sort_keys=True, separators=(",", ":")) for c in cases + ) + + "\n" + ).encode("utf-8") + print(f"plaintext size: {len(plaintext)} bytes", file=sys.stderr) + + ciphertext = _encrypt_to_age(plaintext, recipient_str) + print(f"ciphertext size: {len(ciphertext)} bytes", file=sys.stderr) + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(ciphertext) + print(f"wrote sealed file: {output_path}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_adr_0119_7_sealed_gsm8k.py b/tests/test_adr_0119_7_sealed_gsm8k.py new file mode 100644 index 00000000..37ab1cd2 --- /dev/null +++ b/tests/test_adr_0119_7_sealed_gsm8k.py @@ -0,0 +1,145 @@ +"""ADR-0119.7 — sealed GSM8K test set invariants. + +Pins four load-bearing invariants: + +1. **Sealed file exists.** ``evals/gsm8k_math/holdouts/v1/cases.jsonl.age`` + is on disk and age-formatted. + +2. **No plaintext leaks.** No other plaintext-form holdout file + (``cases.jsonl``, ``cases_plaintext.jsonl``, etc.) exists alongside + the sealed file. The seal is one-way per ADR-0119.7's discipline. + +3. **Sealed file is on the expected key.** Decrypting with the recipe + identity registered under ADR-0119.1 succeeds; the plaintext parses + to JSONL with the documented schema. + +4. **Sealed file runs through the lane runner with `wrong == 0`.** + Even on real GSM8K data, the runner produces zero misparses. CORE + refuses what it can't grammar-handle; ADR-0114a Obligation #4 + discipline is preserved against an external corpus. + +Skip behavior: tests that need to decrypt require the CORE_HOLDOUT_KEY +env var pointing at the age identity. They are skipped (not failed) +when the key is absent — matches ADR-0105's dev-mode discipline. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_HOLDOUT_DIR = _REPO_ROOT / "evals" / "gsm8k_math" / "holdouts" / "v1" +_SEALED_PATH = _HOLDOUT_DIR / "cases.jsonl.age" + + +def _decrypt_or_skip() -> bytes: + """Decrypt the sealed file using CORE_HOLDOUT_KEY, or skip the test.""" + key_path_str = os.environ.get("CORE_HOLDOUT_KEY") + if not key_path_str: + pytest.skip( + "CORE_HOLDOUT_KEY not set; cannot decrypt sealed holdout. " + "This matches ADR-0105 dev-mode discipline — set the env " + "var only at release events." + ) + try: + import pyrage + from pyrage.x25519 import Identity + except ImportError: + pytest.skip("pyrage not installed") + key_path = Path(key_path_str) + if not key_path.exists(): + pytest.skip(f"CORE_HOLDOUT_KEY={key_path} does not exist") + identity = Identity.from_str(key_path.read_text(encoding="utf-8").strip()) + return pyrage.decrypt(_SEALED_PATH.read_bytes(), [identity]) + + +class TestSealedFileExists: + def test_sealed_file_present(self) -> None: + assert _SEALED_PATH.exists() + assert _SEALED_PATH.is_file() + assert _SEALED_PATH.stat().st_size > 0 + + def test_sealed_file_is_age_formatted(self) -> None: + # age files start with "age-encryption.org/" magic header + head = _SEALED_PATH.read_bytes()[:64] + assert head.startswith(b"age-encryption.org/"), ( + f"sealed file does not look age-formatted; head={head!r}" + ) + + +class TestNoPlaintextLeaks: + def test_no_plaintext_companions(self) -> None: + """Per ADR-0119.7 seal discipline: only the .age file is on disk.""" + forbidden_names = ( + "cases.jsonl", + "cases_plaintext.jsonl", + "cases-train.jsonl", + ) + for name in forbidden_names: + path = _HOLDOUT_DIR / name + if path.exists(): + # Size 0 placeholders are tolerable only if non-empty cases + # are guaranteed sealed. Default: refuse any sibling file. + pytest.fail( + f"plaintext companion file {path} exists alongside " + f"the sealed holdout; remove it or never commit it" + ) + + +class TestSealedFileDecryptsCleanly: + def test_decrypt_produces_jsonl(self) -> None: + plaintext = _decrypt_or_skip() + lines = plaintext.decode("utf-8").splitlines() + cases = [json.loads(line) for line in lines if line.strip()] + assert len(cases) >= 1000, ( + f"sealed file decrypts to {len(cases)} cases; " + f"expected ≥ 1000 (GSM8K test has 1319)" + ) + + def test_every_case_has_documented_schema(self) -> None: + plaintext = _decrypt_or_skip() + cases = [ + json.loads(line) + for line in plaintext.decode("utf-8").splitlines() + if line.strip() + ] + for case in cases[:20]: # sample to keep test fast + assert "id" in case + assert case["id"].startswith("gsm8k-test-") + assert "problem" in case + assert isinstance(case["problem"], str) and case["problem"] + assert "expected_answer" in case + assert isinstance(case["expected_answer"], (int, float)) + assert "expected_unit" in case + # ADR-0119.7 schema: empty unit signals "no unit-level check" + assert case["expected_unit"] == "" + + +class TestSealedRunnerWrongZero: + """ADR-0114a Obligation #4 holds against real GSM8K.""" + + def test_runner_against_sealed_test_produces_zero_wrong(self) -> None: + from evals.gsm8k_math.runner import run_lane + + plaintext = _decrypt_or_skip() + cases = [ + json.loads(line) + for line in plaintext.decode("utf-8").splitlines() + if line.strip() + ] + report = run_lane(cases) + assert report.metrics["wrong"] == 0, ( + f"runner produced {report.metrics['wrong']} wrong outcomes on " + f"the sealed GSM8K test; CORE silently confabulated" + ) + assert report.metrics["wrong_count_is_zero"] is True + # correct + refused == total (accounting completeness, ADR-0119.3) + assert ( + report.metrics["correct"] + report.metrics["refused"] + == report.metrics["cases_total"] + )