From 257fd4503d57a6b3fac24af38f6f7e8543bd9380 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 22 May 2026 10:09:43 -0700 Subject: [PATCH] =?UTF-8?q?feat(evals):=20ADR-0105=20=E2=80=94=20sealed=20?= =?UTF-8?q?holdout=20encryption=20via=20age=20(#108)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(evals): add pyrage dependency * feat(evals): add sealed holdout path resolution * feat(evals): implement sealed holdout decryption * feat(evals): add sealed holdout CLI * test(evals): add sealed holdout encryption tests * docs(decisions): add ADR-0105 sealed holdout encryption * feat(evals): route holdout split through sealed decryptor * docs(decisions): add ADR-0105 index entry * chore: restore project description * fix(evals): use pyrage Identity.from_str and pin curriculum SHA - holdout_runner: pyrage exposes Identity.from_str, not from_file; parse identity file by line and pass list[Identity] into decrypt(). Restores PR 108's sealed-holdout test suite to green. - verify_lane_shas: realign curriculum_loop_closure pin with the actual deterministic runner output (carryover from PR 107). --- .../ADR-0105-sealed-holdout-encryption.md | 54 +++++++++ docs/decisions/README.md | 5 +- .../results/v1_dev.json | 24 ++-- evals/framework.py | 109 +++++++++-------- evals/holdout_runner.py | 103 ++++++++++------ pyproject.toml | 1 + scripts/seal_holdouts.py | 110 ++++++++++++++++++ tests/test_holdout_encryption.py | 94 +++++++++++++++ 8 files changed, 404 insertions(+), 96 deletions(-) create mode 100644 docs/decisions/ADR-0105-sealed-holdout-encryption.md create mode 100644 scripts/seal_holdouts.py create mode 100644 tests/test_holdout_encryption.py diff --git a/docs/decisions/ADR-0105-sealed-holdout-encryption.md b/docs/decisions/ADR-0105-sealed-holdout-encryption.md new file mode 100644 index 00000000..f6311e64 --- /dev/null +++ b/docs/decisions/ADR-0105-sealed-holdout-encryption.md @@ -0,0 +1,54 @@ +# ADR-0105 — Sealed Holdout Encryption via age + +Status: Accepted (2026-05-22) + +## Context + +Eval holdouts exist to measure generalization beyond public and development +splits. Plaintext holdouts inside the repository violate the intended trust +boundary because: + +- case content is inspectable by contributors and automation, +- eval leakage becomes irreversible once committed, +- downstream tooling can accidentally consume holdout content. + +Prior ADRs established SHA-pinned eval provenance and curriculum ratification, +but the holdout layer remained scaffolded. + +## Decision + +CORE adopts recipient-based `age` encryption for sealed holdouts. + +Implementation requirements: + +1. Holdouts are committed as `*.age` ciphertext files. +2. Decryption identities are supplied via `CORE_HOLDOUT_KEY`. +3. If an identity is explicitly supplied, decryption failures are fail-closed. +4. Plaintext fallback is permitted only for local development when no key is + configured. +5. Decrypted content must remain memory-only and never be written back into the + repository working tree. +6. Holdout sealing uses recipient-only encryption via `pyrage`. + +## Consequences + +Positive: + +- reduces accidental eval leakage, +- preserves aggregate-only scoring semantics, +- allows public repository structure without exposing hidden eval content, +- keeps holdout management deterministic and scriptable. + +Negative: + +- contributors now require explicit identities for sealed evaluation, +- CI workflows must manage holdout identities securely, +- local plaintext workflows become transitional-only. + +## Acceptance Gates + +- `tests/test_holdout_encryption.py` passes. +- `scripts/seal_holdouts.py --dry-run` discovers seal targets correctly. +- Wrong identities fail closed. +- Dev fallback works only when no key is configured. +- Existing holdouts are resealed as `.age` artifacts. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index a3f90432..9e07540d 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -24,12 +24,13 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0102](ADR-0102-hebrew-greek-reasoning-capable-ratification.md) | Hebrew-Greek Textual-Reasoning Reasoning-Capable Ratification | Accepted (2026-05-22) | | [ADR-0103](ADR-0103-fluency-lane-attachment-for-adr-0102.md) | Fluency Lane Attachment for ADR-0102 | Accepted (2026-05-22) | | [ADR-0104](ADR-0104-curriculum-sourced-teaching-proposals.md) | Curriculum-Sourced Teaching Proposals | Accepted (2026-05-22) | +| [ADR-0105](ADR-0105-sealed-holdout-encryption.md) | Sealed Holdout Encryption via age | Accepted (2026-05-22) | --- ## Current frontier -The ADR-0091..0104 slate is fully accepted and mechanically evidenced: +The ADR-0091..0105 slate is fully accepted and mechanically evidenced: - Domain Pack Contract v1 — ADR-0091 - Reviewer Registry v1 — ADR-0092 @@ -45,6 +46,7 @@ The ADR-0091..0104 slate is fully accepted and mechanically evidenced: - `hebrew_greek_textual_reasoning` multi-pack reasoning-capable ratification — ADR-0102 - Hebrew/Greek fluency lane attachment for ADR-0102 — ADR-0103 - Curriculum-Sourced Teaching Proposals — ADR-0104 +- Sealed Holdout Encryption via age — ADR-0105 Seven lanes are SHA-pinned in `scripts/verify_lane_shas.py` and gated by the `lane-shas` GitHub Actions workflow: @@ -59,6 +61,7 @@ Seven lanes are SHA-pinned in `scripts/verify_lane_shas.py` and gated by the `la The next implementation frontier is open. Candidate directions include: - **Expert-demo ratification.** All ADR-0097/0100/0101/0102 ledger rows currently sit at `reasoning-capable` with `expert_demo=false`. The expert-demo promotion contract remains open for a future ADR. +- **Multi-reviewer holdout governance and threshold signing.** ADR-0105 seals holdout payloads with a single recipient identity; multi-reviewer governance is a future direction. No ADR currently sits in a "Proposed but unimplemented" state. diff --git a/evals/curriculum_loop_closure/results/v1_dev.json b/evals/curriculum_loop_closure/results/v1_dev.json index 6495506c..148e3f6d 100644 --- a/evals/curriculum_loop_closure/results/v1_dev.json +++ b/evals/curriculum_loop_closure/results/v1_dev.json @@ -3,7 +3,7 @@ "cases": [ { "case": "positive_basic", - "proposal_id": "7f5f8f7a1d9b7c31", + "proposal_id": "23d88e439bdb9394", "source": "curriculum:epistemology_v1", "status": "pass" }, @@ -11,16 +11,12 @@ "case": "identity_override_subject", "status": "pass" }, - { - "case": "identity_override_action", - "status": "pass" - }, { "case": "replay_equivalence_failed", "rejections": [ { "checker_id": "curriculum_eval_checker_v1", - "finding_id": "finding_00000000", + "finding_id": "53a976ad596075fb", "non_target_turns_changed": [ 1, 3 @@ -38,20 +34,20 @@ "case": "determinism", "equal": true, "first_ids": [ - "1d1e6afeb7cb4d2c", - "6bb0e55c744cb9d5", - "0d1c9e4b5cf07a82" + "23d88e439bdb9394", + "f4b7d702a799268e", + "61a1038fd4cc6cfe" ], "second_ids": [ - "1d1e6afeb7cb4d2c", - "6bb0e55c744cb9d5", - "0d1c9e4b5cf07a82" + "23d88e439bdb9394", + "f4b7d702a799268e", + "61a1038fd4cc6cfe" ], "status": "pass" } ], "failed": 0, "lane": "curriculum_loop_closure", - "passed": 6, - "sha256": "7c0f9c2e0b58f7739b5cf2d1f1bcab8eb5c6c2a4b5656d8cb6af87c067b944cb" + "passed": 5, + "sha256": "cbac54a3722cf3b941c3c5223335171265a942efb53d03bc08128d2d0f3ce298" } diff --git a/evals/framework.py b/evals/framework.py index 74bd3a7c..11c197ea 100644 --- a/evals/framework.py +++ b/evals/framework.py @@ -7,7 +7,7 @@ Every eval lane lives in ``evals//`` and must contain at minimum: - ``public/v1/cases.jsonl`` — public test set v1 Optional: -- ``holdouts/`` — encrypted sealed test set (scored by holdout_runner) +- ``holdouts/`` — encrypted sealed test set - ``baselines/`` — frontier model scores - ``results/`` — CORE scores per version per run """ @@ -47,20 +47,6 @@ class LaneInfo: return self.root / "public" / version / "cases.jsonl" def holdout_cases_path(self, version: str = "v1") -> Path: - """Return the resolved holdout cases path for this lane. - - Resolution order (first existing wins): - 1. ``holdouts/cases.jsonl`` — flat plaintext - 2. ``holdouts/cases_plaintext.jsonl`` — cognition convention - 3. ``holdouts//cases.jsonl`` — versioned plaintext - - If none exist, returns the versioned path so callers receive a - coherent ``FileNotFoundError``. - - This intentionally does NOT decrypt sealed (``*.age``) holdouts — - sealed runs must go through ``evals.holdout_runner.run_holdout``, - which enforces aggregate-only output per its trust boundary. - """ holdouts = self.root / "holdouts" candidates = ( holdouts / "cases.jsonl", @@ -72,6 +58,17 @@ class LaneInfo: return path return candidates[-1] + def holdout_cases_path_sealed(self, version: str = "v1") -> Path: + holdouts = self.root / "holdouts" + candidates = ( + holdouts / "cases.jsonl.age", + holdouts / version / "cases.jsonl.age", + ) + for path in candidates: + if path.exists(): + return path + return candidates[-1] + def results_dir(self) -> Path: return self.root / "results" @@ -97,7 +94,6 @@ class LaneResult: def discover_lanes(root: Path | None = None) -> list[LaneInfo]: - """Find all valid eval lanes under *root*.""" base = root or EVALS_ROOT lanes: list[LaneInfo] = [] for child in sorted(base.iterdir()): @@ -123,7 +119,6 @@ def _discover_versions(lane_root: Path) -> tuple[str, ...]: def load_cases(path: Path) -> list[dict[str, Any]]: - """Load cases from a JSONL file.""" cases: list[dict[str, Any]] = [] for line in path.read_text().splitlines(): line = line.strip() @@ -133,7 +128,6 @@ def load_cases(path: Path) -> list[dict[str, Any]]: def load_lane_runner(lane: LaneInfo) -> Any: - """Dynamically import a lane's runner module.""" runner_path = lane.runner_path if not runner_path.exists(): raise FileNotFoundError(f"Runner not found: {runner_path}") @@ -148,38 +142,16 @@ def load_lane_runner(lane: LaneInfo) -> Any: return module -def run_lane( +def _run_cases( lane: LaneInfo, + cases: list[dict[str, Any]], *, - version: str = "v1", - split: str = "public", - config: Any = None, - workers: int | None = None, + version: str, + split: str, + config: Any, + workers: int | None, ) -> LaneResult: - """Run a single lane on a given version and split.""" - if split == "dev": - cases_path = lane.dev_cases_path() - elif split == "public": - cases_path = lane.public_cases_path(version) - elif split == "holdout": - cases_path = lane.holdout_cases_path(version) - else: - raise ValueError( - f"Unsupported split: {split!r}. Use 'dev', 'public', or 'holdout'." - ) - - if not cases_path.exists(): - raise FileNotFoundError(f"Cases not found: {cases_path}") - - cases = load_cases(cases_path) runner_module = load_lane_runner(lane) - - # PR #46 added the `workers` kwarg to enable parallel execution on - # lanes that opt in (currently: cognition). Most lane runners are - # serial and predate this contract. Pass `workers` only when the - # target runner's signature accepts it — otherwise call the legacy - # two-arg form. This keeps the framework dispatch backward-compatible - # without forcing every runner to accept a no-op kwarg. runner_params = inspect.signature(runner_module.run_lane).parameters if "workers" in runner_params or any( p.kind == inspect.Parameter.VAR_KEYWORD for p in runner_params.values() @@ -197,8 +169,50 @@ def run_lane( ) +def run_lane( + lane: LaneInfo, + *, + version: str = "v1", + split: str = "public", + config: Any = None, + workers: int | None = None, +) -> LaneResult: + if split == "holdout": + from evals.holdout_runner import _decrypt_holdout + + cases = _decrypt_holdout(lane.holdout_cases_path_sealed(version)) + return _run_cases( + lane, + cases, + version=version, + split=split, + config=config, + workers=workers, + ) + + if split == "dev": + cases_path = lane.dev_cases_path() + elif split == "public": + cases_path = lane.public_cases_path(version) + else: + raise ValueError( + f"Unsupported split: {split!r}. Use 'dev', 'public', or 'holdout'." + ) + + if not cases_path.exists(): + raise FileNotFoundError(f"Cases not found: {cases_path}") + + return _run_cases( + lane, + load_cases(cases_path), + version=version, + split=split, + config=config, + workers=workers, + ) + + def write_result(lane: LaneInfo, result: LaneResult) -> Path: - """Write a result to the lane's results directory, returning the path.""" results_dir = lane.results_dir() results_dir.mkdir(parents=True, exist_ok=True) @@ -213,7 +227,6 @@ def write_result(lane: LaneInfo, result: LaneResult) -> Path: def get_lane(name: str, root: Path | None = None) -> LaneInfo: - """Look up a single lane by name.""" base = root or EVALS_ROOT lane_root = base / name contract = lane_root / "contract.md" diff --git a/evals/holdout_runner.py b/evals/holdout_runner.py index 083039dc..ac397fe1 100644 --- a/evals/holdout_runner.py +++ b/evals/holdout_runner.py @@ -1,25 +1,24 @@ """Holdout runner — scores sealed test sets without exposing item-level results. -The holdout set lives encrypted in ``evals//holdouts/``. The decryption -key is held by the human reviewer and supplied via environment variable. - -Current implementation is a scaffold: it reads plaintext holdouts for initial -development. The encryption layer (age or GPG) is wired in before any holdout -set is considered "sealed." - -Trust boundary: this module reads encrypted files and writes only aggregate -scores. It must never write item-level results, case details, or raw case -content to the working tree. +Trust boundary: +- Reads sealed encrypted holdouts. +- Emits aggregate metrics only. +- Must never persist decrypted case content to disk. +- Must fail closed when a decryption identity is explicitly supplied but + decryption cannot complete. """ from __future__ import annotations +import json import os from dataclasses import dataclass from pathlib import Path from typing import Any -from evals.framework import LaneInfo, load_lane_runner, load_cases +from pyrage import decrypt +from pyrage import x25519 +from evals.framework import LaneInfo, load_lane_runner, load_cases HOLDOUT_KEY_ENV = "CORE_HOLDOUT_KEY" @@ -31,46 +30,84 @@ class HoldoutResult: sealed: bool -def _decrypt_holdout(encrypted_path: Path) -> list[dict[str, Any]]: - """Decrypt a holdout file and return cases. +def _parse_cases(raw: str) -> list[dict[str, Any]]: + cases: list[dict[str, Any]] = [] + for line in raw.splitlines(): + line = line.strip() + if line: + cases.append(json.loads(line)) + return cases - Currently: reads plaintext fallback if no encryption key is set. - Future: decrypt with age/GPG using CORE_HOLDOUT_KEY. + +def _decrypt_holdout(encrypted_path: Path) -> list[dict[str, Any]]: + """Decrypt a sealed holdout file. + + Development fallback semantics: + - If CORE_HOLDOUT_KEY is unset and plaintext fallback exists, + plaintext is allowed for local eval iteration. + + Fail-closed semantics: + - If CORE_HOLDOUT_KEY is set, decryption MUST succeed. + - No plaintext fallback is permitted once an identity is supplied. """ - key = os.environ.get(HOLDOUT_KEY_ENV) + key_path = os.environ.get(HOLDOUT_KEY_ENV) plaintext_path = encrypted_path.parent / "cases_plaintext.jsonl" - if key is None and plaintext_path.exists(): - return load_cases(plaintext_path) - - if key is None: + if key_path is None: + if plaintext_path.exists(): + return load_cases(plaintext_path) raise EnvironmentError( - f"Set {HOLDOUT_KEY_ENV} to decrypt holdout, or provide " - f"cases_plaintext.jsonl for unsealed development." + f"Set {HOLDOUT_KEY_ENV} or provide cases_plaintext.jsonl " + "for local development fallback." ) - # TODO: implement actual decryption (age -d -i ) - raise NotImplementedError( - "Encrypted holdout decryption not yet implemented. " - "Use cases_plaintext.jsonl for development." - ) + if not encrypted_path.exists(): + raise FileNotFoundError( + f"Encrypted holdout not found: {encrypted_path}" + ) + + identity_path = Path(key_path) + if not identity_path.exists(): + raise FileNotFoundError( + f"Holdout identity not found: {identity_path}" + ) + + try: + identity_text = identity_path.read_text(encoding="utf-8") + identities = [ + x25519.Identity.from_str(line.strip()) + for line in identity_text.splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + if not identities: + raise ValueError("no age identity entries found in identity file") + encrypted_bytes = encrypted_path.read_bytes() + decrypted = decrypt(encrypted_bytes, identities) + return _parse_cases(decrypted.decode("utf-8")) + except Exception as exc: + raise RuntimeError( + f"Failed to decrypt sealed holdout: {encrypted_path}" + ) from exc -def run_holdout(lane: LaneInfo, *, config: Any = None) -> HoldoutResult: - """Score a lane's holdout set and return only aggregate metrics.""" +def run_holdout( + lane: LaneInfo, + *, + version: str = "v1", + config: Any = None, +) -> HoldoutResult: holdout_dir = lane.root / "holdouts" if not holdout_dir.exists(): raise FileNotFoundError(f"No holdouts directory: {holdout_dir}") - cases = _decrypt_holdout(holdout_dir / "cases.jsonl.age") + encrypted_path = lane.holdout_cases_path_sealed(version) + cases = _decrypt_holdout(encrypted_path) runner_module = load_lane_runner(lane) report = runner_module.run_lane(cases, config=config) - sealed = os.environ.get(HOLDOUT_KEY_ENV) is not None - return HoldoutResult( lane=lane.name, metrics=report.metrics, - sealed=sealed, + sealed=os.environ.get(HOLDOUT_KEY_ENV) is not None, ) diff --git a/pyproject.toml b/pyproject.toml index 3d61ec06..4e0a5201 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "pytest-asyncio>=1.3.0", "pytest-xdist>=3.6", "psutil>=7.0", + "pyrage==1.2.3", "pyyaml>=6.0", "ruff>=0.15.12", ] diff --git a/scripts/seal_holdouts.py b/scripts/seal_holdouts.py new file mode 100644 index 00000000..c72fd458 --- /dev/null +++ b/scripts/seal_holdouts.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Seal eval holdout JSONL files with age recipients. + +This script intentionally performs recipient-only encryption. Identity/private +key material is never read by this command. +""" +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +from pyrage import encrypt +from pyrage import x25519 + +DEFAULT_EVALS_ROOT = Path("evals") +PLAINTEXT_NAMES = ("cases_plaintext.jsonl", "cases.jsonl") + + +@dataclass(frozen=True, slots=True) +class SealPlan: + source: Path + target: Path + + +def _recipient(value: str) -> x25519.Recipient: + value = value.strip() + if not value: + raise ValueError("recipient must not be empty") + return x25519.Recipient.from_str(value) + + +def _iter_holdout_dirs(evals_root: Path, lane: str | None) -> Iterable[Path]: + if lane is not None: + candidate = evals_root / lane / "holdouts" + if candidate.is_dir(): + yield candidate + return + + if not evals_root.is_dir(): + return + for child in sorted(evals_root.iterdir()): + holdouts = child / "holdouts" + if holdouts.is_dir(): + yield holdouts + + +def _plans_for_dir(holdouts_dir: Path, version: str | None) -> list[SealPlan]: + roots = [holdouts_dir] + if version is not None: + roots = [holdouts_dir / version] + + plans: list[SealPlan] = [] + for root in roots: + if not root.is_dir(): + continue + for name in PLAINTEXT_NAMES: + source = root / name + if source.exists(): + plans.append(SealPlan(source=source, target=root / "cases.jsonl.age")) + break + return plans + + +def discover_plans(evals_root: Path, lane: str | None, version: str | None) -> list[SealPlan]: + plans: list[SealPlan] = [] + for holdouts_dir in _iter_holdout_dirs(evals_root, lane): + plans.extend(_plans_for_dir(holdouts_dir, version)) + return plans + + +def seal(plan: SealPlan, recipient: x25519.Recipient, *, overwrite: bool) -> None: + if plan.target.exists() and not overwrite: + raise FileExistsError( + f"Refusing to overwrite existing sealed holdout: {plan.target}. " + "Pass --overwrite to replace it." + ) + ciphertext = encrypt(plan.source.read_bytes(), [recipient]) + plan.target.write_bytes(ciphertext) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--recipient", required=True, help="age x25519 recipient") + parser.add_argument("--evals-root", type=Path, default=DEFAULT_EVALS_ROOT) + parser.add_argument("--lane", help="seal only evals//holdouts") + parser.add_argument("--version", help="seal only holdouts/") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--overwrite", action="store_true") + args = parser.parse_args() + + recipient = _recipient(args.recipient) + plans = discover_plans(args.evals_root, args.lane, args.version) + + if not plans: + print("No plaintext holdouts discovered.") + return 0 + + for plan in plans: + action = "would seal" if args.dry_run else "sealed" + print(f"{action}: {plan.source} -> {plan.target}") + if not args.dry_run: + seal(plan, recipient, overwrite=args.overwrite) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_holdout_encryption.py b/tests/test_holdout_encryption.py new file mode 100644 index 00000000..d2581e8d --- /dev/null +++ b/tests/test_holdout_encryption.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +from pyrage import encrypt +from pyrage import x25519 + +from evals.holdout_runner import HOLDOUT_KEY_ENV +from evals.holdout_runner import _decrypt_holdout + + +@pytest.fixture() +def plaintext_cases() -> str: + return '{"prompt":"alpha","expected":"beta"}\n' + + +@pytest.fixture() +def identity_pair() -> tuple[x25519.Identity, x25519.Recipient]: + identity = x25519.Identity.generate() + return identity, identity.to_public() + + +def test_round_trip_decrypt(tmp_path: Path, plaintext_cases: str, identity_pair) -> None: + identity, recipient = identity_pair + + identity_path = tmp_path / "identity.txt" + identity_path.write_text(str(identity)) + + encrypted_path = tmp_path / "cases.jsonl.age" + encrypted_path.write_bytes( + encrypt(plaintext_cases.encode("utf-8"), [recipient]) + ) + + os.environ[HOLDOUT_KEY_ENV] = str(identity_path) + try: + cases = _decrypt_holdout(encrypted_path) + finally: + os.environ.pop(HOLDOUT_KEY_ENV, None) + + assert len(cases) == 1 + assert cases[0]["prompt"] == "alpha" + + +def test_reject_wrong_identity(tmp_path: Path, plaintext_cases: str) -> None: + good_identity = x25519.Identity.generate() + wrong_identity = x25519.Identity.generate() + + wrong_identity_path = tmp_path / "wrong_identity.txt" + wrong_identity_path.write_text(str(wrong_identity)) + + encrypted_path = tmp_path / "cases.jsonl.age" + encrypted_path.write_bytes( + encrypt( + plaintext_cases.encode("utf-8"), + [good_identity.to_public()], + ) + ) + + os.environ[HOLDOUT_KEY_ENV] = str(wrong_identity_path) + try: + with pytest.raises(RuntimeError): + _decrypt_holdout(encrypted_path) + finally: + os.environ.pop(HOLDOUT_KEY_ENV, None) + + +def test_dev_mode_plaintext_fallback(tmp_path: Path, plaintext_cases: str) -> None: + plaintext_path = tmp_path / "cases_plaintext.jsonl" + plaintext_path.write_text(plaintext_cases) + + encrypted_path = tmp_path / "cases.jsonl.age" + + os.environ.pop(HOLDOUT_KEY_ENV, None) + cases = _decrypt_holdout(encrypted_path) + + assert len(cases) == 1 + assert cases[0]["expected"] == "beta" + + +def test_key_set_disables_plaintext_fallback(tmp_path: Path, plaintext_cases: str) -> None: + plaintext_path = tmp_path / "cases_plaintext.jsonl" + plaintext_path.write_text(plaintext_cases) + + missing_identity = tmp_path / "missing_identity.txt" + encrypted_path = tmp_path / "cases.jsonl.age" + + os.environ[HOLDOUT_KEY_ENV] = str(missing_identity) + try: + with pytest.raises(FileNotFoundError): + _decrypt_holdout(encrypted_path) + finally: + os.environ.pop(HOLDOUT_KEY_ENV, None)