From db081fa5b30bb5c1477f57851bcd914f7623c67f Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 23 Jun 2026 07:40:53 -0700 Subject: [PATCH 1/3] feat(evals): add GSM1K local audit adapter --- evals/generalization/__init__.py | 2 + evals/generalization/adapters/__init__.py | 9 + evals/generalization/adapters/gsm1k.py | 150 ++++++++++ .../benchmarks/run_generalization_audit.py | 156 ++++++++++- tests/test_generalization_audit_runner.py | 4 +- tests/test_generalization_gsm1k_adapter.py | 258 ++++++++++++++++++ 6 files changed, 573 insertions(+), 6 deletions(-) create mode 100644 evals/generalization/adapters/__init__.py create mode 100644 evals/generalization/adapters/gsm1k.py create mode 100644 tests/test_generalization_gsm1k_adapter.py diff --git a/evals/generalization/__init__.py b/evals/generalization/__init__.py index d35ef0d5..f2a031fc 100644 --- a/evals/generalization/__init__.py +++ b/evals/generalization/__init__.py @@ -21,6 +21,7 @@ from evals.generalization.item_schema import ( GeneralizationAuditReport, ) from evals.generalization.audit_runner import run_generalization_audit +from evals.generalization.adapters.gsm1k import load_gsm1k_items __all__ = [ "GeneralizationBenchmarkManifest", @@ -36,4 +37,5 @@ __all__ = [ "GeneralizationAuditOutcome", "GeneralizationAuditReport", "run_generalization_audit", + "load_gsm1k_items", ] diff --git a/evals/generalization/adapters/__init__.py b/evals/generalization/adapters/__init__.py new file mode 100644 index 00000000..2b05c88f --- /dev/null +++ b/evals/generalization/adapters/__init__.py @@ -0,0 +1,9 @@ +"""Adapters for loading local generalization benchmark datasets.""" + +from __future__ import annotations + +from evals.generalization.adapters.gsm1k import load_gsm1k_items + +__all__ = [ + "load_gsm1k_items", +] diff --git a/evals/generalization/adapters/gsm1k.py b/evals/generalization/adapters/gsm1k.py new file mode 100644 index 00000000..945ff2a4 --- /dev/null +++ b/evals/generalization/adapters/gsm1k.py @@ -0,0 +1,150 @@ +"""Adapter for loading local GSM1K benchmark files.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from evals.generalization.item_schema import GeneralizationAuditItem + + +def load_gsm1k_items( + *, + local_cache: Path, + split: str, + max_items: int | None = None, +) -> tuple[GeneralizationAuditItem, ...]: + """Load and normalize GSM1K items from a local cache directory or file. + + Args: + local_cache: Path to the cache directory or file. + split: The requested dataset split (e.g. 'test'). + max_items: Optional maximum number of items to load. + + Returns: + A tuple of loaded GeneralizationAuditItem records. + """ + if not local_cache.exists(): + raise FileNotFoundError( + f"Local cache path does not exist: {local_cache}" + ) + + target_file = None + if local_cache.is_file(): + target_file = local_cache + elif local_cache.is_dir(): + # Check standard file naming conventions for local cache + candidate_jsonl = local_cache / f"{split}.jsonl" + candidate_json = local_cache / f"{split}.json" + if candidate_jsonl.is_file(): + target_file = candidate_jsonl + elif candidate_json.is_file(): + target_file = candidate_json + else: + # Fallback search for files containing the split name in their filename + all_files = sorted(local_cache.glob("*")) + for f in all_files: + if ( + f.is_file() + and split in f.name + and f.suffix in (".jsonl", ".json") + ): + target_file = f + break + + if not target_file: + raise FileNotFoundError( + f"No JSON or JSONL file found for split {split!r} in {local_cache}" + ) + + try: + content = target_file.read_text(encoding="utf-8") + except OSError as exc: + raise ValueError( + f"Failed to read local GSM1K file {target_file}: {exc}" + ) from exc + + # Parse as JSONL first, fallback to JSON array + raw_records = [] + lines = content.strip().splitlines() + is_jsonl = True + for idx, line in enumerate(lines): + if not line.strip(): + continue + try: + parsed = json.loads(line) + # JSONL lines must be objects (mappings) + if not isinstance(parsed, dict): + is_jsonl = False + break + raw_records.append(parsed) + except Exception: + is_jsonl = False + break + + if not is_jsonl or not raw_records: + try: + data = json.loads(content) + if isinstance(data, list): + raw_records = data + # Since it parsed successfully as a JSON array, it's not JSONL format + is_jsonl = False + elif isinstance(data, dict): + raw_records = [data] + is_jsonl = False + else: + raise ValueError("JSON root must be a list or dictionary.") + except Exception as exc: + raise ValueError( + f"Failed to parse GSM1K file {target_file} as JSON/JSONL: {exc}" + ) from exc + + items: list[GeneralizationAuditItem] = [] + for idx, rec in enumerate(raw_records): + if max_items is not None and len(items) >= max_items: + break + + # Extract question (query) + question = rec.get("question") or rec.get("prompt") + if question is None: + raise ValueError( + f"GSM1K record at index {idx} in {target_file.name} is missing 'question' or 'prompt' field." + ) + + # Extract answer + answer = rec.get("answer") or rec.get("grade") or rec.get("label") + if answer is None: + raise ValueError( + f"GSM1K record at index {idx} in {target_file.name} is missing 'answer', 'grade', or 'label' field." + ) + + # Resolve stable item ID + item_id = str(rec.get("id") or rec.get("item_id") or idx) + + # Build stable opaque prompt reference + prompt_ref = f"gsm1k:{split}:{item_id}" + + # Setup metadata safely storing prompt/answer content out of aggregate report + metadata_list = [ + ("source_format", "jsonl" if is_jsonl else "json"), + ("question", str(question)), + ("answer", str(answer)), + ] + + if "grade" in rec: + metadata_list.append(("grade", str(rec["grade"]))) + if "domain" in rec: + metadata_list.append(("domain", str(rec["domain"]))) + + items.append( + GeneralizationAuditItem( + dataset="GSM1K", + split=split, + item_id=item_id, + prompt_ref=prompt_ref, + answer_kind="numeric_text", + metadata=tuple(metadata_list), + ) + ) + + return tuple(items) diff --git a/scripts/benchmarks/run_generalization_audit.py b/scripts/benchmarks/run_generalization_audit.py index 3c01dfa4..db526016 100755 --- a/scripts/benchmarks/run_generalization_audit.py +++ b/scripts/benchmarks/run_generalization_audit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""CLI script to run generalization audit (skeleton).""" +"""CLI script to run generalization audit.""" from __future__ import annotations @@ -15,6 +15,7 @@ repo_root = script_path.parent.parent.parent if str(repo_root) not in sys.path: sys.path.insert(0, str(repo_root)) +from evals.generalization.adapters.gsm1k import load_gsm1k_items # noqa: E402 from evals.generalization.audit_runner import run_generalization_audit # noqa: E402 from evals.generalization.item_schema import ( # noqa: E402 GeneralizationAuditItem, @@ -33,6 +34,21 @@ def main() -> None: parser.add_argument( "--split", type=str, default="test", help="Dataset split to audit." ) + parser.add_argument( + "--local-cache", + type=str, + help="Explicit path to local cache (overrides manifest).", + ) + parser.add_argument( + "--max-items", + type=int, + help="Maximum number of items to load.", + ) + parser.add_argument( + "--metadata-only", + action="store_true", + help="Run in metadata-only mode, bypassing gate failures.", + ) parser.add_argument( "--json", action="store_true", help="Print report in deterministic JSON format." ) @@ -46,10 +62,142 @@ def main() -> None: ) sys.exit(1) + ADAPTERS = { + "gsm1k": load_gsm1k_items, + } + if args.dataset: - # In PR-2, no real dataset adapters exist, so we fail with the required error code - print("Error: dataset_adapter_unavailable", file=sys.stderr) - sys.exit(1) + dataset_key = args.dataset.lower() + if dataset_key not in ADAPTERS: + print("Error: dataset_adapter_unavailable", file=sys.stderr) + sys.exit(1) + + # 1. Run the verifier from #887 to check manifest gates + from evals.generalization.cache_verifier import ( + verify_local_generalization_cache, + ) + + manifests_dir = repo_root / "evals" / "generalization" / "manifests" + try: + report = verify_local_generalization_cache( + repo_root=repo_root, + manifests_dir=manifests_dir, + require_present=False, + ) + except Exception as exc: + print(f"Manifest validation failed: {exc}", file=sys.stderr) + sys.exit(1) + + # Find the record for the dataset + record = None + for r in report.records: + if r.dataset.lower() == dataset_key: + record = r + break + + if not record: + print( + f"Error: dataset_adapter_unavailable (no manifest for {args.dataset})", + file=sys.stderr, + ) + sys.exit(1) + + # Fail closed on unresolved gates unless in metadata-only mode + if not args.metadata_only: + if not record.license_ready or not record.checksum_ready: + print("Error: benchmark_manifest_unresolved", file=sys.stderr) + sys.exit(1) + + # Resolve local cache path + if args.local_cache: + cache_path = Path(args.local_cache) + else: + from evals.generalization.manifest_schema import ( + load_and_validate_manifest, + ) + + manifest_path = manifests_dir / f"{dataset_key}.yaml" + try: + manifest = load_and_validate_manifest(manifest_path) + cache_path = repo_root / manifest.local_cache + except Exception as exc: + print(f"Error reading manifest: {exc}", file=sys.stderr) + sys.exit(1) + + # Check existence if not in metadata-only mode + if not args.metadata_only and not cache_path.exists(): + print( + f"Error: Cache path does not exist: {cache_path}", + file=sys.stderr, + ) + sys.exit(1) + + # If metadata-only and cache doesn't exist, we can exit gracefully + if args.metadata_only and not cache_path.exists(): + print("Metadata-only validation passed (cache absent).") + sys.exit(0) + + # Load items + adapter_fn = ADAPTERS[dataset_key] + try: + items = adapter_fn( + local_cache=cache_path, + split=args.split, + max_items=args.max_items, + ) + except Exception as exc: + print(f"Failed to load items: {exc}", file=sys.stderr) + sys.exit(1) + + # In PR-3, use a dummy evaluator for testing the adapter composition + def dummy_evaluator( + item: GeneralizationAuditItem, + ) -> GeneralizationAuditOutcome: + return GeneralizationAuditOutcome( + item_id=item.item_id, + disposition="correct", + residual_kinds=(), + candidate_attempt_count=1, + binding_failure_count=0, + replay_refusal_count=0, + sealed_trace_dispositions=("success",), + reason_codes=(), + ) + + try: + report = run_generalization_audit( + dataset=record.dataset, + split=args.split, + items=items, + evaluator=dummy_evaluator, + ) + except Exception as exc: + print(f"Audit Failed: {exc}", file=sys.stderr) + sys.exit(1) + + if args.json: + print(json.dumps(asdict(report), indent=2, sort_keys=True)) + else: + print( + f"Generalization Audit Report (Policy: {report.policy_version})" + ) + print("=" * 80) + print(f"Dataset: {report.dataset}") + print(f"Split: {report.split}") + print(f"Total Items: {report.n_items}") + print(f"Correct: {report.correct}") + print(f"Wrong: {report.wrong}") + print(f"Refused: {report.refused}") + print(f"Unsupported: {report.unsupported}") + print(f"Candidate Attempts: {report.candidate_attempts}") + print(f"Binding Failures: {report.binding_failures}") + print(f"Replay Refusals: {report.replay_refusals}") + print(f"Sealed Trace Dispositions: {report.sealed_trace_dispositions}") + print(f"Dominant Residual Kinds: {report.dominant_residual_kinds}") + print(f"Reason Codes: {', '.join(report.reason_codes)}") + print("=" * 80) + + sys.exit(0) if args.synthetic_smoke: # Generate synthetic items and run smoke audit diff --git a/tests/test_generalization_audit_runner.py b/tests/test_generalization_audit_runner.py index e342addf..f63f2877 100644 --- a/tests/test_generalization_audit_runner.py +++ b/tests/test_generalization_audit_runner.py @@ -262,13 +262,13 @@ def test_cli_synthetic_smoke() -> None: def test_cli_real_dataset_refuses() -> None: - """Verify that requesting a real dataset fails with dataset_adapter_unavailable.""" + """Verify that requesting a dataset with no adapter fails with dataset_adapter_unavailable.""" result = subprocess.run( [ sys.executable, "scripts/benchmarks/run_generalization_audit.py", "--dataset", - "gsm1k", + "asdiv", ], capture_output=True, text=True, diff --git a/tests/test_generalization_gsm1k_adapter.py b/tests/test_generalization_gsm1k_adapter.py new file mode 100644 index 00000000..dc4fbd3e --- /dev/null +++ b/tests/test_generalization_gsm1k_adapter.py @@ -0,0 +1,258 @@ +"""Tests for the GSM1K local cache adapter.""" + +from __future__ import annotations + +import json +import socket +import subprocess +import sys +from pathlib import Path +import pytest +from evals.generalization.adapters.gsm1k import load_gsm1k_items + + + +def write_synthetic_jsonl(path: Path, records: list[dict]) -> None: + lines = [json.dumps(r) for r in records] + path.write_text("\n".join(lines), encoding="utf-8") + + +def write_synthetic_json(path: Path, data: any) -> None: + path.write_text(json.dumps(data), encoding="utf-8") + + +def test_loads_synthetic_jsonl_records(tmp_path: Path) -> None: + """GSM1K adapter loads synthetic JSONL records and produces stable item IDs.""" + cache_dir = tmp_path / "gsm1k_cache" + cache_dir.mkdir() + records = [ + {"question": "Alice has 2 apples.", "answer": "2", "id": "q1"}, + {"question": "Bob has 3 apples.", "answer": "3", "id": "q2"}, + ] + write_synthetic_jsonl(cache_dir / "test.jsonl", records) + + items = load_gsm1k_items(local_cache=cache_dir, split="test") + + assert len(items) == 2 + assert items[0].dataset == "GSM1K" + assert items[0].split == "test" + assert items[0].item_id == "q1" + assert items[0].prompt_ref == "gsm1k:test:q1" + assert items[0].answer_kind == "numeric_text" + + # Verify opaque prompt_ref and question/answer content only in metadata + assert items[0].prompt_ref == "gsm1k:test:q1" + metadata_dict = dict(items[0].metadata) + assert metadata_dict["question"] == "Alice has 2 apples." + assert metadata_dict["answer"] == "2" + + +def test_loads_synthetic_json_records(tmp_path: Path) -> None: + """GSM1K adapter loads synthetic JSON array records.""" + cache_dir = tmp_path / "gsm1k_cache" + cache_dir.mkdir() + records = [ + {"question": "Charlie has 5 coins.", "answer": "5", "id": "c1"}, + {"question": "David has 10 coins.", "answer": "10", "id": "c2"}, + ] + write_synthetic_json(cache_dir / "test.json", records) + + items = load_gsm1k_items(local_cache=cache_dir, split="test") + + assert len(items) == 2 + assert items[0].item_id == "c1" + assert items[1].item_id == "c2" + + +def test_honors_max_items(tmp_path: Path) -> None: + """GSM1K adapter honors max_items argument.""" + cache_dir = tmp_path / "gsm1k_cache" + cache_dir.mkdir() + records = [ + {"question": "Q1", "answer": "A1"}, + {"question": "Q2", "answer": "A2"}, + {"question": "Q3", "answer": "A3"}, + ] + write_synthetic_jsonl(cache_dir / "test.jsonl", records) + + items = load_gsm1k_items(local_cache=cache_dir, split="test", max_items=2) + assert len(items) == 2 + assert items[0].item_id == "0" + assert items[1].item_id == "1" + + +def test_refuses_missing_cache_path() -> None: + """GSM1K adapter raises FileNotFoundError if cache path is missing.""" + with pytest.raises( + FileNotFoundError, + match="Cache directory does not exist|Local cache path does not exist", + ): + load_gsm1k_items( + local_cache=Path("non_existent_directory_1234"), split="test" + ) + + +def test_refuses_unsupported_split(tmp_path: Path) -> None: + """GSM1K adapter raises FileNotFoundError if split file is not found.""" + cache_dir = tmp_path / "gsm1k_cache" + cache_dir.mkdir() + write_synthetic_jsonl( + cache_dir / "test.jsonl", [{"question": "Q", "answer": "A"}] + ) + + with pytest.raises( + FileNotFoundError, match="No JSON or JSONL file found for split" + ): + load_gsm1k_items(local_cache=cache_dir, split="train") + + +def test_refuses_malformed_record_missing_question(tmp_path: Path) -> None: + """GSM1K adapter raises ValueError if a record is missing the question field.""" + cache_dir = tmp_path / "gsm1k_cache" + cache_dir.mkdir() + records = [ + {"answer": "5", "id": "1"}, # missing question/prompt + ] + write_synthetic_jsonl(cache_dir / "test.jsonl", records) + + with pytest.raises(ValueError, match="missing 'question' or 'prompt'"): + load_gsm1k_items(local_cache=cache_dir, split="test") + + +def test_refuses_malformed_record_missing_answer(tmp_path: Path) -> None: + """GSM1K adapter raises ValueError if a record is missing the answer field.""" + cache_dir = tmp_path / "gsm1k_cache" + cache_dir.mkdir() + records = [ + {"question": "Q?", "id": "1"}, # missing answer/grade/label + ] + write_synthetic_jsonl(cache_dir / "test.jsonl", records) + + with pytest.raises( + ValueError, match="missing 'answer', 'grade', or 'label'" + ): + load_gsm1k_items(local_cache=cache_dir, split="test") + + +def test_does_not_download_or_write_cache( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """GSM1K adapter does not use network or write files to disk during load.""" + cache_dir = tmp_path / "gsm1k_cache" + cache_dir.mkdir() + write_synthetic_jsonl( + cache_dir / "test.jsonl", [{"question": "Q", "answer": "A"}] + ) + + # Intercept socket calls to block network + def blocked_socket(*args: any, **kwargs: any) -> any: + raise RuntimeError("Network calls are forbidden during loading!") + + monkeypatch.setattr(socket, "socket", blocked_socket) + + # Intercept write/open actions on paths that are not the test file + orig_write_text = Path.write_text + + def blocked_write_text(self: Path, *args: any, **kwargs: any) -> any: + if self.parent.resolve() != cache_dir.resolve(): + raise RuntimeError( + f"Writing to disk outside test cache is forbidden: {self}" + ) + return orig_write_text(self, *args, **kwargs) + + monkeypatch.setattr(Path, "write_text", blocked_write_text) + + items = load_gsm1k_items(local_cache=cache_dir, split="test") + assert len(items) == 1 + + +def test_repository_contains_no_committed_gsm1k_examples() -> None: + """Check that the repository contains no committed GSM1K examples or dataset files.""" + repo_root = Path(__file__).parent.parent.parent.parent + + try: + result = subprocess.run( + ["git", "ls-files"], + cwd=str(repo_root), + capture_output=True, + text=True, + check=True, + ) + tracked_files = result.stdout.strip().splitlines() + except (subprocess.SubprocessError, FileNotFoundError): + tracked_files = [] + benchmarks_dir = repo_root / ".data" / "benchmarks" + if benchmarks_dir.exists(): + tracked_files = [ + str(p) for p in benchmarks_dir.rglob("*") if p.is_file() + ] + + # Make sure no tracked files under .data/benchmarks/ exist except .gitkeep + for f in tracked_files: + if ".data/benchmarks" in f and not f.endswith(".gitkeep"): + pytest.fail(f"Committed benchmark cache file found: {f}") + + # Check that no manifest YAML file contains actual benchmark questions/answers + manifest_dir = repo_root / "evals" / "generalization" / "manifests" + gsm1k_manifest = manifest_dir / "gsm1k.yaml" + if gsm1k_manifest.exists(): + content = gsm1k_manifest.read_text(encoding="utf-8") + assert ( + "question" not in content.lower() + ), "GSM1K manifest contains raw question data!" + assert ( + "answer" not in content.lower() or "grade" in content.lower() + ), "GSM1K manifest contains raw answer data!" + + +def test_cli_refuses_unresolved_manifest_gates() -> None: + """The CLI refuses to run if the manifest has unresolved license/checksum gates.""" + result = subprocess.run( + [ + sys.executable, + "scripts/benchmarks/run_generalization_audit.py", + "--dataset", + "gsm1k", + "--split", + "test", + ], + capture_output=True, + text=True, + ) + assert result.returncode != 0 + assert "benchmark_manifest_unresolved" in result.stderr + + +def test_cli_local_adapter_works_with_temp_cache_and_metadata_only( + tmp_path: Path, +) -> None: + """The CLI run works when --metadata-only is passed with local-cache.""" + cache_dir = tmp_path / "gsm1k_cache" + cache_dir.mkdir() + records = [ + {"question": "Alice has 2 apples.", "answer": "2", "id": "q1"}, + ] + write_synthetic_jsonl(cache_dir / "test.jsonl", records) + + result = subprocess.run( + [ + sys.executable, + "scripts/benchmarks/run_generalization_audit.py", + "--dataset", + "gsm1k", + "--split", + "test", + "--local-cache", + str(cache_dir), + "--metadata-only", + "--json", + ], + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0 + report = json.loads(result.stdout) + assert report["dataset"] == "GSM1K" + assert report["n_items"] == 1 + assert report["correct"] == 1 From d34883b61a23f32cadad63e9f033149f63282aeb Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 23 Jun 2026 07:52:32 -0700 Subject: [PATCH 2/3] feat(evals): implement safe metadata and fail-closed evaluator check for GSM1K --- evals/generalization/adapters/gsm1k.py | 19 ++-- .../benchmarks/run_generalization_audit.py | 72 ++++------------ tests/test_generalization_gsm1k_adapter.py | 86 +++++++++++++++++-- 3 files changed, 109 insertions(+), 68 deletions(-) diff --git a/evals/generalization/adapters/gsm1k.py b/evals/generalization/adapters/gsm1k.py index 945ff2a4..549b9d52 100644 --- a/evals/generalization/adapters/gsm1k.py +++ b/evals/generalization/adapters/gsm1k.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json from pathlib import Path @@ -124,18 +125,20 @@ def load_gsm1k_items( # Build stable opaque prompt reference prompt_ref = f"gsm1k:{split}:{item_id}" - # Setup metadata safely storing prompt/answer content out of aggregate report + q_sha256 = hashlib.sha256(str(question).encode("utf-8")).hexdigest() + a_sha256 = hashlib.sha256(str(answer).encode("utf-8")).hexdigest() + metadata_list = [ ("source_format", "jsonl" if is_jsonl else "json"), - ("question", str(question)), - ("answer", str(answer)), + ("source_index", str(idx)), + ("source_file_name", target_file.name), + ("source_record_id", item_id), + ("question_sha256", q_sha256), + ("answer_sha256", a_sha256), + ("question_length", str(len(str(question)))), + ("answer_kind", "numeric_text"), ] - if "grade" in rec: - metadata_list.append(("grade", str(rec["grade"]))) - if "domain" in rec: - metadata_list.append(("domain", str(rec["domain"]))) - items.append( GeneralizationAuditItem( dataset="GSM1K", diff --git a/scripts/benchmarks/run_generalization_audit.py b/scripts/benchmarks/run_generalization_audit.py index db526016..16e8ee19 100755 --- a/scripts/benchmarks/run_generalization_audit.py +++ b/scripts/benchmarks/run_generalization_audit.py @@ -107,6 +107,8 @@ def main() -> None: if not record.license_ready or not record.checksum_ready: print("Error: benchmark_manifest_unresolved", file=sys.stderr) sys.exit(1) + print("Error: dataset_evaluator_unavailable", file=sys.stderr) + sys.exit(1) # Resolve local cache path if args.local_cache: @@ -124,20 +126,12 @@ def main() -> None: print(f"Error reading manifest: {exc}", file=sys.stderr) sys.exit(1) - # Check existence if not in metadata-only mode - if not args.metadata_only and not cache_path.exists(): - print( - f"Error: Cache path does not exist: {cache_path}", - file=sys.stderr, - ) - sys.exit(1) - - # If metadata-only and cache doesn't exist, we can exit gracefully - if args.metadata_only and not cache_path.exists(): + # Check existence since we are in metadata-only mode + if not cache_path.exists(): print("Metadata-only validation passed (cache absent).") sys.exit(0) - # Load items + # Load items (metadata-only summary) adapter_fn = ADAPTERS[dataset_key] try: items = adapter_fn( @@ -149,52 +143,22 @@ def main() -> None: print(f"Failed to load items: {exc}", file=sys.stderr) sys.exit(1) - # In PR-3, use a dummy evaluator for testing the adapter composition - def dummy_evaluator( - item: GeneralizationAuditItem, - ) -> GeneralizationAuditOutcome: - return GeneralizationAuditOutcome( - item_id=item.item_id, - disposition="correct", - residual_kinds=(), - candidate_attempt_count=1, - binding_failure_count=0, - replay_refusal_count=0, - sealed_trace_dispositions=("success",), - reason_codes=(), - ) - - try: - report = run_generalization_audit( - dataset=record.dataset, - split=args.split, - items=items, - evaluator=dummy_evaluator, - ) - except Exception as exc: - print(f"Audit Failed: {exc}", file=sys.stderr) - sys.exit(1) - if args.json: - print(json.dumps(asdict(report), indent=2, sort_keys=True)) + summary = { + "dataset": record.dataset, + "split": args.split, + "n_items": len(items), + "local_cache": str(cache_path), + "metadata_only": True, + } + print(json.dumps(summary, indent=2, sort_keys=True)) else: - print( - f"Generalization Audit Report (Policy: {report.policy_version})" - ) + print("Generalization Adapter Summary (Metadata-only)") print("=" * 80) - print(f"Dataset: {report.dataset}") - print(f"Split: {report.split}") - print(f"Total Items: {report.n_items}") - print(f"Correct: {report.correct}") - print(f"Wrong: {report.wrong}") - print(f"Refused: {report.refused}") - print(f"Unsupported: {report.unsupported}") - print(f"Candidate Attempts: {report.candidate_attempts}") - print(f"Binding Failures: {report.binding_failures}") - print(f"Replay Refusals: {report.replay_refusals}") - print(f"Sealed Trace Dispositions: {report.sealed_trace_dispositions}") - print(f"Dominant Residual Kinds: {report.dominant_residual_kinds}") - print(f"Reason Codes: {', '.join(report.reason_codes)}") + print(f"Dataset: {record.dataset}") + print(f"Split: {args.split}") + print(f"Total Items: {len(items)}") + print(f"Cache Path: {cache_path}") print("=" * 80) sys.exit(0) diff --git a/tests/test_generalization_gsm1k_adapter.py b/tests/test_generalization_gsm1k_adapter.py index dc4fbd3e..8579770e 100644 --- a/tests/test_generalization_gsm1k_adapter.py +++ b/tests/test_generalization_gsm1k_adapter.py @@ -8,8 +8,14 @@ import subprocess import sys from pathlib import Path import pytest -from evals.generalization.adapters.gsm1k import load_gsm1k_items +import evals.generalization.cache_verifier +from evals.generalization.adapters.gsm1k import load_gsm1k_items +from evals.generalization.cache_verifier import ( + CacheVerificationRecord, + CacheVerificationReport, +) +from scripts.benchmarks.run_generalization_audit import main as cli_main def write_synthetic_jsonl(path: Path, records: list[dict]) -> None: @@ -40,11 +46,18 @@ def test_loads_synthetic_jsonl_records(tmp_path: Path) -> None: assert items[0].prompt_ref == "gsm1k:test:q1" assert items[0].answer_kind == "numeric_text" - # Verify opaque prompt_ref and question/answer content only in metadata - assert items[0].prompt_ref == "gsm1k:test:q1" + # Verify opaque metadata metadata_dict = dict(items[0].metadata) - assert metadata_dict["question"] == "Alice has 2 apples." - assert metadata_dict["answer"] == "2" + assert "question" not in metadata_dict + assert "prompt" not in metadata_dict + assert "answer" not in metadata_dict + assert "grade" not in metadata_dict + assert "label" not in metadata_dict + + assert "question_sha256" in metadata_dict + assert "answer_sha256" in metadata_dict + assert "question_length" in metadata_dict + assert metadata_dict["source_record_id"] == "q1" def test_loads_synthetic_json_records(tmp_path: Path) -> None: @@ -255,4 +268,65 @@ def test_cli_local_adapter_works_with_temp_cache_and_metadata_only( report = json.loads(result.stdout) assert report["dataset"] == "GSM1K" assert report["n_items"] == 1 - assert report["correct"] == 1 + assert report["metadata_only"] is True + # The metadata-only path does not claim correct/wrong + assert "correct" not in report + assert "wrong" not in report + + +def test_cli_real_gsm1k_without_evaluator_refuses( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """The CLI refuses to run a full audit without an evaluator, failing with dataset_evaluator_unavailable.""" + cache_dir = tmp_path / "gsm1k_cache" + cache_dir.mkdir() + records = [ + {"question": "Alice has 2 apples.", "answer": "2", "id": "q1"}, + ] + write_synthetic_jsonl(cache_dir / "test.jsonl", records) + + # Monkeypatch verify_local_generalization_cache to report resolved gates + def mock_verify(*args: any, **kwargs: any) -> CacheVerificationReport: + record = CacheVerificationRecord( + dataset="GSM1K", + manifest_path="gsm1k.yaml", + local_cache=str(cache_dir), + exists=True, + license_ready=True, + checksum_ready=True, + runnable=True, + reason_codes=(), + ) + return CacheVerificationReport( + policy_version="test.v1", + records=(record,), + all_runnable=True, + reason_codes=(), + ) + + monkeypatch.setattr( + evals.generalization.cache_verifier, + "verify_local_generalization_cache", + mock_verify, + ) + + # Setup sys.argv to run CLI without --metadata-only + monkeypatch.setattr( + sys, + "argv", + [ + "run_generalization_audit.py", + "--dataset", + "gsm1k", + "--local-cache", + str(cache_dir), + ], + ) + + with pytest.raises(SystemExit) as excinfo: + cli_main() + assert excinfo.value.code != 0 + captured = capsys.readouterr() + assert "dataset_evaluator_unavailable" in captured.err From f8fe6e22ca80acdd07aa8e3add435795cb70891c Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 23 Jun 2026 07:57:34 -0700 Subject: [PATCH 3/3] test(evals): strengthen GSM1K repo-root guard --- tests/test_generalization_gsm1k_adapter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_generalization_gsm1k_adapter.py b/tests/test_generalization_gsm1k_adapter.py index 8579770e..08ad9367 100644 --- a/tests/test_generalization_gsm1k_adapter.py +++ b/tests/test_generalization_gsm1k_adapter.py @@ -181,7 +181,9 @@ def test_does_not_download_or_write_cache( def test_repository_contains_no_committed_gsm1k_examples() -> None: """Check that the repository contains no committed GSM1K examples or dataset files.""" - repo_root = Path(__file__).parent.parent.parent.parent + repo_root = Path(__file__).resolve().parents[1] + assert (repo_root / "evals" / "generalization").is_dir() + assert (repo_root / ".git").exists() try: result = subprocess.run(