Merge pull request #889 from AssetOverflow/feat/gsm1k-local-audit-adapter

evals/generalization/__init__.py

@@ -21,6 +21,7 @@ from evals.generalization.item_schema import (
     GeneralizationAuditReport,
 )
 from evals.generalization.audit_runner import run_generalization_audit
+from evals.generalization.adapters.gsm1k import load_gsm1k_items
 
 __all__ = [
     "GeneralizationBenchmarkManifest",
@@ -36,4 +37,5 @@ __all__ = [
     "GeneralizationAuditOutcome",
     "GeneralizationAuditReport",
     "run_generalization_audit",
+    "load_gsm1k_items",
 ]

evals/generalization/adapters/__init__.py

@@ -0,0 +1,9 @@
+"""Adapters for loading local generalization benchmark datasets."""
+
+from __future__ import annotations
+
+from evals.generalization.adapters.gsm1k import load_gsm1k_items
+
+__all__ = [
+    "load_gsm1k_items",
+]

evals/generalization/adapters/gsm1k.py

@@ -0,0 +1,153 @@
+"""Adapter for loading local GSM1K benchmark files."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+from evals.generalization.item_schema import GeneralizationAuditItem
+
+
+def load_gsm1k_items(
+    *,
+    local_cache: Path,
+    split: str,
+    max_items: int | None = None,
+) -> tuple[GeneralizationAuditItem, ...]:
+    """Load and normalize GSM1K items from a local cache directory or file.
+
+    Args:
+        local_cache: Path to the cache directory or file.
+        split: The requested dataset split (e.g. 'test').
+        max_items: Optional maximum number of items to load.
+
+    Returns:
+        A tuple of loaded GeneralizationAuditItem records.
+    """
+    if not local_cache.exists():
+        raise FileNotFoundError(
+            f"Local cache path does not exist: {local_cache}"
+        )
+
+    target_file = None
+    if local_cache.is_file():
+        target_file = local_cache
+    elif local_cache.is_dir():
+        # Check standard file naming conventions for local cache
+        candidate_jsonl = local_cache / f"{split}.jsonl"
+        candidate_json = local_cache / f"{split}.json"
+        if candidate_jsonl.is_file():
+            target_file = candidate_jsonl
+        elif candidate_json.is_file():
+            target_file = candidate_json
+        else:
+            # Fallback search for files containing the split name in their filename
+            all_files = sorted(local_cache.glob("*"))
+            for f in all_files:
+                if (
+                    f.is_file()
+                    and split in f.name
+                    and f.suffix in (".jsonl", ".json")
+                ):
+                    target_file = f
+                    break
+
+    if not target_file:
+        raise FileNotFoundError(
+            f"No JSON or JSONL file found for split {split!r} in {local_cache}"
+        )
+
+    try:
+        content = target_file.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise ValueError(
+            f"Failed to read local GSM1K file {target_file}: {exc}"
+        ) from exc
+
+    # Parse as JSONL first, fallback to JSON array
+    raw_records = []
+    lines = content.strip().splitlines()
+    is_jsonl = True
+    for idx, line in enumerate(lines):
+        if not line.strip():
+            continue
+        try:
+            parsed = json.loads(line)
+            # JSONL lines must be objects (mappings)
+            if not isinstance(parsed, dict):
+                is_jsonl = False
+                break
+            raw_records.append(parsed)
+        except Exception:
+            is_jsonl = False
+            break
+
+    if not is_jsonl or not raw_records:
+        try:
+            data = json.loads(content)
+            if isinstance(data, list):
+                raw_records = data
+                # Since it parsed successfully as a JSON array, it's not JSONL format
+                is_jsonl = False
+            elif isinstance(data, dict):
+                raw_records = [data]
+                is_jsonl = False
+            else:
+                raise ValueError("JSON root must be a list or dictionary.")
+        except Exception as exc:
+            raise ValueError(
+                f"Failed to parse GSM1K file {target_file} as JSON/JSONL: {exc}"
+            ) from exc
+
+    items: list[GeneralizationAuditItem] = []
+    for idx, rec in enumerate(raw_records):
+        if max_items is not None and len(items) >= max_items:
+            break
+
+        # Extract question (query)
+        question = rec.get("question") or rec.get("prompt")
+        if question is None:
+            raise ValueError(
+                f"GSM1K record at index {idx} in {target_file.name} is missing 'question' or 'prompt' field."
+            )
+
+        # Extract answer
+        answer = rec.get("answer") or rec.get("grade") or rec.get("label")
+        if answer is None:
+            raise ValueError(
+                f"GSM1K record at index {idx} in {target_file.name} is missing 'answer', 'grade', or 'label' field."
+            )
+
+        # Resolve stable item ID
+        item_id = str(rec.get("id") or rec.get("item_id") or idx)
+
+        # Build stable opaque prompt reference
+        prompt_ref = f"gsm1k:{split}:{item_id}"
+
+        q_sha256 = hashlib.sha256(str(question).encode("utf-8")).hexdigest()
+        a_sha256 = hashlib.sha256(str(answer).encode("utf-8")).hexdigest()
+
+        metadata_list = [
+            ("source_format", "jsonl" if is_jsonl else "json"),
+            ("source_index", str(idx)),
+            ("source_file_name", target_file.name),
+            ("source_record_id", item_id),
+            ("question_sha256", q_sha256),
+            ("answer_sha256", a_sha256),
+            ("question_length", str(len(str(question)))),
+            ("answer_kind", "numeric_text"),
+        ]
+
+        items.append(
+            GeneralizationAuditItem(
+                dataset="GSM1K",
+                split=split,
+                item_id=item_id,
+                prompt_ref=prompt_ref,
+                answer_kind="numeric_text",
+                metadata=tuple(metadata_list),
+            )
+        )
+
+    return tuple(items)

scripts/benchmarks/run_generalization_audit.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""CLI script to run generalization audit (skeleton)."""
+"""CLI script to run generalization audit."""
 
 from __future__ import annotations
 
@@ -15,6 +15,7 @@ repo_root = script_path.parent.parent.parent
 if str(repo_root) not in sys.path:
     sys.path.insert(0, str(repo_root))
 
+from evals.generalization.adapters.gsm1k import load_gsm1k_items  # noqa: E402
 from evals.generalization.audit_runner import run_generalization_audit  # noqa: E402
 from evals.generalization.item_schema import (  # noqa: E402
     GeneralizationAuditItem,
@@ -33,6 +34,21 @@ def main() -> None:
     parser.add_argument(
         "--split", type=str, default="test", help="Dataset split to audit."
     )
+    parser.add_argument(
+        "--local-cache",
+        type=str,
+        help="Explicit path to local cache (overrides manifest).",
+    )
+    parser.add_argument(
+        "--max-items",
+        type=int,
+        help="Maximum number of items to load.",
+    )
+    parser.add_argument(
+        "--metadata-only",
+        action="store_true",
+        help="Run in metadata-only mode, bypassing gate failures.",
+    )
     parser.add_argument(
         "--json", action="store_true", help="Print report in deterministic JSON format."
     )
@@ -46,10 +62,106 @@ def main() -> None:
         )
         sys.exit(1)
 
+    ADAPTERS = {
+        "gsm1k": load_gsm1k_items,
+    }
+
     if args.dataset:
-        # In PR-2, no real dataset adapters exist, so we fail with the required error code
-        print("Error: dataset_adapter_unavailable", file=sys.stderr)
-        sys.exit(1)
+        dataset_key = args.dataset.lower()
+        if dataset_key not in ADAPTERS:
+            print("Error: dataset_adapter_unavailable", file=sys.stderr)
+            sys.exit(1)
+
+        # 1. Run the verifier from #887 to check manifest gates
+        from evals.generalization.cache_verifier import (
+            verify_local_generalization_cache,
+        )
+
+        manifests_dir = repo_root / "evals" / "generalization" / "manifests"
+        try:
+            report = verify_local_generalization_cache(
+                repo_root=repo_root,
+                manifests_dir=manifests_dir,
+                require_present=False,
+            )
+        except Exception as exc:
+            print(f"Manifest validation failed: {exc}", file=sys.stderr)
+            sys.exit(1)
+
+        # Find the record for the dataset
+        record = None
+        for r in report.records:
+            if r.dataset.lower() == dataset_key:
+                record = r
+                break
+
+        if not record:
+            print(
+                f"Error: dataset_adapter_unavailable (no manifest for {args.dataset})",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        # Fail closed on unresolved gates unless in metadata-only mode
+        if not args.metadata_only:
+            if not record.license_ready or not record.checksum_ready:
+                print("Error: benchmark_manifest_unresolved", file=sys.stderr)
+                sys.exit(1)
+            print("Error: dataset_evaluator_unavailable", file=sys.stderr)
+            sys.exit(1)
+
+        # Resolve local cache path
+        if args.local_cache:
+            cache_path = Path(args.local_cache)
+        else:
+            from evals.generalization.manifest_schema import (
+                load_and_validate_manifest,
+            )
+
+            manifest_path = manifests_dir / f"{dataset_key}.yaml"
+            try:
+                manifest = load_and_validate_manifest(manifest_path)
+                cache_path = repo_root / manifest.local_cache
+            except Exception as exc:
+                print(f"Error reading manifest: {exc}", file=sys.stderr)
+                sys.exit(1)
+
+        # Check existence since we are in metadata-only mode
+        if not cache_path.exists():
+            print("Metadata-only validation passed (cache absent).")
+            sys.exit(0)
+
+        # Load items (metadata-only summary)
+        adapter_fn = ADAPTERS[dataset_key]
+        try:
+            items = adapter_fn(
+                local_cache=cache_path,
+                split=args.split,
+                max_items=args.max_items,
+            )
+        except Exception as exc:
+            print(f"Failed to load items: {exc}", file=sys.stderr)
+            sys.exit(1)
+
+        if args.json:
+            summary = {
+                "dataset": record.dataset,
+                "split": args.split,
+                "n_items": len(items),
+                "local_cache": str(cache_path),
+                "metadata_only": True,
+            }
+            print(json.dumps(summary, indent=2, sort_keys=True))
+        else:
+            print("Generalization Adapter Summary (Metadata-only)")
+            print("=" * 80)
+            print(f"Dataset:     {record.dataset}")
+            print(f"Split:       {args.split}")
+            print(f"Total Items: {len(items)}")
+            print(f"Cache Path:  {cache_path}")
+            print("=" * 80)
+
+        sys.exit(0)
 
     if args.synthetic_smoke:
         # Generate synthetic items and run smoke audit

tests/test_generalization_audit_runner.py

@@ -262,13 +262,13 @@ def test_cli_synthetic_smoke() -> None:
 
 
 def test_cli_real_dataset_refuses() -> None:
-    """Verify that requesting a real dataset fails with dataset_adapter_unavailable."""
+    """Verify that requesting a dataset with no adapter fails with dataset_adapter_unavailable."""
     result = subprocess.run(
         [
             sys.executable,
             "scripts/benchmarks/run_generalization_audit.py",
             "--dataset",
-            "gsm1k",
+            "asdiv",
         ],
         capture_output=True,
         text=True,

tests/test_generalization_gsm1k_adapter.py

@@ -0,0 +1,334 @@
+"""Tests for the GSM1K local cache adapter."""
+
+from __future__ import annotations
+
+import json
+import socket
+import subprocess
+import sys
+from pathlib import Path
+import pytest
+
+import evals.generalization.cache_verifier
+from evals.generalization.adapters.gsm1k import load_gsm1k_items
+from evals.generalization.cache_verifier import (
+    CacheVerificationRecord,
+    CacheVerificationReport,
+)
+from scripts.benchmarks.run_generalization_audit import main as cli_main
+
+
+def write_synthetic_jsonl(path: Path, records: list[dict]) -> None:
+    lines = [json.dumps(r) for r in records]
+    path.write_text("\n".join(lines), encoding="utf-8")
+
+
+def write_synthetic_json(path: Path, data: any) -> None:
+    path.write_text(json.dumps(data), encoding="utf-8")
+
+
+def test_loads_synthetic_jsonl_records(tmp_path: Path) -> None:
+    """GSM1K adapter loads synthetic JSONL records and produces stable item IDs."""
+    cache_dir = tmp_path / "gsm1k_cache"
+    cache_dir.mkdir()
+    records = [
+        {"question": "Alice has 2 apples.", "answer": "2", "id": "q1"},
+        {"question": "Bob has 3 apples.", "answer": "3", "id": "q2"},
+    ]
+    write_synthetic_jsonl(cache_dir / "test.jsonl", records)
+
+    items = load_gsm1k_items(local_cache=cache_dir, split="test")
+
+    assert len(items) == 2
+    assert items[0].dataset == "GSM1K"
+    assert items[0].split == "test"
+    assert items[0].item_id == "q1"
+    assert items[0].prompt_ref == "gsm1k:test:q1"
+    assert items[0].answer_kind == "numeric_text"
+
+    # Verify opaque metadata
+    metadata_dict = dict(items[0].metadata)
+    assert "question" not in metadata_dict
+    assert "prompt" not in metadata_dict
+    assert "answer" not in metadata_dict
+    assert "grade" not in metadata_dict
+    assert "label" not in metadata_dict
+
+    assert "question_sha256" in metadata_dict
+    assert "answer_sha256" in metadata_dict
+    assert "question_length" in metadata_dict
+    assert metadata_dict["source_record_id"] == "q1"
+
+
+def test_loads_synthetic_json_records(tmp_path: Path) -> None:
+    """GSM1K adapter loads synthetic JSON array records."""
+    cache_dir = tmp_path / "gsm1k_cache"
+    cache_dir.mkdir()
+    records = [
+        {"question": "Charlie has 5 coins.", "answer": "5", "id": "c1"},
+        {"question": "David has 10 coins.", "answer": "10", "id": "c2"},
+    ]
+    write_synthetic_json(cache_dir / "test.json", records)
+
+    items = load_gsm1k_items(local_cache=cache_dir, split="test")
+
+    assert len(items) == 2
+    assert items[0].item_id == "c1"
+    assert items[1].item_id == "c2"
+
+
+def test_honors_max_items(tmp_path: Path) -> None:
+    """GSM1K adapter honors max_items argument."""
+    cache_dir = tmp_path / "gsm1k_cache"
+    cache_dir.mkdir()
+    records = [
+        {"question": "Q1", "answer": "A1"},
+        {"question": "Q2", "answer": "A2"},
+        {"question": "Q3", "answer": "A3"},
+    ]
+    write_synthetic_jsonl(cache_dir / "test.jsonl", records)
+
+    items = load_gsm1k_items(local_cache=cache_dir, split="test", max_items=2)
+    assert len(items) == 2
+    assert items[0].item_id == "0"
+    assert items[1].item_id == "1"
+
+
+def test_refuses_missing_cache_path() -> None:
+    """GSM1K adapter raises FileNotFoundError if cache path is missing."""
+    with pytest.raises(
+        FileNotFoundError,
+        match="Cache directory does not exist|Local cache path does not exist",
+    ):
+        load_gsm1k_items(
+            local_cache=Path("non_existent_directory_1234"), split="test"
+        )
+
+
+def test_refuses_unsupported_split(tmp_path: Path) -> None:
+    """GSM1K adapter raises FileNotFoundError if split file is not found."""
+    cache_dir = tmp_path / "gsm1k_cache"
+    cache_dir.mkdir()
+    write_synthetic_jsonl(
+        cache_dir / "test.jsonl", [{"question": "Q", "answer": "A"}]
+    )
+
+    with pytest.raises(
+        FileNotFoundError, match="No JSON or JSONL file found for split"
+    ):
+        load_gsm1k_items(local_cache=cache_dir, split="train")
+
+
+def test_refuses_malformed_record_missing_question(tmp_path: Path) -> None:
+    """GSM1K adapter raises ValueError if a record is missing the question field."""
+    cache_dir = tmp_path / "gsm1k_cache"
+    cache_dir.mkdir()
+    records = [
+        {"answer": "5", "id": "1"},  # missing question/prompt
+    ]
+    write_synthetic_jsonl(cache_dir / "test.jsonl", records)
+
+    with pytest.raises(ValueError, match="missing 'question' or 'prompt'"):
+        load_gsm1k_items(local_cache=cache_dir, split="test")
+
+
+def test_refuses_malformed_record_missing_answer(tmp_path: Path) -> None:
+    """GSM1K adapter raises ValueError if a record is missing the answer field."""
+    cache_dir = tmp_path / "gsm1k_cache"
+    cache_dir.mkdir()
+    records = [
+        {"question": "Q?", "id": "1"},  # missing answer/grade/label
+    ]
+    write_synthetic_jsonl(cache_dir / "test.jsonl", records)
+
+    with pytest.raises(
+        ValueError, match="missing 'answer', 'grade', or 'label'"
+    ):
+        load_gsm1k_items(local_cache=cache_dir, split="test")
+
+
+def test_does_not_download_or_write_cache(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """GSM1K adapter does not use network or write files to disk during load."""
+    cache_dir = tmp_path / "gsm1k_cache"
+    cache_dir.mkdir()
+    write_synthetic_jsonl(
+        cache_dir / "test.jsonl", [{"question": "Q", "answer": "A"}]
+    )
+
+    # Intercept socket calls to block network
+    def blocked_socket(*args: any, **kwargs: any) -> any:
+        raise RuntimeError("Network calls are forbidden during loading!")
+
+    monkeypatch.setattr(socket, "socket", blocked_socket)
+
+    # Intercept write/open actions on paths that are not the test file
+    orig_write_text = Path.write_text
+
+    def blocked_write_text(self: Path, *args: any, **kwargs: any) -> any:
+        if self.parent.resolve() != cache_dir.resolve():
+            raise RuntimeError(
+                f"Writing to disk outside test cache is forbidden: {self}"
+            )
+        return orig_write_text(self, *args, **kwargs)
+
+    monkeypatch.setattr(Path, "write_text", blocked_write_text)
+
+    items = load_gsm1k_items(local_cache=cache_dir, split="test")
+    assert len(items) == 1
+
+
+def test_repository_contains_no_committed_gsm1k_examples() -> None:
+    """Check that the repository contains no committed GSM1K examples or dataset files."""
+    repo_root = Path(__file__).resolve().parents[1]
+    assert (repo_root / "evals" / "generalization").is_dir()
+    assert (repo_root / ".git").exists()
+
+    try:
+        result = subprocess.run(
+            ["git", "ls-files"],
+            cwd=str(repo_root),
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        tracked_files = result.stdout.strip().splitlines()
+    except (subprocess.SubprocessError, FileNotFoundError):
+        tracked_files = []
+        benchmarks_dir = repo_root / ".data" / "benchmarks"
+        if benchmarks_dir.exists():
+            tracked_files = [
+                str(p) for p in benchmarks_dir.rglob("*") if p.is_file()
+            ]
+
+    # Make sure no tracked files under .data/benchmarks/ exist except .gitkeep
+    for f in tracked_files:
+        if ".data/benchmarks" in f and not f.endswith(".gitkeep"):
+            pytest.fail(f"Committed benchmark cache file found: {f}")
+
+    # Check that no manifest YAML file contains actual benchmark questions/answers
+    manifest_dir = repo_root / "evals" / "generalization" / "manifests"
+    gsm1k_manifest = manifest_dir / "gsm1k.yaml"
+    if gsm1k_manifest.exists():
+        content = gsm1k_manifest.read_text(encoding="utf-8")
+        assert (
+            "question" not in content.lower()
+        ), "GSM1K manifest contains raw question data!"
+        assert (
+            "answer" not in content.lower() or "grade" in content.lower()
+        ), "GSM1K manifest contains raw answer data!"
+
+
+def test_cli_refuses_unresolved_manifest_gates() -> None:
+    """The CLI refuses to run if the manifest has unresolved license/checksum gates."""
+    result = subprocess.run(
+        [
+            sys.executable,
+            "scripts/benchmarks/run_generalization_audit.py",
+            "--dataset",
+            "gsm1k",
+            "--split",
+            "test",
+        ],
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode != 0
+    assert "benchmark_manifest_unresolved" in result.stderr
+
+
+def test_cli_local_adapter_works_with_temp_cache_and_metadata_only(
+    tmp_path: Path,
+) -> None:
+    """The CLI run works when --metadata-only is passed with local-cache."""
+    cache_dir = tmp_path / "gsm1k_cache"
+    cache_dir.mkdir()
+    records = [
+        {"question": "Alice has 2 apples.", "answer": "2", "id": "q1"},
+    ]
+    write_synthetic_jsonl(cache_dir / "test.jsonl", records)
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            "scripts/benchmarks/run_generalization_audit.py",
+            "--dataset",
+            "gsm1k",
+            "--split",
+            "test",
+            "--local-cache",
+            str(cache_dir),
+            "--metadata-only",
+            "--json",
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    assert result.returncode == 0
+    report = json.loads(result.stdout)
+    assert report["dataset"] == "GSM1K"
+    assert report["n_items"] == 1
+    assert report["metadata_only"] is True
+    # The metadata-only path does not claim correct/wrong
+    assert "correct" not in report
+    assert "wrong" not in report
+
+
+def test_cli_real_gsm1k_without_evaluator_refuses(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    """The CLI refuses to run a full audit without an evaluator, failing with dataset_evaluator_unavailable."""
+    cache_dir = tmp_path / "gsm1k_cache"
+    cache_dir.mkdir()
+    records = [
+        {"question": "Alice has 2 apples.", "answer": "2", "id": "q1"},
+    ]
+    write_synthetic_jsonl(cache_dir / "test.jsonl", records)
+
+    # Monkeypatch verify_local_generalization_cache to report resolved gates
+    def mock_verify(*args: any, **kwargs: any) -> CacheVerificationReport:
+        record = CacheVerificationRecord(
+            dataset="GSM1K",
+            manifest_path="gsm1k.yaml",
+            local_cache=str(cache_dir),
+            exists=True,
+            license_ready=True,
+            checksum_ready=True,
+            runnable=True,
+            reason_codes=(),
+        )
+        return CacheVerificationReport(
+            policy_version="test.v1",
+            records=(record,),
+            all_runnable=True,
+            reason_codes=(),
+        )
+
+    monkeypatch.setattr(
+        evals.generalization.cache_verifier,
+        "verify_local_generalization_cache",
+        mock_verify,
+    )
+
+    # Setup sys.argv to run CLI without --metadata-only
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "run_generalization_audit.py",
+            "--dataset",
+            "gsm1k",
+            "--local-cache",
+            str(cache_dir),
+        ],
+    )
+
+    with pytest.raises(SystemExit) as excinfo:
+        cli_main()
+    assert excinfo.value.code != 0
+    captured = capsys.readouterr()
+    assert "dataset_evaluator_unavailable" in captured.err