feat(evals): implement safe metadata and fail-closed evaluator check for GSM1K

evals/generalization/adapters/gsm1k.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import json
 from pathlib import Path
 
@@ -124,18 +125,20 @@ def load_gsm1k_items(
         # Build stable opaque prompt reference
         prompt_ref = f"gsm1k:{split}:{item_id}"
 
-        # Setup metadata safely storing prompt/answer content out of aggregate report
+        q_sha256 = hashlib.sha256(str(question).encode("utf-8")).hexdigest()
+        a_sha256 = hashlib.sha256(str(answer).encode("utf-8")).hexdigest()
+
         metadata_list = [
             ("source_format", "jsonl" if is_jsonl else "json"),
-            ("question", str(question)),
+            ("source_index", str(idx)),
-            ("answer", str(answer)),
+            ("source_file_name", target_file.name),
+            ("source_record_id", item_id),
+            ("question_sha256", q_sha256),
+            ("answer_sha256", a_sha256),
+            ("question_length", str(len(str(question)))),
+            ("answer_kind", "numeric_text"),
         ]
 
-        if "grade" in rec:
-            metadata_list.append(("grade", str(rec["grade"])))
-        if "domain" in rec:
-            metadata_list.append(("domain", str(rec["domain"])))
-
         items.append(
             GeneralizationAuditItem(
                 dataset="GSM1K",

scripts/benchmarks/run_generalization_audit.py

@@ -107,6 +107,8 @@ def main() -> None:
             if not record.license_ready or not record.checksum_ready:
                 print("Error: benchmark_manifest_unresolved", file=sys.stderr)
                 sys.exit(1)
+            print("Error: dataset_evaluator_unavailable", file=sys.stderr)
+            sys.exit(1)
 
         # Resolve local cache path
         if args.local_cache:
@@ -124,20 +126,12 @@ def main() -> None:
                 print(f"Error reading manifest: {exc}", file=sys.stderr)
                 sys.exit(1)
 
-        # Check existence if not in metadata-only mode
+        # Check existence since we are in metadata-only mode
-        if not args.metadata_only and not cache_path.exists():
+        if not cache_path.exists():
-            print(
-                f"Error: Cache path does not exist: {cache_path}",
-                file=sys.stderr,
-            )
-            sys.exit(1)
-
-        # If metadata-only and cache doesn't exist, we can exit gracefully
-        if args.metadata_only and not cache_path.exists():
             print("Metadata-only validation passed (cache absent).")
             sys.exit(0)
 
-        # Load items
+        # Load items (metadata-only summary)
         adapter_fn = ADAPTERS[dataset_key]
         try:
             items = adapter_fn(
@@ -149,52 +143,22 @@ def main() -> None:
             print(f"Failed to load items: {exc}", file=sys.stderr)
             sys.exit(1)
 
-        # In PR-3, use a dummy evaluator for testing the adapter composition
-        def dummy_evaluator(
-            item: GeneralizationAuditItem,
-        ) -> GeneralizationAuditOutcome:
-            return GeneralizationAuditOutcome(
-                item_id=item.item_id,
-                disposition="correct",
-                residual_kinds=(),
-                candidate_attempt_count=1,
-                binding_failure_count=0,
-                replay_refusal_count=0,
-                sealed_trace_dispositions=("success",),
-                reason_codes=(),
-            )
-
-        try:
-            report = run_generalization_audit(
-                dataset=record.dataset,
-                split=args.split,
-                items=items,
-                evaluator=dummy_evaluator,
-            )
-        except Exception as exc:
-            print(f"Audit Failed: {exc}", file=sys.stderr)
-            sys.exit(1)
-
         if args.json:
-            print(json.dumps(asdict(report), indent=2, sort_keys=True))
+            summary = {
+                "dataset": record.dataset,
+                "split": args.split,
+                "n_items": len(items),
+                "local_cache": str(cache_path),
+                "metadata_only": True,
+            }
+            print(json.dumps(summary, indent=2, sort_keys=True))
         else:
-            print(
+            print("Generalization Adapter Summary (Metadata-only)")
-                f"Generalization Audit Report (Policy: {report.policy_version})"
-            )
             print("=" * 80)
-            print(f"Dataset:                  {report.dataset}")
+            print(f"Dataset:     {record.dataset}")
-            print(f"Split:                    {report.split}")
+            print(f"Split:       {args.split}")
-            print(f"Total Items:              {report.n_items}")
+            print(f"Total Items: {len(items)}")
-            print(f"Correct:                  {report.correct}")
+            print(f"Cache Path:  {cache_path}")
-            print(f"Wrong:                    {report.wrong}")
-            print(f"Refused:                  {report.refused}")
-            print(f"Unsupported:              {report.unsupported}")
-            print(f"Candidate Attempts:       {report.candidate_attempts}")
-            print(f"Binding Failures:         {report.binding_failures}")
-            print(f"Replay Refusals:          {report.replay_refusals}")
-            print(f"Sealed Trace Dispositions: {report.sealed_trace_dispositions}")
-            print(f"Dominant Residual Kinds:  {report.dominant_residual_kinds}")
-            print(f"Reason Codes:             {', '.join(report.reason_codes)}")
             print("=" * 80)
 
         sys.exit(0)

tests/test_generalization_gsm1k_adapter.py

@@ -8,8 +8,14 @@ import subprocess
 import sys
 from pathlib import Path
 import pytest
-from evals.generalization.adapters.gsm1k import load_gsm1k_items
 
+import evals.generalization.cache_verifier
+from evals.generalization.adapters.gsm1k import load_gsm1k_items
+from evals.generalization.cache_verifier import (
+    CacheVerificationRecord,
+    CacheVerificationReport,
+)
+from scripts.benchmarks.run_generalization_audit import main as cli_main
 
 
 def write_synthetic_jsonl(path: Path, records: list[dict]) -> None:
@@ -40,11 +46,18 @@ def test_loads_synthetic_jsonl_records(tmp_path: Path) -> None:
     assert items[0].prompt_ref == "gsm1k:test:q1"
     assert items[0].answer_kind == "numeric_text"
 
-    # Verify opaque prompt_ref and question/answer content only in metadata
+    # Verify opaque metadata
-    assert items[0].prompt_ref == "gsm1k:test:q1"
     metadata_dict = dict(items[0].metadata)
-    assert metadata_dict["question"] == "Alice has 2 apples."
+    assert "question" not in metadata_dict
-    assert metadata_dict["answer"] == "2"
+    assert "prompt" not in metadata_dict
+    assert "answer" not in metadata_dict
+    assert "grade" not in metadata_dict
+    assert "label" not in metadata_dict
+
+    assert "question_sha256" in metadata_dict
+    assert "answer_sha256" in metadata_dict
+    assert "question_length" in metadata_dict
+    assert metadata_dict["source_record_id"] == "q1"
 
 
 def test_loads_synthetic_json_records(tmp_path: Path) -> None:
@@ -255,4 +268,65 @@ def test_cli_local_adapter_works_with_temp_cache_and_metadata_only(
     report = json.loads(result.stdout)
     assert report["dataset"] == "GSM1K"
     assert report["n_items"] == 1
-    assert report["correct"] == 1
+    assert report["metadata_only"] is True
+    # The metadata-only path does not claim correct/wrong
+    assert "correct" not in report
+    assert "wrong" not in report
+
+
+def test_cli_real_gsm1k_without_evaluator_refuses(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    """The CLI refuses to run a full audit without an evaluator, failing with dataset_evaluator_unavailable."""
+    cache_dir = tmp_path / "gsm1k_cache"
+    cache_dir.mkdir()
+    records = [
+        {"question": "Alice has 2 apples.", "answer": "2", "id": "q1"},
+    ]
+    write_synthetic_jsonl(cache_dir / "test.jsonl", records)
+
+    # Monkeypatch verify_local_generalization_cache to report resolved gates
+    def mock_verify(*args: any, **kwargs: any) -> CacheVerificationReport:
+        record = CacheVerificationRecord(
+            dataset="GSM1K",
+            manifest_path="gsm1k.yaml",
+            local_cache=str(cache_dir),
+            exists=True,
+            license_ready=True,
+            checksum_ready=True,
+            runnable=True,
+            reason_codes=(),
+        )
+        return CacheVerificationReport(
+            policy_version="test.v1",
+            records=(record,),
+            all_runnable=True,
+            reason_codes=(),
+        )
+
+    monkeypatch.setattr(
+        evals.generalization.cache_verifier,
+        "verify_local_generalization_cache",
+        mock_verify,
+    )
+
+    # Setup sys.argv to run CLI without --metadata-only
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "run_generalization_audit.py",
+            "--dataset",
+            "gsm1k",
+            "--local-cache",
+            str(cache_dir),
+        ],
+    )
+
+    with pytest.raises(SystemExit) as excinfo:
+        cli_main()
+    assert excinfo.value.code != 0
+    captured = capsys.readouterr()
+    assert "dataset_evaluator_unavailable" in captured.err