diff --git a/evals/generalization/adapters/gsm1k.py b/evals/generalization/adapters/gsm1k.py index 945ff2a4..549b9d52 100644 --- a/evals/generalization/adapters/gsm1k.py +++ b/evals/generalization/adapters/gsm1k.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json from pathlib import Path @@ -124,18 +125,20 @@ def load_gsm1k_items( # Build stable opaque prompt reference prompt_ref = f"gsm1k:{split}:{item_id}" - # Setup metadata safely storing prompt/answer content out of aggregate report + q_sha256 = hashlib.sha256(str(question).encode("utf-8")).hexdigest() + a_sha256 = hashlib.sha256(str(answer).encode("utf-8")).hexdigest() + metadata_list = [ ("source_format", "jsonl" if is_jsonl else "json"), - ("question", str(question)), - ("answer", str(answer)), + ("source_index", str(idx)), + ("source_file_name", target_file.name), + ("source_record_id", item_id), + ("question_sha256", q_sha256), + ("answer_sha256", a_sha256), + ("question_length", str(len(str(question)))), + ("answer_kind", "numeric_text"), ] - if "grade" in rec: - metadata_list.append(("grade", str(rec["grade"]))) - if "domain" in rec: - metadata_list.append(("domain", str(rec["domain"]))) - items.append( GeneralizationAuditItem( dataset="GSM1K", diff --git a/scripts/benchmarks/run_generalization_audit.py b/scripts/benchmarks/run_generalization_audit.py index db526016..16e8ee19 100755 --- a/scripts/benchmarks/run_generalization_audit.py +++ b/scripts/benchmarks/run_generalization_audit.py @@ -107,6 +107,8 @@ def main() -> None: if not record.license_ready or not record.checksum_ready: print("Error: benchmark_manifest_unresolved", file=sys.stderr) sys.exit(1) + print("Error: dataset_evaluator_unavailable", file=sys.stderr) + sys.exit(1) # Resolve local cache path if args.local_cache: @@ -124,20 +126,12 @@ def main() -> None: print(f"Error reading manifest: {exc}", file=sys.stderr) sys.exit(1) - # Check existence if not in metadata-only mode - if not args.metadata_only and not cache_path.exists(): - print( - f"Error: Cache path does not exist: {cache_path}", - file=sys.stderr, - ) - sys.exit(1) - - # If metadata-only and cache doesn't exist, we can exit gracefully - if args.metadata_only and not cache_path.exists(): + # Check existence since we are in metadata-only mode + if not cache_path.exists(): print("Metadata-only validation passed (cache absent).") sys.exit(0) - # Load items + # Load items (metadata-only summary) adapter_fn = ADAPTERS[dataset_key] try: items = adapter_fn( @@ -149,52 +143,22 @@ def main() -> None: print(f"Failed to load items: {exc}", file=sys.stderr) sys.exit(1) - # In PR-3, use a dummy evaluator for testing the adapter composition - def dummy_evaluator( - item: GeneralizationAuditItem, - ) -> GeneralizationAuditOutcome: - return GeneralizationAuditOutcome( - item_id=item.item_id, - disposition="correct", - residual_kinds=(), - candidate_attempt_count=1, - binding_failure_count=0, - replay_refusal_count=0, - sealed_trace_dispositions=("success",), - reason_codes=(), - ) - - try: - report = run_generalization_audit( - dataset=record.dataset, - split=args.split, - items=items, - evaluator=dummy_evaluator, - ) - except Exception as exc: - print(f"Audit Failed: {exc}", file=sys.stderr) - sys.exit(1) - if args.json: - print(json.dumps(asdict(report), indent=2, sort_keys=True)) + summary = { + "dataset": record.dataset, + "split": args.split, + "n_items": len(items), + "local_cache": str(cache_path), + "metadata_only": True, + } + print(json.dumps(summary, indent=2, sort_keys=True)) else: - print( - f"Generalization Audit Report (Policy: {report.policy_version})" - ) + print("Generalization Adapter Summary (Metadata-only)") print("=" * 80) - print(f"Dataset: {report.dataset}") - print(f"Split: {report.split}") - print(f"Total Items: {report.n_items}") - print(f"Correct: {report.correct}") - print(f"Wrong: {report.wrong}") - print(f"Refused: {report.refused}") - print(f"Unsupported: {report.unsupported}") - print(f"Candidate Attempts: {report.candidate_attempts}") - print(f"Binding Failures: {report.binding_failures}") - print(f"Replay Refusals: {report.replay_refusals}") - print(f"Sealed Trace Dispositions: {report.sealed_trace_dispositions}") - print(f"Dominant Residual Kinds: {report.dominant_residual_kinds}") - print(f"Reason Codes: {', '.join(report.reason_codes)}") + print(f"Dataset: {record.dataset}") + print(f"Split: {args.split}") + print(f"Total Items: {len(items)}") + print(f"Cache Path: {cache_path}") print("=" * 80) sys.exit(0) diff --git a/tests/test_generalization_gsm1k_adapter.py b/tests/test_generalization_gsm1k_adapter.py index dc4fbd3e..8579770e 100644 --- a/tests/test_generalization_gsm1k_adapter.py +++ b/tests/test_generalization_gsm1k_adapter.py @@ -8,8 +8,14 @@ import subprocess import sys from pathlib import Path import pytest -from evals.generalization.adapters.gsm1k import load_gsm1k_items +import evals.generalization.cache_verifier +from evals.generalization.adapters.gsm1k import load_gsm1k_items +from evals.generalization.cache_verifier import ( + CacheVerificationRecord, + CacheVerificationReport, +) +from scripts.benchmarks.run_generalization_audit import main as cli_main def write_synthetic_jsonl(path: Path, records: list[dict]) -> None: @@ -40,11 +46,18 @@ def test_loads_synthetic_jsonl_records(tmp_path: Path) -> None: assert items[0].prompt_ref == "gsm1k:test:q1" assert items[0].answer_kind == "numeric_text" - # Verify opaque prompt_ref and question/answer content only in metadata - assert items[0].prompt_ref == "gsm1k:test:q1" + # Verify opaque metadata metadata_dict = dict(items[0].metadata) - assert metadata_dict["question"] == "Alice has 2 apples." - assert metadata_dict["answer"] == "2" + assert "question" not in metadata_dict + assert "prompt" not in metadata_dict + assert "answer" not in metadata_dict + assert "grade" not in metadata_dict + assert "label" not in metadata_dict + + assert "question_sha256" in metadata_dict + assert "answer_sha256" in metadata_dict + assert "question_length" in metadata_dict + assert metadata_dict["source_record_id"] == "q1" def test_loads_synthetic_json_records(tmp_path: Path) -> None: @@ -255,4 +268,65 @@ def test_cli_local_adapter_works_with_temp_cache_and_metadata_only( report = json.loads(result.stdout) assert report["dataset"] == "GSM1K" assert report["n_items"] == 1 - assert report["correct"] == 1 + assert report["metadata_only"] is True + # The metadata-only path does not claim correct/wrong + assert "correct" not in report + assert "wrong" not in report + + +def test_cli_real_gsm1k_without_evaluator_refuses( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """The CLI refuses to run a full audit without an evaluator, failing with dataset_evaluator_unavailable.""" + cache_dir = tmp_path / "gsm1k_cache" + cache_dir.mkdir() + records = [ + {"question": "Alice has 2 apples.", "answer": "2", "id": "q1"}, + ] + write_synthetic_jsonl(cache_dir / "test.jsonl", records) + + # Monkeypatch verify_local_generalization_cache to report resolved gates + def mock_verify(*args: any, **kwargs: any) -> CacheVerificationReport: + record = CacheVerificationRecord( + dataset="GSM1K", + manifest_path="gsm1k.yaml", + local_cache=str(cache_dir), + exists=True, + license_ready=True, + checksum_ready=True, + runnable=True, + reason_codes=(), + ) + return CacheVerificationReport( + policy_version="test.v1", + records=(record,), + all_runnable=True, + reason_codes=(), + ) + + monkeypatch.setattr( + evals.generalization.cache_verifier, + "verify_local_generalization_cache", + mock_verify, + ) + + # Setup sys.argv to run CLI without --metadata-only + monkeypatch.setattr( + sys, + "argv", + [ + "run_generalization_audit.py", + "--dataset", + "gsm1k", + "--local-cache", + str(cache_dir), + ], + ) + + with pytest.raises(SystemExit) as excinfo: + cli_main() + assert excinfo.value.code != 0 + captured = capsys.readouterr() + assert "dataset_evaluator_unavailable" in captured.err