98 lines
3.1 KiB
Python
Executable file
98 lines
3.1 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""CLI script to verify local generalization benchmark cache directories."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from dataclasses import asdict
|
|
from pathlib import Path
|
|
|
|
# Resolve repository root and add to sys.path to support imports
|
|
script_path = Path(__file__).resolve()
|
|
repo_root = script_path.parent.parent.parent
|
|
if str(repo_root) not in sys.path:
|
|
sys.path.insert(0, str(repo_root))
|
|
|
|
from evals.generalization.cache_verifier import verify_local_generalization_cache # noqa: E402
|
|
|
|
|
|
|
|
def main() -> None:
|
|
# Explicit rejection check for download/fetch flags before argparse
|
|
forbidden = {"--download", "--fetch", "--pull"}
|
|
for arg in sys.argv[1:]:
|
|
if arg in forbidden:
|
|
print(
|
|
f"Error: Flag {arg} is not supported. Fetching or downloading dataset content is forbidden by policy.",
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(1)
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="Verify local generalization benchmark cache."
|
|
)
|
|
parser.add_argument(
|
|
"--metadata-only",
|
|
action="store_true",
|
|
help="Only validate manifests metadata without requiring cache presence.",
|
|
)
|
|
parser.add_argument(
|
|
"--require-present",
|
|
action="store_true",
|
|
help="Require all local cache directories to exist.",
|
|
)
|
|
parser.add_argument(
|
|
"--json",
|
|
action="store_true",
|
|
help="Print report in deterministic JSON format.",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
script_path = Path(__file__).resolve()
|
|
repo_root = script_path.parent.parent.parent
|
|
manifests_dir = repo_root / "evals" / "generalization" / "manifests"
|
|
|
|
try:
|
|
report = verify_local_generalization_cache(
|
|
repo_root=repo_root,
|
|
manifests_dir=manifests_dir,
|
|
require_present=args.require_present,
|
|
)
|
|
except Exception as exc:
|
|
print(f"Verification Failed: {exc}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
if args.json:
|
|
report_dict = asdict(report)
|
|
print(json.dumps(report_dict, indent=2, sort_keys=True))
|
|
else:
|
|
print(
|
|
f"Generalization Cache Verification Report (Policy: {report.policy_version})"
|
|
)
|
|
print("=" * 80)
|
|
for record in report.records:
|
|
status = "RUNNABLE" if record.runnable else "NOT RUNNABLE"
|
|
print(f"Dataset: {record.dataset}")
|
|
print(f" Manifest: {record.manifest_path}")
|
|
print(f" Cache Path: {record.local_cache}")
|
|
print(f" Exists: {record.exists}")
|
|
print(f" License Ready: {record.license_ready}")
|
|
print(f" Checksum Ready: {record.checksum_ready}")
|
|
print(f" Status: {status}")
|
|
if record.reason_codes:
|
|
print(f" Reasons: {', '.join(record.reason_codes)}")
|
|
print("-" * 80)
|
|
print(f"All datasets runnable: {report.all_runnable}")
|
|
|
|
# If --require-present is set, exit non-zero if any dataset is not runnable
|
|
if args.require_present and not report.all_runnable:
|
|
sys.exit(1)
|
|
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|