From 83ba9a2e9b1cbc569eb155d4eac271cb8ad18309 Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 23 Jun 2026 06:30:12 -0700 Subject: [PATCH] feat(evals): implement manifest schema and policy linter (#886) --- evals/generalization/__init__.py | 17 ++ evals/generalization/manifest_schema.py | 212 +++++++++++++++++++ tests/test_generalization_manifest_policy.py | 158 ++++++++++++++ 3 files changed, 387 insertions(+) create mode 100644 evals/generalization/__init__.py create mode 100644 evals/generalization/manifest_schema.py create mode 100644 tests/test_generalization_manifest_policy.py diff --git a/evals/generalization/__init__.py b/evals/generalization/__init__.py new file mode 100644 index 00000000..d310e46b --- /dev/null +++ b/evals/generalization/__init__.py @@ -0,0 +1,17 @@ +"""Generalization benchmark audit infrastructure.""" + +from __future__ import annotations + +from evals.generalization.manifest_schema import ( + GeneralizationBenchmarkManifest, + ManifestValidationError, + load_and_validate_manifest, + validate_manifest_data, +) + +__all__ = [ + "GeneralizationBenchmarkManifest", + "ManifestValidationError", + "load_and_validate_manifest", + "validate_manifest_data", +] diff --git a/evals/generalization/manifest_schema.py b/evals/generalization/manifest_schema.py new file mode 100644 index 00000000..edb3e1e1 --- /dev/null +++ b/evals/generalization/manifest_schema.py @@ -0,0 +1,212 @@ +"""Schema and validation policies for generalization benchmark manifests.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Optional +import yaml + + +@dataclass(frozen=True) +class GeneralizationBenchmarkManifest: + """Represents a validated generalization benchmark manifest.""" + + dataset: str + purpose: str + description: str + source: str + license: str + version: str + split: str + sha256: str + local_cache: str + repo_policy: str + inspection_policy: str + mutation_policy: str + smoke_fixture: Optional[str] = None + notes: Optional[str] = None + hf_config: Optional[str] = None + + +class ManifestValidationError(Exception): + """Raised when a manifest violates schema or policy constraints.""" + + +REQUIRED_FIELDS = { + "dataset", + "purpose", + "description", + "source", + "license", + "version", + "split", + "sha256", + "local_cache", + "repo_policy", + "inspection_policy", + "mutation_policy", +} + +ALLOWED_FIELDS = REQUIRED_FIELDS | {"smoke_fixture", "notes", "hf_config"} + +SHA256_RE = re.compile(r"^[a-fA-F0-9]{64}$") +VIBE_LICENSE_WORDS = {"typically", "probably", "confirm;", "likely", "check"} + + +def validate_manifest_data(data: Mapping[str, Any]) -> GeneralizationBenchmarkManifest: + """Validate a loaded manifest dictionary against policy constraints.""" + if not isinstance(data, Mapping): + raise ManifestValidationError("Manifest root must be a mapping.") + + # 1. Field existence checks + missing_fields = REQUIRED_FIELDS - set(data.keys()) + if missing_fields: + raise ManifestValidationError( + f"Missing required fields: {sorted(missing_fields)}" + ) + + extra_fields = set(data.keys()) - ALLOWED_FIELDS + if extra_fields: + raise ManifestValidationError(f"Unknown fields found: {sorted(extra_fields)}") + + # 2. Extract and check types + dataset = data["dataset"] + purpose = data["purpose"] + description = data["description"] + source = data["source"] + lic = data["license"] + version = data["version"] + split = data["split"] + sha256 = data["sha256"] + local_cache = data["local_cache"] + repo_policy = data["repo_policy"] + inspection_policy = data["inspection_policy"] + mutation_policy = data["mutation_policy"] + smoke_fixture = data.get("smoke_fixture") + notes = data.get("notes") + hf_config = data.get("hf_config") + + # Type validation + for name, val in [ + ("dataset", dataset), + ("purpose", purpose), + ("description", description), + ("source", source), + ("license", lic), + ("version", version), + ("split", split), + ("sha256", sha256), + ("local_cache", local_cache), + ("repo_policy", repo_policy), + ("inspection_policy", inspection_policy), + ("mutation_policy", mutation_policy), + ]: + if not isinstance(val, str): + raise ManifestValidationError(f"Field '{name}' must be a string.") + + if smoke_fixture is not None and not isinstance(smoke_fixture, str): + raise ManifestValidationError("Field 'smoke_fixture' must be null or a string.") + if notes is not None and not isinstance(notes, str): + raise ManifestValidationError("Field 'notes' must be null or a string.") + if hf_config is not None and not isinstance(hf_config, str): + raise ManifestValidationError("Field 'hf_config' must be null or a string.") + + # 3. Policy constraint validation + if purpose != "sealed_audit_not_training": + raise ManifestValidationError( + f"Field 'purpose' must be 'sealed_audit_not_training'; got {purpose!r}" + ) + + if repo_policy != "manifest_only": + raise ManifestValidationError( + f"Field 'repo_policy' must be 'manifest_only'; got {repo_policy!r}" + ) + + if inspection_policy != "aggregate_reports_only": + raise ManifestValidationError( + f"Field 'inspection_policy' must be 'aggregate_reports_only'; got {inspection_policy!r}" + ) + + if mutation_policy != "no_direct_pack_policy_operator_mutation": + raise ManifestValidationError( + f"Field 'mutation_policy' must be 'no_direct_pack_policy_operator_mutation'; got {mutation_policy!r}" + ) + + if not local_cache.startswith(".data/benchmarks/"): + raise ManifestValidationError( + f"Field 'local_cache' must start with '.data/benchmarks/'; got {local_cache!r}" + ) + if not local_cache.endswith("/"): + raise ManifestValidationError( + f"Field 'local_cache' must end with '/' to denote a directory path; got {local_cache!r}" + ) + + # sha256 checks + if sha256 != "TODO_AFTER_DOWNLOAD" and not SHA256_RE.match(sha256): + raise ManifestValidationError( + f"Field 'sha256' must be 'TODO_AFTER_DOWNLOAD' or a valid 64-char hex SHA256; got {sha256!r}" + ) + + # license checks + lic_lower = lic.lower() + if "todo" in lic_lower and lic != "TODO_VERIFY_BEFORE_CACHE": + raise ManifestValidationError( + f"Field 'license' containing 'TODO' must be exactly 'TODO_VERIFY_BEFORE_CACHE'; got {lic!r}" + ) + + for word in VIBE_LICENSE_WORDS: + if word in lic_lower: + raise ManifestValidationError( + f"Field 'license' contains non-permitted vague/vibe word '{word}': {lic!r}" + ) + + # smoke_fixture checks + if smoke_fixture is not None: + if lic == "TODO_VERIFY_BEFORE_CACHE": + raise ManifestValidationError( + f"Dataset has smoke_fixture {smoke_fixture!r} but license is still 'TODO_VERIFY_BEFORE_CACHE'" + ) + + return GeneralizationBenchmarkManifest( + dataset=dataset, + purpose=purpose, + description=description, + source=source, + license=lic, + version=version, + split=split, + sha256=sha256, + local_cache=local_cache, + repo_policy=repo_policy, + inspection_policy=inspection_policy, + mutation_policy=mutation_policy, + smoke_fixture=smoke_fixture, + notes=notes, + hf_config=hf_config, + ) + + +def load_and_validate_manifest(path: Path) -> GeneralizationBenchmarkManifest: + """Load a manifest from a YAML file and validate it.""" + try: + content = path.read_text(encoding="utf-8") + except OSError as exc: + raise ManifestValidationError( + f"Failed to read manifest file at {path}: {exc}" + ) from exc + + try: + data = yaml.safe_load(content) + except yaml.YAMLError as exc: + raise ManifestValidationError( + f"YAML parse error in manifest at {path}: {exc}" + ) from exc + + try: + return validate_manifest_data(data) + except ManifestValidationError as exc: + raise ManifestValidationError( + f"Validation error in manifest {path.name}: {exc}" + ) from exc diff --git a/tests/test_generalization_manifest_policy.py b/tests/test_generalization_manifest_policy.py new file mode 100644 index 00000000..053d3c64 --- /dev/null +++ b/tests/test_generalization_manifest_policy.py @@ -0,0 +1,158 @@ +"""Tests to enforce generalization benchmark manifest policy constraints.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path +import pytest + +from evals.generalization.manifest_schema import ( + load_and_validate_manifest, + ManifestValidationError, + validate_manifest_data, +) + +MANIFESTS_DIR = Path(__file__).parent.parent / "evals" / "generalization" / "manifests" + + +def test_all_manifests_valid() -> None: + """Ensure all committed generalization manifests parse and satisfy policy checks.""" + manifest_files = list(MANIFESTS_DIR.glob("*.yaml")) + assert len(manifest_files) > 0, "No manifests found in generalization/manifests/" + + for path in manifest_files: + try: + load_and_validate_manifest(path) + except ManifestValidationError as exc: + pytest.fail(f"Manifest {path.name} failed policy validation: {exc}") + + +def test_no_sealed_slice_in_comments() -> None: + """Ensure no manifest comments contain the words 'sealed slice' to respect holdout rules.""" + manifest_files = list(MANIFESTS_DIR.glob("*.yaml")) + for path in manifest_files: + content = path.read_text(encoding="utf-8") + for line in content.splitlines(): + comment_idx = line.find("#") + if comment_idx != -1: + comment_text = line[comment_idx:] + if "sealed slice" in comment_text.lower(): + pytest.fail( + f"Manifest {path.name} contains forbidden comment text " + f"'sealed slice': {line!r}" + ) + + +def test_manifest_validation_failures() -> None: + """Test validation errors for invalid data cases.""" + base_valid = { + "dataset": "TEST", + "purpose": "sealed_audit_not_training", + "description": "Test description", + "source": "hf://test/test", + "license": "TODO_VERIFY_BEFORE_CACHE", + "version": "pinned", + "split": "test", + "sha256": "TODO_AFTER_DOWNLOAD", + "local_cache": ".data/benchmarks/test/", + "repo_policy": "manifest_only", + "inspection_policy": "aggregate_reports_only", + "mutation_policy": "no_direct_pack_policy_operator_mutation", + } + + # Wrong purpose + bad_purpose = base_valid.copy() + bad_purpose["purpose"] = "training" + with pytest.raises(ManifestValidationError, match="purpose"): + validate_manifest_data(bad_purpose) + + # Missing field + missing_field = base_valid.copy() + del missing_field["dataset"] + with pytest.raises(ManifestValidationError, match="Missing required fields"): + validate_manifest_data(missing_field) + + # Extra field + extra_field = base_valid.copy() + extra_field["extra_key"] = "forbidden" + with pytest.raises(ManifestValidationError, match="Unknown fields found"): + validate_manifest_data(extra_field) + + # Invalid local_cache path structure + bad_cache = base_valid.copy() + bad_cache["local_cache"] = ".data/benchmarks/test" # missing ending slash + with pytest.raises(ManifestValidationError, match="end with '/'"): + validate_manifest_data(bad_cache) + + # Invalid sha256 + bad_sha = base_valid.copy() + bad_sha["sha256"] = "not-a-sha256" + with pytest.raises(ManifestValidationError, match="sha256"): + validate_manifest_data(bad_sha) + + # Vague license string + bad_lic = base_valid.copy() + bad_lic["license"] = "Typically Apache 2.0" + with pytest.raises( + ManifestValidationError, match="contains non-permitted vague/vibe word" + ): + validate_manifest_data(bad_lic) + + # License TODO mismatch + bad_lic_todo = base_valid.copy() + bad_lic_todo["license"] = "TODO verify" + with pytest.raises( + ManifestValidationError, + match="license.*must be exactly 'TODO_VERIFY_BEFORE_CACHE'", + ): + validate_manifest_data(bad_lic_todo) + + # Smoke fixture with TODO license + bad_smoke = base_valid.copy() + bad_smoke["license"] = "TODO_VERIFY_BEFORE_CACHE" + bad_smoke["smoke_fixture"] = "evals/generalization/smoke/test.jsonl" + with pytest.raises( + ManifestValidationError, match="has smoke_fixture.*but license is still" + ): + validate_manifest_data(bad_smoke) + + +def test_repo_leak_guard() -> None: + """Ensure no files under .data/benchmarks/ are tracked in Git except .gitkeep.""" + repo_root = Path(__file__).parent.parent + try: + result = subprocess.run( + ["git", "ls-files", ".data/benchmarks/"], + cwd=str(repo_root), + capture_output=True, + text=True, + check=True, + ) + except (subprocess.SubprocessError, FileNotFoundError): + # Fallback if git is not available in the test environment (e.g. sandbox limits) + benchmarks_dir = repo_root / ".data" / "benchmarks" + if benchmarks_dir.exists(): + tracked_files = [ + str(p.relative_to(repo_root)) + for p in benchmarks_dir.rglob("*") + if p.is_file() and p.name != ".gitkeep" + ] + assert not tracked_files, ( + f"Found untracked/local benchmark files: {tracked_files}" + ) + return + + tracked_files = result.stdout.strip().splitlines() + # Normalize path separators + tracked_files = [f.replace(os.sep, "/") for f in tracked_files] + + forbidden_files = [ + f + for f in tracked_files + if not f.endswith(".data/benchmarks/.gitkeep") + and f != ".data/benchmarks/.gitkeep" + ] + assert not forbidden_files, ( + f"Committed files under .data/benchmarks/ detected: {forbidden_files}" + )