feat(evals): implement manifest schema and policy linter (#886)

evals/generalization/__init__.py

@@ -0,0 +1,17 @@
+"""Generalization benchmark audit infrastructure."""
+
+from __future__ import annotations
+
+from evals.generalization.manifest_schema import (
+    GeneralizationBenchmarkManifest,
+    ManifestValidationError,
+    load_and_validate_manifest,
+    validate_manifest_data,
+)
+
+__all__ = [
+    "GeneralizationBenchmarkManifest",
+    "ManifestValidationError",
+    "load_and_validate_manifest",
+    "validate_manifest_data",
+]

evals/generalization/manifest_schema.py

@@ -0,0 +1,212 @@
+"""Schema and validation policies for generalization benchmark manifests."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Mapping, Optional
+import yaml
+
+
+@dataclass(frozen=True)
+class GeneralizationBenchmarkManifest:
+    """Represents a validated generalization benchmark manifest."""
+
+    dataset: str
+    purpose: str
+    description: str
+    source: str
+    license: str
+    version: str
+    split: str
+    sha256: str
+    local_cache: str
+    repo_policy: str
+    inspection_policy: str
+    mutation_policy: str
+    smoke_fixture: Optional[str] = None
+    notes: Optional[str] = None
+    hf_config: Optional[str] = None
+
+
+class ManifestValidationError(Exception):
+    """Raised when a manifest violates schema or policy constraints."""
+
+
+REQUIRED_FIELDS = {
+    "dataset",
+    "purpose",
+    "description",
+    "source",
+    "license",
+    "version",
+    "split",
+    "sha256",
+    "local_cache",
+    "repo_policy",
+    "inspection_policy",
+    "mutation_policy",
+}
+
+ALLOWED_FIELDS = REQUIRED_FIELDS | {"smoke_fixture", "notes", "hf_config"}
+
+SHA256_RE = re.compile(r"^[a-fA-F0-9]{64}$")
+VIBE_LICENSE_WORDS = {"typically", "probably", "confirm;", "likely", "check"}
+
+
+def validate_manifest_data(data: Mapping[str, Any]) -> GeneralizationBenchmarkManifest:
+    """Validate a loaded manifest dictionary against policy constraints."""
+    if not isinstance(data, Mapping):
+        raise ManifestValidationError("Manifest root must be a mapping.")
+
+    # 1. Field existence checks
+    missing_fields = REQUIRED_FIELDS - set(data.keys())
+    if missing_fields:
+        raise ManifestValidationError(
+            f"Missing required fields: {sorted(missing_fields)}"
+        )
+
+    extra_fields = set(data.keys()) - ALLOWED_FIELDS
+    if extra_fields:
+        raise ManifestValidationError(f"Unknown fields found: {sorted(extra_fields)}")
+
+    # 2. Extract and check types
+    dataset = data["dataset"]
+    purpose = data["purpose"]
+    description = data["description"]
+    source = data["source"]
+    lic = data["license"]
+    version = data["version"]
+    split = data["split"]
+    sha256 = data["sha256"]
+    local_cache = data["local_cache"]
+    repo_policy = data["repo_policy"]
+    inspection_policy = data["inspection_policy"]
+    mutation_policy = data["mutation_policy"]
+    smoke_fixture = data.get("smoke_fixture")
+    notes = data.get("notes")
+    hf_config = data.get("hf_config")
+
+    # Type validation
+    for name, val in [
+        ("dataset", dataset),
+        ("purpose", purpose),
+        ("description", description),
+        ("source", source),
+        ("license", lic),
+        ("version", version),
+        ("split", split),
+        ("sha256", sha256),
+        ("local_cache", local_cache),
+        ("repo_policy", repo_policy),
+        ("inspection_policy", inspection_policy),
+        ("mutation_policy", mutation_policy),
+    ]:
+        if not isinstance(val, str):
+            raise ManifestValidationError(f"Field '{name}' must be a string.")
+
+    if smoke_fixture is not None and not isinstance(smoke_fixture, str):
+        raise ManifestValidationError("Field 'smoke_fixture' must be null or a string.")
+    if notes is not None and not isinstance(notes, str):
+        raise ManifestValidationError("Field 'notes' must be null or a string.")
+    if hf_config is not None and not isinstance(hf_config, str):
+        raise ManifestValidationError("Field 'hf_config' must be null or a string.")
+
+    # 3. Policy constraint validation
+    if purpose != "sealed_audit_not_training":
+        raise ManifestValidationError(
+            f"Field 'purpose' must be 'sealed_audit_not_training'; got {purpose!r}"
+        )
+
+    if repo_policy != "manifest_only":
+        raise ManifestValidationError(
+            f"Field 'repo_policy' must be 'manifest_only'; got {repo_policy!r}"
+        )
+
+    if inspection_policy != "aggregate_reports_only":
+        raise ManifestValidationError(
+            f"Field 'inspection_policy' must be 'aggregate_reports_only'; got {inspection_policy!r}"
+        )
+
+    if mutation_policy != "no_direct_pack_policy_operator_mutation":
+        raise ManifestValidationError(
+            f"Field 'mutation_policy' must be 'no_direct_pack_policy_operator_mutation'; got {mutation_policy!r}"
+        )
+
+    if not local_cache.startswith(".data/benchmarks/"):
+        raise ManifestValidationError(
+            f"Field 'local_cache' must start with '.data/benchmarks/'; got {local_cache!r}"
+        )
+    if not local_cache.endswith("/"):
+        raise ManifestValidationError(
+            f"Field 'local_cache' must end with '/' to denote a directory path; got {local_cache!r}"
+        )
+
+    # sha256 checks
+    if sha256 != "TODO_AFTER_DOWNLOAD" and not SHA256_RE.match(sha256):
+        raise ManifestValidationError(
+            f"Field 'sha256' must be 'TODO_AFTER_DOWNLOAD' or a valid 64-char hex SHA256; got {sha256!r}"
+        )
+
+    # license checks
+    lic_lower = lic.lower()
+    if "todo" in lic_lower and lic != "TODO_VERIFY_BEFORE_CACHE":
+        raise ManifestValidationError(
+            f"Field 'license' containing 'TODO' must be exactly 'TODO_VERIFY_BEFORE_CACHE'; got {lic!r}"
+        )
+
+    for word in VIBE_LICENSE_WORDS:
+        if word in lic_lower:
+            raise ManifestValidationError(
+                f"Field 'license' contains non-permitted vague/vibe word '{word}': {lic!r}"
+            )
+
+    # smoke_fixture checks
+    if smoke_fixture is not None:
+        if lic == "TODO_VERIFY_BEFORE_CACHE":
+            raise ManifestValidationError(
+                f"Dataset has smoke_fixture {smoke_fixture!r} but license is still 'TODO_VERIFY_BEFORE_CACHE'"
+            )
+
+    return GeneralizationBenchmarkManifest(
+        dataset=dataset,
+        purpose=purpose,
+        description=description,
+        source=source,
+        license=lic,
+        version=version,
+        split=split,
+        sha256=sha256,
+        local_cache=local_cache,
+        repo_policy=repo_policy,
+        inspection_policy=inspection_policy,
+        mutation_policy=mutation_policy,
+        smoke_fixture=smoke_fixture,
+        notes=notes,
+        hf_config=hf_config,
+    )
+
+
+def load_and_validate_manifest(path: Path) -> GeneralizationBenchmarkManifest:
+    """Load a manifest from a YAML file and validate it."""
+    try:
+        content = path.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise ManifestValidationError(
+            f"Failed to read manifest file at {path}: {exc}"
+        ) from exc
+
+    try:
+        data = yaml.safe_load(content)
+    except yaml.YAMLError as exc:
+        raise ManifestValidationError(
+            f"YAML parse error in manifest at {path}: {exc}"
+        ) from exc
+
+    try:
+        return validate_manifest_data(data)
+    except ManifestValidationError as exc:
+        raise ManifestValidationError(
+            f"Validation error in manifest {path.name}: {exc}"
+        ) from exc

tests/test_generalization_manifest_policy.py

@@ -0,0 +1,158 @@
+"""Tests to enforce generalization benchmark manifest policy constraints."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from pathlib import Path
+import pytest
+
+from evals.generalization.manifest_schema import (
+    load_and_validate_manifest,
+    ManifestValidationError,
+    validate_manifest_data,
+)
+
+MANIFESTS_DIR = Path(__file__).parent.parent / "evals" / "generalization" / "manifests"
+
+
+def test_all_manifests_valid() -> None:
+    """Ensure all committed generalization manifests parse and satisfy policy checks."""
+    manifest_files = list(MANIFESTS_DIR.glob("*.yaml"))
+    assert len(manifest_files) > 0, "No manifests found in generalization/manifests/"
+
+    for path in manifest_files:
+        try:
+            load_and_validate_manifest(path)
+        except ManifestValidationError as exc:
+            pytest.fail(f"Manifest {path.name} failed policy validation: {exc}")
+
+
+def test_no_sealed_slice_in_comments() -> None:
+    """Ensure no manifest comments contain the words 'sealed slice' to respect holdout rules."""
+    manifest_files = list(MANIFESTS_DIR.glob("*.yaml"))
+    for path in manifest_files:
+        content = path.read_text(encoding="utf-8")
+        for line in content.splitlines():
+            comment_idx = line.find("#")
+            if comment_idx != -1:
+                comment_text = line[comment_idx:]
+                if "sealed slice" in comment_text.lower():
+                    pytest.fail(
+                        f"Manifest {path.name} contains forbidden comment text "
+                        f"'sealed slice': {line!r}"
+                    )
+
+
+def test_manifest_validation_failures() -> None:
+    """Test validation errors for invalid data cases."""
+    base_valid = {
+        "dataset": "TEST",
+        "purpose": "sealed_audit_not_training",
+        "description": "Test description",
+        "source": "hf://test/test",
+        "license": "TODO_VERIFY_BEFORE_CACHE",
+        "version": "pinned",
+        "split": "test",
+        "sha256": "TODO_AFTER_DOWNLOAD",
+        "local_cache": ".data/benchmarks/test/",
+        "repo_policy": "manifest_only",
+        "inspection_policy": "aggregate_reports_only",
+        "mutation_policy": "no_direct_pack_policy_operator_mutation",
+    }
+
+    # Wrong purpose
+    bad_purpose = base_valid.copy()
+    bad_purpose["purpose"] = "training"
+    with pytest.raises(ManifestValidationError, match="purpose"):
+        validate_manifest_data(bad_purpose)
+
+    # Missing field
+    missing_field = base_valid.copy()
+    del missing_field["dataset"]
+    with pytest.raises(ManifestValidationError, match="Missing required fields"):
+        validate_manifest_data(missing_field)
+
+    # Extra field
+    extra_field = base_valid.copy()
+    extra_field["extra_key"] = "forbidden"
+    with pytest.raises(ManifestValidationError, match="Unknown fields found"):
+        validate_manifest_data(extra_field)
+
+    # Invalid local_cache path structure
+    bad_cache = base_valid.copy()
+    bad_cache["local_cache"] = ".data/benchmarks/test"  # missing ending slash
+    with pytest.raises(ManifestValidationError, match="end with '/'"):
+        validate_manifest_data(bad_cache)
+
+    # Invalid sha256
+    bad_sha = base_valid.copy()
+    bad_sha["sha256"] = "not-a-sha256"
+    with pytest.raises(ManifestValidationError, match="sha256"):
+        validate_manifest_data(bad_sha)
+
+    # Vague license string
+    bad_lic = base_valid.copy()
+    bad_lic["license"] = "Typically Apache 2.0"
+    with pytest.raises(
+        ManifestValidationError, match="contains non-permitted vague/vibe word"
+    ):
+        validate_manifest_data(bad_lic)
+
+    # License TODO mismatch
+    bad_lic_todo = base_valid.copy()
+    bad_lic_todo["license"] = "TODO verify"
+    with pytest.raises(
+        ManifestValidationError,
+        match="license.*must be exactly 'TODO_VERIFY_BEFORE_CACHE'",
+    ):
+        validate_manifest_data(bad_lic_todo)
+
+    # Smoke fixture with TODO license
+    bad_smoke = base_valid.copy()
+    bad_smoke["license"] = "TODO_VERIFY_BEFORE_CACHE"
+    bad_smoke["smoke_fixture"] = "evals/generalization/smoke/test.jsonl"
+    with pytest.raises(
+        ManifestValidationError, match="has smoke_fixture.*but license is still"
+    ):
+        validate_manifest_data(bad_smoke)
+
+
+def test_repo_leak_guard() -> None:
+    """Ensure no files under .data/benchmarks/ are tracked in Git except .gitkeep."""
+    repo_root = Path(__file__).parent.parent
+    try:
+        result = subprocess.run(
+            ["git", "ls-files", ".data/benchmarks/"],
+            cwd=str(repo_root),
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+    except (subprocess.SubprocessError, FileNotFoundError):
+        # Fallback if git is not available in the test environment (e.g. sandbox limits)
+        benchmarks_dir = repo_root / ".data" / "benchmarks"
+        if benchmarks_dir.exists():
+            tracked_files = [
+                str(p.relative_to(repo_root))
+                for p in benchmarks_dir.rglob("*")
+                if p.is_file() and p.name != ".gitkeep"
+            ]
+            assert not tracked_files, (
+                f"Found untracked/local benchmark files: {tracked_files}"
+            )
+        return
+
+    tracked_files = result.stdout.strip().splitlines()
+    # Normalize path separators
+    tracked_files = [f.replace(os.sep, "/") for f in tracked_files]
+
+    forbidden_files = [
+        f
+        for f in tracked_files
+        if not f.endswith(".data/benchmarks/.gitkeep")
+        and f != ".data/benchmarks/.gitkeep"
+    ]
+    assert not forbidden_files, (
+        f"Committed files under .data/benchmarks/ detected: {forbidden_files}"
+    )