diff --git a/.gitignore b/.gitignore index aae64fbb..740a0374 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,10 @@ uv.lock reports/ frontier_wave1.json +# Benchmark local cache — manifests + fetch script only; benchmark data never committed +# (extends ADR-0119.7 sealed-holdout discipline to generalization audits) +.data/benchmarks/ + # Workbench UI browser-test artifacts workbench-ui/test-results/ workbench-ui/playwright-report/ diff --git a/evals/generalization/manifests/clutrr.yaml b/evals/generalization/manifests/clutrr.yaml index 4e25fdc1..0a3a4249 100644 --- a/evals/generalization/manifests/clutrr.yaml +++ b/evals/generalization/manifests/clutrr.yaml @@ -13,7 +13,7 @@ local_cache: .data/benchmarks/clutrr/ repo_policy: manifest_only inspection_policy: aggregate_reports_only mutation_policy: no_direct_pack_policy_operator_mutation -smoke_fixture: null # PR-2: sealed relational slice +smoke_fixture: null # PR-2: tiny public smoke slice only where license-permissive notes: | Best alignment to CORE's relation/field work. Kinship graph inference from stories; variable-length reasoning chains (k=2..10). diff --git a/evals/generalization/manifests/openbookqa.yaml b/evals/generalization/manifests/openbookqa.yaml index d4da4d98..23352705 100644 --- a/evals/generalization/manifests/openbookqa.yaml +++ b/evals/generalization/manifests/openbookqa.yaml @@ -5,7 +5,7 @@ description: > science facts. Requires combining a retrieved fact with broader commonsense/world knowledge. See arXiv:1809.02789. source: hf://allenai/openbookqa -license: Apache-2.0 # confirm; Allenai typically Apache-2.0 +license: TODO_VERIFY_BEFORE_CACHE # HF reports unknown; verify upstream AllenAI terms before cache version: pinned_release_or_commit split: test sha256: TODO_AFTER_DOWNLOAD diff --git a/evals/generalization/manifests/para_mawps.yaml b/evals/generalization/manifests/para_mawps.yaml index 49ba97f2..144e2e61 100644 --- a/evals/generalization/manifests/para_mawps.yaml +++ b/evals/generalization/manifests/para_mawps.yaml @@ -4,8 +4,8 @@ description: > Paraphrased, adversarial, and inverse variants of MAWPS-style math word problems. Good perturbation/robustness check for CORE organs. See arXiv:2306.13899. -source: hf://TODO_CONFIRM_HF_ID # verify canonical HF dataset ID -license: TODO_VERIFY_BEFORE_CACHE +source: hf://Starscream-11813/ParaMAWPS +license: CC-BY-NC-SA-4.0 version: pinned_release_or_commit split: test sha256: TODO_AFTER_DOWNLOAD @@ -18,3 +18,5 @@ notes: | Paraphrase + adversarial + inverse variant structure. Pull local cache only at first; confirm HF dataset ID before fetch script in PR-2. Also covers base MAWPS variants. + CC-BY-NC-SA-4.0; respect noncommercial/share-alike terms. + Local audit cache only unless license review authorizes committed smoke fixtures. diff --git a/evals/generalization/manifests/svamp.yaml b/evals/generalization/manifests/svamp.yaml index d05a38fa..f0bdc179 100644 --- a/evals/generalization/manifests/svamp.yaml +++ b/evals/generalization/manifests/svamp.yaml @@ -13,7 +13,7 @@ local_cache: .data/benchmarks/svamp/ repo_policy: manifest_only inspection_policy: aggregate_reports_only mutation_policy: no_direct_pack_policy_operator_mutation -smoke_fixture: null # PR-2: add sealed audit slice +smoke_fixture: null # PR-2: tiny public smoke slice only where license-permissive notes: | Specifically targets shallow heuristic solvers. Keep sealed. Variation structure makes it a strong adversarial math lane.