diff --git a/.data/benchmarks/.gitkeep b/.data/benchmarks/.gitkeep new file mode 100644 index 00000000..38549f60 --- /dev/null +++ b/.data/benchmarks/.gitkeep @@ -0,0 +1,3 @@ +# This directory is gitignored (see .gitignore: .data/benchmarks/). +# Only this .gitkeep and manifest YAMLs under evals/generalization/manifests/ are committed. +# Raw dataset files must never be committed to the repo. diff --git a/evals/generalization/BENCHMARK_POLICY.md b/evals/generalization/BENCHMARK_POLICY.md new file mode 100644 index 00000000..690f5269 --- /dev/null +++ b/evals/generalization/BENCHMARK_POLICY.md @@ -0,0 +1,41 @@ +# Generalization Benchmark Policy + +> **Rule:** These datasets are audit/test-only instruments. +> They must never be used as training material, capability patches, +> or fine-tuning targets for any CORE organ. + +## Core constraints + +| Constraint | Rule | +|---|---| +| Vendoring | Do **not** commit full dataset files to the repo | +| Sealed splits | Do **not** inspect sealed holdout examples during implementation | +| Patch prohibition | Do **not** create capability patches from benchmark item failures | +| Mutation prohibition | Do **not** apply `pack_policy_operator` mutations derived from benchmark errors | +| License gate | Verify license before caching; record in manifest `license` field | +| Checksum gate | Record `sha256` in manifest after download; re-verify before each eval run | + +## Directory layout + +``` +.data/benchmarks// # gitignored local cache (never committed) +evals/generalization/manifests/ # manifest YAMLs (committed) +evals/generalization/smoke/ # tiny public smoke fixtures (committed, PR-2) +evals/generalization/reports/ # aggregate score reports (committed, no raw items) +``` + +## Two-PR plan + +- **PR-1 (this PR):** Policy + manifests for datasets 1–8 +- **PR-2:** Fetch/verify scripts + smoke fixtures where licenses allow; + manifests for datasets 9–12 (MathQA, MATH L1-2, ARC-DA, CommonsenseQA) + +## Inspection policy + +Aggregate score reports only. No raw items from sealed slices may appear +in commit diffs, eval logs, or any CORE source file. + +## Mutation policy + +`no_direct_pack_policy_operator_mutation` — benchmark failures are +diagnosis signals, not direct targets for operator rewrite. diff --git a/evals/generalization/manifests/arc_challenge.yaml b/evals/generalization/manifests/arc_challenge.yaml new file mode 100644 index 00000000..302b18ff --- /dev/null +++ b/evals/generalization/manifests/arc_challenge.yaml @@ -0,0 +1,20 @@ +dataset: ARC-Challenge +purpose: sealed_audit_not_training +description: > + AI2 Reasoning Challenge hard partition. Questions that simple retrieval + and word-correlation methods fail on. Pairs with ARC-Easy for a + difficulty-stratified science audit. See arXiv:1803.05457. +source: hf://allenai/ai2_arc +hf_config: ARC-Challenge +license: CC-BY-SA-4.0 +version: pinned_release_or_commit +split: test +sha256: TODO_AFTER_DOWNLOAD +local_cache: .data/benchmarks/arc_challenge/ +repo_policy: manifest_only +inspection_policy: aggregate_reports_only +mutation_policy: no_direct_pack_policy_operator_mutation +smoke_fixture: null # PR-2: small public smoke slice +notes: | + Hard science reasoning. Only questions that retrieval-based methods + failed on are included. Key non-math audit lane for CORE. diff --git a/evals/generalization/manifests/arc_easy.yaml b/evals/generalization/manifests/arc_easy.yaml new file mode 100644 index 00000000..043aaaa4 --- /dev/null +++ b/evals/generalization/manifests/arc_easy.yaml @@ -0,0 +1,20 @@ +dataset: ARC-Easy +purpose: sealed_audit_not_training +description: > + AI2 Reasoning Challenge Easy partition. Grade-school science reasoning; + part of the 7,787-question ARC suite split into Easy and Challenge sets. + Good non-math generalization lane. See arXiv:1803.05457. +source: hf://allenai/ai2_arc +hf_config: ARC-Easy +license: CC-BY-SA-4.0 # confirmed public domain science questions +version: pinned_release_or_commit +split: test +sha256: TODO_AFTER_DOWNLOAD +local_cache: .data/benchmarks/arc_easy/ +repo_policy: manifest_only +inspection_policy: aggregate_reports_only +mutation_policy: no_direct_pack_policy_operator_mutation +smoke_fixture: null # PR-2: small public smoke slice +notes: | + Non-math science reasoning lane. Pairs with arc_challenge for a + difficulty-stratified view. License is CC-BY-SA-4.0. diff --git a/evals/generalization/manifests/asdiv.yaml b/evals/generalization/manifests/asdiv.yaml new file mode 100644 index 00000000..f41efe21 --- /dev/null +++ b/evals/generalization/manifests/asdiv.yaml @@ -0,0 +1,19 @@ +dataset: ASDiv +purpose: sealed_audit_not_training +description: > + A Diverse Corpus for Evaluating and Developing English Math Word Problem + Solvers. 2,305 problems covering quantity, entity, rate, and other + problem types with more diverse language than GSM8K. See arXiv:2106.15772. +source: hf://EleutherAI/asdiv +license: TODO_VERIFY_BEFORE_CACHE # check original Miao et al. license +version: pinned_release_or_commit +split: test +sha256: TODO_AFTER_DOWNLOAD +local_cache: .data/benchmarks/asdiv/ +repo_policy: manifest_only +inspection_policy: aggregate_reports_only +mutation_policy: no_direct_pack_policy_operator_mutation +smoke_fixture: null # PR-2: add small public smoke slice +notes: | + Good lane for checking quantity/entity/rate generalization beyond + GSM8K phrasing. 2,305 problems total. diff --git a/evals/generalization/manifests/clutrr.yaml b/evals/generalization/manifests/clutrr.yaml new file mode 100644 index 00000000..4e25fdc1 --- /dev/null +++ b/evals/generalization/manifests/clutrr.yaml @@ -0,0 +1,20 @@ +dataset: CLUTRR +purpose: sealed_audit_not_training +description: > + Diagnostic Benchmark for Inductive Reasoning from Text. Kinship and + entity relation inference from short stories. Maps directly onto CORE's + relational reasoning work. See arXiv:1908.06177. +source: hf://CLUTRR/v1 # confirm canonical HF path +license: TODO_VERIFY_BEFORE_CACHE # check Sinha et al. / McGill license +version: pinned_release_or_commit +split: test +sha256: TODO_AFTER_DOWNLOAD +local_cache: .data/benchmarks/clutrr/ +repo_policy: manifest_only +inspection_policy: aggregate_reports_only +mutation_policy: no_direct_pack_policy_operator_mutation +smoke_fixture: null # PR-2: sealed relational slice +notes: | + Best alignment to CORE's relation/field work. Kinship graph inference + from stories; variable-length reasoning chains (k=2..10). + Keep sealed slice for hold-out relational audit. diff --git a/evals/generalization/manifests/gsm1k.yaml b/evals/generalization/manifests/gsm1k.yaml new file mode 100644 index 00000000..5c149670 --- /dev/null +++ b/evals/generalization/manifests/gsm1k.yaml @@ -0,0 +1,19 @@ +dataset: GSM1K +purpose: sealed_audit_not_training +description: > + Direct generalization check for GSM8K overfit. Commissioned to mirror + GSM8K style and complexity while reducing contamination risk. + ~1,000 problems. See arXiv:2405.00332. +source: hf://openai/gsm1k +license: TODO_VERIFY_BEFORE_CACHE # likely MIT or CC-BY; confirm on HuggingFace +version: pinned_release_or_commit # TODO: pin after first pull +split: test +sha256: TODO_AFTER_DOWNLOAD +local_cache: .data/benchmarks/gsm1k/ +repo_policy: manifest_only +inspection_policy: aggregate_reports_only +mutation_policy: no_direct_pack_policy_operator_mutation +smoke_fixture: null # PR-2: add tiny public smoke slice +notes: | + Best first pull. Directly audits GSM8K-style generalization. + Do not inspect sealed examples during organ development. diff --git a/evals/generalization/manifests/openbookqa.yaml b/evals/generalization/manifests/openbookqa.yaml new file mode 100644 index 00000000..d4da4d98 --- /dev/null +++ b/evals/generalization/manifests/openbookqa.yaml @@ -0,0 +1,19 @@ +dataset: OpenBookQA +purpose: sealed_audit_not_training +description: > + Elementary open-book science QA: ~6,000 questions over 1,329 core + science facts. Requires combining a retrieved fact with broader + commonsense/world knowledge. See arXiv:1809.02789. +source: hf://allenai/openbookqa +license: Apache-2.0 # confirm; Allenai typically Apache-2.0 +version: pinned_release_or_commit +split: test +sha256: TODO_AFTER_DOWNLOAD +local_cache: .data/benchmarks/openbookqa/ +repo_policy: manifest_only +inspection_policy: aggregate_reports_only +mutation_policy: no_direct_pack_policy_operator_mutation +smoke_fixture: null # PR-2: small smoke slice +notes: | + Evidence + commonsense audit. 5,957 questions, 1,329 core science facts. + Good probe for fact retrieval + inference composition in CORE. diff --git a/evals/generalization/manifests/para_mawps.yaml b/evals/generalization/manifests/para_mawps.yaml new file mode 100644 index 00000000..49ba97f2 --- /dev/null +++ b/evals/generalization/manifests/para_mawps.yaml @@ -0,0 +1,20 @@ +dataset: PARA-MAWPS +purpose: sealed_audit_not_training +description: > + Paraphrased, adversarial, and inverse variants of MAWPS-style math + word problems. Good perturbation/robustness check for CORE organs. + See arXiv:2306.13899. +source: hf://TODO_CONFIRM_HF_ID # verify canonical HF dataset ID +license: TODO_VERIFY_BEFORE_CACHE +version: pinned_release_or_commit +split: test +sha256: TODO_AFTER_DOWNLOAD +local_cache: .data/benchmarks/para_mawps/ +repo_policy: manifest_only +inspection_policy: aggregate_reports_only +mutation_policy: no_direct_pack_policy_operator_mutation +smoke_fixture: null +notes: | + Paraphrase + adversarial + inverse variant structure. Pull local cache + only at first; confirm HF dataset ID before fetch script in PR-2. + Also covers base MAWPS variants. diff --git a/evals/generalization/manifests/svamp.yaml b/evals/generalization/manifests/svamp.yaml new file mode 100644 index 00000000..d05a38fa --- /dev/null +++ b/evals/generalization/manifests/svamp.yaml @@ -0,0 +1,19 @@ +dataset: SVAMP +purpose: sealed_audit_not_training +description: > + Simple Variations on Arithmetic Math word Problems. Designed specifically + to expose shallow heuristics in simple math word-problem solvers. + See arXiv:2103.07191. +source: hf://ChilleD/SVAMP +license: TODO_VERIFY_BEFORE_CACHE # check Patel et al. license +version: pinned_release_or_commit +split: test +sha256: TODO_AFTER_DOWNLOAD +local_cache: .data/benchmarks/svamp/ +repo_policy: manifest_only +inspection_policy: aggregate_reports_only +mutation_policy: no_direct_pack_policy_operator_mutation +smoke_fixture: null # PR-2: add sealed audit slice +notes: | + Specifically targets shallow heuristic solvers. Keep sealed. + Variation structure makes it a strong adversarial math lane.