From 1bff5689db2abd29ddc1ffe3b26b2c96e96acc77 Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 26 May 2026 11:52:23 -0700 Subject: [PATCH] =?UTF-8?q?feat(ADR-0163.B.1):=20exemplar=20corpora=20?= =?UTF-8?q?=E2=80=94=20descriptive=5Fsetup=5Fno=5Fquantity,=20temporal=5Fa?= =?UTF-8?q?ggregation,=20rate=5Fwith=5Fcurrency=20(#298)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 1 of ADR-0163 Phase B: hand-author seed exemplars for the top three refusal shape categories surfaced by the Phase A histogram. These corpora are INPUT to the Phase C contemplation runner, which will derive DerivedRecognizer proposals from them; this PR ships no recognizer logic, no proposal logging, and no runtime change. Per-category breakdown: - descriptive_setup_no_quantity_v1.jsonl — 20 exemplars (5 train + 12 novel + 3 edge) - temporal_aggregation_v1.jsonl — 20 exemplars (4 train + 13 novel + 3 edge) - rate_with_currency_v1.jsonl — 20 exemplars (3 train + 14 novel + 3 edge) Train-sample citations resolve against evals/gsm8k_math/train_sample/v1/report.json (the 50-case sample only; public/holdout/full splits NOT mined per ADR-0163 §Constraints). Each file is sorted by exemplar_id, byte-canonical, and disjoint from the others. Statements are surface-preserved verbatim from the train sample where cited. Validation: - tests/test_admissibility_exemplars.py: 20/20 passed (schema, enum binding, per-category quantity_anchor dispatch, cross-file disjointness, >=3 train-sample citations per category, sort/byte-canonical determinism, read-only import invariant) - tests/test_adr_0131_*.py: 224 passed / 3 skipped — capability axes G1..G5 + S1 remain wrong=0 - core test --suite smoke: 67 passed - core eval refusal_taxonomy: case_digest unchanged (d030f826cb0f4088771d90c52c8be2ff75054ab27c7d47eae8dbfe1225b2eea1) - Phase A categorize() agrees with the file's category for all 60 statements (sanity check; not pinned in tests since the rules-only categorizer is coarser than the recognizer Phase C will derive) Author notes on quantity_anchor annotation calls flagged for operator review are embedded in provenance.author_note where ambiguous (notably: 'in N minutes' / 'over N hours' window framings collapsed to window_quantifier='per', 'every other day' approximated as 'every', day-of-week labels not captured in the schema, 'for one X' / slash-form per-unit framings, non-USD currencies, and discrete-occurrence per_unit values like 'event' and 'session'). Refs: ADR-0163 §Phase B; depends on the Phase A lane shipped in #297. Cross-refs: ADR-0057 (proposal review), ADR-0149/0154 (recognizer pipeline), ADR-0161 (HITL queue), [[thesis-decoding-not-generating]]. --- docs/admissibility-exemplars.md | 57 +++ teaching/admissibility_exemplars/__init__.py | 0 teaching/admissibility_exemplars/contract.md | 178 ++++++++ .../descriptive_setup_no_quantity_v1.jsonl | 20 + .../rate_with_currency_v1.jsonl | 20 + .../temporal_aggregation_v1.jsonl | 20 + tests/test_admissibility_exemplars.py | 379 ++++++++++++++++++ 7 files changed, 674 insertions(+) create mode 100644 docs/admissibility-exemplars.md create mode 100644 teaching/admissibility_exemplars/__init__.py create mode 100644 teaching/admissibility_exemplars/contract.md create mode 100644 teaching/admissibility_exemplars/descriptive_setup_no_quantity_v1.jsonl create mode 100644 teaching/admissibility_exemplars/rate_with_currency_v1.jsonl create mode 100644 teaching/admissibility_exemplars/temporal_aggregation_v1.jsonl create mode 100644 tests/test_admissibility_exemplars.py diff --git a/docs/admissibility-exemplars.md b/docs/admissibility-exemplars.md new file mode 100644 index 00000000..74696bf2 --- /dev/null +++ b/docs/admissibility-exemplars.md @@ -0,0 +1,57 @@ +# Admissibility Exemplars (ADR-0163 Phase B) + +Operator-facing overview of the hand-authored exemplar corpora that feed the +Phase C contemplation runner. For the full schema, sourcing rules, and +forward reference, see +[`teaching/admissibility_exemplars/contract.md`](../teaching/admissibility_exemplars/contract.md). + +## What this is + +For each shape category surfaced by the Phase A refusal-taxonomy lane, the +operator hand-authors a small JSONL corpus of canonical exemplars. Phase C +(contemplation runner) ingests these corpora and emits recognizer proposals; +Phase D ratifies; Phase E re-baselines GSM8K. + +Phase B is the **only** phase where the engine learns from operator-authored +statements. Every dimension of "what shape did the operator think was +canonical?" propagates into the recognizer Phase C derives and the gates +Phase D ratifies. Therefore: canonical over comprehensive, surface +preservation over normalization, distinguishing over similar. + +## Round 1 — categories and counts + +The Phase A histogram (`evals/refusal_taxonomy/v1/report.json`) selected +these three categories: + +| Rank | Category | Phase A count | Exemplars (this round) | +|---|---|---|---| +| 1 | `descriptive_setup_no_quantity` | 17 | 20 | +| 2 | `temporal_aggregation` | 4 | 20 | +| 3 | `rate_with_currency` | 3 | 20 | + +Total: **60 hand-authored exemplars** across three files in +`teaching/admissibility_exemplars/`. + +Per-category breakdown of sourcing: + +| Category | Train-sample citations | Novel (operator-authored) | Edge cases | +|---|---|---|---| +| `descriptive_setup_no_quantity` | 5 | 12 | 3 | +| `temporal_aggregation` | 4 | 13 | 3 | +| `rate_with_currency` | 3 | 14 | 3 | + +(Edge cases overlap with novel; counts above split them out.) + +## Files + +- [`teaching/admissibility_exemplars/descriptive_setup_no_quantity_v1.jsonl`](../teaching/admissibility_exemplars/descriptive_setup_no_quantity_v1.jsonl) +- [`teaching/admissibility_exemplars/temporal_aggregation_v1.jsonl`](../teaching/admissibility_exemplars/temporal_aggregation_v1.jsonl) +- [`teaching/admissibility_exemplars/rate_with_currency_v1.jsonl`](../teaching/admissibility_exemplars/rate_with_currency_v1.jsonl) +- [`teaching/admissibility_exemplars/contract.md`](../teaching/admissibility_exemplars/contract.md) + +## Cross-references + +- [ADR-0163 — Path to GSM8K mastery](decisions/ADR-0163-gsm8k-path-to-mastery.md) +- [Phase A refusal taxonomy contract](../evals/refusal_taxonomy/contract.md) +- [ADR-0057 — Proposal review + replay-equivalence](decisions/ADR-0057-teaching-chain-proposal-review.md) +- [ADR-0161 — HITL async queue](decisions/ADR-0161-hitl-async-queue.md) diff --git a/teaching/admissibility_exemplars/__init__.py b/teaching/admissibility_exemplars/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/teaching/admissibility_exemplars/contract.md b/teaching/admissibility_exemplars/contract.md new file mode 100644 index 00000000..746dd2a6 --- /dev/null +++ b/teaching/admissibility_exemplars/contract.md @@ -0,0 +1,178 @@ +# Admissibility Exemplars Contract (ADR-0163 Phase B) + +## Purpose + +`teaching/admissibility_exemplars/` holds operator-authored exemplar corpora for +refusal shape categories surfaced by the Phase A refusal-taxonomy lane. + +Each exemplar is a **seed**, not a sample. Phase C (contemplation runner) will +ingest these files as a candidate source and derive `DerivedRecognizer` +proposals that generalize the shape. The seeds must therefore be the cleanest, +most canonical instances of their shape — ambiguous seeds produce ambiguous +recognizers. + +See [ADR-0163 §Phase B](../../docs/decisions/ADR-0163-gsm8k-path-to-mastery.md) +for the contract this file implements. + +## Round 1 — categories + +The Phase A histogram (`evals/refusal_taxonomy/v1/report.json`) surfaced this +distribution: + +``` +descriptive_setup_no_quantity 17 <- selected (rank 1) +uncategorized 14 +temporal_aggregation 4 <- selected (rank 2) +rate_with_currency 3 <- selected (rank 3, operator pick) +comparative_with_unit 3 +fractional_rate_of_change 3 +indefinite_quantity 3 +nested_question_target 2 +unit_partition 1 +conditional_quantity 0 +``` + +`rate_with_currency` was selected over the other three-count categories +(`comparative_with_unit`, `fractional_rate_of_change`, `indefinite_quantity`) +by operator decision: GSM8K is heavy on money/rate framings and the lift +compounds with `temporal_aggregation` (per-time-unit framings). + +`uncategorized` is intentionally not addressed in round 1 — Phase B writes +exemplars for *named* shapes; the uncategorized tail is the next-round work +once a categorizer rule surfaces what shape these statements actually carry. + +## Exemplar schema + +Each line in a `*.jsonl` file is a JSON object: + +```json +{ + "exemplar_id": "-v1-", + "shape_category": "", + "statement": "", + "expected_graph": { + "subject": "", + "quantity_anchors": [ ... ], + "graph_intent": "", + "outcome": "" + }, + "provenance": { + "source": "phase_b_seed", + "author": "", + "round": 1, + "category_rank": <1|2|3>, + "train_case_id": "", + "author_note": "" + } +} +``` + +`shape_category` MUST be a valid member of `ShapeCategory` in +`evals/refusal_taxonomy/shape_categories.py`, and it MUST equal the category +this file is named for. + +## Per-category `quantity_anchors` schema + +### `descriptive_setup_no_quantity` + +```json +"quantity_anchors": [] +"graph_intent": "setup" +"outcome": "inadmissible_by_design" +``` + +These statements have no extractable quantity. Phase C's recognizer will +produce a *setup admission* verdict for them — they are context that should be +admitted as setup-only, not refused outright. + +### `temporal_aggregation` + +```json +"quantity_anchors": [ + { + "kind": "event_count_per_window", + "count_token": "", + "window_unit": "", + "window_quantifier": "", + "subject_role": "" + }, + ... +] +"graph_intent": "aggregate" +"outcome": "admissible" +``` + +Multiple anchors may appear when a statement enumerates several events (e.g., +day-of-week enumeration). + +### `rate_with_currency` + +```json +"quantity_anchors": [ + { + "kind": "currency_per_unit_rate", + "currency_symbol": "<$|£|€|¥>", + "amount": "", + "amount_kind": "", + "per_unit": "", + "subject_role": "" + } +] +"graph_intent": "rate" +"outcome": "admissible" +``` + +## Sourcing rules + +For each category, the corpus MUST satisfy: + +- **At least 3 verbatim train-sample citations.** These cite a real + `case_id` from `evals/gsm8k_math/train_sample/v1/report.json` via the + `provenance.train_case_id` field. The statement string MUST equal the + refused statement in that case verbatim — no normalization, no punctuation + edits, no contraction expansion. +- **At least 12 operator-authored novel statements** that instantiate the + shape canonically and were not mined from GSM8K. +- **At least 2 edge cases** that exercise the shape's boundary (alternative + surface forms, threshold-of-rule instances, currency variants). +- **No duplicate statements** within a file. +- **No statement shared across files** — every statement belongs to exactly + one category. + +## Disjointness and category fidelity + +Every exemplar MUST belong unambiguously to its named category, where +"unambiguously" is operationalized as: `categorize(statement)` from +`evals/refusal_taxonomy/shape_categories.py` returns the file's category. + +This is not enforced by tests in this PR (the categorizer is a coarser +rules-only filter than the recognizer Phase C will derive), but it is the +authoring guideline that produces clean seeds. Where a statement could +plausibly belong to a different category, it is excluded from this corpus. + +## Determinism + +Each `*.jsonl` file is sorted by `exemplar_id` (lexicographic) and committed +in that order. Lines have no trailing whitespace and a single trailing +newline. The file is byte-stable across re-sorts. + +## Holdout / split discipline + +Train-sample citations come only from +`evals/gsm8k_math/train_sample/v1/report.json` (the 50-case sample). The +public, holdout, and full GSM8K splits MUST NOT be mined for exemplars — +doing so would tune against the benchmark we are honestly measuring. + +## Forward reference — Phase C + +Phase C will: + +1. Read each `*_v1.jsonl` as a candidate source alongside + `teaching/discovery/discovery_candidates.jsonl`. +2. Decompose statements into recognizer patterns. +3. Emit `DerivedRecognizer` proposals to + `teaching/proposals/proposals.jsonl` via the standard ADR-0057 path. +4. Surface the proposals in the HITL queue (ADR-0161) for operator review. + +Phase B ships inputs only. No recognizer logic, no proposal logging, no +runtime change lands with this corpus. diff --git a/teaching/admissibility_exemplars/descriptive_setup_no_quantity_v1.jsonl b/teaching/admissibility_exemplars/descriptive_setup_no_quantity_v1.jsonl new file mode 100644 index 00000000..9fcc1ed1 --- /dev/null +++ b/teaching/admissibility_exemplars/descriptive_setup_no_quantity_v1.jsonl @@ -0,0 +1,20 @@ +{"exemplar_id": "dsnq-v1-0001", "shape_category": "descriptive_setup_no_quantity", "statement": "The student council sells scented erasers in the morning before school starts to help raise money for school dances.", "expected_graph": {"subject": null, "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1, "train_case_id": "gsm8k-train-sample-v1-0003"}} +{"exemplar_id": "dsnq-v1-0002", "shape_category": "descriptive_setup_no_quantity", "statement": "Marnie makes bead bracelets.", "expected_graph": {"subject": "Marnie", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1, "train_case_id": "gsm8k-train-sample-v1-0008"}} +{"exemplar_id": "dsnq-v1-0003", "shape_category": "descriptive_setup_no_quantity", "statement": "John adopts a dog from a shelter.", "expected_graph": {"subject": "John", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1, "train_case_id": "gsm8k-train-sample-v1-0019"}} +{"exemplar_id": "dsnq-v1-0004", "shape_category": "descriptive_setup_no_quantity", "statement": "John is lifting weights.", "expected_graph": {"subject": "John", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1, "train_case_id": "gsm8k-train-sample-v1-0021"}} +{"exemplar_id": "dsnq-v1-0005", "shape_category": "descriptive_setup_no_quantity", "statement": "Jed collects stamp cards.", "expected_graph": {"subject": "Jed", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1, "train_case_id": "gsm8k-train-sample-v1-0048"}} +{"exemplar_id": "dsnq-v1-0006", "shape_category": "descriptive_setup_no_quantity", "statement": "Maria runs a bookshop in town.", "expected_graph": {"subject": "Maria", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1}} +{"exemplar_id": "dsnq-v1-0007", "shape_category": "descriptive_setup_no_quantity", "statement": "Liam volunteers at the community garden on weekends.", "expected_graph": {"subject": "Liam", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1}} +{"exemplar_id": "dsnq-v1-0008", "shape_category": "descriptive_setup_no_quantity", "statement": "The library hosts story time for children.", "expected_graph": {"subject": null, "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1}} +{"exemplar_id": "dsnq-v1-0009", "shape_category": "descriptive_setup_no_quantity", "statement": "Priya bakes sourdough bread for her neighbors.", "expected_graph": {"subject": "Priya", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1}} +{"exemplar_id": "dsnq-v1-0010", "shape_category": "descriptive_setup_no_quantity", "statement": "Carlos teaches piano lessons after work.", "expected_graph": {"subject": "Carlos", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1}} +{"exemplar_id": "dsnq-v1-0011", "shape_category": "descriptive_setup_no_quantity", "statement": "Anika writes poetry in her free time.", "expected_graph": {"subject": "Anika", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1}} +{"exemplar_id": "dsnq-v1-0012", "shape_category": "descriptive_setup_no_quantity", "statement": "The choir rehearses in the church basement.", "expected_graph": {"subject": null, "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1}} +{"exemplar_id": "dsnq-v1-0013", "shape_category": "descriptive_setup_no_quantity", "statement": "Diego paints murals downtown.", "expected_graph": {"subject": "Diego", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1}} +{"exemplar_id": "dsnq-v1-0014", "shape_category": "descriptive_setup_no_quantity", "statement": "Sophie photographs weddings on weekends.", "expected_graph": {"subject": "Sophie", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1}} +{"exemplar_id": "dsnq-v1-0015", "shape_category": "descriptive_setup_no_quantity", "statement": "Hiroshi sells handmade pottery at the farmer's market.", "expected_graph": {"subject": "Hiroshi", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1}} +{"exemplar_id": "dsnq-v1-0016", "shape_category": "descriptive_setup_no_quantity", "statement": "Beatrice tends the rose garden behind her cottage.", "expected_graph": {"subject": "Beatrice", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1}} +{"exemplar_id": "dsnq-v1-0017", "shape_category": "descriptive_setup_no_quantity", "statement": "The mailman delivers packages along the avenue.", "expected_graph": {"subject": null, "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1}} +{"exemplar_id": "dsnq-v1-0018", "shape_category": "descriptive_setup_no_quantity", "statement": "Marco walks to the corner store after dinner.", "expected_graph": {"subject": "Marco", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1, "author_note": "Edge case: includes a time-adjacency phrase ('after dinner') but carries no quantity marker; should not trip temporal_aggregation."}} +{"exemplar_id": "dsnq-v1-0019", "shape_category": "descriptive_setup_no_quantity", "statement": "Emma volunteers at the animal shelter on Saturdays.", "expected_graph": {"subject": "Emma", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1, "author_note": "Edge case: a single day-of-week token ('Saturdays') is present but day-of-week enumeration requires >=2 distinct days, so it must not trip temporal_aggregation."}} +{"exemplar_id": "dsnq-v1-0020", "shape_category": "descriptive_setup_no_quantity", "statement": "Renee delivers groceries during her lunch break.", "expected_graph": {"subject": "Renee", "quantity_anchors": [], "graph_intent": "setup", "outcome": "inadmissible_by_design"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 1, "author_note": "Edge case: contains a time-unit-adjacent phrase ('lunch break') but no numeric or word-form count."}} diff --git a/teaching/admissibility_exemplars/rate_with_currency_v1.jsonl b/teaching/admissibility_exemplars/rate_with_currency_v1.jsonl new file mode 100644 index 00000000..0e71fc4a --- /dev/null +++ b/teaching/admissibility_exemplars/rate_with_currency_v1.jsonl @@ -0,0 +1,20 @@ +{"exemplar_id": "rwc-v1-0001", "shape_category": "rate_with_currency", "statement": "Tina makes $18.00 an hour.", "expected_graph": {"subject": "Tina", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "18.00", "amount_kind": "decimal", "per_unit": "hour", "subject_role": "Tina"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3, "train_case_id": "gsm8k-train-sample-v1-0001"}} +{"exemplar_id": "rwc-v1-0002", "shape_category": "rate_with_currency", "statement": "Alexa has a lemonade stand where she sells lemonade for $2 for one cup.", "expected_graph": {"subject": "Alexa", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "2", "amount_kind": "integer", "per_unit": "cup", "subject_role": "lemonade"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3, "train_case_id": "gsm8k-train-sample-v1-0011", "author_note": "Non-canonical 'for one X' framing rather than 'per X'. The recognizer must accept both surface forms."}} +{"exemplar_id": "rwc-v1-0003", "shape_category": "rate_with_currency", "statement": "Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.", "expected_graph": {"subject": "Erica", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "20", "amount_kind": "integer", "per_unit": "kg", "subject_role": "fish"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3, "train_case_id": "gsm8k-train-sample-v1-0022"}} +{"exemplar_id": "rwc-v1-0004", "shape_category": "rate_with_currency", "statement": "Marcus charges $25 per hour for tutoring math students.", "expected_graph": {"subject": "Marcus", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "25", "amount_kind": "integer", "per_unit": "hour", "subject_role": "Marcus"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3}} +{"exemplar_id": "rwc-v1-0005", "shape_category": "rate_with_currency", "statement": "The cafe charges $4 per cup of coffee.", "expected_graph": {"subject": "the cafe", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "4", "amount_kind": "integer", "per_unit": "cup", "subject_role": "coffee"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3}} +{"exemplar_id": "rwc-v1-0006", "shape_category": "rate_with_currency", "statement": "Yuki earns $15 an hour at the bookstore.", "expected_graph": {"subject": "Yuki", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "15", "amount_kind": "integer", "per_unit": "hour", "subject_role": "Yuki"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3}} +{"exemplar_id": "rwc-v1-0007", "shape_category": "rate_with_currency", "statement": "Ravi pays $1200 a month for his apartment.", "expected_graph": {"subject": "Ravi", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "1200", "amount_kind": "integer", "per_unit": "month", "subject_role": "Ravi"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3}} +{"exemplar_id": "rwc-v1-0008", "shape_category": "rate_with_currency", "statement": "Greta sells handmade soap for $6 per bar.", "expected_graph": {"subject": "Greta", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "6", "amount_kind": "integer", "per_unit": "bar", "subject_role": "soap"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3}} +{"exemplar_id": "rwc-v1-0009", "shape_category": "rate_with_currency", "statement": "The farm sells eggs at $3 per dozen.", "expected_graph": {"subject": "the farm", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "3", "amount_kind": "integer", "per_unit": "dozen", "subject_role": "eggs"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3, "author_note": "per_unit='dozen' is a multiplicative unit (12-pack), not a base unit; flagged for Phase C disambiguation."}} +{"exemplar_id": "rwc-v1-0010", "shape_category": "rate_with_currency", "statement": "Anders works as a freelancer for $45/hour.", "expected_graph": {"subject": "Anders", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "45", "amount_kind": "integer", "per_unit": "hour", "subject_role": "Anders"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3, "author_note": "Slash-form per-unit framing ('$45/hour') rather than 'per hour'; recognizer must accept both."}} +{"exemplar_id": "rwc-v1-0011", "shape_category": "rate_with_currency", "statement": "The market buys apples from growers at $2 per pound.", "expected_graph": {"subject": "the market", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "2", "amount_kind": "integer", "per_unit": "pound", "subject_role": "apples"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3}} +{"exemplar_id": "rwc-v1-0012", "shape_category": "rate_with_currency", "statement": "Niko's gym membership costs $50 a month.", "expected_graph": {"subject": "Niko", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "50", "amount_kind": "integer", "per_unit": "month", "subject_role": "Niko"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3}} +{"exemplar_id": "rwc-v1-0013", "shape_category": "rate_with_currency", "statement": "Lila tutors French for $30 an hour.", "expected_graph": {"subject": "Lila", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "30", "amount_kind": "integer", "per_unit": "hour", "subject_role": "Lila"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3}} +{"exemplar_id": "rwc-v1-0014", "shape_category": "rate_with_currency", "statement": "The catering company charges $200 per event for service.", "expected_graph": {"subject": "the catering company", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "200", "amount_kind": "integer", "per_unit": "event", "subject_role": "the catering company"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3, "author_note": "per_unit='event' is a discrete-occurrence unit, not a base measurement unit; flagged for Phase C disambiguation."}} +{"exemplar_id": "rwc-v1-0015", "shape_category": "rate_with_currency", "statement": "Vikram earns $85000 a year as a software engineer.", "expected_graph": {"subject": "Vikram", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "85000", "amount_kind": "integer", "per_unit": "year", "subject_role": "Vikram"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3}} +{"exemplar_id": "rwc-v1-0016", "shape_category": "rate_with_currency", "statement": "Tomas's lawn-care service charges $40 for one yard.", "expected_graph": {"subject": "Tomas", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "40", "amount_kind": "integer", "per_unit": "yard", "subject_role": "Tomas"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3, "author_note": "'for one X' alternative framing, matching the surface of train case 0011."}} +{"exemplar_id": "rwc-v1-0017", "shape_category": "rate_with_currency", "statement": "The dog walker charges $20 for each walk.", "expected_graph": {"subject": "the dog walker", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "$", "amount": "20", "amount_kind": "integer", "per_unit": "walk", "subject_role": "the dog walker"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3, "author_note": "'for each X' alternative framing."}} +{"exemplar_id": "rwc-v1-0018", "shape_category": "rate_with_currency", "statement": "Nina earns £15 an hour as a barista in London.", "expected_graph": {"subject": "Nina", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "£", "amount": "15", "amount_kind": "integer", "per_unit": "hour", "subject_role": "Nina"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3, "author_note": "Edge case: non-USD currency (pound sterling). Tests that the recognizer generalizes the currency-symbol slot."}} +{"exemplar_id": "rwc-v1-0019", "shape_category": "rate_with_currency", "statement": "Klaus pays €800 per month for his Berlin studio.", "expected_graph": {"subject": "Klaus", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "€", "amount": "800", "amount_kind": "integer", "per_unit": "month", "subject_role": "Klaus"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3, "author_note": "Edge case: non-USD currency (euro)."}} +{"exemplar_id": "rwc-v1-0020", "shape_category": "rate_with_currency", "statement": "Akari sells tea ceremony lessons for ¥3000 per session.", "expected_graph": {"subject": "Akari", "quantity_anchors": [{"kind": "currency_per_unit_rate", "currency_symbol": "¥", "amount": "3000", "amount_kind": "integer", "per_unit": "session", "subject_role": "Akari"}], "graph_intent": "rate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 3, "author_note": "Edge case: non-USD currency (yen) and discrete-occurrence per_unit ('session')."}} diff --git a/teaching/admissibility_exemplars/temporal_aggregation_v1.jsonl b/teaching/admissibility_exemplars/temporal_aggregation_v1.jsonl new file mode 100644 index 00000000..c5e8afc1 --- /dev/null +++ b/teaching/admissibility_exemplars/temporal_aggregation_v1.jsonl @@ -0,0 +1,20 @@ +{"exemplar_id": "ta-v1-0001", "shape_category": "temporal_aggregation", "statement": "Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.", "expected_graph": {"subject": "Allison", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "10", "window_unit": "day", "window_quantifier": "each", "subject_role": "Allison"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0013", "author_note": "The 'one-hour' detail describes video length, not the aggregation window; it is intentionally omitted from quantity_anchors so Phase C generalizes the window-axis cleanly."}} +{"exemplar_id": "ta-v1-0002", "shape_category": "temporal_aggregation", "statement": "Bob can shuck 10 oysters in 5 minutes.", "expected_graph": {"subject": "Bob", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "10", "window_unit": "minute", "window_quantifier": "per", "subject_role": "Bob"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0014", "author_note": "The window is 5 minutes (composite count + unit); the schema only encodes window_unit/quantifier, so 'per' is the closest canonical quantifier and the '5' multiplier is implicit. Phase C may need a window_count extension to express '10 per 5 minutes' versus '10 per minute' precisely."}} +{"exemplar_id": "ta-v1-0003", "shape_category": "temporal_aggregation", "statement": "Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday, and 50 on Thursday.", "expected_graph": {"subject": "Sidney", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "20", "window_unit": "day", "window_quantifier": "each", "subject_role": "Sidney"}, {"kind": "event_count_per_window", "count_token": "36", "window_unit": "day", "window_quantifier": "each", "subject_role": "Sidney"}, {"kind": "event_count_per_window", "count_token": "40", "window_unit": "day", "window_quantifier": "each", "subject_role": "Sidney"}, {"kind": "event_count_per_window", "count_token": "50", "window_unit": "day", "window_quantifier": "each", "subject_role": "Sidney"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0024", "author_note": "Day-of-week enumeration is encoded as four day-windowed event_count anchors; specific day-of-week labels (Mon/Tue/Wed/Thu) are not captured because the schema lacks a day-of-week field."}} +{"exemplar_id": "ta-v1-0004", "shape_category": "temporal_aggregation", "statement": "Mark does a gig every other day for 2 weeks.", "expected_graph": {"subject": "Mark", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "1", "window_unit": "day", "window_quantifier": "every", "subject_role": "Mark"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0050", "author_note": "'a gig' implies count=1; 'every other day' is approximated as window_quantifier='every' with unit 'day'; the outer 'for 2 weeks' duration bound is not captured by the current schema."}} +{"exemplar_id": "ta-v1-0005", "shape_category": "temporal_aggregation", "statement": "Olivia practices violin for 2 hours each day before school.", "expected_graph": {"subject": "Olivia", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "2", "window_unit": "day", "window_quantifier": "each", "subject_role": "Olivia"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2, "author_note": "count_token here refers to the per-day quantity (2 hours of practice); the inner unit 'hours' is part of the counted thing, not the window."}} +{"exemplar_id": "ta-v1-0006", "shape_category": "temporal_aggregation", "statement": "Daniel runs 5 miles every day at sunrise.", "expected_graph": {"subject": "Daniel", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "5", "window_unit": "day", "window_quantifier": "every", "subject_role": "Daniel"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ta-v1-0007", "shape_category": "temporal_aggregation", "statement": "Priya solves 12 algebra problems per day during summer break.", "expected_graph": {"subject": "Priya", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "12", "window_unit": "day", "window_quantifier": "per", "subject_role": "Priya"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ta-v1-0008", "shape_category": "temporal_aggregation", "statement": "The bakery sells 50 croissants daily before noon.", "expected_graph": {"subject": "the bakery", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "50", "window_unit": "day", "window_quantifier": "per", "subject_role": "the bakery"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2, "author_note": "'daily' is treated as window_unit='day' with quantifier='per'; the adverbial form has no explicit quantifier word."}} +{"exemplar_id": "ta-v1-0009", "shape_category": "temporal_aggregation", "statement": "Tomas writes 1000 words each week for his blog.", "expected_graph": {"subject": "Tomas", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "1000", "window_unit": "week", "window_quantifier": "each", "subject_role": "Tomas"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ta-v1-0010", "shape_category": "temporal_aggregation", "statement": "Helena reads 3 chapters per week from her novel pile.", "expected_graph": {"subject": "Helena", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "3", "window_unit": "week", "window_quantifier": "per", "subject_role": "Helena"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ta-v1-0011", "shape_category": "temporal_aggregation", "statement": "The factory produces 200 widgets each hour during peak shifts.", "expected_graph": {"subject": "the factory", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "200", "window_unit": "hour", "window_quantifier": "each", "subject_role": "the factory"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ta-v1-0012", "shape_category": "temporal_aggregation", "statement": "Robin walks 4 dogs every other day around the park.", "expected_graph": {"subject": "Robin", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "4", "window_unit": "day", "window_quantifier": "every", "subject_role": "Robin"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2, "author_note": "'every other day' approximated as window_quantifier='every'; the 'other' (skip-one) cadence is not represented in the schema."}} +{"exemplar_id": "ta-v1-0013", "shape_category": "temporal_aggregation", "statement": "Aiko swims 30 laps daily at the community pool.", "expected_graph": {"subject": "Aiko", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "30", "window_unit": "day", "window_quantifier": "per", "subject_role": "Aiko"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ta-v1-0014", "shape_category": "temporal_aggregation", "statement": "The shop receives 75 packages every week from its supplier.", "expected_graph": {"subject": "the shop", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "75", "window_unit": "week", "window_quantifier": "every", "subject_role": "the shop"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ta-v1-0015", "shape_category": "temporal_aggregation", "statement": "Tahlia paints 2 portraits every month for her clients.", "expected_graph": {"subject": "Tahlia", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "2", "window_unit": "month", "window_quantifier": "every", "subject_role": "Tahlia"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ta-v1-0016", "shape_category": "temporal_aggregation", "statement": "Joaquin practices karate for 90 minutes each day after school.", "expected_graph": {"subject": "Joaquin", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "90", "window_unit": "day", "window_quantifier": "each", "subject_role": "Joaquin"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2, "author_note": "count_token=90 refers to minutes-of-practice per day; the inner unit 'minutes' is part of the counted thing."}} +{"exemplar_id": "ta-v1-0017", "shape_category": "temporal_aggregation", "statement": "Felix mows 6 lawns weekly during the summer.", "expected_graph": {"subject": "Felix", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "6", "window_unit": "week", "window_quantifier": "per", "subject_role": "Felix"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ta-v1-0018", "shape_category": "temporal_aggregation", "statement": "Greta bakes 24 cookies in 30 minutes.", "expected_graph": {"subject": "Greta", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "24", "window_unit": "minute", "window_quantifier": "per", "subject_role": "Greta"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: 'in N minutes' window framing (no 'each/every/per' word). Mirrors train case 0014 but with a different count/unit pairing; window_quantifier is mapped to 'per' and the '30' multiplier is implicit."}} +{"exemplar_id": "ta-v1-0019", "shape_category": "temporal_aggregation", "statement": "The pump fills the tank with 80 gallons over 6 hours.", "expected_graph": {"subject": "the pump", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "80", "window_unit": "hour", "window_quantifier": "per", "subject_role": "the pump"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: 'over N hours' temporal-window framing. As with case 0014, the schema collapses '80 per 6 hours' to window_unit='hour' with quantifier='per'."}} +{"exemplar_id": "ta-v1-0020", "shape_category": "temporal_aggregation", "statement": "Wendy runs 3 miles on Monday and 5 miles on Wednesday.", "expected_graph": {"subject": "Wendy", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "3", "window_unit": "day", "window_quantifier": "each", "subject_role": "Wendy"}, {"kind": "event_count_per_window", "count_token": "5", "window_unit": "day", "window_quantifier": "each", "subject_role": "Wendy"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: minimal day-of-week enumeration (exactly two distinct day names — the threshold for the rule). Distinct from case 0024 which enumerates four days."}} diff --git a/tests/test_admissibility_exemplars.py b/tests/test_admissibility_exemplars.py new file mode 100644 index 00000000..a2469423 --- /dev/null +++ b/tests/test_admissibility_exemplars.py @@ -0,0 +1,379 @@ +"""ADR-0163 Phase B — admissibility exemplar corpora tests. + +Validates the operator-authored exemplar corpora under +``teaching/admissibility_exemplars/`` against the schema specified in +``teaching/admissibility_exemplars/contract.md``. + +The tests are pure, deterministic, and read-only — they import no runtime +module beyond ``evals.refusal_taxonomy.shape_categories`` (for enum binding) +and never mutate any file under ``generate/``, ``evals/``, or +``teaching/proposals/``. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from evals.refusal_taxonomy.shape_categories import ShapeCategory + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_EXEMPLARS_ROOT = _REPO_ROOT / "teaching" / "admissibility_exemplars" +_GSM8K_TRAIN_REPORT = ( + _REPO_ROOT / "evals" / "gsm8k_math" / "train_sample" / "v1" / "report.json" +) + +# Round 1 categories, with their file stem and expected category-rank. +_ROUND_1: tuple[tuple[str, ShapeCategory, int], ...] = ( + ( + "descriptive_setup_no_quantity_v1", + ShapeCategory.DESCRIPTIVE_SETUP_NO_QUANTITY, + 1, + ), + ("temporal_aggregation_v1", ShapeCategory.TEMPORAL_AGGREGATION, 2), + ("rate_with_currency_v1", ShapeCategory.RATE_WITH_CURRENCY, 3), +) + +_REQUIRED_TOP_KEYS: frozenset[str] = frozenset( + {"exemplar_id", "shape_category", "statement", "expected_graph", "provenance"} +) +_REQUIRED_GRAPH_KEYS: frozenset[str] = frozenset( + {"subject", "quantity_anchors", "graph_intent", "outcome"} +) +_REQUIRED_PROVENANCE_KEYS: frozenset[str] = frozenset( + {"source", "author", "round", "category_rank"} +) + +_VALID_WINDOW_UNITS: frozenset[str] = frozenset( + {"day", "week", "month", "year", "hour", "minute", "second"} +) +_VALID_WINDOW_QUANTIFIERS: frozenset[str] = frozenset({"each", "every", "per"}) +_VALID_CURRENCY_SYMBOLS: frozenset[str] = frozenset({"$", "£", "€", "¥"}) +_VALID_AMOUNT_KINDS: frozenset[str] = frozenset({"integer", "decimal", "word"}) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _load_jsonl(path: Path) -> list[dict[str, Any]]: + raw = path.read_text(encoding="utf-8") + if not raw.endswith("\n"): + raise AssertionError(f"{path} must end with a single trailing newline") + if raw.endswith("\n\n"): + raise AssertionError(f"{path} must not have multiple trailing newlines") + lines = raw.splitlines() + records: list[dict[str, Any]] = [] + for idx, line in enumerate(lines, start=1): + if line != line.rstrip(): + raise AssertionError( + f"{path}:{idx} has trailing whitespace" + ) + records.append(json.loads(line)) + return records + + +def _train_sample_case_ids() -> set[str]: + report = json.loads(_GSM8K_TRAIN_REPORT.read_text(encoding="utf-8")) + return {entry["case_id"] for entry in report.get("per_case", [])} + + +# --------------------------------------------------------------------------- +# File presence +# --------------------------------------------------------------------------- + + +def test_exemplars_root_exists_and_marker_is_empty(): + assert _EXEMPLARS_ROOT.is_dir(), _EXEMPLARS_ROOT + init = _EXEMPLARS_ROOT / "__init__.py" + assert init.is_file() + assert init.read_text(encoding="utf-8") == "" + + +def test_contract_exists(): + assert (_EXEMPLARS_ROOT / "contract.md").is_file() + + +@pytest.mark.parametrize(("stem", "_category", "_rank"), _ROUND_1) +def test_corpus_file_exists(stem: str, _category: ShapeCategory, _rank: int): + path = _EXEMPLARS_ROOT / f"{stem}.jsonl" + assert path.is_file(), path + + +# --------------------------------------------------------------------------- +# Schema validation +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize(("stem", "category", "rank"), _ROUND_1) +def test_records_schema(stem: str, category: ShapeCategory, rank: int): + path = _EXEMPLARS_ROOT / f"{stem}.jsonl" + records = _load_jsonl(path) + assert 1 <= len(records) <= 20, f"{path} has {len(records)} records" + + seen_ids: set[str] = set() + for idx, record in enumerate(records, start=1): + missing = _REQUIRED_TOP_KEYS - set(record) + assert not missing, f"{path}:{idx} missing top-level keys: {missing}" + + eid = record["exemplar_id"] + assert isinstance(eid, str) and eid, f"{path}:{idx} bad exemplar_id" + assert eid not in seen_ids, f"{path}:{idx} duplicate exemplar_id {eid}" + seen_ids.add(eid) + + # exemplar_id format: "-v1-". The prefix is per-file. + parts = eid.rsplit("-", 2) + assert len(parts) == 3 and parts[1] == "v1", ( + f"{path}:{idx} exemplar_id {eid!r} must match -v1-" + ) + assert parts[2].isdigit() and len(parts[2]) == 4, ( + f"{path}:{idx} exemplar_id suffix {parts[2]!r} must be 4 digits" + ) + + # shape_category binds to the file's category. + assert record["shape_category"] == category.value, ( + f"{path}:{idx} shape_category mismatch: " + f"{record['shape_category']!r} != {category.value!r}" + ) + + # Enum binding: every shape_category value is a valid ShapeCategory. + assert any( + record["shape_category"] == m.value for m in ShapeCategory + ), f"{path}:{idx} shape_category not in ShapeCategory" + + # Statement: non-empty string. + assert isinstance(record["statement"], str) and record["statement"].strip(), ( + f"{path}:{idx} statement empty" + ) + + # expected_graph keys. + graph = record["expected_graph"] + missing_g = _REQUIRED_GRAPH_KEYS - set(graph) + assert not missing_g, f"{path}:{idx} expected_graph missing: {missing_g}" + + # provenance keys. + prov = record["provenance"] + missing_p = _REQUIRED_PROVENANCE_KEYS - set(prov) + assert not missing_p, f"{path}:{idx} provenance missing: {missing_p}" + + assert prov["source"] == "phase_b_seed", f"{path}:{idx} provenance.source" + assert prov["round"] == 1, f"{path}:{idx} provenance.round" + assert prov["category_rank"] == rank, ( + f"{path}:{idx} provenance.category_rank {prov['category_rank']} " + f"!= {rank}" + ) + assert isinstance(prov["author"], str) and prov["author"], ( + f"{path}:{idx} provenance.author" + ) + + # Per-category dispatch for quantity_anchors + graph_intent + outcome. + _validate_per_category(path, idx, category, graph) + + +def _validate_per_category( + path: Path, + idx: int, + category: ShapeCategory, + graph: dict[str, Any], +) -> None: + anchors = graph["quantity_anchors"] + assert isinstance(anchors, list), f"{path}:{idx} quantity_anchors must be list" + + if category is ShapeCategory.DESCRIPTIVE_SETUP_NO_QUANTITY: + assert anchors == [], ( + f"{path}:{idx} descriptive_setup_no_quantity requires empty anchors" + ) + assert graph["graph_intent"] == "setup", f"{path}:{idx} graph_intent" + assert graph["outcome"] == "inadmissible_by_design", ( + f"{path}:{idx} outcome" + ) + return + + if category is ShapeCategory.TEMPORAL_AGGREGATION: + assert len(anchors) >= 1, f"{path}:{idx} temporal_aggregation needs anchors" + for a in anchors: + _check_keys(path, idx, a, { + "kind", "count_token", "window_unit", + "window_quantifier", "subject_role", + }) + assert a["kind"] == "event_count_per_window", ( + f"{path}:{idx} anchor kind" + ) + assert a["window_unit"] in _VALID_WINDOW_UNITS, ( + f"{path}:{idx} window_unit {a['window_unit']!r}" + ) + assert a["window_quantifier"] in _VALID_WINDOW_QUANTIFIERS, ( + f"{path}:{idx} window_quantifier {a['window_quantifier']!r}" + ) + assert isinstance(a["count_token"], str) and a["count_token"], ( + f"{path}:{idx} count_token" + ) + assert isinstance(a["subject_role"], str) and a["subject_role"], ( + f"{path}:{idx} subject_role" + ) + assert graph["graph_intent"] == "aggregate", f"{path}:{idx} graph_intent" + assert graph["outcome"] == "admissible", f"{path}:{idx} outcome" + return + + if category is ShapeCategory.RATE_WITH_CURRENCY: + assert len(anchors) >= 1, f"{path}:{idx} rate_with_currency needs anchors" + for a in anchors: + _check_keys(path, idx, a, { + "kind", "currency_symbol", "amount", "amount_kind", + "per_unit", "subject_role", + }) + assert a["kind"] == "currency_per_unit_rate", ( + f"{path}:{idx} anchor kind" + ) + assert a["currency_symbol"] in _VALID_CURRENCY_SYMBOLS, ( + f"{path}:{idx} currency_symbol {a['currency_symbol']!r}" + ) + assert a["amount_kind"] in _VALID_AMOUNT_KINDS, ( + f"{path}:{idx} amount_kind {a['amount_kind']!r}" + ) + assert isinstance(a["amount"], str) and a["amount"], ( + f"{path}:{idx} amount" + ) + assert isinstance(a["per_unit"], str) and a["per_unit"], ( + f"{path}:{idx} per_unit" + ) + assert isinstance(a["subject_role"], str) and a["subject_role"], ( + f"{path}:{idx} subject_role" + ) + assert graph["graph_intent"] == "rate", f"{path}:{idx} graph_intent" + assert graph["outcome"] == "admissible", f"{path}:{idx} outcome" + return + + raise AssertionError(f"unhandled category in dispatch: {category!r}") + + +def _check_keys( + path: Path, idx: int, mapping: dict[str, Any], required: set[str] +) -> None: + missing = required - set(mapping) + assert not missing, f"{path}:{idx} anchor missing keys: {missing}" + + +# --------------------------------------------------------------------------- +# Cross-file invariants +# --------------------------------------------------------------------------- + + +def test_no_statement_appears_in_more_than_one_file(): + seen: dict[str, str] = {} + for stem, _cat, _rank in _ROUND_1: + records = _load_jsonl(_EXEMPLARS_ROOT / f"{stem}.jsonl") + for rec in records: + s = rec["statement"] + assert s not in seen, ( + f"statement appears in {seen[s]} and {stem}: {s!r}" + ) + seen[s] = stem + + +def test_no_duplicate_statement_within_file(): + for stem, _cat, _rank in _ROUND_1: + path = _EXEMPLARS_ROOT / f"{stem}.jsonl" + records = _load_jsonl(path) + statements = [r["statement"] for r in records] + assert len(statements) == len(set(statements)), ( + f"{path} contains duplicate statements" + ) + + +@pytest.mark.parametrize(("stem", "_category", "_rank"), _ROUND_1) +def test_train_sample_binding_minimum( + stem: str, _category: ShapeCategory, _rank: int +): + path = _EXEMPLARS_ROOT / f"{stem}.jsonl" + records = _load_jsonl(path) + valid_case_ids = _train_sample_case_ids() + cited: set[str] = set() + for rec in records: + case_id = rec["provenance"].get("train_case_id") + if case_id is None: + continue + assert case_id in valid_case_ids, ( + f"{path} cites unknown train case_id: {case_id!r}" + ) + cited.add(case_id) + assert len(cited) >= 3, ( + f"{path} cites only {len(cited)} train case_ids; need >= 3" + ) + + +# --------------------------------------------------------------------------- +# Determinism +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize(("stem", "_category", "_rank"), _ROUND_1) +def test_records_sorted_by_exemplar_id( + stem: str, _category: ShapeCategory, _rank: int +): + path = _EXEMPLARS_ROOT / f"{stem}.jsonl" + records = _load_jsonl(path) + ids = [r["exemplar_id"] for r in records] + assert ids == sorted(ids), f"{path} not sorted by exemplar_id" + + +@pytest.mark.parametrize(("stem", "_category", "_rank"), _ROUND_1) +def test_file_canonical_byte_form( + stem: str, _category: ShapeCategory, _rank: int +): + """Each file ends with a single newline and no trailing whitespace per line. + + Re-walks the file bytes since ``_load_jsonl`` would have already raised on + these conditions; this exists as an explicit, named assertion the brief + asks for. + """ + + path = _EXEMPLARS_ROOT / f"{stem}.jsonl" + raw = path.read_text(encoding="utf-8") + assert raw, f"{path} empty" + assert raw.endswith("\n"), f"{path} missing trailing newline" + assert not raw.endswith("\n\n"), f"{path} extra trailing newline" + for idx, line in enumerate(raw.splitlines(), start=1): + assert line == line.rstrip(), f"{path}:{idx} trailing whitespace" + + +# --------------------------------------------------------------------------- +# Read-only invariant — importing this module must not mutate runtime trees. +# --------------------------------------------------------------------------- + + +def test_runtime_trees_not_mutated_by_import(): + """A weak but useful check: importing the exemplar package adds no files. + + The exemplar package is a marker only; importing it must not write to + ``generate/``, ``teaching/proposals/``, or any ``evals/`` artifact. We + snapshot the relevant directory listings before and after import. + """ + + snapshots: dict[Path, list[str]] = {} + sensitive = ( + _REPO_ROOT / "generate", + _REPO_ROOT / "teaching" / "proposals", + _REPO_ROOT / "evals" / "refusal_taxonomy" / "v1", + ) + def _snapshot(root: Path) -> list[str]: + return sorted(p.name for p in root.iterdir() if p.name != "__pycache__") + + for root in sensitive: + if root.is_dir(): + snapshots[root] = _snapshot(root) + + import importlib + + importlib.import_module("teaching.admissibility_exemplars") + + for root, before in snapshots.items(): + after = _snapshot(root) + assert after == before, ( + f"importing teaching.admissibility_exemplars mutated {root}: " + f"before={before} after={after}" + )