diff --git a/docs/admissibility-exemplars.md b/docs/admissibility-exemplars.md index 74696bf2..aca83159 100644 --- a/docs/admissibility-exemplars.md +++ b/docs/admissibility-exemplars.md @@ -20,19 +20,16 @@ preservation over normalization, distinguishing over similar. ## Round 1 — categories and counts -The Phase A histogram (`evals/refusal_taxonomy/v1/report.json`) selected -these three categories: +The original Phase A histogram (pre-round-2-extension) selected these three +categories: -| Rank | Category | Phase A count | Exemplars (this round) | +| Rank | Category | Phase A count | Exemplars | |---|---|---|---| | 1 | `descriptive_setup_no_quantity` | 17 | 20 | | 2 | `temporal_aggregation` | 4 | 20 | | 3 | `rate_with_currency` | 3 | 20 | -Total: **60 hand-authored exemplars** across three files in -`teaching/admissibility_exemplars/`. - -Per-category breakdown of sourcing: +Round 1 sourcing breakdown: | Category | Train-sample citations | Novel (operator-authored) | Edge cases | |---|---|---|---| @@ -40,13 +37,54 @@ Per-category breakdown of sourcing: | `temporal_aggregation` | 4 | 13 | 3 | | `rate_with_currency` | 3 | 14 | 3 | -(Edge cases overlap with novel; counts above split them out.) +## Round 2 — categories and counts + +Round 2 was driven by categorizing the post-#304 GSM8K `train_sample` +still-refused 47 set: 23 had been UNCATEGORIZED under the round-1 +categorizer; the categorization sweep surfaced three coherent +sub-shapes, plus five ratified-but-unmatched temporal cases that called +for a v2 widening. + +| Rank | Category | Phase A round-2 count (public 50) | Exemplars | +|---|---|---|---| +| — | `discrete_count_statement` (new) | 10 | 20 | +| — | `multiplicative_aggregation` (new) | 2 | 20 | +| — | `currency_amount` (new) | 1 | 20 | +| — | `temporal_aggregation` v2 (widening) | — | 10 | + +The "Phase A round-2 count" column is the number of the public 50-case +sample now categorized into the new category by the extended Phase A +categorizer; it is the empirical signal the categorization actually +worked. Pre-round-2 the public sample carried 14 UNCATEGORIZED cases; +post-round-2 only 1 remains (case 0044, "10% simple interest" with no +change verb — an honest residual outside the three sub-shapes). + +Round 2 sourcing breakdown: + +| Category | Train-sample citations | Novel (operator-authored) | Edge cases | +|---|---|---|---| +| `discrete_count_statement` | 6 | 11 | 3 | +| `multiplicative_aggregation` | 3 | 13 | 4 | +| `currency_amount` | 3 | 14 | 3 | +| `temporal_aggregation` v2 | 4 | 5 | 1 | ## Files +Round 1: + - [`teaching/admissibility_exemplars/descriptive_setup_no_quantity_v1.jsonl`](../teaching/admissibility_exemplars/descriptive_setup_no_quantity_v1.jsonl) - [`teaching/admissibility_exemplars/temporal_aggregation_v1.jsonl`](../teaching/admissibility_exemplars/temporal_aggregation_v1.jsonl) - [`teaching/admissibility_exemplars/rate_with_currency_v1.jsonl`](../teaching/admissibility_exemplars/rate_with_currency_v1.jsonl) + +Round 2: + +- [`teaching/admissibility_exemplars/discrete_count_statement_v1.jsonl`](../teaching/admissibility_exemplars/discrete_count_statement_v1.jsonl) +- [`teaching/admissibility_exemplars/multiplicative_aggregation_v1.jsonl`](../teaching/admissibility_exemplars/multiplicative_aggregation_v1.jsonl) +- [`teaching/admissibility_exemplars/currency_amount_v1.jsonl`](../teaching/admissibility_exemplars/currency_amount_v1.jsonl) +- [`teaching/admissibility_exemplars/temporal_aggregation_v2.jsonl`](../teaching/admissibility_exemplars/temporal_aggregation_v2.jsonl) + +Contract: + - [`teaching/admissibility_exemplars/contract.md`](../teaching/admissibility_exemplars/contract.md) ## Cross-references diff --git a/evals/refusal_taxonomy/shape_categories.py b/evals/refusal_taxonomy/shape_categories.py index ab872ca0..bc2311f3 100644 --- a/evals/refusal_taxonomy/shape_categories.py +++ b/evals/refusal_taxonomy/shape_categories.py @@ -18,6 +18,15 @@ Doctrine (non-negotiable): Adding a new category requires ≥ 3 refused statements as evidence, cited inline. ADR-0163 §Risks enforces this. + +Phase B round 2 extension +~~~~~~~~~~~~~~~~~~~~~~~~~ +Three new categories surfaced from categorizing the post-#304 GSM8K +train_sample's still-refused 47 set: ``CURRENCY_AMOUNT``, +``MULTIPLICATIVE_AGGREGATION``, and ``DISCRETE_COUNT_STATEMENT``. Each is +inserted into the dispatch order *after* the existing more-specific +categories so that statements matching both (e.g. currency + per-unit +rate framing) still resolve to the more-specific existing category. """ from __future__ import annotations @@ -29,9 +38,10 @@ from enum import Enum class ShapeCategory(str, Enum): """Disjoint shape categories surfaced from the GSM8K train-sample report. - The nine non-``UNCATEGORIZED`` values are the baseline set named in - ADR-0163 §Phase A. Every value listed here has a rule below; if no - rule fires, ``UNCATEGORIZED`` is returned. + The non-``UNCATEGORIZED`` values are the baseline set named in + ADR-0163 §Phase A plus the round-2 extensions; every value listed + here has a rule below, and if no rule fires ``UNCATEGORIZED`` is + returned. """ RATE_WITH_CURRENCY = "rate_with_currency" @@ -43,6 +53,9 @@ class ShapeCategory(str, Enum): NESTED_QUESTION_TARGET = "nested_question_target" TEMPORAL_AGGREGATION = "temporal_aggregation" CONDITIONAL_QUANTITY = "conditional_quantity" + CURRENCY_AMOUNT = "currency_amount" + MULTIPLICATIVE_AGGREGATION = "multiplicative_aggregation" + DISCRETE_COUNT_STATEMENT = "discrete_count_statement" UNCATEGORIZED = "uncategorized" @@ -66,6 +79,15 @@ _NUMBER_WORDS: frozenset[str] = frozenset({ "dozen", "dozens", }) +# Subset of NUMBER_WORDS that are *integer* count words — used by the +# discrete-count and multiplicative rules so a bare fraction word +# ("half", "third") does not misroute to "discrete count of half things". +_COUNT_NUMBER_WORDS: frozenset[str] = _NUMBER_WORDS - { + "half", "halves", "third", "thirds", "quarter", "quarters", + "fourth", "fourths", "fifth", "fifths", "sixth", "sixths", +} +_COUNT_NUMBER_WORDS_RE_SRC = "|".join(sorted(_COUNT_NUMBER_WORDS)) + # Indefinite quantifiers — "amount unspecified" signals. _INDEFINITE_TOKENS: tuple[str, ...] = ( " some ", " several ", " a few ", " many ", " any ", @@ -149,6 +171,52 @@ _CONDITIONAL_RE = re.compile( re.IGNORECASE | re.DOTALL, ) +# --- Round-2 extension primitives ------------------------------------------ + +# Container nouns whose presence with a count + ("of"|"with"|...) signals +# multiplicative aggregation ("N of " or +# "N with N "). Kept tight so a bare "house" or +# "room" doesn't promote unrelated counts. +_CONTAINER_NOUNS: tuple[str, ...] = ( + "bag", "bags", "basket", "baskets", "box", "boxes", + "case", "cases", "crate", "crates", "carton", "cartons", + "package", "packages", "bundle", "bundles", "set", "sets", + "pack", "packs", "tray", "trays", "jar", "jars", + "bottle", "bottles", "bin", "bins", "shelf", "shelves", + "pallet", "pallets", "container", "containers", +) +_CONTAINER_NOUNS_RE_SRC = "|".join(_CONTAINER_NOUNS) + +# " (with|of|holding|containing|having)" +# Allow up to two adjective words between the count and the container noun +# ("five full boxes of crayons" → "five full boxes of"). +_MULTIPLICATIVE_CONTAINER_RE = re.compile( + r"\b(?:\d+(?:,\d{3})*|" + _COUNT_NUMBER_WORDS_RE_SRC + r")\s+" + r"(?:\w+\s+){0,2}" + r"(?:" + _CONTAINER_NOUNS_RE_SRC + r")\s+" + r"(?:of|with|holding|containing|having)\b", + re.IGNORECASE, +) +# "each ... " within a ~4-word window. Catches +# "each weighing 5 ounces", "each basket holds 50", "each contain six". +_MULTIPLICATIVE_EACH_RE = re.compile( + r"\beach\s+(?:\w+\s+){0,4}(?:\d+|" + _COUNT_NUMBER_WORDS_RE_SRC + r")\b", + re.IGNORECASE, +) + +# Discrete-count noun pair: a digit (optionally comma-grouped) immediately +# followed by a counted noun, OR an integer number-word immediately +# followed by a counted noun. The "immediately" matters: "10% simple" +# is *not* a digit+noun pair because "%" intervenes, so it stays +# UNCATEGORIZED (an honest residual). +_DISCRETE_COUNT_DIGIT_NOUN_RE = re.compile( + r"\b\d+(?:,\d{3})*\s+[A-Za-z][A-Za-z\-]*\b" +) +_DISCRETE_COUNT_WORD_NOUN_RE = re.compile( + r"\b(?:" + _COUNT_NUMBER_WORDS_RE_SRC + r")\s+[A-Za-z][A-Za-z\-]*\b", + re.IGNORECASE, +) + # --------------------------------------------------------------------------- # Helpers — pure string predicates. @@ -257,6 +325,33 @@ def _is_rate_with_currency(padded_lower: str) -> bool: return _has_any_substring(padded_lower, _RATE_PER_UNIT_TOKENS) +def _is_currency_amount(padded_lower: str) -> bool: + """Currency symbol without a per-unit framing — a bare amount. + + Dispatched AFTER ``_is_rate_with_currency`` so any "$N per X" + statement is already claimed; what reaches this predicate is a + currency mention without rate framing. + + Examples (≥ 3 cites, drawn from the post-#304 still-refused 47 + GSM8K train_sample set): + - case 0019 "The dog ends up having health problems and this + requires 3 vet appointments, which cost $400 each." + - case 0026 "Aaron and his brother Carson each saved up $40 + to go to dinner." + - case 0028 "It cost $100,000 to open initially." + + Alternative considered: only fire when a currency-bearing token is + paired with a verb of payment ("cost", "paid", "owes"). + Rejected — the bare currency presence is the load-bearing signal, + and the rate-with-currency rule already claims any per-unit form + above this dispatch slot, so widening the verb whitelist would + only add false negatives (e.g., "the donation jar collected + $87.50 at the festival" should still admit). + """ + + return bool(_CURRENCY_RE.search(padded_lower)) + + def _is_comparative_with_unit(padded_lower: str) -> bool: """Comparative quantity phrases. @@ -333,6 +428,66 @@ def _is_indefinite_quantity(padded_lower: str) -> bool: return _has_any_substring(padded_lower, _INDEFINITE_TOKENS) +def _is_multiplicative_aggregation(padded_lower: str) -> bool: + """N outer-units each carrying M inner-units — spatial / per-container. + + Two surface forms compose: (a) ``N of/with/holding/... + `` and (b) ``each `` within a short window. + Either form is sufficient. + + Examples (≥ 3 cites, drawn from the post-#304 still-refused 47): + - case 0025 "Lilibeth fills 6 baskets where each basket holds 50 + strawberries." + - case 0045 "Each survey has 10 questions." + - case 0047 "John bakes 12 coconut macaroons, each weighing 5 ounces." + + Discriminator vs ``TEMPORAL_AGGREGATION``: multiplicative is + spatial/per-container ("baskets ... strawberries"); temporal is + per-time-window ("per day", "every week"). Dispatch order places + temporal first so an "each day" framing wins where both apply. + Discriminator vs ``DISCRETE_COUNT_STATEMENT``: multiplicative carries + an inner aggregate dimension (the count-of-counted-things), while + discrete-count carries one count-noun pair; the inner aggregate is + the more-specific reading. + """ + + if _MULTIPLICATIVE_EACH_RE.search(padded_lower): + return True + if _MULTIPLICATIVE_CONTAINER_RE.search(padded_lower): + return True + return False + + +def _is_discrete_count_statement(padded_lower: str) -> bool: + """A subject paired with one or more bare count-noun anchors. + + The canonical surface is `` ``; + "Nicole collected 400 Pokemon cards" and "A school has 100 students" + are central instances. Lists of count-noun pairs ("2 horses, 5 dogs") + are admitted by the same predicate because each pair matches. + + Examples (≥ 3 cites, drawn from the post-#304 still-refused 47): + - case 0023 "Nicole collected 400 Pokemon cards." + - case 0027 "Malcolm has 240 followers on Instagram and 500 + followers on Facebook." + - case 0046 "A school has 100 students." + + Discriminator vs ``RATE_WITH_CURRENCY`` and ``CURRENCY_AMOUNT``: + discrete-count statements carry no currency symbol; the currency + rules above this dispatch slot already claim those. Discriminator + vs ``MULTIPLICATIVE_AGGREGATION``: discrete-count has no + inner-aggregate dimension; multiplicative above this slot already + claims "each ... N" and "N of N" forms. Anything left + is a bare count-noun pair. + """ + + if _DISCRETE_COUNT_DIGIT_NOUN_RE.search(padded_lower): + return True + if _DISCRETE_COUNT_WORD_NOUN_RE.search(padded_lower): + return True + return False + + def _is_descriptive_setup_no_quantity(statement: str, padded_lower: str) -> bool: """Pure context with no extractable measurement. @@ -368,6 +523,8 @@ def categorize(statement: str) -> ShapeCategory: return ShapeCategory.UNIT_PARTITION if _is_rate_with_currency(padded_lower): return ShapeCategory.RATE_WITH_CURRENCY + if _is_currency_amount(padded_lower): + return ShapeCategory.CURRENCY_AMOUNT if _is_comparative_with_unit(padded_lower): return ShapeCategory.COMPARATIVE_WITH_UNIT if _is_fractional_rate_of_change(padded_lower): @@ -378,6 +535,10 @@ def categorize(statement: str) -> ShapeCategory: return ShapeCategory.TEMPORAL_AGGREGATION if _is_conditional_quantity(statement): return ShapeCategory.CONDITIONAL_QUANTITY + if _is_multiplicative_aggregation(padded_lower): + return ShapeCategory.MULTIPLICATIVE_AGGREGATION + if _is_discrete_count_statement(padded_lower): + return ShapeCategory.DISCRETE_COUNT_STATEMENT if _is_descriptive_setup_no_quantity(statement, padded_lower): return ShapeCategory.DESCRIPTIVE_SETUP_NO_QUANTITY return ShapeCategory.UNCATEGORIZED @@ -389,11 +550,14 @@ SHAPE_CATEGORY_ORDER: tuple[ShapeCategory, ...] = ( ShapeCategory.NESTED_QUESTION_TARGET, ShapeCategory.UNIT_PARTITION, ShapeCategory.RATE_WITH_CURRENCY, + ShapeCategory.CURRENCY_AMOUNT, ShapeCategory.COMPARATIVE_WITH_UNIT, ShapeCategory.FRACTIONAL_RATE_OF_CHANGE, ShapeCategory.INDEFINITE_QUANTITY, ShapeCategory.TEMPORAL_AGGREGATION, ShapeCategory.CONDITIONAL_QUANTITY, + ShapeCategory.MULTIPLICATIVE_AGGREGATION, + ShapeCategory.DISCRETE_COUNT_STATEMENT, ShapeCategory.DESCRIPTIVE_SETUP_NO_QUANTITY, ShapeCategory.UNCATEGORIZED, ) diff --git a/evals/refusal_taxonomy/v1/report.json b/evals/refusal_taxonomy/v1/report.json index ff13b991..34ab63c1 100644 --- a/evals/refusal_taxonomy/v1/report.json +++ b/evals/refusal_taxonomy/v1/report.json @@ -33,13 +33,13 @@ { "case_id": "gsm8k-train-sample-v1-0006", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Mandy started reading books with only 8 pages when she was 6 years old.'", - "shape_category": "uncategorized", + "shape_category": "discrete_count_statement", "statement": "Mandy started reading books with only 8 pages when she was 6 years old." }, { "case_id": "gsm8k-train-sample-v1-0007", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Francine has five full boxes of crayons and 5 loose crayons, and her friend has 27 loose crayons.'", - "shape_category": "uncategorized", + "shape_category": "multiplicative_aggregation", "statement": "Francine has five full boxes of crayons and 5 loose crayons, and her friend has 27 loose crayons." }, { @@ -57,7 +57,7 @@ { "case_id": "gsm8k-train-sample-v1-0010", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Yun had 20 paperclips initially, but then lost 12.'", - "shape_category": "uncategorized", + "shape_category": "discrete_count_statement", "statement": "Yun had 20 paperclips initially, but then lost 12." }, { @@ -117,7 +117,7 @@ { "case_id": "gsm8k-train-sample-v1-0020", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Two puppies, two kittens, and three parakeets were for sale at the pet shop.'", - "shape_category": "uncategorized", + "shape_category": "discrete_count_statement", "statement": "Two puppies, two kittens, and three parakeets were for sale at the pet shop." }, { @@ -135,7 +135,7 @@ { "case_id": "gsm8k-train-sample-v1-0023", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Nicole collected 400 Pokemon cards.'", - "shape_category": "uncategorized", + "shape_category": "discrete_count_statement", "statement": "Nicole collected 400 Pokemon cards." }, { @@ -153,13 +153,13 @@ { "case_id": "gsm8k-train-sample-v1-0026", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Aaron and his brother Carson each saved up $40 to go to dinner.'", - "shape_category": "uncategorized", + "shape_category": "currency_amount", "statement": "Aaron and his brother Carson each saved up $40 to go to dinner." }, { "case_id": "gsm8k-train-sample-v1-0027", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Malcolm has 240 followers on Instagram and 500 followers on Facebook.'", - "shape_category": "uncategorized", + "shape_category": "discrete_count_statement", "statement": "Malcolm has 240 followers on Instagram and 500 followers on Facebook." }, { @@ -183,7 +183,7 @@ { "case_id": "gsm8k-train-sample-v1-0031", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Jeremie wants to go to an amusement park with 3 friends at the end of summer.'", - "shape_category": "uncategorized", + "shape_category": "discrete_count_statement", "statement": "Jeremie wants to go to an amusement park with 3 friends at the end of summer." }, { @@ -219,13 +219,13 @@ { "case_id": "gsm8k-train-sample-v1-0037", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Michael wants to lose 10 pounds by June.'", - "shape_category": "uncategorized", + "shape_category": "discrete_count_statement", "statement": "Michael wants to lose 10 pounds by June." }, { "case_id": "gsm8k-train-sample-v1-0038", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'In a building, there are a hundred ladies on the first-floor studying.'", - "shape_category": "uncategorized", + "shape_category": "discrete_count_statement", "statement": "In a building, there are a hundred ladies on the first-floor studying." }, { @@ -273,13 +273,13 @@ { "case_id": "gsm8k-train-sample-v1-0046", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'A school has 100 students.'", - "shape_category": "uncategorized", + "shape_category": "discrete_count_statement", "statement": "A school has 100 students." }, { "case_id": "gsm8k-train-sample-v1-0047", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'John bakes 12 coconut macaroons, each weighing 5 ounces.'", - "shape_category": "uncategorized", + "shape_category": "multiplicative_aggregation", "statement": "John bakes 12 coconut macaroons, each weighing 5 ounces." }, { @@ -291,7 +291,7 @@ { "case_id": "gsm8k-train-sample-v1-0049", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Malcolm is trying to find the fastest walk to school and is currently comparing two routes.'", - "shape_category": "uncategorized", + "shape_category": "discrete_count_statement", "statement": "Malcolm is trying to find the fastest walk to school and is currently comparing two routes." }, { @@ -306,21 +306,24 @@ "by_category": { "comparative_with_unit": 3, "conditional_quantity": 0, + "currency_amount": 1, "descriptive_setup_no_quantity": 17, + "discrete_count_statement": 10, "fractional_rate_of_change": 3, "indefinite_quantity": 3, + "multiplicative_aggregation": 2, "nested_question_target": 2, "rate_with_currency": 3, "temporal_aggregation": 4, - "uncategorized": 14, + "uncategorized": 1, "unit_partition": 1 }, - "case_digest": "d030f826cb0f4088771d90c52c8be2ff75054ab27c7d47eae8dbfe1225b2eea1", - "categorized_rate": 0.72, + "case_digest": "7be4897842ff6530a84506c095c44db91f15d13a13c622d17b8f52daa5d16970", + "categorized_rate": 0.98, "total": 50, - "uncategorized": 14 + "uncategorized": 1 }, - "source_cases": "/Users/kaizenpro/Projects/core-adr-0163-A-taxonomy/evals/refusal_taxonomy/public/v1/cases.jsonl", + "source_cases": "evals/refusal_taxonomy/public/v1/cases.jsonl", "split": "public", "version": "v1" } diff --git a/teaching/admissibility_exemplars/contract.md b/teaching/admissibility_exemplars/contract.md index 746dd2a6..688a4291 100644 --- a/teaching/admissibility_exemplars/contract.md +++ b/teaching/admissibility_exemplars/contract.md @@ -122,6 +122,101 @@ day-of-week enumeration). "outcome": "admissible" ``` +### `discrete_count_statement` (Round 2) + +```json +"quantity_anchors": [ + { + "kind": "discrete_count", + "subject_role": "", + "count_token": "", + "count_kind": "", + "counted_noun": "" + }, + ... +] +"graph_intent": "count" +"outcome": "admissible" +``` + +Multiple anchors when a statement enumerates several count-noun pairs. + +**Discriminator vs the currency rules**: discrete-count statements carry +no currency symbol — Phase A's dispatch resolves the rare overlap with +`rate_with_currency` and `currency_amount` by placing the currency rules +first. Near-miss example *not* in this corpus: `"He earns $5 per +apple"` — currency-bearing, with per-unit framing, so it belongs in +`rate_with_currency`, not here. + +### `multiplicative_aggregation` (Round 2) + +```json +"quantity_anchors": [ + { + "kind": "multiplicative_aggregate", + "outer_count": "", + "outer_unit": "", + "inner_count": "", + "inner_unit": "", + "subject_role": "" + }, + ... +] +"graph_intent": "aggregate" +"outcome": "admissible" +``` + +Multiple anchors per statement when a joined aggregation enumerates +several container-of pairs (e.g., "4 bags with 20 apples and 6 bags +with 25 apples"). + +**Discriminator vs `temporal_aggregation`**: multiplicative is spatial +or per-container ("baskets ... strawberries"); temporal is +per-time-window ("per day", "every week"). Where both could apply the +temporal framing wins via dispatch order. Near-miss example *not* in +this corpus: `"10 oysters per 5 minutes"` — per-time, so it belongs +to `temporal_aggregation`. + +### `currency_amount` (Round 2) + +```json +"quantity_anchors": [ + { + "kind": "currency_amount", + "currency_symbol": "<$|£|€|¥>", + "amount": "", + "amount_kind": "", + "subject_role": "" + }, + ... +] +"graph_intent": "amount" +"outcome": "admissible" +``` + +**The load-bearing discriminator**: `rate_with_currency` carries a +per-unit framing ("per X", "for one X", "/X", "an hour"); this +category does NOT. Phase A dispatch resolves this by running +`rate_with_currency` first. Near-miss example *not* in this corpus: +`"Tina makes $18.00 an hour"` — currency + per-time, so it belongs in +`rate_with_currency`. + +### `temporal_aggregation` v2 — widening + +The v2 corpus uses the SAME schema as v1 (`event_count_per_window`); no +schema extension lands in this round. v2 widens the surface forms +seeded for the Phase C recognizer: v1 covered `{each, every, per}` +window quantifiers and trailing-clause time framings, v2 adds the +`for`-window and `within`-window quantifier variants plus the +leading-clause `Every ,` position. + +The v2 corpus becomes a SEPARATE Phase C proposal (its own +`recognizer_spec`, distinct `exemplar_digest`, distinct +`proposal_id`). The operator decides whether to ratify v2 alongside +v1 (both specs admit via first-match-wins over the registry) OR to +ratify v2 + withdraw v1 (clean replacement). This is a meta-decision +deferred to the Phase C/D review path. + ## Sourcing rules For each category, the corpus MUST satisfy: diff --git a/teaching/admissibility_exemplars/currency_amount_v1.jsonl b/teaching/admissibility_exemplars/currency_amount_v1.jsonl new file mode 100644 index 00000000..b55d21a2 --- /dev/null +++ b/teaching/admissibility_exemplars/currency_amount_v1.jsonl @@ -0,0 +1,20 @@ +{"exemplar_id": "ca-v1-0001", "shape_category": "currency_amount", "statement": "The dog ends up having health problems and this requires 3 vet appointments, which cost $400 each.", "expected_graph": {"subject": "the dog", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "400", "amount_kind": "integer", "subject_role": "vet appointments"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0019", "author_note": "Source statement carries a double-space between 'appointments,' and 'which'; preserved verbatim. '$400 each' is a bare amount (per-item, not per-time), so it routes to CURRENCY_AMOUNT rather than RATE_WITH_CURRENCY."}} +{"exemplar_id": "ca-v1-0002", "shape_category": "currency_amount", "statement": "Aaron and his brother Carson each saved up $40 to go to dinner.", "expected_graph": {"subject": "Aaron and Carson", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "40", "amount_kind": "integer", "subject_role": "savings"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0026", "author_note": "'each saved up $40' could imply two distinct $40 savings; the exemplar records a single amount anchor because the surface form does not commit to per-person duplication."}} +{"exemplar_id": "ca-v1-0003", "shape_category": "currency_amount", "statement": "It cost $100,000 to open initially.", "expected_graph": {"subject": "it", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "100,000", "amount_kind": "integer", "subject_role": "opening cost"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0028", "author_note": "Subject is the pronominal antecedent of an earlier sentence; preserved as 'it' so Phase C can resolve via prior context."}} +{"exemplar_id": "ca-v1-0004", "shape_category": "currency_amount", "statement": "Yuki paid $35 at the concert ticket booth.", "expected_graph": {"subject": "Yuki", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "35", "amount_kind": "integer", "subject_role": "ticket"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ca-v1-0005", "shape_category": "currency_amount", "statement": "The bakery donated $200 to the school fundraiser.", "expected_graph": {"subject": "the bakery", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "200", "amount_kind": "integer", "subject_role": "donation"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ca-v1-0006", "shape_category": "currency_amount", "statement": "Marcus inherited $5,500 from his grandfather.", "expected_graph": {"subject": "Marcus", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "5,500", "amount_kind": "integer", "subject_role": "inheritance"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ca-v1-0007", "shape_category": "currency_amount", "statement": "Liam owes $89 on his library fines.", "expected_graph": {"subject": "Liam", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "89", "amount_kind": "integer", "subject_role": "library fines"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ca-v1-0008", "shape_category": "currency_amount", "statement": "The vintage lamp was priced at $1,250 at the auction.", "expected_graph": {"subject": "the lamp", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "1,250", "amount_kind": "integer", "subject_role": "lamp"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ca-v1-0009", "shape_category": "currency_amount", "statement": "Sofia found $20 tucked inside the book.", "expected_graph": {"subject": "Sofia", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "20", "amount_kind": "integer", "subject_role": "found money"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ca-v1-0010", "shape_category": "currency_amount", "statement": "The cafe charged £15 at the brunch counter.", "expected_graph": {"subject": "the cafe", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "£", "amount": "15", "amount_kind": "integer", "subject_role": "brunch"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Pound sterling currency; Phase C should derive a recognizer that admits all four symbols in the schema's enum."}} +{"exemplar_id": "ca-v1-0011", "shape_category": "currency_amount", "statement": "Tomás spent €60 at the train station.", "expected_graph": {"subject": "Tomás", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "€", "amount": "60", "amount_kind": "integer", "subject_role": "train fare"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Euro currency."}} +{"exemplar_id": "ca-v1-0012", "shape_category": "currency_amount", "statement": "Akemi exchanged ¥3,000 at the airport kiosk.", "expected_graph": {"subject": "Akemi", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "¥", "amount": "3,000", "amount_kind": "integer", "subject_role": "currency exchange"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Yen currency."}} +{"exemplar_id": "ca-v1-0013", "shape_category": "currency_amount", "statement": "Devika contributed $45 to the office gift pool.", "expected_graph": {"subject": "Devika", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "45", "amount_kind": "integer", "subject_role": "gift contribution"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ca-v1-0014", "shape_category": "currency_amount", "statement": "The film grossed $12,000,000 at the box office last summer.", "expected_graph": {"subject": "the film", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "12,000,000", "amount_kind": "integer", "subject_role": "gross revenue"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ca-v1-0015", "shape_category": "currency_amount", "statement": "Helena saved $800 toward the security deposit.", "expected_graph": {"subject": "Helena", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "800", "amount_kind": "integer", "subject_role": "deposit savings"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ca-v1-0016", "shape_category": "currency_amount", "statement": "Jonah lent $250 to his cousin last summer.", "expected_graph": {"subject": "Jonah", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "250", "amount_kind": "integer", "subject_role": "loan"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ca-v1-0017", "shape_category": "currency_amount", "statement": "The diner's lunch special costs $7.95.", "expected_graph": {"subject": "the lunch special", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "7.95", "amount_kind": "decimal", "subject_role": "lunch special"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Decimal amount."}} +{"exemplar_id": "ca-v1-0018", "shape_category": "currency_amount", "statement": "The thrift store priced the jacket at $12 even.", "expected_graph": {"subject": "the jacket", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "12", "amount_kind": "integer", "subject_role": "jacket"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: 'even' is a colloquial marker of round amount; the schema ignores it."}} +{"exemplar_id": "ca-v1-0019", "shape_category": "currency_amount", "statement": "Maeve saved up $1,000 to fund her trip overseas.", "expected_graph": {"subject": "Maeve", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "1,000", "amount_kind": "integer", "subject_role": "trip savings"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: 'to fund her trip' is a purpose modifier, not a per-unit framing; categorizer routes to CURRENCY_AMOUNT."}} +{"exemplar_id": "ca-v1-0020", "shape_category": "currency_amount", "statement": "The donation jar collected $87.50 at the spring festival.", "expected_graph": {"subject": "the donation jar", "quantity_anchors": [{"kind": "currency_amount", "currency_symbol": "$", "amount": "87.50", "amount_kind": "decimal", "subject_role": "donations"}], "graph_intent": "amount", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: decimal amount with a temporal modifier 'at the spring festival' that is not a window-unit; categorizer keeps CURRENCY_AMOUNT."}} diff --git a/teaching/admissibility_exemplars/discrete_count_statement_v1.jsonl b/teaching/admissibility_exemplars/discrete_count_statement_v1.jsonl new file mode 100644 index 00000000..0dd14c2b --- /dev/null +++ b/teaching/admissibility_exemplars/discrete_count_statement_v1.jsonl @@ -0,0 +1,20 @@ +{"exemplar_id": "dcs-v1-0001", "shape_category": "discrete_count_statement", "statement": "Yun had 20 paperclips initially, but then lost 12.", "expected_graph": {"subject": "Yun", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Yun", "count_token": "20", "count_kind": "integer", "counted_noun": "paperclips"}, {"kind": "discrete_count", "subject_role": "Yun", "count_token": "12", "count_kind": "integer", "counted_noun": "paperclips"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0010", "author_note": "Two count anchors share the counted_noun lemma ('paperclips') across the elided second mention; the schema records each occurrence rather than collapsing them, so Phase C can derive subtractive intent if it chooses."}} +{"exemplar_id": "dcs-v1-0002", "shape_category": "discrete_count_statement", "statement": "Two puppies, two kittens, and three parakeets were for sale at the pet shop.", "expected_graph": {"subject": "the pet shop", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "pet shop", "count_token": "two", "count_kind": "word", "counted_noun": "puppies"}, {"kind": "discrete_count", "subject_role": "pet shop", "count_token": "two", "count_kind": "word", "counted_noun": "kittens"}, {"kind": "discrete_count", "subject_role": "pet shop", "count_token": "three", "count_kind": "word", "counted_noun": "parakeets"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0020", "author_note": "Enumeration of three count-noun pairs; subject is implicit ('the pet shop')."}} +{"exemplar_id": "dcs-v1-0003", "shape_category": "discrete_count_statement", "statement": "Nicole collected 400 Pokemon cards.", "expected_graph": {"subject": "Nicole", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Nicole", "count_token": "400", "count_kind": "integer", "counted_noun": "Pokemon cards"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0023"}} +{"exemplar_id": "dcs-v1-0004", "shape_category": "discrete_count_statement", "statement": "Malcolm has 240 followers on Instagram and 500 followers on Facebook.", "expected_graph": {"subject": "Malcolm", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Malcolm", "count_token": "240", "count_kind": "integer", "counted_noun": "followers"}, {"kind": "discrete_count", "subject_role": "Malcolm", "count_token": "500", "count_kind": "integer", "counted_noun": "followers"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0027"}} +{"exemplar_id": "dcs-v1-0005", "shape_category": "discrete_count_statement", "statement": "Michael wants to lose 10 pounds by June.", "expected_graph": {"subject": "Michael", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Michael", "count_token": "10", "count_kind": "integer", "counted_noun": "pounds"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0037", "author_note": "'pounds' here is the counted-noun under a discrete-count reading; the temporal modifier 'by June' is not encoded because the schema only captures count + counted_noun + subject_role."}} +{"exemplar_id": "dcs-v1-0006", "shape_category": "discrete_count_statement", "statement": "He now has 2 horses, 5 dogs, 7 cats, 3 turtles, and 1 goat.", "expected_graph": {"subject": "he", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "he", "count_token": "2", "count_kind": "integer", "counted_noun": "horses"}, {"kind": "discrete_count", "subject_role": "he", "count_token": "5", "count_kind": "integer", "counted_noun": "dogs"}, {"kind": "discrete_count", "subject_role": "he", "count_token": "7", "count_kind": "integer", "counted_noun": "cats"}, {"kind": "discrete_count", "subject_role": "he", "count_token": "3", "count_kind": "integer", "counted_noun": "turtles"}, {"kind": "discrete_count", "subject_role": "he", "count_token": "1", "count_kind": "integer", "counted_noun": "goat"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0040", "author_note": "Five count-noun anchors share the same subject (pronominal 'he'); this is the canonical multi-item enumeration form."}} +{"exemplar_id": "dcs-v1-0007", "shape_category": "discrete_count_statement", "statement": "Carlos owns 7 vintage motorcycles.", "expected_graph": {"subject": "Carlos", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Carlos", "count_token": "7", "count_kind": "integer", "counted_noun": "motorcycles"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "dcs-v1-0008", "shape_category": "discrete_count_statement", "statement": "The library returned 32 overdue books.", "expected_graph": {"subject": "the library", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "library", "count_token": "32", "count_kind": "integer", "counted_noun": "books"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "dcs-v1-0009", "shape_category": "discrete_count_statement", "statement": "Priya has 18 paintbrushes in her studio.", "expected_graph": {"subject": "Priya", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Priya", "count_token": "18", "count_kind": "integer", "counted_noun": "paintbrushes"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "dcs-v1-0010", "shape_category": "discrete_count_statement", "statement": "Marisol counted 56 seashells along the shoreline.", "expected_graph": {"subject": "Marisol", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Marisol", "count_token": "56", "count_kind": "integer", "counted_noun": "seashells"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "dcs-v1-0011", "shape_category": "discrete_count_statement", "statement": "Devon adopted 3 rescue cats from the shelter.", "expected_graph": {"subject": "Devon", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Devon", "count_token": "3", "count_kind": "integer", "counted_noun": "cats"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "dcs-v1-0012", "shape_category": "discrete_count_statement", "statement": "The garden contains 24 sunflowers near the back fence.", "expected_graph": {"subject": "the garden", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "garden", "count_token": "24", "count_kind": "integer", "counted_noun": "sunflowers"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "dcs-v1-0013", "shape_category": "discrete_count_statement", "statement": "Alma keeps 12 chickens behind the barn.", "expected_graph": {"subject": "Alma", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Alma", "count_token": "12", "count_kind": "integer", "counted_noun": "chickens"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "dcs-v1-0014", "shape_category": "discrete_count_statement", "statement": "Greta owns 8 antique typewriters on display.", "expected_graph": {"subject": "Greta", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Greta", "count_token": "8", "count_kind": "integer", "counted_noun": "typewriters"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "dcs-v1-0015", "shape_category": "discrete_count_statement", "statement": "Felipe brought 15 balloons to the party.", "expected_graph": {"subject": "Felipe", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Felipe", "count_token": "15", "count_kind": "integer", "counted_noun": "balloons"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "dcs-v1-0016", "shape_category": "discrete_count_statement", "statement": "Nora collected 22 marbles from the playground.", "expected_graph": {"subject": "Nora", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Nora", "count_token": "22", "count_kind": "integer", "counted_noun": "marbles"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "dcs-v1-0017", "shape_category": "discrete_count_statement", "statement": "Tariq has 4 nephews who visit on weekends.", "expected_graph": {"subject": "Tariq", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Tariq", "count_token": "4", "count_kind": "integer", "counted_noun": "nephews"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "dcs-v1-0018", "shape_category": "discrete_count_statement", "statement": "A hundred swallows perched on the wire above the meadow.", "expected_graph": {"subject": "swallows", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "swallows", "count_token": "hundred", "count_kind": "word", "counted_noun": "swallows"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: word-form count ('a hundred'); the count_kind='word' preserves the surface representation for Phase C."}} +{"exemplar_id": "dcs-v1-0019", "shape_category": "discrete_count_statement", "statement": "Olamide bought 1,250 stickers at the wholesale market.", "expected_graph": {"subject": "Olamide", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "Olamide", "count_token": "1,250", "count_kind": "integer", "counted_noun": "stickers"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: comma-grouped large integer. The count_token preserves the surface ',' so Phase C can decide whether to strip or honor the grouping."}} +{"exemplar_id": "dcs-v1-0020", "shape_category": "discrete_count_statement", "statement": "A dozen apples sit in the basket on the counter.", "expected_graph": {"subject": "apples", "quantity_anchors": [{"kind": "discrete_count", "subject_role": "apples", "count_token": "dozen", "count_kind": "word", "counted_noun": "apples"}], "graph_intent": "count", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: 'dozen' is an integer-scale word that resolves to 12 in convention; the schema preserves it as count_token to let Phase C decide whether to expand."}} diff --git a/teaching/admissibility_exemplars/multiplicative_aggregation_v1.jsonl b/teaching/admissibility_exemplars/multiplicative_aggregation_v1.jsonl new file mode 100644 index 00000000..77810680 --- /dev/null +++ b/teaching/admissibility_exemplars/multiplicative_aggregation_v1.jsonl @@ -0,0 +1,20 @@ +{"exemplar_id": "ma-v1-0001", "shape_category": "multiplicative_aggregation", "statement": "Lilibeth fills 6 baskets where each basket holds 50 strawberries.", "expected_graph": {"subject": "Lilibeth", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "6", "outer_unit": "baskets", "inner_count": "50", "inner_unit": "strawberries", "subject_role": "Lilibeth"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0025", "author_note": "Canonical multiplicative form: outer container count + each + inner count. Phase C should derive a recognizer that admits both the 'where each holds N' and the comma-only form (e.g., '6 baskets, each holding 50 strawberries')."}} +{"exemplar_id": "ma-v1-0002", "shape_category": "multiplicative_aggregation", "statement": "Each survey has 10 questions.", "expected_graph": {"subject": "survey", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "1", "outer_unit": "survey", "inner_count": "10", "inner_unit": "questions", "subject_role": "survey"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0045", "author_note": "Minimal-form multiplicative: leading 'each' with implicit outer_count=1. The exemplar records '1' so the schema's outer_count field stays a string token, not null."}} +{"exemplar_id": "ma-v1-0003", "shape_category": "multiplicative_aggregation", "statement": "John bakes 12 coconut macaroons, each weighing 5 ounces.", "expected_graph": {"subject": "John", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "12", "outer_unit": "macaroons", "inner_count": "5", "inner_unit": "ounces", "subject_role": "John"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0047", "author_note": "The inner_unit 'ounces' is a weight-per-item, not a count of inner items. Phase C may distinguish 'inner_kind=weight' vs 'inner_kind=count' but the current schema collapses both into the inner_count+inner_unit pair."}} +{"exemplar_id": "ma-v1-0004", "shape_category": "multiplicative_aggregation", "statement": "Ella packs 4 bags with 20 apples each.", "expected_graph": {"subject": "Ella", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "4", "outer_unit": "bags", "inner_count": "20", "inner_unit": "apples", "subject_role": "Ella"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ma-v1-0005", "shape_category": "multiplicative_aggregation", "statement": "Six bags with 25 apples were loaded onto the truck.", "expected_graph": {"subject": "bags", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "six", "outer_unit": "bags", "inner_count": "25", "inner_unit": "apples", "subject_role": "bags"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Outer count is word-form ('six') while inner is integer; the schema preserves both surface representations."}} +{"exemplar_id": "ma-v1-0006", "shape_category": "multiplicative_aggregation", "statement": "The farm sells 8 crates of oranges, each containing 30 fruits.", "expected_graph": {"subject": "the farm", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "8", "outer_unit": "crates", "inner_count": "30", "inner_unit": "fruits", "subject_role": "farm"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ma-v1-0007", "shape_category": "multiplicative_aggregation", "statement": "Mira owns 12 jars holding 50 marbles each.", "expected_graph": {"subject": "Mira", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "12", "outer_unit": "jars", "inner_count": "50", "inner_unit": "marbles", "subject_role": "Mira"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ma-v1-0008", "shape_category": "multiplicative_aggregation", "statement": "There are 7 trays of cupcakes with 12 cupcakes each on the rack.", "expected_graph": {"subject": "trays", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "7", "outer_unit": "trays", "inner_count": "12", "inner_unit": "cupcakes", "subject_role": "trays"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ma-v1-0009", "shape_category": "multiplicative_aggregation", "statement": "Carlos delivers 5 boxes containing 24 doughnuts apiece.", "expected_graph": {"subject": "Carlos", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "5", "outer_unit": "boxes", "inner_count": "24", "inner_unit": "doughnuts", "subject_role": "Carlos"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "'apiece' is treated as the same axis as 'each' for inner-count framing; the recognizer Phase C derives should admit both."}} +{"exemplar_id": "ma-v1-0010", "shape_category": "multiplicative_aggregation", "statement": "The bookshop displays 9 shelves of novels with 40 novels each.", "expected_graph": {"subject": "the bookshop", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "9", "outer_unit": "shelves", "inner_count": "40", "inner_unit": "novels", "subject_role": "bookshop"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ma-v1-0011", "shape_category": "multiplicative_aggregation", "statement": "Yara prepared 6 lunch boxes each holding 3 sandwiches.", "expected_graph": {"subject": "Yara", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "6", "outer_unit": "boxes", "inner_count": "3", "inner_unit": "sandwiches", "subject_role": "Yara"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ma-v1-0012", "shape_category": "multiplicative_aggregation", "statement": "Diego owns 4 cartons of eggs holding 12 eggs each.", "expected_graph": {"subject": "Diego", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "4", "outer_unit": "cartons", "inner_count": "12", "inner_unit": "eggs", "subject_role": "Diego"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ma-v1-0013", "shape_category": "multiplicative_aggregation", "statement": "The orchard yielded 10 baskets of peaches with 15 peaches each.", "expected_graph": {"subject": "the orchard", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "10", "outer_unit": "baskets", "inner_count": "15", "inner_unit": "peaches", "subject_role": "orchard"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ma-v1-0014", "shape_category": "multiplicative_aggregation", "statement": "Priya tied 8 bundles with 25 sticks each.", "expected_graph": {"subject": "Priya", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "8", "outer_unit": "bundles", "inner_count": "25", "inner_unit": "sticks", "subject_role": "Priya"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ma-v1-0015", "shape_category": "multiplicative_aggregation", "statement": "The factory packs 20 cases of bottles, each containing 24 bottles.", "expected_graph": {"subject": "the factory", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "20", "outer_unit": "cases", "inner_count": "24", "inner_unit": "bottles", "subject_role": "factory"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ma-v1-0016", "shape_category": "multiplicative_aggregation", "statement": "Anika filled 3 jars with 50 marbles each.", "expected_graph": {"subject": "Anika", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "3", "outer_unit": "jars", "inner_count": "50", "inner_unit": "marbles", "subject_role": "Anika"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2}} +{"exemplar_id": "ma-v1-0017", "shape_category": "multiplicative_aggregation", "statement": "Sundar packs five boxes of pencils, each holding 12 pencils.", "expected_graph": {"subject": "Sundar", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "five", "outer_unit": "boxes", "inner_count": "12", "inner_unit": "pencils", "subject_role": "Sundar"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: word-form outer count combined with integer inner."}} +{"exemplar_id": "ma-v1-0018", "shape_category": "multiplicative_aggregation", "statement": "Each bag holds 15 candies on the shelf.", "expected_graph": {"subject": "bag", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "1", "outer_unit": "bag", "inner_count": "15", "inner_unit": "candies", "subject_role": "bag"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: minimal 'each' with no outer count; outer_count='1' is the canonical default for the schema."}} +{"exemplar_id": "ma-v1-0019", "shape_category": "multiplicative_aggregation", "statement": "The warehouse stores 100 pallets of bricks with 200 bricks per pallet.", "expected_graph": {"subject": "the warehouse", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "100", "outer_unit": "pallets", "inner_count": "200", "inner_unit": "bricks", "subject_role": "warehouse"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: 'per pallet' uses spatial 'per' (not temporal); the categorizer routes correctly because no time-unit follows."}} +{"exemplar_id": "ma-v1-0020", "shape_category": "multiplicative_aggregation", "statement": "Two dozen donut boxes each contain six donuts.", "expected_graph": {"subject": "donut boxes", "quantity_anchors": [{"kind": "multiplicative_aggregate", "outer_count": "two dozen", "outer_unit": "boxes", "inner_count": "six", "inner_unit": "donuts", "subject_role": "boxes"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: composite word-form outer count ('two dozen') and word-form inner count ('six'). Phase C should derive a recognizer admitting word-form composites in both positions."}} diff --git a/teaching/admissibility_exemplars/temporal_aggregation_v2.jsonl b/teaching/admissibility_exemplars/temporal_aggregation_v2.jsonl new file mode 100644 index 00000000..46c31268 --- /dev/null +++ b/teaching/admissibility_exemplars/temporal_aggregation_v2.jsonl @@ -0,0 +1,10 @@ +{"exemplar_id": "tav2-v1-0001", "shape_category": "temporal_aggregation", "statement": "Eric wants to rent the house for 20 days.", "expected_graph": {"subject": "Eric", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "1", "window_unit": "day", "window_quantifier": "per", "subject_role": "Eric"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0017", "author_note": "Widening target: 'for N days' window-quantifier form not in v1 (v1's window_quantifier set was {each, every, per}). The schema collapses 'for 20 days' to window_unit='day' with quantifier='per' and an implicit '20' multiplier."}} +{"exemplar_id": "tav2-v1-0002", "shape_category": "temporal_aggregation", "statement": "He can run 40 yards within 5 seconds.", "expected_graph": {"subject": "he", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "40", "window_unit": "second", "window_quantifier": "per", "subject_role": "he"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0034", "author_note": "Widening target: 'within N seconds' window form. Mirrors v1 case ta-v1-0002 in shape but with the 'within' quantifier variant absent from v1."}} +{"exemplar_id": "tav2-v1-0003", "shape_category": "temporal_aggregation", "statement": "She studied for 2 hours on Wednesday and three times as long on Thursday.", "expected_graph": {"subject": "she", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "2", "window_unit": "day", "window_quantifier": "each", "subject_role": "she"}, {"kind": "event_count_per_window", "count_token": "2", "window_unit": "day", "window_quantifier": "each", "subject_role": "she"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0036", "author_note": "Widening target: 'for N hours on ' composite. Two day-windowed anchors (Wednesday, Thursday); the comparative inner 'three times as long' is not captured by the schema and is flagged for operator review."}} +{"exemplar_id": "tav2-v1-0004", "shape_category": "temporal_aggregation", "statement": "Every week, he gets 6 cards.", "expected_graph": {"subject": "he", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "6", "window_unit": "week", "window_quantifier": "every", "subject_role": "he"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "train_case_id": "gsm8k-train-sample-v1-0048", "author_note": "Widening target: leading-clause 'Every week,' form (v1 covered only trailing 'every week' position)."}} +{"exemplar_id": "tav2-v1-0005", "shape_category": "temporal_aggregation", "statement": "Lina sketched for 3 hours during the workshop.", "expected_graph": {"subject": "Lina", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "3", "window_unit": "hour", "window_quantifier": "per", "subject_role": "Lina"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Novel 'for N hours' widening (matches case 0036/0017 quantifier)."}} +{"exemplar_id": "tav2-v1-0006", "shape_category": "temporal_aggregation", "statement": "The kettle boils within 2 minutes on full power.", "expected_graph": {"subject": "the kettle", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "1", "window_unit": "minute", "window_quantifier": "per", "subject_role": "kettle"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Novel 'within N minutes' widening. count_token='1' encodes the single boil event."}} +{"exemplar_id": "tav2-v1-0007", "shape_category": "temporal_aggregation", "statement": "Every month, the council reviews 4 applications.", "expected_graph": {"subject": "the council", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "4", "window_unit": "month", "window_quantifier": "every", "subject_role": "council"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Novel leading 'Every month,' form."}} +{"exemplar_id": "tav2-v1-0008", "shape_category": "temporal_aggregation", "statement": "Khalil practiced violin for 45 minutes during the recital prep.", "expected_graph": {"subject": "Khalil", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "45", "window_unit": "minute", "window_quantifier": "per", "subject_role": "Khalil"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Novel 'for N minutes' widening."}} +{"exemplar_id": "tav2-v1-0009", "shape_category": "temporal_aggregation", "statement": "The marathon runner crossed the finish line within 3 hours of the starting gun.", "expected_graph": {"subject": "the runner", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "1", "window_unit": "hour", "window_quantifier": "per", "subject_role": "runner"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Novel 'within N hours' widening; count_token='1' encodes the single crossing event."}} +{"exemplar_id": "tav2-v1-0010", "shape_category": "temporal_aggregation", "statement": "Every year, the festival hosts 12 musicians on opening night.", "expected_graph": {"subject": "the festival", "quantity_anchors": [{"kind": "event_count_per_window", "count_token": "12", "window_unit": "year", "window_quantifier": "every", "subject_role": "festival"}], "graph_intent": "aggregate", "outcome": "admissible"}, "provenance": {"source": "phase_b_seed", "author": "Claude (Phase B round 2 agent)", "round": 1, "category_rank": 2, "author_note": "Edge case: leading 'Every year,' with a year window — the longest window unit in the schema."}} diff --git a/tests/test_admissibility_exemplars.py b/tests/test_admissibility_exemplars.py index a2469423..3f1f24cf 100644 --- a/tests/test_admissibility_exemplars.py +++ b/tests/test_admissibility_exemplars.py @@ -18,7 +18,7 @@ from typing import Any import pytest -from evals.refusal_taxonomy.shape_categories import ShapeCategory +from evals.refusal_taxonomy.shape_categories import ShapeCategory, categorize _REPO_ROOT = Path(__file__).resolve().parent.parent _EXEMPLARS_ROOT = _REPO_ROOT / "teaching" / "admissibility_exemplars" @@ -26,7 +26,10 @@ _GSM8K_TRAIN_REPORT = ( _REPO_ROOT / "evals" / "gsm8k_math" / "train_sample" / "v1" / "report.json" ) -# Round 1 categories, with their file stem and expected category-rank. +# Round 1 + Round 2 categories, with their file stem, expected category, and +# category-rank. Round 2 introduces three new categories plus a v2 widening +# corpus for the existing TEMPORAL_AGGREGATION category. Per-file record +# ceiling is 20 for new corpora and 10 for the v2 widening. _ROUND_1: tuple[tuple[str, ShapeCategory, int], ...] = ( ( "descriptive_setup_no_quantity_v1", @@ -36,6 +39,22 @@ _ROUND_1: tuple[tuple[str, ShapeCategory, int], ...] = ( ("temporal_aggregation_v1", ShapeCategory.TEMPORAL_AGGREGATION, 2), ("rate_with_currency_v1", ShapeCategory.RATE_WITH_CURRENCY, 3), ) +_ROUND_2: tuple[tuple[str, ShapeCategory, int], ...] = ( + ( + "discrete_count_statement_v1", + ShapeCategory.DISCRETE_COUNT_STATEMENT, + 2, + ), + ( + "multiplicative_aggregation_v1", + ShapeCategory.MULTIPLICATIVE_AGGREGATION, + 2, + ), + ("currency_amount_v1", ShapeCategory.CURRENCY_AMOUNT, 2), + ("temporal_aggregation_v2", ShapeCategory.TEMPORAL_AGGREGATION, 2), +) +_ALL_CORPORA: tuple[tuple[str, ShapeCategory, int], ...] = _ROUND_1 + _ROUND_2 +_TEN_RECORD_CEILING_STEMS: frozenset[str] = frozenset({"temporal_aggregation_v2"}) _REQUIRED_TOP_KEYS: frozenset[str] = frozenset( {"exemplar_id", "shape_category", "statement", "expected_graph", "provenance"} @@ -98,7 +117,7 @@ def test_contract_exists(): assert (_EXEMPLARS_ROOT / "contract.md").is_file() -@pytest.mark.parametrize(("stem", "_category", "_rank"), _ROUND_1) +@pytest.mark.parametrize(("stem", "_category", "_rank"), _ALL_CORPORA) def test_corpus_file_exists(stem: str, _category: ShapeCategory, _rank: int): path = _EXEMPLARS_ROOT / f"{stem}.jsonl" assert path.is_file(), path @@ -109,11 +128,14 @@ def test_corpus_file_exists(stem: str, _category: ShapeCategory, _rank: int): # --------------------------------------------------------------------------- -@pytest.mark.parametrize(("stem", "category", "rank"), _ROUND_1) +@pytest.mark.parametrize(("stem", "category", "rank"), _ALL_CORPORA) def test_records_schema(stem: str, category: ShapeCategory, rank: int): path = _EXEMPLARS_ROOT / f"{stem}.jsonl" records = _load_jsonl(path) - assert 1 <= len(records) <= 20, f"{path} has {len(records)} records" + ceiling = 10 if stem in _TEN_RECORD_CEILING_STEMS else 20 + assert 1 <= len(records) <= ceiling, ( + f"{path} has {len(records)} records (ceiling {ceiling})" + ) seen_ids: set[str] = set() for idx, record in enumerate(records, start=1): @@ -248,6 +270,72 @@ def _validate_per_category( assert graph["outcome"] == "admissible", f"{path}:{idx} outcome" return + if category is ShapeCategory.CURRENCY_AMOUNT: + assert len(anchors) >= 1, f"{path}:{idx} currency_amount needs anchors" + for a in anchors: + _check_keys(path, idx, a, { + "kind", "currency_symbol", "amount", "amount_kind", + "subject_role", + }) + assert a["kind"] == "currency_amount", f"{path}:{idx} anchor kind" + assert a["currency_symbol"] in _VALID_CURRENCY_SYMBOLS, ( + f"{path}:{idx} currency_symbol {a['currency_symbol']!r}" + ) + assert a["amount_kind"] in _VALID_AMOUNT_KINDS, ( + f"{path}:{idx} amount_kind {a['amount_kind']!r}" + ) + assert isinstance(a["amount"], str) and a["amount"], ( + f"{path}:{idx} amount" + ) + assert isinstance(a["subject_role"], str) and a["subject_role"], ( + f"{path}:{idx} subject_role" + ) + assert graph["graph_intent"] == "amount", f"{path}:{idx} graph_intent" + assert graph["outcome"] == "admissible", f"{path}:{idx} outcome" + return + + if category is ShapeCategory.MULTIPLICATIVE_AGGREGATION: + assert len(anchors) >= 1, ( + f"{path}:{idx} multiplicative_aggregation needs anchors" + ) + for a in anchors: + _check_keys(path, idx, a, { + "kind", "outer_count", "outer_unit", + "inner_count", "inner_unit", "subject_role", + }) + assert a["kind"] == "multiplicative_aggregate", ( + f"{path}:{idx} anchor kind" + ) + for field in ("outer_count", "outer_unit", + "inner_count", "inner_unit", "subject_role"): + assert isinstance(a[field], str) and a[field], ( + f"{path}:{idx} {field}" + ) + assert graph["graph_intent"] == "aggregate", f"{path}:{idx} graph_intent" + assert graph["outcome"] == "admissible", f"{path}:{idx} outcome" + return + + if category is ShapeCategory.DISCRETE_COUNT_STATEMENT: + assert len(anchors) >= 1, ( + f"{path}:{idx} discrete_count_statement needs anchors" + ) + for a in anchors: + _check_keys(path, idx, a, { + "kind", "subject_role", "count_token", + "count_kind", "counted_noun", + }) + assert a["kind"] == "discrete_count", f"{path}:{idx} anchor kind" + assert a["count_kind"] in {"integer", "word"}, ( + f"{path}:{idx} count_kind {a['count_kind']!r}" + ) + for field in ("subject_role", "count_token", "counted_noun"): + assert isinstance(a[field], str) and a[field], ( + f"{path}:{idx} {field}" + ) + assert graph["graph_intent"] == "count", f"{path}:{idx} graph_intent" + assert graph["outcome"] == "admissible", f"{path}:{idx} outcome" + return + raise AssertionError(f"unhandled category in dispatch: {category!r}") @@ -265,7 +353,7 @@ def _check_keys( def test_no_statement_appears_in_more_than_one_file(): seen: dict[str, str] = {} - for stem, _cat, _rank in _ROUND_1: + for stem, _cat, _rank in _ALL_CORPORA: records = _load_jsonl(_EXEMPLARS_ROOT / f"{stem}.jsonl") for rec in records: s = rec["statement"] @@ -276,7 +364,7 @@ def test_no_statement_appears_in_more_than_one_file(): def test_no_duplicate_statement_within_file(): - for stem, _cat, _rank in _ROUND_1: + for stem, _cat, _rank in _ALL_CORPORA: path = _EXEMPLARS_ROOT / f"{stem}.jsonl" records = _load_jsonl(path) statements = [r["statement"] for r in records] @@ -285,7 +373,29 @@ def test_no_duplicate_statement_within_file(): ) -@pytest.mark.parametrize(("stem", "_category", "_rank"), _ROUND_1) +@pytest.mark.parametrize(("stem", "category", "_rank"), _ALL_CORPORA) +def test_phase_a_categorizer_agrees_with_file( + stem: str, category: ShapeCategory, _rank: int +): + """Every exemplar statement categorizes to its file's category. + + This is the load-bearing fidelity check for the Phase B → Phase C + handoff: if Phase A's categorizer disagrees with the operator's + file assignment, the seed is ambiguous and the recognizer Phase C + derives will be ambiguous too. + """ + + path = _EXEMPLARS_ROOT / f"{stem}.jsonl" + for rec in _load_jsonl(path): + observed = categorize(rec["statement"]) + assert observed is category, ( + f"{path}: {rec['exemplar_id']!r} categorizes as " + f"{observed.value!r} but file declares {category.value!r}: " + f"{rec['statement']!r}" + ) + + +@pytest.mark.parametrize(("stem", "_category", "_rank"), _ALL_CORPORA) def test_train_sample_binding_minimum( stem: str, _category: ShapeCategory, _rank: int ): @@ -311,7 +421,7 @@ def test_train_sample_binding_minimum( # --------------------------------------------------------------------------- -@pytest.mark.parametrize(("stem", "_category", "_rank"), _ROUND_1) +@pytest.mark.parametrize(("stem", "_category", "_rank"), _ALL_CORPORA) def test_records_sorted_by_exemplar_id( stem: str, _category: ShapeCategory, _rank: int ): @@ -321,7 +431,7 @@ def test_records_sorted_by_exemplar_id( assert ids == sorted(ids), f"{path} not sorted by exemplar_id" -@pytest.mark.parametrize(("stem", "_category", "_rank"), _ROUND_1) +@pytest.mark.parametrize(("stem", "_category", "_rank"), _ALL_CORPORA) def test_file_canonical_byte_form( stem: str, _category: ShapeCategory, _rank: int ): diff --git a/tests/test_refusal_taxonomy_lane.py b/tests/test_refusal_taxonomy_lane.py index acdf9db3..ade06c37 100644 --- a/tests/test_refusal_taxonomy_lane.py +++ b/tests/test_refusal_taxonomy_lane.py @@ -72,7 +72,11 @@ def test_lane_run_via_framework(): lane = get_lane("refusal_taxonomy") result = run_lane(lane, version="v1", split="public") assert result.metrics["total"] == 50 - assert result.metrics["categorized_rate"] == pytest.approx(0.72) + # Phase B round 2 extended the categorizer with three new shape + # categories; the post-extension histogram leaves only the residual + # uncategorized tail. The exact rate is asserted against the + # committed report by ``test_committed_report_matches_categorizer``. + assert result.metrics["categorized_rate"] >= 0.95 # --------------------------------------------------------------------------- @@ -106,8 +110,14 @@ def test_every_category_value_reachable_by_a_rule(): "If she had two more, she would have plenty.", ShapeCategory.DESCRIPTIVE_SETUP_NO_QUANTITY: "Marnie makes bead bracelets.", - ShapeCategory.UNCATEGORIZED: + ShapeCategory.CURRENCY_AMOUNT: + "It cost $100,000 to open initially.", + ShapeCategory.MULTIPLICATIVE_AGGREGATION: + "Each survey has 10 questions.", + ShapeCategory.DISCRETE_COUNT_STATEMENT: "Nicole collected 400 Pokemon cards.", + ShapeCategory.UNCATEGORIZED: + "John invests in a bank and gets 10% simple interest.", } for category, probe in probes.items(): assert categorize(probe) is category, (