feat(ADR-0163.A): refusal taxonomy lane — shape categorization of GSM8K admissibility gaps (#297)
ADR-0163 Phase A measurement. Reads the GSM8K train-sample refusal report
(50 cases, all refused on candidate-graph admissibility) and emits a
histogram of statement shapes. Read-only: no corpus, pack, or proposal
mutation; the categorizer is rules-only with no LLM, embedding, or
learned model.
Lane: evals/refusal_taxonomy/ (auto-discovered by evals.framework)
- shape_categories.py — ShapeCategory enum + deterministic categorizer
(9 ADR-mandated baseline categories + UNCATEGORIZED, first-match-wins)
- runner.py — pure run_lane(cases) -> LaneReport
- contract.md — purpose, doctrine, schema, ADR compatibility
- public/v1/cases.jsonl — 50 refused statements (sorted by case_id)
- v1/report.json — first run output (categorized_rate=72%)
CLI: core teaching refusal-taxonomy [--input PATH] [--json] [--save]
Accepts a cases JSONL or a raw GSM8K eval report.json directly.
Helper: scripts/build_refusal_taxonomy_cases.py rebuilds the v1 case set
from the GSM8K train-sample report deterministically.
Tests: tests/test_refusal_taxonomy_lane.py (21 passing) cover schema
integrity, lane auto-discovery, enum exhaustiveness, categorizer
determinism + purity + no-ML-imports, histogram correctness, replay
byte-identity, committed report match, helper extraction, and a
read-only invariant snapshot over teaching/, packs/, language_packs/data/.
v1 histogram (50-case sample):
17 descriptive_setup_no_quantity
14 uncategorized
4 temporal_aggregation
3 rate_with_currency
3 fractional_rate_of_change
3 indefinite_quantity
3 comparative_with_unit
2 nested_question_target
1 unit_partition
0 conditional_quantity
total=50 categorized_rate=72% uncategorized=28% (below 50% target)
Top three by count (Phase B candidates):
1. descriptive_setup_no_quantity (17)
2. temporal_aggregation (4)
3. tie at 3 — operator selects from {rate_with_currency,
fractional_rate_of_change, indefinite_quantity, comparative_with_unit}
Phase B is not started in this PR — the ADR explicitly requires the
operator to ratify the top-N selection before any exemplar corpus is
authored.
Invariants verified:
- tests/test_adr_0131_*.py: 224 passed, 0 wrong on G1..G5 + S1
- core test --suite smoke -q: 67 passed
- The refusal_taxonomy/__init__.py and runner do not import openai,
anthropic, transformers, torch, sklearn, sentence_transformers,
requests, or httpx — verified by test_categorizer_no_llm_or_ml_imports.
Cross-references: ADR-0163 (parent), ADR-0114a (capability obligations),
ADR-0149 (recognizer pipeline substrate that Phases C–E build on).
Refs: [[thesis-decoding-not-generating]] — the rules-only categorizer
honors the doctrine: the engine learns to find better shapes; this PR
does not stuff it with another found pattern.
This commit is contained in:
Shay
GitHub
parent
8b3314f060
commit
5b4dcb17ca
10 changed files with 1653 additions and 0 deletions
90
core/cli.py
90
core/cli.py
|
|
@ -1524,6 +1524,78 @@ def cmd_teaching_supersede(args: argparse.Namespace) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
def cmd_teaching_refusal_taxonomy(args: argparse.Namespace) -> int:
|
||||
"""ADR-0163 Phase A — categorise refused statements by shape.
|
||||
|
||||
Read-only. Reads a JSONL of refused cases (defaults to the v1
|
||||
refusal_taxonomy case set) and emits a histogram of shape categories.
|
||||
Per ADR-0163, the categorizer is rules-only: no LLM call, no
|
||||
embedding, no learned model. --save writes the report to
|
||||
``evals/refusal_taxonomy/v1/report.json``.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from evals.framework import load_cases
|
||||
from evals.refusal_taxonomy.runner import run_lane
|
||||
from scripts.build_refusal_taxonomy_cases import build_cases
|
||||
|
||||
input_path = Path(args.input) if args.input else (
|
||||
_REPO_ROOT / "evals" / "refusal_taxonomy" / "public" / "v1" / "cases.jsonl"
|
||||
)
|
||||
if not input_path.exists():
|
||||
print(f"input not found: {input_path}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
# Accept either a cases JSONL (one record per line) or a GSM8K-style
|
||||
# eval report.json with a top-level ``per_case`` list of refusals.
|
||||
if input_path.suffix == ".jsonl":
|
||||
cases = load_cases(input_path)
|
||||
else:
|
||||
cases = build_cases(input_path)
|
||||
report = run_lane(cases)
|
||||
metrics = report.metrics
|
||||
|
||||
if args.json:
|
||||
payload = {
|
||||
"lane": "refusal_taxonomy",
|
||||
"input": str(input_path),
|
||||
"metrics": metrics,
|
||||
"cases": report.case_details,
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
else:
|
||||
print(f"input : {input_path}")
|
||||
print(f"total : {metrics['total']}")
|
||||
print(f"categorized_rate : {metrics['categorized_rate']:.3f}")
|
||||
print(f"uncategorized : {metrics['uncategorized']}")
|
||||
print(f"case_digest : {metrics['case_digest']}")
|
||||
print("histogram:")
|
||||
for category, count in sorted(
|
||||
metrics["by_category"].items(), key=lambda kv: (-kv[1], kv[0]),
|
||||
):
|
||||
print(f" {count:3d} {category}")
|
||||
|
||||
if args.save:
|
||||
out = _REPO_ROOT / "evals" / "refusal_taxonomy" / "v1" / "report.json"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"lane": "refusal_taxonomy",
|
||||
"version": "v1",
|
||||
"split": "public",
|
||||
"source_cases": str(input_path),
|
||||
"metrics": metrics,
|
||||
"cases": report.case_details,
|
||||
}
|
||||
out.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
+ "\n"
|
||||
)
|
||||
print(f"saved : {out}", file=sys.stderr)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_pack_validate(args: argparse.Namespace) -> int:
|
||||
"""Run executable source-pack validation gates."""
|
||||
pack_id = _safe_pack_id(args.pack_id)
|
||||
|
|
@ -3668,6 +3740,24 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
)
|
||||
teaching_supersede.set_defaults(func=cmd_teaching_supersede)
|
||||
|
||||
teaching_refusal_taxonomy = teaching_sub.add_parser(
|
||||
"refusal-taxonomy",
|
||||
help="ADR-0163 Phase A — categorise refused statements by shape",
|
||||
)
|
||||
teaching_refusal_taxonomy.add_argument(
|
||||
"--input", default=None,
|
||||
help="path to refused-cases JSONL (default: evals/refusal_taxonomy/public/v1/cases.jsonl)",
|
||||
)
|
||||
teaching_refusal_taxonomy.add_argument(
|
||||
"--json", action="store_true",
|
||||
help="emit machine-readable JSON",
|
||||
)
|
||||
teaching_refusal_taxonomy.add_argument(
|
||||
"--save", action="store_true",
|
||||
help="write report to evals/refusal_taxonomy/v1/report.json",
|
||||
)
|
||||
teaching_refusal_taxonomy.set_defaults(func=cmd_teaching_refusal_taxonomy)
|
||||
|
||||
teaching_supersessions = teaching_sub.add_parser(
|
||||
"supersessions",
|
||||
help="pair each retired chain with its active replacement (derived view)",
|
||||
|
|
|
|||
92
docs/refusal-taxonomy.md
Normal file
92
docs/refusal-taxonomy.md
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
# Refusal Taxonomy — ADR-0163 Phase A
|
||||
|
||||
The refusal-taxonomy lane categorises every refused GSM8K statement by
|
||||
*statement shape*, not by content. It is the load-bearing measurement
|
||||
that gates Phase B of [ADR-0163](decisions/ADR-0163-gsm8k-path-to-mastery.md):
|
||||
the top categories by count become the operator's input list for
|
||||
hand-authored exemplar corpora.
|
||||
|
||||
The lane is **read-only**. It does not mutate the active corpus, packs,
|
||||
language packs, or proposal log. The categorizer is rules-only — no LLM
|
||||
call, no embedding, no learned classifier — per ADR-0163 §Constraint #4
|
||||
and CLAUDE.md.
|
||||
|
||||
## v1 histogram (50-case GSM8K train sample)
|
||||
|
||||
| Count | Category |
|
||||
|------:|----------|
|
||||
| 17 | `descriptive_setup_no_quantity` |
|
||||
| 14 | `uncategorized` |
|
||||
| 4 | `temporal_aggregation` |
|
||||
| 3 | `rate_with_currency` |
|
||||
| 3 | `fractional_rate_of_change` |
|
||||
| 3 | `indefinite_quantity` |
|
||||
| 3 | `comparative_with_unit` |
|
||||
| 2 | `nested_question_target` |
|
||||
| 1 | `unit_partition` |
|
||||
| 0 | `conditional_quantity` |
|
||||
|
||||
Total: 50. Uncategorized rate: 28%. Categorized rate: 72%.
|
||||
|
||||
Case digest: `d030f826cb0f4088771d90c52c8be2ff75054ab27c7d47eae8dbfe1225b2eea1`.
|
||||
|
||||
## Top three by count (Phase B candidates)
|
||||
|
||||
1. **`descriptive_setup_no_quantity` (17)** — statements with no
|
||||
extractable measurement at all. The candidate-graph needs to admit
|
||||
pure-context lines as setup rather than refusing them.
|
||||
2. **`temporal_aggregation` (4)** — repeated/aggregated time framing
|
||||
("each day", "every other day for 2 weeks", "in 5 minutes",
|
||||
day-of-week enumeration).
|
||||
3. **Tie at 3** — `rate_with_currency`, `fractional_rate_of_change`,
|
||||
`indefinite_quantity`, `comparative_with_unit`. The operator
|
||||
selects which member of the tie (or combination) to seed in Phase
|
||||
B's first round; the agent does not make that call.
|
||||
|
||||
ADR-0163 §Phase B ratchet is "three categories per round". The 17/4
|
||||
spread suggests Round 1 should anchor on
|
||||
`descriptive_setup_no_quantity` plus two of the tied trio.
|
||||
|
||||
## Reading the uncategorized tail (14)
|
||||
|
||||
`uncategorized` is honest measurement, not failure. The 14 statements
|
||||
that no rule fires for share these emergent sub-shapes (not yet promoted
|
||||
to first-class categories — none has ≥ 3 instances individually):
|
||||
|
||||
- **bare declarative quantity** — "Nicole collected 400 Pokemon cards."
|
||||
/ "A school has 100 students." / "Malcolm has 240 followers on
|
||||
Instagram and 500 followers on Facebook."
|
||||
- **distributive** — "each saved up $40" / "each weighing 5 ounces"
|
||||
- **sequential change** — "had 20 paperclips initially, lost 12"
|
||||
- **word-number enumeration** — "Two puppies, two kittens, and three
|
||||
parakeets" / "a hundred ladies"
|
||||
- **percentage-rate without change verb** — "10% simple interest"
|
||||
- **quantity embedded in narrative** — "3 friends at the end of summer"
|
||||
/ "lose 10 pounds by June"
|
||||
|
||||
The operator decides whether any of these warrant promotion to a
|
||||
first-class category in v2 of the taxonomy (each promotion requires
|
||||
≥ 3 cited refused statements per ADR-0163 §Risks).
|
||||
|
||||
## How to re-run
|
||||
|
||||
```bash
|
||||
# Run via the eval framework (uses the standard public/v1/cases.jsonl)
|
||||
uv run core eval refusal_taxonomy
|
||||
|
||||
# Run via the teaching CLI on an arbitrary refused-case set
|
||||
uv run core teaching refusal-taxonomy \
|
||||
--input evals/gsm8k_math/train_sample/v1/report.json
|
||||
|
||||
# Regenerate the v1 cases.jsonl from the source GSM8K report
|
||||
uv run python scripts/build_refusal_taxonomy_cases.py
|
||||
```
|
||||
|
||||
## Phase A boundary (what this lane does NOT do)
|
||||
|
||||
- It does not add or modify recognizers.
|
||||
- It does not author exemplar corpora (that is Phase B).
|
||||
- It does not emit recognizer proposals (that is Phase C).
|
||||
- It does not change the GSM8K `correct/refused/wrong` counts. Per
|
||||
ADR-0114a, the `wrong = 0` invariant on G1..G5 and S1 is unchanged
|
||||
by this work.
|
||||
0
evals/refusal_taxonomy/__init__.py
Normal file
0
evals/refusal_taxonomy/__init__.py
Normal file
125
evals/refusal_taxonomy/contract.md
Normal file
125
evals/refusal_taxonomy/contract.md
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
# Refusal Taxonomy Eval Contract (ADR-0163 Phase A)
|
||||
|
||||
## Purpose
|
||||
|
||||
`refusal-taxonomy` is a read-only evaluation lane that categorises refused
|
||||
GSM8K statements by *statement shape*, not by content. The histogram it
|
||||
emits is the load-bearing measurement that gates Phase B of ADR-0163: the
|
||||
top categories by count become the input list for hand-authored exemplar
|
||||
corpora.
|
||||
|
||||
Phase A produces no recognizers and no corpus changes. Its sole job is to
|
||||
turn a flat refusal list into a measured distribution of shapes so the
|
||||
operator can choose what to teach next.
|
||||
|
||||
## Source of truth
|
||||
|
||||
The v1 case set is derived from:
|
||||
|
||||
```
|
||||
evals/gsm8k_math/train_sample/v1/report.json
|
||||
```
|
||||
|
||||
— every record whose `verdict == "refused"`, with the embedded statement
|
||||
extracted out of the `reason` envelope. The case set is rebuilt
|
||||
deterministically by `scripts/build_refusal_taxonomy_cases.py`; the
|
||||
script's output is sorted by `case_id` and committed to
|
||||
`evals/refusal_taxonomy/public/v1/cases.jsonl` (the framework-standard
|
||||
public-split location). The Phase A v1 report artifact lives at
|
||||
`evals/refusal_taxonomy/v1/report.json`.
|
||||
|
||||
## Non-goals
|
||||
|
||||
This lane MUST NOT:
|
||||
|
||||
- mutate any corpus, pack, or language pack
|
||||
- write to `engine_state`
|
||||
- create, accept, or reject proposals
|
||||
- call an LLM, embedding model, or any learned classifier
|
||||
- alter the GSM8K refusal counts elsewhere in the repo
|
||||
|
||||
The lane only reads its own `cases.jsonl` (or an operator-supplied
|
||||
refused-case set) and emits a histogram.
|
||||
|
||||
## Categorizer doctrine
|
||||
|
||||
Per ADR-0163 §Constraint #4 and CLAUDE.md, the categorizer is rules-only:
|
||||
|
||||
- No LLM call, no embedding, no learned model.
|
||||
- Deterministic — the output is a pure function of the input string.
|
||||
- No hidden normalization — lowercasing/padding is only for substring
|
||||
word-boundary safety; the original statement is never mutated for
|
||||
downstream consumers.
|
||||
- First-match-wins. Priority order is fixed in `shape_categories.py`.
|
||||
- `UNCATEGORIZED` is a first-class outcome. It is honest measurement,
|
||||
not a failure.
|
||||
|
||||
Adding a new category requires citing ≥ 3 refused statements as evidence
|
||||
in the category's docstring. This is enforced by `tests/test_refusal_taxonomy_lane.py`.
|
||||
|
||||
## Shape categories (v1)
|
||||
|
||||
The nine baseline categories named in ADR-0163 §Phase A, plus
|
||||
`uncategorized`:
|
||||
|
||||
| Category | Definition |
|
||||
|---|---|
|
||||
| `nested_question_target` | "If X, how many/much Y …?" |
|
||||
| `unit_partition` | hyphenated unit, e.g. "25-foot sections" |
|
||||
| `rate_with_currency` | `$N` + per-unit framing (per hour, /kg, for one cup) |
|
||||
| `comparative_with_unit` | "more than", "twice as", "N times", "as many as" |
|
||||
| `fractional_rate_of_change` | fraction or `%` paired with a change-of-state verb |
|
||||
| `indefinite_quantity` | "some", "several", "a few", "many", "any" |
|
||||
| `temporal_aggregation` | "each day", "every X", "in N minutes", day-of-week enumeration |
|
||||
| `conditional_quantity` | bare "If X, would/will Y" without a `how`-target |
|
||||
| `descriptive_setup_no_quantity` | no digit, no number-word, no quantifier |
|
||||
| `uncategorized` | none of the above |
|
||||
|
||||
## Case set schema
|
||||
|
||||
Each line in `cases.jsonl` is a JSON object:
|
||||
|
||||
```json
|
||||
{"case_id": "gsm8k-train-sample-v1-NNNN",
|
||||
"statement": "<the refused statement>",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for ..."}
|
||||
```
|
||||
|
||||
## Lane output
|
||||
|
||||
`run_lane(cases)` returns a `LaneReport` with:
|
||||
|
||||
```python
|
||||
metrics = {
|
||||
"total": <int>,
|
||||
"by_category": {category_value: count, ...}, # every category present
|
||||
"uncategorized": <int>,
|
||||
"categorized_rate": <float in [0, 1]>,
|
||||
"case_digest": <sha256 of canonical case_details>,
|
||||
}
|
||||
case_details = [
|
||||
{"case_id", "statement", "shape_category", "refusal_reason"},
|
||||
...
|
||||
]
|
||||
```
|
||||
|
||||
The histogram includes every category from `SHAPE_CATEGORY_ORDER` — counts
|
||||
of zero are reported explicitly, not omitted.
|
||||
|
||||
## Replay & determinism
|
||||
|
||||
For a fixed `cases.jsonl` at a fixed commit SHA, `run_lane` returns the
|
||||
same metrics and case_details bit-for-bit. `case_digest` is a sha256 over
|
||||
the canonical-JSON serialization of `case_details` and acts as a single
|
||||
integrity hash for downstream tooling.
|
||||
|
||||
## ADR compatibility
|
||||
|
||||
This lane preserves:
|
||||
|
||||
- ADR-0163 Phase A boundary — measurement only, no corpus mutation
|
||||
- ADR-0114a `wrong = 0` invariant — the lane scores shape, not
|
||||
correctness, so the GSM8K and capability-axis `wrong` counts elsewhere
|
||||
in the repo are unaffected
|
||||
- CLAUDE.md non-negotiables — no hidden normalization, no LLM fallback,
|
||||
deterministic replay
|
||||
50
evals/refusal_taxonomy/public/v1/cases.jsonl
Normal file
50
evals/refusal_taxonomy/public/v1/cases.jsonl
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
{"case_id": "gsm8k-train-sample-v1-0001", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Tina makes $18.00 an hour.'", "statement": "Tina makes $18.00 an hour."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0002", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'She splits it up into 25-foot sections.'", "statement": "She splits it up into 25-foot sections."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0003", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'The student council sells scented erasers in the morning before school starts to help raise money for school dances.'", "statement": "The student council sells scented erasers in the morning before school starts to help raise money for school dances."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0004", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'There are some kids in camp.'", "statement": "There are some kids in camp."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0005", "refusal_reason": "candidate_graph: no admissible candidate for statement: \"In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature.\"", "statement": "In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0006", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Mandy started reading books with only 8 pages when she was 6 years old.'", "statement": "Mandy started reading books with only 8 pages when she was 6 years old."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0007", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Francine has five full boxes of crayons and 5 loose crayons, and her friend has 27 loose crayons.'", "statement": "Francine has five full boxes of crayons and 5 loose crayons, and her friend has 27 loose crayons."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0008", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Marnie makes bead bracelets.'", "statement": "Marnie makes bead bracelets."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0009", "refusal_reason": "candidate_graph: no admissible candidate for question: 'If Jen has 150 ducks, how many total birds does she have?'", "statement": "If Jen has 150 ducks, how many total birds does she have?"}
|
||||
{"case_id": "gsm8k-train-sample-v1-0010", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Yun had 20 paperclips initially, but then lost 12.'", "statement": "Yun had 20 paperclips initially, but then lost 12."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0011", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Alexa has a lemonade stand where she sells lemonade for $2 for one cup.'", "statement": "Alexa has a lemonade stand where she sells lemonade for $2 for one cup."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0012", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'He put all of them in his aquarium but his fish ate half of them.'", "statement": "He put all of them in his aquarium but his fish ate half of them."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0013", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.'", "statement": "Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0014", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Bob can shuck 10 oysters in 5 minutes.'", "statement": "Bob can shuck 10 oysters in 5 minutes."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0015", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours.'", "statement": "Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0016", "refusal_reason": "candidate_graph: no admissible candidate for statement: \"On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs.\"", "statement": "On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0017", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Jason has a carriage house that he rents out.'", "statement": "Jason has a carriage house that he rents out."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0018", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Xavier plays football with his friends.'", "statement": "Xavier plays football with his friends."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0019", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'John adopts a dog from a shelter.'", "statement": "John adopts a dog from a shelter."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0020", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Two puppies, two kittens, and three parakeets were for sale at the pet shop.'", "statement": "Two puppies, two kittens, and three parakeets were for sale at the pet shop."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0021", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'John is lifting weights.'", "statement": "John is lifting weights."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0022", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.'", "statement": "Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0023", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Nicole collected 400 Pokemon cards.'", "statement": "Nicole collected 400 Pokemon cards."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0024", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday, and 50 on Thursday.'", "statement": "Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday, and 50 on Thursday."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0025", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Lilibeth and her friends go strawberry picking.'", "statement": "Lilibeth and her friends go strawberry picking."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0026", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Aaron and his brother Carson each saved up $40 to go to dinner.'", "statement": "Aaron and his brother Carson each saved up $40 to go to dinner."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0027", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Malcolm has 240 followers on Instagram and 500 followers on Facebook.'", "statement": "Malcolm has 240 followers on Instagram and 500 followers on Facebook."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0028", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Tom opens an amusement park.'", "statement": "Tom opens an amusement park."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0029", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Fabian bought a brand new computer mouse and keyboard to be able to work from home.'", "statement": "Fabian bought a brand new computer mouse and keyboard to be able to work from home."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0030", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Jake decides to go to the beach for a fun day.'", "statement": "Jake decides to go to the beach for a fun day."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0031", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Jeremie wants to go to an amusement park with 3 friends at the end of summer.'", "statement": "Jeremie wants to go to an amusement park with 3 friends at the end of summer."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0032", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'John decides to take up illustration.'", "statement": "John decides to take up illustration."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0033", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Rachel is 12 years old, and her grandfather is 7 times her age.'", "statement": "Rachel is 12 years old, and her grandfather is 7 times her age."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0034", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Georgie is a varsity player on a football team.'", "statement": "Georgie is a varsity player on a football team."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0035", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'She decided to split them among her friends.'", "statement": "She decided to split them among her friends."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0036", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Monica way studying for an exam.'", "statement": "Monica way studying for an exam."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0037", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Michael wants to lose 10 pounds by June.'", "statement": "Michael wants to lose 10 pounds by June."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0038", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'In a building, there are a hundred ladies on the first-floor studying.'", "statement": "In a building, there are a hundred ladies on the first-floor studying."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0039", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'At the family reunion, everyone ate too much food and gained weight.'", "statement": "At the family reunion, everyone ate too much food and gained weight."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0040", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Over several years, Daniel has adopted any stray animals he sees on the side of the road.'", "statement": "Over several years, Daniel has adopted any stray animals he sees on the side of the road."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0041", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'The guests eat all of 1 pan, and 75% of the 2nd pan.'", "statement": "The guests eat all of 1 pan, and 75% of the 2nd pan."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0042", "refusal_reason": "candidate_graph: no admissible candidate for question: 'If Ella sells 200 apples, how many apples does Ella has left?'", "statement": "If Ella sells 200 apples, how many apples does Ella has left?"}
|
||||
{"case_id": "gsm8k-train-sample-v1-0043", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Sandra wants to buy some sweets.'", "statement": "Sandra wants to buy some sweets."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0044", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'John invests in a bank and gets 10% simple interest.'", "statement": "John invests in a bank and gets 10% simple interest."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0045", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Bart fills out surveys to earn money.'", "statement": "Bart fills out surveys to earn money."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0046", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'A school has 100 students.'", "statement": "A school has 100 students."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0047", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'John bakes 12 coconut macaroons, each weighing 5 ounces.'", "statement": "John bakes 12 coconut macaroons, each weighing 5 ounces."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0048", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Jed collects stamp cards.'", "statement": "Jed collects stamp cards."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0049", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Malcolm is trying to find the fastest walk to school and is currently comparing two routes.'", "statement": "Malcolm is trying to find the fastest walk to school and is currently comparing two routes."}
|
||||
{"case_id": "gsm8k-train-sample-v1-0050", "refusal_reason": "candidate_graph: no admissible candidate for statement: 'Mark does a gig every other day for 2 weeks.'", "statement": "Mark does a gig every other day for 2 weeks."}
|
||||
141
evals/refusal_taxonomy/runner.py
Normal file
141
evals/refusal_taxonomy/runner.py
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
"""ADR-0163 Phase A — refusal-taxonomy lane runner.
|
||||
|
||||
Read-only. The lane categorises refused statements by *statement shape*
|
||||
and emits a histogram. It never mutates corpora, packs, language packs,
|
||||
proposals, or engine state.
|
||||
|
||||
Per ADR-0163 §Constraint #4 and CLAUDE.md, the categorizer is rules-only:
|
||||
no LLM call, no embedding, no learned classifier, no normalization beyond
|
||||
lowercasing for substring matching.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from evals.refusal_taxonomy.shape_categories import (
|
||||
SHAPE_CATEGORY_ORDER,
|
||||
ShapeCategory,
|
||||
categorize,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CategorizedCase:
|
||||
"""One refused case decorated with its shape category."""
|
||||
|
||||
case_id: str
|
||||
statement: str
|
||||
shape_category: ShapeCategory
|
||||
refusal_reason: str
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"case_id": self.case_id,
|
||||
"statement": self.statement,
|
||||
"shape_category": self.shape_category.value,
|
||||
"refusal_reason": self.refusal_reason,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LaneReport:
|
||||
"""Adapter shape expected by ``evals.framework.run_lane``."""
|
||||
|
||||
metrics: dict[str, Any]
|
||||
case_details: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
def _empty_histogram() -> dict[str, int]:
|
||||
return {category.value: 0 for category in SHAPE_CATEGORY_ORDER}
|
||||
|
||||
|
||||
def _digest(records: list[dict[str, Any]]) -> str:
|
||||
payload = json.dumps(records, ensure_ascii=False, sort_keys=True,
|
||||
separators=(",", ":"))
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def categorize_cases(cases: list[dict[str, Any]]) -> list[CategorizedCase]:
|
||||
"""Pure helper — categorise a list of refused-case dicts."""
|
||||
|
||||
out: list[CategorizedCase] = []
|
||||
for case in cases:
|
||||
if not isinstance(case, dict):
|
||||
raise TypeError("each case must be a dictionary")
|
||||
case_id = str(case.get("case_id", "")).strip()
|
||||
statement = case.get("statement", "")
|
||||
refusal_reason = str(case.get("refusal_reason", "")).strip()
|
||||
if not case_id:
|
||||
raise ValueError("case missing case_id")
|
||||
if not isinstance(statement, str) or not statement.strip():
|
||||
raise ValueError(f"case {case_id!r} has empty statement")
|
||||
out.append(
|
||||
CategorizedCase(
|
||||
case_id=case_id,
|
||||
statement=statement,
|
||||
shape_category=categorize(statement),
|
||||
refusal_reason=refusal_reason,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def build_report(cases: list[dict[str, Any]]) -> LaneReport:
|
||||
"""Build a ``LaneReport`` from a refused-case dict list.
|
||||
|
||||
The report is the lane's full deterministic output: the histogram over
|
||||
all shape categories, the categorized-rate (1 - uncategorized share),
|
||||
and per-case details.
|
||||
"""
|
||||
|
||||
categorized = categorize_cases(cases)
|
||||
histogram = _empty_histogram()
|
||||
for record in categorized:
|
||||
histogram[record.shape_category.value] += 1
|
||||
|
||||
total = len(categorized)
|
||||
uncategorized = histogram[ShapeCategory.UNCATEGORIZED.value]
|
||||
categorized_rate = (
|
||||
(total - uncategorized) / total if total else 0.0
|
||||
)
|
||||
|
||||
case_details = [record.as_dict() for record in categorized]
|
||||
metrics: dict[str, Any] = {
|
||||
"total": total,
|
||||
"by_category": histogram,
|
||||
"uncategorized": uncategorized,
|
||||
"categorized_rate": categorized_rate,
|
||||
"case_digest": _digest(case_details),
|
||||
}
|
||||
return LaneReport(metrics=metrics, case_details=case_details)
|
||||
|
||||
|
||||
def run_lane(
|
||||
cases: list[dict[str, Any]],
|
||||
*,
|
||||
config: Any = None,
|
||||
workers: int | None = None,
|
||||
) -> LaneReport:
|
||||
"""Generic eval-framework entry point.
|
||||
|
||||
``config`` and ``workers`` are accepted for framework compatibility and
|
||||
ignored — the categorizer is pure and synchronous.
|
||||
"""
|
||||
|
||||
del config, workers
|
||||
if not isinstance(cases, list):
|
||||
raise TypeError("cases must be a list of dictionaries")
|
||||
return build_report(cases)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CategorizedCase",
|
||||
"LaneReport",
|
||||
"build_report",
|
||||
"categorize_cases",
|
||||
"run_lane",
|
||||
]
|
||||
406
evals/refusal_taxonomy/shape_categories.py
Normal file
406
evals/refusal_taxonomy/shape_categories.py
Normal file
|
|
@ -0,0 +1,406 @@
|
|||
"""ADR-0163 Phase A — refusal-shape taxonomy.
|
||||
|
||||
Deterministic, rules-only categorizer that maps a refused GSM8K statement
|
||||
to a single ``ShapeCategory``.
|
||||
|
||||
Doctrine (non-negotiable):
|
||||
- No LLM call, no embedding, no learned classifier. The categorizer is a
|
||||
pure function of its input string; the same input always returns the same
|
||||
output, and the function has no side effects, no I/O, no global state.
|
||||
- No hidden normalization. Comparisons are lower-cased and padded with
|
||||
spaces for word-boundary safety; the original statement is never mutated
|
||||
for downstream consumers.
|
||||
- First-match-wins. Priority order is fixed at module level; a statement
|
||||
belongs to exactly one category.
|
||||
- UNCATEGORIZED is a first-class outcome. It is honest measurement, not
|
||||
a failure. Phase A's purpose is to surface the histogram, including the
|
||||
uncategorized tail; the operator selects what Phase B addresses.
|
||||
|
||||
Adding a new category requires ≥ 3 refused statements as evidence, cited
|
||||
inline. ADR-0163 §Risks enforces this.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class ShapeCategory(str, Enum):
|
||||
"""Disjoint shape categories surfaced from the GSM8K train-sample report.
|
||||
|
||||
The nine non-``UNCATEGORIZED`` values are the baseline set named in
|
||||
ADR-0163 §Phase A. Every value listed here has a rule below; if no
|
||||
rule fires, ``UNCATEGORIZED`` is returned.
|
||||
"""
|
||||
|
||||
RATE_WITH_CURRENCY = "rate_with_currency"
|
||||
UNIT_PARTITION = "unit_partition"
|
||||
DESCRIPTIVE_SETUP_NO_QUANTITY = "descriptive_setup_no_quantity"
|
||||
INDEFINITE_QUANTITY = "indefinite_quantity"
|
||||
FRACTIONAL_RATE_OF_CHANGE = "fractional_rate_of_change"
|
||||
COMPARATIVE_WITH_UNIT = "comparative_with_unit"
|
||||
NESTED_QUESTION_TARGET = "nested_question_target"
|
||||
TEMPORAL_AGGREGATION = "temporal_aggregation"
|
||||
CONDITIONAL_QUANTITY = "conditional_quantity"
|
||||
UNCATEGORIZED = "uncategorized"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lexical primitives — small, named sets the rules below compose from.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DIGIT_RE = re.compile(r"\d")
|
||||
|
||||
# Word-form numerals 1..20 plus tens, scale words, common fraction words, and
|
||||
# "dozen". Used only to decide "does the statement contain any quantity at
|
||||
# all". Detecting an exact value is out of scope here.
|
||||
_NUMBER_WORDS: frozenset[str] = frozenset({
|
||||
"one", "two", "three", "four", "five", "six", "seven", "eight", "nine",
|
||||
"ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen",
|
||||
"seventeen", "eighteen", "nineteen", "twenty", "thirty", "forty", "fifty",
|
||||
"sixty", "seventy", "eighty", "ninety",
|
||||
"hundred", "thousand", "million", "billion",
|
||||
"half", "halves", "third", "thirds", "quarter", "quarters",
|
||||
"fourth", "fourths", "fifth", "fifths", "sixth", "sixths",
|
||||
"dozen", "dozens",
|
||||
})
|
||||
|
||||
# Indefinite quantifiers — "amount unspecified" signals.
|
||||
_INDEFINITE_TOKENS: tuple[str, ...] = (
|
||||
" some ", " several ", " a few ", " many ", " any ",
|
||||
)
|
||||
|
||||
# Change-of-state verbs that combine with a fraction/percent to mean
|
||||
# "fractional rate of change". Limited to high-signal lemmas.
|
||||
_CHANGE_VERBS: tuple[str, ...] = (
|
||||
"decrease", "decreased", "decreases", "decreasing",
|
||||
"increase", "increased", "increases", "increasing",
|
||||
"lose", "loses", "lost", "losing",
|
||||
"gain", "gained", "gains", "gaining",
|
||||
"drop", "drops", "dropped", "dropping",
|
||||
"reduce", "reduced", "reduces", "reducing",
|
||||
"eat", "eats", "eaten", "ate", "eating",
|
||||
"shrink", "shrinks", "shrank", "shrunk", "shrinking",
|
||||
"grow", "grows", "grew", "grown", "growing",
|
||||
)
|
||||
|
||||
# Fraction-or-percentage signal. "%" symbol, ``digit/digit`` pattern, or
|
||||
# fraction words.
|
||||
_PERCENT_OR_FRACTION_RE = re.compile(r"%|\b\d+\s*/\s*\d+\b")
|
||||
_FRACTION_WORDS: tuple[str, ...] = (
|
||||
" half ", " halves ", " quarter ", " quarters ",
|
||||
" third ", " thirds ", " fourth ", " fifth ",
|
||||
)
|
||||
|
||||
# Comparative markers. Each is a substring on the padded lower-case form.
|
||||
_COMPARATIVE_TOKENS: tuple[str, ...] = (
|
||||
" more than ", " less than ",
|
||||
" twice as ", " twice the ",
|
||||
" as much as ", " as many as ", " as long as ",
|
||||
" times her ", " times his ", " times their ", " times the ",
|
||||
)
|
||||
# Pattern for "N times" where N is a digit — covers "7 times her age".
|
||||
_DIGIT_TIMES_RE = re.compile(r"\b\d+(?:\.\d+)?\s+times\b")
|
||||
|
||||
# Currency + rate signal. A "$" alone is not a rate; the statement must
|
||||
# also carry a per-unit framing.
|
||||
_CURRENCY_RE = re.compile(r"[\$£€¥]")
|
||||
_RATE_PER_UNIT_TOKENS: tuple[str, ...] = (
|
||||
" per ", " an hour", " a hour", " a day", " a week", " a month", " a year",
|
||||
"/hour", "/day", "/week", "/month", "/year", "/kg", "/lb",
|
||||
" for one ", " for each ", " for a ", " for every ",
|
||||
)
|
||||
|
||||
# Hyphenated unit partition — e.g. "25-foot sections". Pattern stays small.
|
||||
_HYPHENATED_UNIT_RE = re.compile(
|
||||
r"\b\d+(?:\.\d+)?-(?:foot|feet|inch|inches|mile|miles|meter|meters|"
|
||||
r"yard|yards|pound|pounds|ounce|ounces|hour|hours|minute|minutes|"
|
||||
r"second|seconds|gram|grams|kg|liter|liters|gallon|gallons)\b"
|
||||
)
|
||||
# Temporal aggregation cues. Repeated/aggregated time, not incidental "old".
|
||||
_TEMPORAL_TOKENS: tuple[str, ...] = (
|
||||
" each day", " each week", " each month", " each year", " each hour",
|
||||
" each minute", " each second",
|
||||
" every day", " every week", " every month", " every year",
|
||||
" every hour", " every minute", " every other ",
|
||||
" per day", " per week", " per month", " per year", " per hour",
|
||||
" per minute", " per second",
|
||||
" daily", " weekly", " monthly", " yearly", " hourly",
|
||||
)
|
||||
_TEMPORAL_WINDOW_RE = re.compile(
|
||||
r"\b(?:over|for|in|within)\s+\d+(?:\.\d+)?\s+"
|
||||
r"(?:second|seconds|minute|minutes|hour|hours|day|days|week|weeks|"
|
||||
r"month|months|year|years)\b"
|
||||
)
|
||||
_DAY_NAMES: tuple[str, ...] = (
|
||||
" monday", " tuesday", " wednesday", " thursday", " friday",
|
||||
" saturday", " sunday",
|
||||
)
|
||||
|
||||
# Nested question + conditional: "If X, how many/much Y ...?".
|
||||
_NESTED_QUESTION_RE = re.compile(
|
||||
r"^\s*if\b.+?\bhow\s+(?:many|much|long|far|old|tall)\b.*\?\s*$",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
# Bare conditional — "If X, would/will Y" — without a how-target.
|
||||
_CONDITIONAL_RE = re.compile(
|
||||
r"^\s*if\b.+?\b(?:would|will|could|then|gets?|has|have)\b",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers — pure string predicates.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _padded_lower(statement: str) -> str:
|
||||
"""Return a lower-case form padded with spaces for substring word matching.
|
||||
|
||||
Padding lets ``" some "`` reliably miss inside ``"somewhere"``.
|
||||
"""
|
||||
|
||||
return " " + statement.lower().replace("\n", " ") + " "
|
||||
|
||||
|
||||
def _has_any_substring(haystack: str, needles: tuple[str, ...]) -> bool:
|
||||
return any(needle in haystack for needle in needles)
|
||||
|
||||
|
||||
def _has_number_word(padded_lower: str) -> bool:
|
||||
# Walk word boundaries cheaply by splitting on whitespace and stripping
|
||||
# surrounding punctuation. This stays string-only.
|
||||
for raw_token in padded_lower.split():
|
||||
token = raw_token.strip(".,;:!?\"'()[]{}").lower()
|
||||
if token in _NUMBER_WORDS:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _has_any_quantity_marker(statement: str, padded_lower: str) -> bool:
|
||||
if _DIGIT_RE.search(statement):
|
||||
return True
|
||||
if _has_number_word(padded_lower):
|
||||
return True
|
||||
if _has_any_substring(padded_lower, _INDEFINITE_TOKENS):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rule predicates — one per category. Each is independent and pure.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _is_nested_question_target(statement: str) -> bool:
|
||||
"""Conditional question whose target is a ``how`` quantity.
|
||||
|
||||
Baseline category — ADR-0163 §Phase A names this shape explicitly.
|
||||
The 50-case sample carries two instances (case 0009, case 0042); the
|
||||
rule is retained as a reserve for the public/holdout splits where the
|
||||
shape is expected to be more frequent. Alternative chosen over
|
||||
splitting into separate "question" vs "conditional" categories: every
|
||||
observed nested-question case in the 50-sample is also conditional,
|
||||
so collapsing them is honest.
|
||||
"""
|
||||
|
||||
return _NESTED_QUESTION_RE.match(statement) is not None
|
||||
|
||||
|
||||
def _is_conditional_quantity(statement: str) -> bool:
|
||||
"""Conditional setup without a nested ``how``-target.
|
||||
|
||||
Examples surfaced as candidates:
|
||||
- case 0009 (also matches nested) — handled by ordering
|
||||
- case 0042 (also matches nested) — handled by ordering
|
||||
- reserve slot: ADR-0163 §Phase A names this category explicitly with
|
||||
"if she had 2 more, she would have…" as the template. No bare
|
||||
conditional appears in the 50-sample (every conditional was also
|
||||
a nested question), so this rule is conservative and may return
|
||||
``False`` for the entire current corpus. That is the honest
|
||||
outcome; the category is retained because the ADR commits to it.
|
||||
"""
|
||||
|
||||
return _CONDITIONAL_RE.match(statement) is not None
|
||||
|
||||
|
||||
def _is_unit_partition(padded_lower: str) -> bool:
|
||||
"""Partition into unit-bearing pieces.
|
||||
|
||||
Baseline category — ADR-0163 §Phase A names this shape explicitly
|
||||
("She splits it up into 25-foot sections"). The 50-case sample
|
||||
carries one instance (case 0002); the rule is retained as a reserve
|
||||
for the public/holdout splits where partition-into-unit-pieces is
|
||||
expected to be more frequent. Alternative considered: also fold any
|
||||
"split/divide/cut ... into N pieces" without a unit. Rejected —
|
||||
that shape is "enumeration", not "unit partition", and Phase B's
|
||||
exemplars should treat them separately.
|
||||
"""
|
||||
|
||||
if _HYPHENATED_UNIT_RE.search(padded_lower):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_rate_with_currency(padded_lower: str) -> bool:
|
||||
"""Currency + per-unit framing.
|
||||
|
||||
Examples (≥ 3 cites):
|
||||
- case 0001 "Tina makes $18.00 an hour."
|
||||
- case 0011 "Alexa ... sells lemonade for $2 for one cup."
|
||||
- case 0022 "earning $20 per kg of fish."
|
||||
"""
|
||||
|
||||
if not _CURRENCY_RE.search(padded_lower):
|
||||
return False
|
||||
return _has_any_substring(padded_lower, _RATE_PER_UNIT_TOKENS)
|
||||
|
||||
|
||||
def _is_comparative_with_unit(padded_lower: str) -> bool:
|
||||
"""Comparative quantity phrases.
|
||||
|
||||
Examples (≥ 3 cites):
|
||||
- case 0015 "rides for twice as much time as the subway ride"
|
||||
- case 0016 "2 more than 5 miles ... 3 less than 17 stop signs"
|
||||
- case 0033 "her grandfather is 7 times her age"
|
||||
"""
|
||||
|
||||
if _has_any_substring(padded_lower, _COMPARATIVE_TOKENS):
|
||||
return True
|
||||
if _DIGIT_TIMES_RE.search(padded_lower):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_fractional_rate_of_change(padded_lower: str) -> bool:
|
||||
"""Fraction or percentage paired with a change-of-state verb.
|
||||
|
||||
Examples (≥ 3 cites):
|
||||
- case 0005 "decrease to 3/4 of its temperature"
|
||||
- case 0012 "his fish ate half of them"
|
||||
- case 0041 "guests eat all of 1 pan, and 75% of the 2nd pan"
|
||||
Alternative considered: include bare "10% simple interest" (case 0044).
|
||||
Rejected — there is no change verb, so the shape is a percentage-rate,
|
||||
not a percentage-change. That case falls to UNCATEGORIZED honestly.
|
||||
"""
|
||||
|
||||
has_fraction = bool(_PERCENT_OR_FRACTION_RE.search(padded_lower)) or \
|
||||
_has_any_substring(padded_lower, _FRACTION_WORDS)
|
||||
if not has_fraction:
|
||||
return False
|
||||
return any(f" {verb} " in padded_lower for verb in _CHANGE_VERBS)
|
||||
|
||||
|
||||
def _is_temporal_aggregation(padded_lower: str) -> bool:
|
||||
"""Repeated / aggregated time framing with a quantity.
|
||||
|
||||
Examples (≥ 3 cites):
|
||||
- case 0013 "uploads 10 one-hour videos ... each day"
|
||||
- case 0014 "Bob can shuck 10 oysters in 5 minutes"
|
||||
- case 0024 "20 jumping jacks on Monday, 36 on Tuesday ..."
|
||||
- case 0050 "every other day for 2 weeks"
|
||||
Alternative considered: split "in N minutes" (rate) from "each day"
|
||||
(aggregation). Rejected for parsimony — both demand the same shape
|
||||
treatment from the candidate-graph (event repetition over a time axis).
|
||||
"""
|
||||
|
||||
if _has_any_substring(padded_lower, _TEMPORAL_TOKENS):
|
||||
return True
|
||||
if _TEMPORAL_WINDOW_RE.search(padded_lower):
|
||||
return True
|
||||
# Day-of-week enumeration: require at least two day names to avoid
|
||||
# incidental "on Monday" one-offs.
|
||||
day_hits = sum(1 for d in _DAY_NAMES if d in padded_lower)
|
||||
if day_hits >= 2:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_indefinite_quantity(padded_lower: str) -> bool:
|
||||
"""Indefinite quantifier present without a specific count.
|
||||
|
||||
Examples (≥ 3 cites):
|
||||
- case 0004 "There are some kids in camp."
|
||||
- case 0040 "Over several years, Daniel has adopted any stray animals"
|
||||
- case 0043 "Sandra wants to buy some sweets."
|
||||
Alternative considered: route case 0040 to TEMPORAL_AGGREGATION because
|
||||
of "Over several years". Rejected — the load-bearing shape problem is
|
||||
the unbounded quantifier ("any stray animals"), not the time window.
|
||||
Ordering reflects that.
|
||||
"""
|
||||
|
||||
return _has_any_substring(padded_lower, _INDEFINITE_TOKENS)
|
||||
|
||||
|
||||
def _is_descriptive_setup_no_quantity(statement: str, padded_lower: str) -> bool:
|
||||
"""Pure context with no extractable measurement.
|
||||
|
||||
Examples (≥ 3 cites):
|
||||
- case 0003 "The student council sells scented erasers in the morning ..."
|
||||
- case 0008 "Marnie makes bead bracelets."
|
||||
- case 0019 "John adopts a dog from a shelter."
|
||||
"""
|
||||
|
||||
return not _has_any_quantity_marker(statement, padded_lower)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatch — fixed priority order.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def categorize(statement: str) -> ShapeCategory:
|
||||
"""Map a refused statement to exactly one ``ShapeCategory``.
|
||||
|
||||
Deterministic and side-effect free. Returns ``UNCATEGORIZED`` when no
|
||||
rule fires; UNCATEGORIZED is the honest outcome, not a failure.
|
||||
"""
|
||||
|
||||
if not isinstance(statement, str):
|
||||
raise TypeError("statement must be a string")
|
||||
|
||||
padded_lower = _padded_lower(statement)
|
||||
|
||||
if _is_nested_question_target(statement):
|
||||
return ShapeCategory.NESTED_QUESTION_TARGET
|
||||
if _is_unit_partition(padded_lower):
|
||||
return ShapeCategory.UNIT_PARTITION
|
||||
if _is_rate_with_currency(padded_lower):
|
||||
return ShapeCategory.RATE_WITH_CURRENCY
|
||||
if _is_comparative_with_unit(padded_lower):
|
||||
return ShapeCategory.COMPARATIVE_WITH_UNIT
|
||||
if _is_fractional_rate_of_change(padded_lower):
|
||||
return ShapeCategory.FRACTIONAL_RATE_OF_CHANGE
|
||||
if _is_indefinite_quantity(padded_lower):
|
||||
return ShapeCategory.INDEFINITE_QUANTITY
|
||||
if _is_temporal_aggregation(padded_lower):
|
||||
return ShapeCategory.TEMPORAL_AGGREGATION
|
||||
if _is_conditional_quantity(statement):
|
||||
return ShapeCategory.CONDITIONAL_QUANTITY
|
||||
if _is_descriptive_setup_no_quantity(statement, padded_lower):
|
||||
return ShapeCategory.DESCRIPTIVE_SETUP_NO_QUANTITY
|
||||
return ShapeCategory.UNCATEGORIZED
|
||||
|
||||
|
||||
# Ordered enum tuple — exposed for tests asserting exhaustive coverage and
|
||||
# for runners that need a stable iteration order.
|
||||
SHAPE_CATEGORY_ORDER: tuple[ShapeCategory, ...] = (
|
||||
ShapeCategory.NESTED_QUESTION_TARGET,
|
||||
ShapeCategory.UNIT_PARTITION,
|
||||
ShapeCategory.RATE_WITH_CURRENCY,
|
||||
ShapeCategory.COMPARATIVE_WITH_UNIT,
|
||||
ShapeCategory.FRACTIONAL_RATE_OF_CHANGE,
|
||||
ShapeCategory.INDEFINITE_QUANTITY,
|
||||
ShapeCategory.TEMPORAL_AGGREGATION,
|
||||
ShapeCategory.CONDITIONAL_QUANTITY,
|
||||
ShapeCategory.DESCRIPTIVE_SETUP_NO_QUANTITY,
|
||||
ShapeCategory.UNCATEGORIZED,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ShapeCategory",
|
||||
"SHAPE_CATEGORY_ORDER",
|
||||
"categorize",
|
||||
]
|
||||
326
evals/refusal_taxonomy/v1/report.json
Normal file
326
evals/refusal_taxonomy/v1/report.json
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
{
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0001",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Tina makes $18.00 an hour.'",
|
||||
"shape_category": "rate_with_currency",
|
||||
"statement": "Tina makes $18.00 an hour."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0002",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'She splits it up into 25-foot sections.'",
|
||||
"shape_category": "unit_partition",
|
||||
"statement": "She splits it up into 25-foot sections."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0003",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'The student council sells scented erasers in the morning before school starts to help raise money for school dances.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "The student council sells scented erasers in the morning before school starts to help raise money for school dances."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0004",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'There are some kids in camp.'",
|
||||
"shape_category": "indefinite_quantity",
|
||||
"statement": "There are some kids in camp."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0005",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: \"In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature.\"",
|
||||
"shape_category": "fractional_rate_of_change",
|
||||
"statement": "In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0006",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Mandy started reading books with only 8 pages when she was 6 years old.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "Mandy started reading books with only 8 pages when she was 6 years old."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0007",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Francine has five full boxes of crayons and 5 loose crayons, and her friend has 27 loose crayons.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "Francine has five full boxes of crayons and 5 loose crayons, and her friend has 27 loose crayons."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0008",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Marnie makes bead bracelets.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "Marnie makes bead bracelets."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0009",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for question: 'If Jen has 150 ducks, how many total birds does she have?'",
|
||||
"shape_category": "nested_question_target",
|
||||
"statement": "If Jen has 150 ducks, how many total birds does she have?"
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0010",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Yun had 20 paperclips initially, but then lost 12.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "Yun had 20 paperclips initially, but then lost 12."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0011",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Alexa has a lemonade stand where she sells lemonade for $2 for one cup.'",
|
||||
"shape_category": "rate_with_currency",
|
||||
"statement": "Alexa has a lemonade stand where she sells lemonade for $2 for one cup."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0012",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'He put all of them in his aquarium but his fish ate half of them.'",
|
||||
"shape_category": "fractional_rate_of_change",
|
||||
"statement": "He put all of them in his aquarium but his fish ate half of them."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0013",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.'",
|
||||
"shape_category": "temporal_aggregation",
|
||||
"statement": "Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0014",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Bob can shuck 10 oysters in 5 minutes.'",
|
||||
"shape_category": "temporal_aggregation",
|
||||
"statement": "Bob can shuck 10 oysters in 5 minutes."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0015",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours.'",
|
||||
"shape_category": "comparative_with_unit",
|
||||
"statement": "Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0016",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: \"On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs.\"",
|
||||
"shape_category": "comparative_with_unit",
|
||||
"statement": "On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0017",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Jason has a carriage house that he rents out.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "Jason has a carriage house that he rents out."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0018",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Xavier plays football with his friends.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "Xavier plays football with his friends."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0019",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'John adopts a dog from a shelter.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "John adopts a dog from a shelter."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0020",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Two puppies, two kittens, and three parakeets were for sale at the pet shop.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "Two puppies, two kittens, and three parakeets were for sale at the pet shop."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0021",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'John is lifting weights.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "John is lifting weights."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0022",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.'",
|
||||
"shape_category": "rate_with_currency",
|
||||
"statement": "Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0023",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Nicole collected 400 Pokemon cards.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "Nicole collected 400 Pokemon cards."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0024",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday, and 50 on Thursday.'",
|
||||
"shape_category": "temporal_aggregation",
|
||||
"statement": "Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday, and 50 on Thursday."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0025",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Lilibeth and her friends go strawberry picking.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "Lilibeth and her friends go strawberry picking."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0026",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Aaron and his brother Carson each saved up $40 to go to dinner.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "Aaron and his brother Carson each saved up $40 to go to dinner."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0027",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Malcolm has 240 followers on Instagram and 500 followers on Facebook.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "Malcolm has 240 followers on Instagram and 500 followers on Facebook."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0028",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Tom opens an amusement park.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "Tom opens an amusement park."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0029",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Fabian bought a brand new computer mouse and keyboard to be able to work from home.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "Fabian bought a brand new computer mouse and keyboard to be able to work from home."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0030",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Jake decides to go to the beach for a fun day.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "Jake decides to go to the beach for a fun day."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0031",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Jeremie wants to go to an amusement park with 3 friends at the end of summer.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "Jeremie wants to go to an amusement park with 3 friends at the end of summer."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0032",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'John decides to take up illustration.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "John decides to take up illustration."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0033",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Rachel is 12 years old, and her grandfather is 7 times her age.'",
|
||||
"shape_category": "comparative_with_unit",
|
||||
"statement": "Rachel is 12 years old, and her grandfather is 7 times her age."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0034",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Georgie is a varsity player on a football team.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "Georgie is a varsity player on a football team."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0035",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'She decided to split them among her friends.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "She decided to split them among her friends."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0036",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Monica way studying for an exam.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "Monica way studying for an exam."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0037",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Michael wants to lose 10 pounds by June.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "Michael wants to lose 10 pounds by June."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0038",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'In a building, there are a hundred ladies on the first-floor studying.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "In a building, there are a hundred ladies on the first-floor studying."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0039",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'At the family reunion, everyone ate too much food and gained weight.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "At the family reunion, everyone ate too much food and gained weight."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0040",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Over several years, Daniel has adopted any stray animals he sees on the side of the road.'",
|
||||
"shape_category": "indefinite_quantity",
|
||||
"statement": "Over several years, Daniel has adopted any stray animals he sees on the side of the road."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0041",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'The guests eat all of 1 pan, and 75% of the 2nd pan.'",
|
||||
"shape_category": "fractional_rate_of_change",
|
||||
"statement": "The guests eat all of 1 pan, and 75% of the 2nd pan."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0042",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for question: 'If Ella sells 200 apples, how many apples does Ella has left?'",
|
||||
"shape_category": "nested_question_target",
|
||||
"statement": "If Ella sells 200 apples, how many apples does Ella has left?"
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0043",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Sandra wants to buy some sweets.'",
|
||||
"shape_category": "indefinite_quantity",
|
||||
"statement": "Sandra wants to buy some sweets."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0044",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'John invests in a bank and gets 10% simple interest.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "John invests in a bank and gets 10% simple interest."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0045",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Bart fills out surveys to earn money.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "Bart fills out surveys to earn money."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0046",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'A school has 100 students.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "A school has 100 students."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0047",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'John bakes 12 coconut macaroons, each weighing 5 ounces.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "John bakes 12 coconut macaroons, each weighing 5 ounces."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0048",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Jed collects stamp cards.'",
|
||||
"shape_category": "descriptive_setup_no_quantity",
|
||||
"statement": "Jed collects stamp cards."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0049",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Malcolm is trying to find the fastest walk to school and is currently comparing two routes.'",
|
||||
"shape_category": "uncategorized",
|
||||
"statement": "Malcolm is trying to find the fastest walk to school and is currently comparing two routes."
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0050",
|
||||
"refusal_reason": "candidate_graph: no admissible candidate for statement: 'Mark does a gig every other day for 2 weeks.'",
|
||||
"shape_category": "temporal_aggregation",
|
||||
"statement": "Mark does a gig every other day for 2 weeks."
|
||||
}
|
||||
],
|
||||
"lane": "refusal_taxonomy",
|
||||
"metrics": {
|
||||
"by_category": {
|
||||
"comparative_with_unit": 3,
|
||||
"conditional_quantity": 0,
|
||||
"descriptive_setup_no_quantity": 17,
|
||||
"fractional_rate_of_change": 3,
|
||||
"indefinite_quantity": 3,
|
||||
"nested_question_target": 2,
|
||||
"rate_with_currency": 3,
|
||||
"temporal_aggregation": 4,
|
||||
"uncategorized": 14,
|
||||
"unit_partition": 1
|
||||
},
|
||||
"case_digest": "d030f826cb0f4088771d90c52c8be2ff75054ab27c7d47eae8dbfe1225b2eea1",
|
||||
"categorized_rate": 0.72,
|
||||
"total": 50,
|
||||
"uncategorized": 14
|
||||
},
|
||||
"source_cases": "/Users/kaizenpro/Projects/core-adr-0163-A-taxonomy/evals/refusal_taxonomy/public/v1/cases.jsonl",
|
||||
"split": "public",
|
||||
"version": "v1"
|
||||
}
|
||||
96
scripts/build_refusal_taxonomy_cases.py
Normal file
96
scripts/build_refusal_taxonomy_cases.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
"""ADR-0163 Phase A — build the refusal_taxonomy v1 case set.
|
||||
|
||||
Reads ``evals/gsm8k_math/train_sample/v1/report.json`` and emits one JSONL
|
||||
record per refused case, extracting the embedded statement out of the
|
||||
``refusal_reason`` string so the lane operates on the statement itself
|
||||
(not on the reason envelope).
|
||||
|
||||
The output is deterministic: cases are sorted by ``case_id`` and serialized
|
||||
with ``sort_keys=True`` and ``ensure_ascii=False``.
|
||||
|
||||
Usage::
|
||||
|
||||
uv run python scripts/build_refusal_taxonomy_cases.py \\
|
||||
--report evals/gsm8k_math/train_sample/v1/report.json \\
|
||||
--out evals/refusal_taxonomy/v1/cases.jsonl
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_STATEMENT_RE = re.compile(
|
||||
r"^candidate_graph:\s*no admissible candidate for\s+"
|
||||
r"(?:statement|question):\s*['\"](.+)['\"]\s*$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
def extract_statement(reason: str) -> str | None:
|
||||
"""Pull the embedded statement out of a refusal reason.
|
||||
|
||||
Returns ``None`` if the reason does not match the expected envelope.
|
||||
"""
|
||||
|
||||
match = _STATEMENT_RE.match(reason.strip())
|
||||
if not match:
|
||||
return None
|
||||
return match.group(1).strip()
|
||||
|
||||
|
||||
def build_cases(report_path: Path) -> list[dict[str, str]]:
|
||||
payload = json.loads(report_path.read_text())
|
||||
per_case = payload.get("per_case", [])
|
||||
out: list[dict[str, str]] = []
|
||||
for case in per_case:
|
||||
if case.get("verdict") != "refused":
|
||||
continue
|
||||
reason = case.get("reason", "")
|
||||
statement = extract_statement(reason)
|
||||
if statement is None:
|
||||
continue
|
||||
out.append({
|
||||
"case_id": case["case_id"],
|
||||
"statement": statement,
|
||||
"refusal_reason": reason,
|
||||
})
|
||||
out.sort(key=lambda r: r["case_id"])
|
||||
return out
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--report",
|
||||
type=Path,
|
||||
default=Path("evals/gsm8k_math/train_sample/v1/report.json"),
|
||||
help="path to a GSM8K eval report containing refused cases",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--out",
|
||||
type=Path,
|
||||
default=Path("evals/refusal_taxonomy/public/v1/cases.jsonl"),
|
||||
help="output JSONL path",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if not args.report.exists():
|
||||
parser.error(f"report not found: {args.report}")
|
||||
|
||||
cases = build_cases(args.report)
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.out.open("w", encoding="utf-8") as handle:
|
||||
for record in cases:
|
||||
handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True))
|
||||
handle.write("\n")
|
||||
|
||||
print(f"wrote {len(cases)} cases to {args.out}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
327
tests/test_refusal_taxonomy_lane.py
Normal file
327
tests/test_refusal_taxonomy_lane.py
Normal file
|
|
@ -0,0 +1,327 @@
|
|||
"""ADR-0163 Phase A — refusal-taxonomy lane tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from evals.framework import discover_lanes, get_lane, load_cases, run_lane
|
||||
from evals.refusal_taxonomy.runner import build_report, categorize_cases
|
||||
from evals.refusal_taxonomy.shape_categories import (
|
||||
SHAPE_CATEGORY_ORDER,
|
||||
ShapeCategory,
|
||||
categorize,
|
||||
)
|
||||
from scripts.build_refusal_taxonomy_cases import (
|
||||
build_cases,
|
||||
extract_statement,
|
||||
)
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
_LANE_ROOT = _REPO_ROOT / "evals" / "refusal_taxonomy"
|
||||
_CASES_PATH = _LANE_ROOT / "public" / "v1" / "cases.jsonl"
|
||||
_REPORT_PATH = _LANE_ROOT / "v1" / "report.json"
|
||||
_GSM8K_REPORT = (
|
||||
_REPO_ROOT / "evals" / "gsm8k_math" / "train_sample" / "v1" / "report.json"
|
||||
)
|
||||
_SHAPE_CATEGORIES_PATH = _LANE_ROOT / "shape_categories.py"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Case-set integrity
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cases_file_exists_and_nonempty():
|
||||
assert _CASES_PATH.exists(), f"expected case set at {_CASES_PATH}"
|
||||
cases = load_cases(_CASES_PATH)
|
||||
assert len(cases) == 50, "v1 sample should mirror the 50-case GSM8K train sample"
|
||||
|
||||
|
||||
def test_case_schema_valid():
|
||||
cases = load_cases(_CASES_PATH)
|
||||
for case in cases:
|
||||
assert set(case.keys()) >= {"case_id", "statement", "refusal_reason"}
|
||||
assert isinstance(case["case_id"], str) and case["case_id"].strip()
|
||||
assert isinstance(case["statement"], str) and case["statement"].strip()
|
||||
assert isinstance(case["refusal_reason"], str) and case["refusal_reason"].strip()
|
||||
|
||||
|
||||
def test_cases_sorted_by_id():
|
||||
cases = load_cases(_CASES_PATH)
|
||||
ids = [c["case_id"] for c in cases]
|
||||
assert ids == sorted(ids), "cases.jsonl must be deterministically sorted"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lane discovery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_lane_auto_discoverable():
|
||||
names = [lane.name for lane in discover_lanes()]
|
||||
assert "refusal_taxonomy" in names
|
||||
|
||||
|
||||
def test_lane_run_via_framework():
|
||||
lane = get_lane("refusal_taxonomy")
|
||||
result = run_lane(lane, version="v1", split="public")
|
||||
assert result.metrics["total"] == 50
|
||||
assert result.metrics["categorized_rate"] == pytest.approx(0.72)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Enum coverage + exhaustiveness
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_shape_category_order_covers_enum():
|
||||
assert set(SHAPE_CATEGORY_ORDER) == set(ShapeCategory)
|
||||
assert len(SHAPE_CATEGORY_ORDER) == len(ShapeCategory)
|
||||
|
||||
|
||||
def test_every_category_value_reachable_by_a_rule():
|
||||
# For each non-UNCATEGORIZED category, provide a tiny synthetic
|
||||
# statement that the categorizer routes to it. If a future change
|
||||
# eliminates a rule, this test fails loudly.
|
||||
probes: dict[ShapeCategory, str] = {
|
||||
ShapeCategory.NESTED_QUESTION_TARGET:
|
||||
"If Jen has 150 ducks, how many total birds does she have?",
|
||||
ShapeCategory.UNIT_PARTITION:
|
||||
"She splits it up into 25-foot sections.",
|
||||
ShapeCategory.RATE_WITH_CURRENCY: "Tina makes $18.00 an hour.",
|
||||
ShapeCategory.COMPARATIVE_WITH_UNIT:
|
||||
"Her grandfather is 7 times her age.",
|
||||
ShapeCategory.FRACTIONAL_RATE_OF_CHANGE:
|
||||
"His fish ate half of them.",
|
||||
ShapeCategory.INDEFINITE_QUANTITY: "There are some kids in camp.",
|
||||
ShapeCategory.TEMPORAL_AGGREGATION:
|
||||
"Mark does a gig every other day for 2 weeks.",
|
||||
ShapeCategory.CONDITIONAL_QUANTITY:
|
||||
"If she had two more, she would have plenty.",
|
||||
ShapeCategory.DESCRIPTIVE_SETUP_NO_QUANTITY:
|
||||
"Marnie makes bead bracelets.",
|
||||
ShapeCategory.UNCATEGORIZED:
|
||||
"Nicole collected 400 Pokemon cards.",
|
||||
}
|
||||
for category, probe in probes.items():
|
||||
assert categorize(probe) is category, (
|
||||
f"probe for {category.value!r} routed to {categorize(probe).value!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_added_category_cites_three_examples():
|
||||
"""Enforce ADR-0163 §Risks: every category cites ≥ 3 refused statements
|
||||
OR is documented in its docstring as a reserve slot with rationale.
|
||||
|
||||
This test parses the shape_categories.py source and verifies that each
|
||||
rule predicate docstring contains either three "case " citations or the
|
||||
word "reserve".
|
||||
"""
|
||||
|
||||
source = _SHAPE_CATEGORIES_PATH.read_text()
|
||||
tree = ast.parse(source)
|
||||
rule_funcs = [
|
||||
node for node in ast.walk(tree)
|
||||
if isinstance(node, ast.FunctionDef)
|
||||
and node.name.startswith("_is_")
|
||||
]
|
||||
assert rule_funcs, "expected rule predicates named _is_*"
|
||||
case_cite_re = re.compile(r"case\s+\d{4}", re.IGNORECASE)
|
||||
for func in rule_funcs:
|
||||
doc = ast.get_docstring(func) or ""
|
||||
cites = len(case_cite_re.findall(doc))
|
||||
if cites < 3 and "reserve" not in doc.lower():
|
||||
pytest.fail(
|
||||
f"{func.name}: needs ≥ 3 case citations or a 'reserve' note "
|
||||
f"(found {cites} citations)"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Categorizer determinism + purity
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_categorizer_is_deterministic():
|
||||
cases = load_cases(_CASES_PATH)
|
||||
runs = [
|
||||
[categorize(c["statement"]).value for c in cases]
|
||||
for _ in range(5)
|
||||
]
|
||||
assert all(run == runs[0] for run in runs)
|
||||
|
||||
|
||||
def test_categorizer_is_pure_no_io(tmp_path, monkeypatch):
|
||||
# Sentinel — fail if the categorizer touches the filesystem or os.environ.
|
||||
def fail_open(*_a, **_kw):
|
||||
raise AssertionError("categorize() must not perform I/O")
|
||||
monkeypatch.setattr("builtins.open", fail_open)
|
||||
monkeypatch.setattr("os.environ", {})
|
||||
# Drive a handful of probes; any I/O attempt explodes the test.
|
||||
for statement in (
|
||||
"Tina makes $18.00 an hour.",
|
||||
"There are some kids in camp.",
|
||||
"She splits it up into 25-foot sections.",
|
||||
"If Jen has 150 ducks, how many total birds does she have?",
|
||||
):
|
||||
categorize(statement)
|
||||
|
||||
|
||||
def test_categorizer_rejects_non_string():
|
||||
with pytest.raises(TypeError):
|
||||
categorize(None) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_categorizer_no_llm_or_ml_imports():
|
||||
"""Per ADR-0163 §Constraint #4: no LLM call, no embedding, no learned model."""
|
||||
|
||||
source = _SHAPE_CATEGORIES_PATH.read_text()
|
||||
banned = (
|
||||
"openai", "anthropic", "huggingface", "transformers", "torch",
|
||||
"tensorflow", "sklearn", "numpy", "sentence_transformers",
|
||||
"requests", "httpx", "urllib",
|
||||
)
|
||||
for token in banned:
|
||||
assert token not in source.lower(), (
|
||||
f"shape_categories.py must not reference {token!r} — "
|
||||
"rules-only doctrine"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Histogram correctness
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_histogram_synthetic_fixture():
|
||||
cases = [
|
||||
{"case_id": "x1", "statement": "There are some kids in camp.",
|
||||
"refusal_reason": "r"},
|
||||
{"case_id": "x2", "statement": "Marnie makes bead bracelets.",
|
||||
"refusal_reason": "r"},
|
||||
{"case_id": "x3", "statement": "Tina makes $18.00 an hour.",
|
||||
"refusal_reason": "r"},
|
||||
]
|
||||
report = build_report(cases)
|
||||
assert report.metrics["total"] == 3
|
||||
assert report.metrics["by_category"]["indefinite_quantity"] == 1
|
||||
assert report.metrics["by_category"]["descriptive_setup_no_quantity"] == 1
|
||||
assert report.metrics["by_category"]["rate_with_currency"] == 1
|
||||
assert report.metrics["uncategorized"] == 0
|
||||
assert report.metrics["categorized_rate"] == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_histogram_includes_all_categories_even_when_zero():
|
||||
cases = [
|
||||
{"case_id": "x1", "statement": "Marnie makes bead bracelets.",
|
||||
"refusal_reason": "r"},
|
||||
]
|
||||
report = build_report(cases)
|
||||
keys = set(report.metrics["by_category"].keys())
|
||||
assert keys == {c.value for c in SHAPE_CATEGORY_ORDER}
|
||||
|
||||
|
||||
def test_v1_report_uncategorized_under_fifty_percent():
|
||||
payload = json.loads(_REPORT_PATH.read_text())
|
||||
rate = payload["metrics"]["categorized_rate"]
|
||||
assert rate >= 0.5, (
|
||||
f"<50% categorized signals taxonomy/data mismatch; got {rate:.2%}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Replay determinism
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_report_replays_byte_identical():
|
||||
cases = load_cases(_CASES_PATH)
|
||||
r1 = build_report(cases)
|
||||
r2 = build_report(cases)
|
||||
assert r1.metrics == r2.metrics
|
||||
assert r1.case_details == r2.case_details
|
||||
|
||||
|
||||
def test_committed_report_matches_categorizer():
|
||||
cases = load_cases(_CASES_PATH)
|
||||
fresh = build_report(cases)
|
||||
committed = json.loads(_REPORT_PATH.read_text())
|
||||
assert fresh.metrics["case_digest"] == committed["metrics"]["case_digest"]
|
||||
assert fresh.metrics["by_category"] == committed["metrics"]["by_category"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper script
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_helper_extracts_statement_from_reason():
|
||||
reason = (
|
||||
"candidate_graph: no admissible candidate for statement: "
|
||||
"'Tina makes $18.00 an hour.'"
|
||||
)
|
||||
assert extract_statement(reason) == "Tina makes $18.00 an hour."
|
||||
|
||||
|
||||
def test_helper_handles_question_envelope():
|
||||
reason = (
|
||||
"candidate_graph: no admissible candidate for question: "
|
||||
"'If Jen has 150 ducks, how many total birds does she have?'"
|
||||
)
|
||||
assert extract_statement(reason) == (
|
||||
"If Jen has 150 ducks, how many total birds does she have?"
|
||||
)
|
||||
|
||||
|
||||
def test_helper_rebuilds_cases_matching_committed():
|
||||
rebuilt = build_cases(_GSM8K_REPORT)
|
||||
committed = load_cases(_CASES_PATH)
|
||||
assert rebuilt == committed
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Read-only invariant
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _tree_digest(root: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
if not root.exists():
|
||||
return "absent"
|
||||
for path in sorted(p for p in root.rglob("*") if p.is_file()):
|
||||
h.update(str(path.relative_to(root)).encode("utf-8"))
|
||||
h.update(b"\0")
|
||||
h.update(path.read_bytes())
|
||||
h.update(b"\0")
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def test_lane_run_does_not_mutate_protected_trees():
|
||||
teaching = _REPO_ROOT / "teaching"
|
||||
packs = _REPO_ROOT / "packs"
|
||||
lp_data = _REPO_ROOT / "language_packs" / "data"
|
||||
|
||||
before = (
|
||||
_tree_digest(teaching),
|
||||
_tree_digest(packs),
|
||||
_tree_digest(lp_data),
|
||||
)
|
||||
|
||||
cases = load_cases(_CASES_PATH)
|
||||
build_report(cases)
|
||||
|
||||
after = (
|
||||
_tree_digest(teaching),
|
||||
_tree_digest(packs),
|
||||
_tree_digest(lp_data),
|
||||
)
|
||||
assert before == after, (
|
||||
"refusal-taxonomy lane must be read-only over teaching/, packs/, "
|
||||
"and language_packs/data/"
|
||||
)
|
||||
Loading…
Reference in a new issue