From c69f99485d307bfc9ee8b11a551eb67910e5e9c5 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 27 Jul 2026 14:11:07 -0700 Subject: [PATCH] feat(evals): measure the reader/writer construction inventories against each other MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 5 item 1 asked for the overlap to be sized "by measuring the reader's construction set against the writer's rather than by growing corpora blindly". New lane `evals/construction_inventory` does that: it sweeps the writer's whole parameter space (every RhetoricalMove x IntentTag x predicate x quantifier x tense x aspect, both public entry points, 32292 cells), quotients it by the reader's OWN function-word skeleton, and comprehends each construction under three vocabularies. writer constructions 1739 reader constructions 19 (mint-site AST-guarded) overlap, faithful 6 accepts but MIS-READS 22 refuses 1711 vocabulary-dependent 0 Faithfulness, not acceptance, is the criterion — and that reverses two claims in the plan's §6 RESULT, using a metric that was already on the page: * g_read_rate is 1/293 but **g_args_rate is 0.0**. The one surface that "reads", `all molecules are defined as compounds`, is comprehended as `subset(molecule, defined_as_compound)` — the reader chunked the writer's verb phrase into a class name. It accepted; it did not comprehend. * "one construction wide" was wrong both ways: zero on that corpus, six over the writer's actual output space. A corpus cannot report an inventory's size. Three findings re-order Phase 5 item 1: 1. The reader FABRICATES on 22 constructions — neither reads nor refuses. `every dog is a mammal` -> member(every_dog, mammal); `furthermore, all dogs are mammals` -> asserted(furthermore). Both are ordinary user English, not writer artefacts. Root cause: _RESERVED lacks the function words the writer emits, and _parse_propositional accepts any single token as a fact. 2. It reaches SERVED output. deduction_surface recites `Given: furthermore; p implies q; p.` — a premise the user never stated — and chat/runtime.py realizes declarative turns into the held self, so a fabricated atom is vault-writable. Widening the inventory first would widen the fabrication surface with it. 3. ADR-0265's defect class survives inside ADR-0265's designated owner of clause grammar: the four aspect arms of _inflect_predicate bind `negated` to a wildcard and never read it, so `dog has been defined as mammal` serves both the assertion and the denial (10530/16146 points). Unreachable today — no producer sets aspect — so a loaded gun, not a casualty. It survived because ADR-0265's invariant is structural (is `negated` threaded?) and cannot see an arm that receives the flag and ignores it. A behavioural sweep can. The two fixes are written already, as the mutations that turn the defect pins red. They are NOT applied here: they change what CORE comprehends from user input, which is a serving change on the truth path — authorization-gated, ADR first, on the ADR-0261 §5.1 refuse-don't-drop precedent. Guards, so the tables cannot rot: reader constructions are pinned to an AST count of the reader's 10 mint sites; the declared tense/aspect axes are pinned to _inflect_predicate's match arms; the committed corpus is pinned to the fillers in use. 14 pins, 13 mutations observed RED. [Verification]: in-worktree, CPython 3.12.13, uv sync --locked. deductive 517 (was 503, +14 — count moved, registration confirmed) smoke 641 (unchanged; the file is registered in `deductive`) lane SHAs 11/11 match, no pin edited pyright 0 errors on all new files mutations 13/13 red, including both fabrication fixes --- core/cli_test.py | 8 + docs/plans/grammar-unification-2026-07-26.md | 83 +++++ evals/construction_inventory/__init__.py | 1 + evals/construction_inventory/contract.md | 173 ++++++++++ evals/construction_inventory/corpora.py | 46 +++ .../public/v1/cases.jsonl | 4 + evals/construction_inventory/reader_space.py | 128 +++++++ evals/construction_inventory/runner.py | 255 ++++++++++++++ evals/construction_inventory/skeleton.py | 61 ++++ evals/construction_inventory/writer_space.py | 128 +++++++ evals/grammar_roundtrip/contract.md | 11 + tests/test_construction_inventory.py | 316 ++++++++++++++++++ 12 files changed, 1214 insertions(+) create mode 100644 evals/construction_inventory/__init__.py create mode 100644 evals/construction_inventory/contract.md create mode 100644 evals/construction_inventory/corpora.py create mode 100644 evals/construction_inventory/public/v1/cases.jsonl create mode 100644 evals/construction_inventory/reader_space.py create mode 100644 evals/construction_inventory/runner.py create mode 100644 evals/construction_inventory/skeleton.py create mode 100644 evals/construction_inventory/writer_space.py create mode 100644 tests/test_construction_inventory.py diff --git a/core/cli_test.py b/core/cli_test.py index fdb57901..4af5983e 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -270,6 +270,14 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { # reason smoke stayed at 621 across two PRs that added 13 tests. # ~0.4s; it belongs with the other grammar pins. "tests/test_realizer_quantifier_agreement.py", + # Phase 5 item 1 — the reader/writer construction overlap, plus the + # fabrication pins. Gated rather than left to the `full` lane because + # three of its pins are wrong=0 hazards on a serving path: the reader + # accepts `every dog is a mammal` as an assertion about `every_dog`, + # and recites `Given: furthermore; ...` back to the user. Those must + # not be able to worsen silently. ~9.5s (sweeps the writer's whole + # parameter space three times, once per vocabulary). + "tests/test_construction_inventory.py", ), "full": ("tests/",), } diff --git a/docs/plans/grammar-unification-2026-07-26.md b/docs/plans/grammar-unification-2026-07-26.md index d5db0b38..a5f542f8 100644 --- a/docs/plans/grammar-unification-2026-07-26.md +++ b/docs/plans/grammar-unification-2026-07-26.md @@ -914,6 +914,63 @@ a producer exists. Phase 5's *diversity* work (items 1–3 above) remains open and is now correctly scoped by §6's RESULT: the frontier is **construction-inventory overlap**. +**RESULT (second unit) — the inventories measured against each other.** +New lane `evals/construction_inventory` (contract + 14 pins, 13 mutations +observed red). It sweeps the writer's whole parameter space — every +`RhetoricalMove` × `IntentTag` × predicate × quantifier × tense × aspect, both +public entry points, 32292 cells — quotients it by the reader's own function-word +skeleton, and comprehends each construction under three vocabularies. + +| | | +|---|---| +| writer constructions | **1739** | +| reader constructions | **19** (mint-site AST-guarded) | +| **overlap, faithful** | **6** | +| reader accepts but mis-reads | **22** | +| reader refuses | 1711 | +| vocabulary-dependent | **0** | + +The overlap is the same six from both sides — `X is a Y`, `the X is a Y`, and +the four categoricals — and it is **entirely set-theoretic**. The writer cannot +express one ordering, propositional or interrogative construction; the reader +has no template for the bare transitive clause that is the writer's whole +default output. Two of the reader's constructions need a `?`, which no writer +template contains, so they are unreachable **by shape** — corpus work cannot +close that gap, only a template can. + +Three findings change what item 1 should be: + +1. **Growing the inventory is not the first move — the reader fabricates.** On + 22 constructions it neither reads nor refuses: it invents. `every dog is a + mammal` → `member(every_dog, mammal)`; `all dogs are defined as mammals` → + `subset(dog, defined_as_mammal)`; `furthermore, all dogs are mammals` → + `asserted(furthermore)`. One root cause: `_RESERVED` does not contain the + function words the writer emits, and `_parse_propositional` accepts any + single token as a fact. **The first and third are ordinary user English.** + +2. **It reaches served output.** `chat/deduction_surface.py` recites + `Given: furthermore; p implies q; p.` — a premise the user never stated — + and `chat/runtime.py` *realizes* declarative turns into the held self, so a + fabricated atom is vault-writable. Widening the inventory before closing this + widens the fabrication surface with it. + +3. **ADR-0265's defect class survives inside ADR-0265's own designated owner of + clause grammar.** The four aspect arms of `_inflect_predicate` bind `negated` + to a wildcard and never read it, so `dog has been defined as mammal` is + emitted for both the assertion and the denial — 10530 of 16146 parameter + points. Not reachable today (no producer sets `aspect`), so a loaded gun, not + a casualty. It survived because ADR-0265's invariant is *structural* — does + `negated` reach the call? — and cannot see an arm that receives the flag and + ignores it. **A behavioural sweep catches what a threading invariant cannot.** + +**Item 1 is therefore re-ordered: close the fabrications first, then widen.** +Both fixes are small and both are already written as mutations that turn the +defect pins red (`every`/`each` into `_RESERVED`; discourse connectives stripped +in `_split_clauses`). But they change what CORE *comprehends* from user input, +which is a serving change on the truth path — **authorization-gated, and an ADR +before any code**, on the ADR-0261 §5.1 precedent that a reader must refuse +rather than drop or invent. + --- ## 5. What is deliberately not estimated @@ -1008,6 +1065,32 @@ The graph-model ADR is **not** cancelled — §1.8's type mismatch is still real but nothing measured here obliges it, and it should not be opened until a case exists that fails for that reason and no other. +#### CORRECTION to the block above, from the Phase 5 measurement it asked for + +Two claims made here are wrong, and the metric that disproves both was already +on the page when they were written. + +**"the reader independently reads `all molecules are defined as compounds`" — +no, it mis-reads it.** `g_read_rate` is `1/293` but **`g_args_rate` is `0.0`**. +The reader comprehends that surface as `subset(molecule, defined_as_compound)`: +it chunked the writer's verb phrase into a class name. It accepted, it did not +comprehend. The contract for this lane separates `read` from `args` for exactly +this reason and the separation did its job; the RESULT above simply put its +weight on the wrong one of the two. + +**"one construction wide" — wrong in both directions.** On this corpus the +*faithful* overlap is **zero**, because the one case that reads is the mis-read +above. Measured over the writer's whole output space instead of a 293-case +corpus, the true overlap is **six** — the corpus never exercised five of them. +A corpus cannot report the size of an inventory. Only the inventory can, which +is what §Phase 5's RESULT now does. + +The generalisable lesson, and the reason this is kept rather than edited away: +**acceptance is not comprehension.** A reader scored on whether it returned a +graph, rather than on whether the graph is the one it was given, is rewarded for +guessing — a `wrong=0` hazard wearing a green metric. `g_read_rate` should never +again be quoted without `g_args_rate` beside it. + --- ## 7. Verification protocol diff --git a/evals/construction_inventory/__init__.py b/evals/construction_inventory/__init__.py new file mode 100644 index 00000000..7a79e876 --- /dev/null +++ b/evals/construction_inventory/__init__.py @@ -0,0 +1 @@ +"""Construction-inventory lane: how wide is the reader/writer overlap?""" diff --git a/evals/construction_inventory/contract.md b/evals/construction_inventory/contract.md new file mode 100644 index 00000000..994f52a1 --- /dev/null +++ b/evals/construction_inventory/contract.md @@ -0,0 +1,173 @@ +# Construction-Inventory Eval Lane — Contract + +**Lane:** `construction_inventory` +**Version:** v1 +**Created:** 2026-07-27 +**Plan:** `docs/plans/grammar-unification-2026-07-26.md` (Phase 5, item 1) + +## What this lane measures + +How many **constructions** the reader and the writer share — the quantity §6's +RESULT identified as the arc's frontier, and asked to be measured directly +"rather than by growing corpora blindly." + +## Why `grammar_roundtrip` could not answer this + +`grammar_roundtrip` measures a **corpus**. Its G-direction reports +`g_read_rate = 1/293`, and §6 read that one case as the shared construction, +concluding the overlap was "one construction wide." + +Both halves of that were wrong, and the lane's own metrics already said so: + +- **`g_args_rate` is `0.0`.** The one surface that "reads" — + `all molecules are defined as compounds` — is comprehended as + `subset(molecule, defined_as_compound)`. The reader chunked the writer's verb + phrase into a class name. It is a **mis-read**, so on that corpus the faithful + overlap is **zero**, not one. +- **The true overlap is six**, and the corpus never exercised five of them. + 293 cases cannot report the size of an inventory; only the inventory can. + +The lesson generalises: *acceptance is not comprehension*. A lane that counts +`read` without counting `args` will reward a reader for guessing. + +## The construction quotient + +A construction is defined by the reader's own dispatch. `reader.py` reads +"STRUCTURE symbolically ... via templates keyed on FUNCTION WORDS + ORDER", so +replacing every non-function word with `*` yields a form that is, by the +reader's design, indistinguishable to its parser. The function-word set is read +off `reader._RESERVED` — the lane cannot pick a quotient that flatters a number, +and it widens automatically when the reader learns a word. + +**Known limit:** `_chunk_class` refuses on unrecognized plurals, which depends on +the token rather than the shape. So the skeleton determines the verdict *up to +morphology*. The lane sweeps three count-noun vocabularies (regular, `-f/-ves`, +suppletive) and counts a construction as shared only when all three agree; +disagreement is reported as `w2r_filler_dependent`, currently **0**. + +## The two directions + +| direction | question | how | +|---|---|---| +| **W → R** | of everything the writer can emit, what does the reader read *truly*? | sweep the writer's whole parameter space, quotient to constructions, comprehend each | +| **R → W** | of everything the reader can read, what can the writer emit? | instantiate each reader construction, quotient, test membership in the writer's construction set | + +The writer's space is enumerated from live source — `RhetoricalMove`, +`IntentTag`, `PREDICATE_DISPLAY`, `PREDICATIVE_NOMINAL`, `PLURAL_QUANTIFIERS`, +and the tense/aspect literals in `_inflect_predicate`'s match arms (AST-pinned). +Nothing is a hand-copied list of "what the writer can say." + +Both writer entry points are swept: `render_semantic` (the **serving** writer, +called by `core/cognition/pipeline.py`) and `render_step` (the clause owner it +delegates to, also reached by the eval-only `realize_target`). + +## Acceptance is split three ways + +| bucket | meaning | +|---|---| +| `faithful` | the reader recovered the writer's proposition and minted **nothing else** — the only bucket that counts as overlap | +| `fabricating` | accepted, but minted an argument the writer never wrote, or manufactured a query from a declarative | +| `refused` | the honest outcome: no template, and the reader says so | + +## Metrics + +| metric | definition | +|---|---| +| `writer_constructions` | distinct constructions the writer can emit | +| `overlap_constructions` | shared **and faithful** under every count-noun filler — the Phase 5 number | +| `w2r_fabricating` | writer constructions the reader mis-reads — a `wrong=0` hazard, ratcheted **down** | +| `w2r_filler_dependent` | shared under some vocabularies but not others (morphology gaps) | +| `reader_constructions` | the reader's inventory, guarded against rot by mint-site AST count | +| `r2w_expressible` | reader constructions the writer can emit | +| `negation_collapsed_points` | writer parameter points whose denial is byte-identical to its assertion — ADR-0265 sentinel, ratcheted **down** | + +## v1 baseline — measured on `main` @ `8927c563` + +``` +writer_cells 32292 +writer_constructions 1739 +w2r_faithful 6 +w2r_fabricating 22 +w2r_refused 1711 +w2r_filler_dependent 0 +w2r_faithful_rate 0.00345 +cells_faithful 12 +cells_fabricating 150 + +reader_constructions 19 +r2w_expressible 6 +r2w_rate 0.315789 +reader_interrogative_constructions 2 + +overlap_constructions 6 <-- the Phase 5 number +negation_collapsed_points 10530 <-- defect, see below +``` + +## What this lane found + +**1. The overlap is six, and all six are set-theoretic.** +`X is a Y`, `the X is a Y`, `all Xs are Ys`, `no Xs are Ys`, `some Xs are Ys`, +`some Xs are not Ys`. The writer cannot express a single ordering, propositional +or interrogative construction; the reader has no template for the bare +transitive clause (`X supports Y`) that is the writer's entire default output. +The two halves meet only on set membership and subsumption. + +**2. Three of the reader's constructions are unreachable by SHAPE, not by +vocabulary.** `member_query` and `subset_query` require `?`; no writer template +contains one. Corpus work cannot close that gap — a template can. + +**3. The reader fabricates on 22 constructions, and it is on a serving path.** +Three distinct leaks, one root cause: `_RESERVED` does not contain the +function-word vocabulary the writer actually emits. + +| leak | example | minted | +|---|---|---| +| determiner glued to entity | `every dog is a mammal` | `member(every_dog, mammal)` | +| verb phrase chunked as class | `all dogs are defined as mammals` | `subset(dog, defined_as_mammal)` | +| connective promoted to proposition | `furthermore, all dogs are mammals` | `asserted(furthermore)` | + +The first and third are **ordinary user English**, not writer artefacts. And the +third reaches served output: + +``` +"if p then q. p. therefore q." -> Given: p implies q; p. ... +"furthermore, if p then q. p. therefore q." -> Given: furthermore; p implies q; p. ... +``` + +CORE recites a premise the user never stated. `chat/runtime.py` additionally +*realizes* declarative turns into the held self, so `asserted(furthermore)` is +writable to the vault. + +**4. ADR-0265's defect class survives in the module ADR-0265 named as the single +owner of clause grammar.** The four aspect arms of `_inflect_predicate` bind +`negated` to a wildcard and never read it: + +``` +aspect=perfective affirm: dog has been defined as mammal + denied: dog has been defined as mammal <-- identical +``` + +10530 of 16146 writer parameter points collapse this way. It is **not reachable +today** — no producer sets `aspect` on the serving path — so it is a loaded gun, +not a casualty. It survived because ADR-0265's invariant is *structural* (is +`negated` threaded to the call?) and cannot see an arm that receives the flag and +ignores it. This lane's sentinel is behavioural, and does. + +## What this lane does NOT prove + +The overlap is a count of **shapes**, not a quality judgement. Six shared +constructions that are all set-theoretic is a narrow grammar, and nothing here +says the shared six are *well* written — `render_step` could emit stilted +English inside every one of them and this lane would be satisfied. + +It also says nothing about `MeaningGraph`/`PropositionGraph` type compatibility +(§1.8). That question is still open and still un-forced: no case measured here +fails for that reason and no other. + +## Mutation guarantees + +Every pin in `tests/test_construction_inventory.py` has been run against a +mutation of its own subject and observed RED — 12 of 12, recorded in the Phase 5 +RESULT block of the plan. The two fabrication fixes (`every`/`each` into +`_RESERVED`; discourse connectives stripped in `_split_clauses`) are among the +mutations, so the defect pins are known to move when the defect is fixed. diff --git a/evals/construction_inventory/corpora.py b/evals/construction_inventory/corpora.py new file mode 100644 index 00000000..7aa7a428 --- /dev/null +++ b/evals/construction_inventory/corpora.py @@ -0,0 +1,46 @@ +"""Lexical fillers — the controls that keep this a construction measurement. + +The reader refuses for two unrelated reasons: it has no template for a shape +(*construction*), or it cannot singularize a token (*morphology*). Folding those +together produces a number that moves when someone adds a noun, which is not +what Phase 5 needs to size. + +So every construction is probed with several vocabularies. A construction counts +as shared only when **all count-noun fillers agree**; a construction that reads +under one filler and refuses under another is reported as ``filler_dependent``, +which is the signature of a morphology gap rather than a missing template. + +The mass-noun filler is deliberately excluded from the headline and reported on +its own. Mass nouns have no plural, so the writer's own rule keeps them singular +under a quantifier ("all evidence supports truth") — and the reader's +categorical template *requires* a plural head. That is a real inventory +mismatch, but it is a different one from "the reader has no template", and +averaging it into the overlap rate would hide both. +""" + +from __future__ import annotations + +from evals.construction_inventory.reader_space import LexicalFiller + +#: Count-noun fillers — these determine the headline overlap. +COUNT_FILLERS: tuple[LexicalFiller, ...] = ( + LexicalFiller("regular", "dog", "mammal", "dogs", "mammals", "p", "q"), + LexicalFiller("f_plural", "wolf", "canine", "wolves", "canines", "r", "s"), + LexicalFiller("suppletive", "child", "person", "children", "people", "u", "v"), +) + +#: Mass-noun filler — diagnostic only, never part of the headline. +MASS_FILLER = LexicalFiller("mass", "evidence", "truth", "evidence", "truth", "m", "n") + +ALL_FILLERS: tuple[LexicalFiller, ...] = (*COUNT_FILLERS, MASS_FILLER) + +#: Third lexical slot, used by the frame intents that take a ``secondary``. +SECONDARY: dict[str, str] = { + "regular": "cat", + "f_plural": "fox", + "suppletive": "adult", + "mass": "meaning", +} + + +__all__ = ("ALL_FILLERS", "COUNT_FILLERS", "MASS_FILLER", "SECONDARY") diff --git a/evals/construction_inventory/public/v1/cases.jsonl b/evals/construction_inventory/public/v1/cases.jsonl new file mode 100644 index 00000000..7eca92f5 --- /dev/null +++ b/evals/construction_inventory/public/v1/cases.jsonl @@ -0,0 +1,4 @@ +{"id": "filler-regular", "kind": "count_filler", "x": "dog", "y": "mammal", "plural_x": "dogs", "plural_y": "mammals", "secondary": "cat", "note": "regular -s plural; the default vocabulary"} +{"id": "filler-f-plural", "kind": "count_filler", "x": "wolf", "y": "canine", "plural_x": "wolves", "plural_y": "canines", "secondary": "fox", "note": "-f/-ves; exercises IRREGULAR_SINGULARS in both directions"} +{"id": "filler-suppletive", "kind": "count_filler", "x": "child", "y": "person", "plural_x": "children", "plural_y": "people", "secondary": "adult", "note": "suppletive plural; no suffix rule can reach it"} +{"id": "filler-mass", "kind": "mass_filler", "x": "evidence", "y": "truth", "plural_x": "evidence", "plural_y": "truth", "secondary": "meaning", "note": "DIAGNOSTIC ONLY, excluded from the headline: mass nouns have no plural, so the writer keeps them singular under a quantifier and the reader's categorical template requires a plural head. A real inventory mismatch, but a different one from 'no template'."} diff --git a/evals/construction_inventory/reader_space.py b/evals/construction_inventory/reader_space.py new file mode 100644 index 00000000..22b8270c --- /dev/null +++ b/evals/construction_inventory/reader_space.py @@ -0,0 +1,128 @@ +"""The reader's construction inventory, with a structural guard against rot. + +A hand-written table of "what the reader reads" is worth nothing on its own — +it drifts the first time someone adds a template, and then the overlap number +silently measures a stale list. So the table is paired with a guard keyed on +something the reader cannot add a construction without touching: a **mint +site**, one of the ``relations.append`` / ``queries.append`` calls that are the +only ways a proposition or query enters a ``Comprehension``. + +Every construction below names the mint site it flows through. The guard +(``test_construction_inventory.py``) AST-scans ``reader.py``, counts mint sites, +and requires that count to equal the number of distinct sites named here — so a +new template breaks the test until it is entered in this table. That is the +``feedback-pin-registered-in-no-suite`` lesson applied one level up: a table +that cannot go stale beats a table someone promises to update. + +Constructions are keyed on SHAPE, not on predicate: ``all Xs are Ys`` and ``no +Xs are Ys`` are two constructions through one mint site, because the reader +dispatches on the quantifier token and a writer must emit that token to reach +either. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +#: The reader's mint sites — the only routes into a ``Comprehension``. +#: Named, not numbered, so a diff shows which route a change touched. +MINT_RELATION_CATEGORICAL = "relations:categorical" +MINT_RELATION_MEMBER = "relations:member" +MINT_RELATION_LESS = "relations:less" +MINT_RELATION_PROPOSITIONAL = "relations:propositional" +MINT_QUERY_COMPARE = "queries:compare" +MINT_QUERY_SORT = "queries:sort" +MINT_QUERY_MEMBER = "queries:member" +MINT_QUERY_SUBSET = "queries:subset" +MINT_QUERY_THEREFORE_CATEGORICAL = "queries:therefore_categorical" +MINT_QUERY_THEREFORE_PROPOSITIONAL = "queries:therefore_propositional" + + +@dataclass(frozen=True, slots=True) +class ReaderConstruction: + """One surface shape the reader claims to read, plus a canonical instance. + + ``instance`` is a template over the lane's lexical fillers, so the same + construction is probed with several vocabularies and a morphology refusal + shows up as filler-dependence rather than as a missing construction. + """ + + construction_id: str + instance: str + mint_site: str + interrogative: bool = False + + +#: The inventory. Order is stable so report diffs are readable. +READER_CONSTRUCTIONS: tuple[ReaderConstruction, ...] = ( + # --- categorical facts: one mint site, four dispatch tokens ------------- + ReaderConstruction("subset_fact", "all {plural_x} are {plural_y}", MINT_RELATION_CATEGORICAL), + ReaderConstruction("disjoint_fact", "no {plural_x} are {plural_y}", MINT_RELATION_CATEGORICAL), + ReaderConstruction("intersects_fact", "some {plural_x} are {plural_y}", MINT_RELATION_CATEGORICAL), + ReaderConstruction("some_not_fact", "some {plural_x} are not {plural_y}", MINT_RELATION_CATEGORICAL), + # --- membership facts --------------------------------------------------- + ReaderConstruction("member_fact", "{x} is a {y}", MINT_RELATION_MEMBER), + ReaderConstruction("member_fact_definite", "the {x} is a {y}", MINT_RELATION_MEMBER), + # --- ordering facts: one mint site, two comparator directions ----------- + ReaderConstruction("less_fact", "{x} is below {y}", MINT_RELATION_LESS), + ReaderConstruction("greater_fact", "{x} is above {y}", MINT_RELATION_LESS), + # --- propositional facts ------------------------------------------------ + ReaderConstruction("implies_fact", "if {p} then {q}", MINT_RELATION_PROPOSITIONAL), + ReaderConstruction("negated_atom_fact", "not {p}", MINT_RELATION_PROPOSITIONAL), + ReaderConstruction("or_fact", "{p} or {q}", MINT_RELATION_PROPOSITIONAL), + ReaderConstruction("atom_fact", "{p}", MINT_RELATION_PROPOSITIONAL), + # --- queries ------------------------------------------------------------ + ReaderConstruction("member_query", "is {x} a {y}?", MINT_QUERY_MEMBER, interrogative=True), + ReaderConstruction("subset_query", "are all {plural_x} {plural_y}?", MINT_QUERY_SUBSET, interrogative=True), + ReaderConstruction("compare_query", "compare {x} with {y}", MINT_QUERY_COMPARE), + ReaderConstruction("sort_query", "sort ascending", MINT_QUERY_SORT), + ReaderConstruction("sort_range_query", "order from lowest to highest", MINT_QUERY_SORT), + ReaderConstruction( + "therefore_categorical", "therefore all {plural_x} are {plural_y}", + MINT_QUERY_THEREFORE_CATEGORICAL, + ), + ReaderConstruction( + "therefore_propositional", "therefore {q}", MINT_QUERY_THEREFORE_PROPOSITIONAL, + ), +) + +#: Distinct mint sites named above — the quantity the AST guard checks. +READER_MINT_SITES: frozenset[str] = frozenset(c.mint_site for c in READER_CONSTRUCTIONS) + + +def render_instance(construction: ReaderConstruction, filler: "LexicalFiller") -> str: + """Instantiate a construction's canonical surface with *filler*'s vocabulary.""" + return construction.instance.format( + x=filler.x, y=filler.y, + plural_x=filler.plural_x, plural_y=filler.plural_y, + p=filler.p, q=filler.q, + ) + + +@dataclass(frozen=True, slots=True) +class LexicalFiller: + """One vocabulary for probing constructions. + + Held separate from the constructions so that a refusal caused by *morphology* + (an unrecognized plural) is distinguishable from a refusal caused by the + *construction*. A construction counts as readable only when every filler + agrees; disagreement is reported as ``filler_dependent`` rather than being + averaged away. + """ + + name: str + x: str + y: str + plural_x: str + plural_y: str + p: str + q: str + + +__all__ = ( + "LexicalFiller", + "READER_CONSTRUCTIONS", + "READER_MINT_SITES", + "ReaderConstruction", + "render_instance", +) diff --git a/evals/construction_inventory/runner.py b/evals/construction_inventory/runner.py new file mode 100644 index 00000000..3ff9119f --- /dev/null +++ b/evals/construction_inventory/runner.py @@ -0,0 +1,255 @@ +"""Construction-inventory eval lane. + +Sizes Phase 5 by measuring the reader's construction set against the writer's, +in both directions, on the criterion that matters: **faithfulness, not +acceptance**. + +Why acceptance is the wrong criterion +------------------------------------- +``grammar_roundtrip`` reports ``g_read_rate`` alongside ``g_args_rate`` for +exactly this reason, and the two disagree: on ``main`` the G-direction reads +1 of 293 surfaces and recovers **0** argument pairs. The single surface that +"reads" is ``all molecules are defined as compounds``, which the reader accepts +as ``subset(molecule, defined_as_compound)`` — a class it invented from the +writer's verb phrase. Counting that as overlap would have Phase 5 growing an +inventory toward a number that rewards mis-reading. + +So this lane splits acceptance three ways: + +``faithful`` + the reader recovered the proposition the writer emitted, and minted nothing + else. This is the only bucket that counts as overlap. +``fabricating`` + the reader accepted the surface and minted at least one argument the writer + never wrote. This is a ``wrong=0`` hazard on a serving path + (``chat/runtime.py`` realizes declarative turns into the held self; + ``chat/deduction_surface.py`` recites premises back to the user), and it is + ratcheted **downward**. +``refused`` + the honest outcome — the reader has no template and says so. + +Framework contract: ``run_lane(cases, config=None) -> LaneReport``. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +from typing import Any + +from generate.meaning_graph.reader import Refusal, comprehend + +from evals.construction_inventory.corpora import ( + COUNT_FILLERS, + MASS_FILLER, + SECONDARY, +) +from evals.construction_inventory.reader_space import ( + READER_CONSTRUCTIONS, + LexicalFiller, + render_instance, +) +from evals.construction_inventory.skeleton import skeleton +from evals.construction_inventory.writer_space import writer_cells + +@dataclass(frozen=True, slots=True) +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +FAITHFUL = "faithful" +FABRICATING = "fabricating" +REFUSED = "refused" + + +@dataclass(frozen=True, slots=True) +class Verdict: + """What the reader did with one writer construction.""" + + bucket: str + detail: str + + +def _classify(surface: str, filler: LexicalFiller, secondary: str) -> Verdict: + """Read *surface* back and decide whether the reading is faithful. + + The writer emitted a proposition over ``{x, y}`` (or ``{x, secondary}`` for + the frame intents). Any argument outside that set is something the reader + invented — a discourse connective promoted to a proposition, a verb phrase + chunked into a class, a determiner glued onto an entity. Any *query* is also + invention: the writer never emits an interrogative, so a question can only + have been manufactured. + """ + reading = comprehend(surface) + if isinstance(reading, Refusal): + return Verdict(REFUSED, reading.reason) + + permitted = {filler.x, filler.y, secondary} + relations = reading.meaning_graph.relations + + if reading.queries: + minted = ",".join(q.predicate for q in reading.queries) + return Verdict(FABRICATING, f"query_from_declarative:{minted}") + + invented = sorted( + {arg for rel in relations for arg in rel.arguments if arg not in permitted} + ) + if invented: + return Verdict(FABRICATING, f"invented_args:{'|'.join(invented)}") + + recovered = any( + tuple(rel.arguments) in ((filler.x, filler.y), (filler.x, secondary)) + for rel in relations + ) + if not recovered: + return Verdict(FABRICATING, "proposition_lost") + return Verdict(FAITHFUL, ",".join(sorted({r.predicate for r in relations}))) + + +def _sweep_filler(filler: LexicalFiller) -> dict[str, Any]: + """Sweep the whole writer space under one vocabulary.""" + secondary = SECONDARY[filler.name] + by_skeleton: dict[str, str] = {} + cells_per_skeleton: Counter[str] = Counter() + # Negation sentinel (ADR-0265): key each parameter point WITHOUT its polarity + # so the two polarities land in one bucket. A point whose negated surface is + # byte-identical to its affirmative is a proposition served as its own denial. + polarity: dict[str, dict[bool, str]] = {} + n_cells = 0 + + for cell in writer_cells(filler.x, filler.y, secondary): + n_cells += 1 + sk = skeleton(cell.surface) + by_skeleton.setdefault(sk, cell.surface) + cells_per_skeleton[sk] += 1 + point = f"{cell.entry}|{cell.label.replace('neg=True', '').replace('neg=False', '')}" + polarity.setdefault(point, {})[cell.negated] = cell.surface + + negation_collapsed = sorted( + point for point, both in polarity.items() + if len(both) == 2 and both[True] == both[False] + ) + + verdicts = {sk: _classify(s, filler, secondary) for sk, s in by_skeleton.items()} + return { + "filler": filler.name, + "cells": n_cells, + "skeletons": by_skeleton, + "cells_per_skeleton": cells_per_skeleton, + "verdicts": verdicts, + "negation_collapsed": negation_collapsed, + } + + +def _reader_direction(writer_skeletons: frozenset[str]) -> dict[str, Any]: + """Which of the reader's constructions can the writer emit at all?""" + rows = [] + for construction in READER_CONSTRUCTIONS: + instance = render_instance(construction, COUNT_FILLERS[0]) + sk = skeleton(instance) + reading = comprehend(instance) + rows.append({ + "construction_id": construction.construction_id, + "mint_site": construction.mint_site, + "instance": instance, + "skeleton": sk, + "writer_can_emit": sk in writer_skeletons, + "reader_reads_own": not isinstance(reading, Refusal), + "interrogative": construction.interrogative, + }) + return {"rows": rows} + + +def _rate(num: int, den: int) -> float: + return round(num / den, 6) if den else 0.0 + + +def run_lane(cases: Any = None, config: Any = None) -> LaneReport: # noqa: ARG001 + """Measure the two construction inventories against each other.""" + sweeps = [_sweep_filler(f) for f in COUNT_FILLERS] + mass = _sweep_filler(MASS_FILLER) + + # A construction is shared only if EVERY count-noun filler agrees it is. + per_filler_faithful = [ + {sk for sk, v in s["verdicts"].items() if v.bucket == FAITHFUL} for s in sweeps + ] + per_filler_fabricating = [ + {sk for sk, v in s["verdicts"].items() if v.bucket == FABRICATING} for s in sweeps + ] + faithful_all = set.intersection(*per_filler_faithful) + faithful_any = set.union(*per_filler_faithful) + fabricating_any = set.union(*per_filler_fabricating) + filler_dependent = faithful_any - faithful_all + + base = sweeps[0] + all_skeletons = frozenset(base["skeletons"]) + reader_dir = _reader_direction(frozenset().union(*(frozenset(s["skeletons"]) for s in sweeps))) + + n_constructions = len(all_skeletons) + refusal_census = Counter( + v.detail for v in base["verdicts"].values() if v.bucket == REFUSED + ) + fabrication_census = Counter( + v.detail.split(":")[0] for v in base["verdicts"].values() if v.bucket == FABRICATING + ) + + cells_fabricating = sum( + base["cells_per_skeleton"][sk] for sk in fabricating_any if sk in base["cells_per_skeleton"] + ) + cells_faithful = sum( + base["cells_per_skeleton"][sk] for sk in faithful_all if sk in base["cells_per_skeleton"] + ) + + reader_rows = reader_dir["rows"] + expressible = [r for r in reader_rows if r["writer_can_emit"]] + unreadable_own = [r for r in reader_rows if not r["reader_reads_own"]] + interrogative = [r for r in reader_rows if r["interrogative"]] + + metrics: dict[str, Any] = { + # --- W -> R: the writer's space, judged by the reader --------------- + "writer_cells": base["cells"], + "writer_constructions": n_constructions, + "w2r_faithful": len(faithful_all), + "w2r_fabricating": len(fabricating_any), + "w2r_refused": n_constructions - len(faithful_any) - len(fabricating_any), + "w2r_filler_dependent": len(filler_dependent), + "w2r_faithful_rate": _rate(len(faithful_all), n_constructions), + "w2r_fabricating_rate": _rate(len(fabricating_any), n_constructions), + "cells_faithful": cells_faithful, + "cells_fabricating": cells_fabricating, + # --- R -> W: the reader's inventory, judged by the writer ------------ + "reader_constructions": len(reader_rows), + "r2w_expressible": len(expressible), + "r2w_rate": _rate(len(expressible), len(reader_rows)), + "reader_constructions_unreadable_by_reader": len(unreadable_own), + "reader_interrogative_constructions": len(interrogative), + # --- the headline ----------------------------------------------------- + "overlap_constructions": len(faithful_all), + # --- sentinels -------------------------------------------------------- + "negation_collapsed_points": len(base["negation_collapsed"]), + "mass_noun_faithful": len( + [1 for v in mass["verdicts"].values() if v.bucket == FAITHFUL] + ), + } + + details: list[dict[str, Any]] = [ + { + "kind": "writer_construction", + "skeleton": sk, + "example": base["skeletons"][sk], + "cells": base["cells_per_skeleton"][sk], + "bucket": base["verdicts"][sk].bucket, + "detail": base["verdicts"][sk].detail, + "shared_by_all_fillers": sk in faithful_all, + } + for sk in sorted(all_skeletons, key=lambda s: -base["cells_per_skeleton"][s]) + ] + details.extend({"kind": "reader_construction", **row} for row in reader_rows) + details.append({"kind": "refusal_census", **dict(refusal_census)}) + details.append({"kind": "fabrication_census", **dict(fabrication_census)}) + + return LaneReport(metrics=metrics, case_details=details) + + +__all__ = ("FABRICATING", "FAITHFUL", "REFUSED", "LaneReport", "Verdict", "run_lane") diff --git a/evals/construction_inventory/skeleton.py b/evals/construction_inventory/skeleton.py new file mode 100644 index 00000000..544d2e9f --- /dev/null +++ b/evals/construction_inventory/skeleton.py @@ -0,0 +1,61 @@ +"""The construction quotient: a surface reduced to what the reader parses by. + +The reader is documented as reading "STRUCTURE symbolically from the token +sequence via domain-agnostic templates keyed on FUNCTION WORDS + ORDER". Take +that literally and it defines an equivalence relation on surfaces: replace every +token the reader does *not* key on with a placeholder, and two surfaces with the +same result are — by the reader's own design — indistinguishable to its parser. + +That relation is what makes "how many constructions?" a countable question +instead of a judgement call. The quotient is not chosen by this module; it is +read off ``reader._RESERVED``, so it cannot be gerrymandered to flatter a +number, and it widens automatically when the reader learns a new function word. + +**The one place the abstraction leaks**, stated here because a silent leak in a +measurement instrument is worse than the defect it measures: ``_chunk_class`` +refuses on an unrecognized plural, which depends on the *token*, not on whether +the token is reserved. So the skeleton determines the reader's verdict **up to +morphology**. The lane handles this by sweeping several lexical fillers per +construction and only counting a construction as shared when every filler +agrees — see ``runner.py``. Morphology refusals are then visible as +filler-dependent disagreement rather than being silently folded into the +construction count. +""" + +from __future__ import annotations + +from generate.meaning_graph.reader import _RESERVED + +#: Function words the reader keys on that ``_RESERVED`` does not list. +#: +#: ``_parse_propositional`` dispatches on ``if`` and ``then``, but neither is in +#: ``_RESERVED`` — so ``_chunk`` will happily swallow them into a noun phrase. +#: That asymmetry is a reader defect (see contract.md, "What this lane found"), +#: not an artefact of this module; it is recorded here so the quotient reflects +#: the tokens the parser *actually* branches on rather than the ones it declares. +_UNDECLARED_KEYWORDS: frozenset[str] = frozenset({"if", "then"}) + +#: The reader's full function-word vocabulary, derived from source. +READER_FUNCTION_WORDS: frozenset[str] = frozenset(_RESERVED) | _UNDECLARED_KEYWORDS + +#: Stands in for any token the reader treats as content. +CONTENT = "*" + +_PUNCT = ".,;:?!" + + +def skeleton(surface: str) -> str: + """Reduce *surface* to its reader-visible construction. + + Content tokens collapse to ``*``; function words and attached punctuation + survive, because both change how the reader splits and dispatches. + """ + out: list[str] = [] + for token in surface.lower().split(): + core = token.strip(_PUNCT) + trailing = token[len(core):] if core and token.startswith(core) else "" + out.append((core if core in READER_FUNCTION_WORDS else CONTENT) + trailing) + return " ".join(out) + + +__all__ = ("CONTENT", "READER_FUNCTION_WORDS", "skeleton") diff --git a/evals/construction_inventory/writer_space.py b/evals/construction_inventory/writer_space.py new file mode 100644 index 00000000..e8c905e5 --- /dev/null +++ b/evals/construction_inventory/writer_space.py @@ -0,0 +1,128 @@ +"""The writer's construction space, enumerated from live source. + +Every axis below is pulled from the table or enum the writer actually consults, +so the space grows the moment the writer grows. Nothing here is a hand-copied +list of what the writer "can say" — that is the failure mode this lane exists to +avoid, and the reason ``docs/plans/grammar-unification-2026-07-26.md`` §6 could +report a rate without knowing what the rate was about. + +Two axes cannot be read off a table because the writer does not keep one: + +* **quantifier** — ``render_step`` accepts any string. ``PLURAL_QUANTIFIERS`` is + authoritative for the plural half; the singular determiners are a *probe set*, + labelled as such, not an inventory claim. +* **tense / aspect** — these are literals in ``_inflect_predicate``'s ``match`` + arms. They are declared here for readability and pinned against the source by + ``test_construction_inventory.py``, which AST-scans those arms and fails if a + value appears there and not here. +""" + +from __future__ import annotations + +import itertools +from dataclasses import dataclass +from typing import Iterator + +from generate.graph_planner import RhetoricalMove +from generate.intent import IntentTag +from generate.lexicon import PLURAL_QUANTIFIERS, PREDICATE_DISPLAY, PREDICATIVE_NOMINAL +from generate.semantic_templates import render_semantic +from generate.templates import render_step + +#: Every predicate the writer can humanize. ``PREDICATIVE_NOMINAL`` contributes +#: ``is_a``, which has no ``PREDICATE_DISPLAY`` entry (it humanizes by the +#: underscore rule) and is the *only* predicate that reaches the reader's +#: membership template, so omitting it would erase the overlap being measured. +WRITER_PREDICATES: tuple[str, ...] = tuple( + sorted(set(PREDICATE_DISPLAY) | set(PREDICATIVE_NOMINAL)) +) + +#: Singular determiners — a probe, not an inventory (see module docstring). +SINGULAR_DETERMINER_PROBES: tuple[str, ...] = ("the", "every", "each", "a") + +WRITER_QUANTIFIERS: tuple[str | None, ...] = ( + None, + *sorted(PLURAL_QUANTIFIERS), + *SINGULAR_DETERMINER_PROBES, +) + +#: Pinned against ``_inflect_predicate``'s match arms by AST scan. +WRITER_TENSES: tuple[str | None, ...] = (None, "past", "future") +WRITER_ASPECTS: tuple[str | None, ...] = (None, "perfective", "imperfective") + + +@dataclass(frozen=True, slots=True) +class WriterCell: + """One point in the writer's parameter space, and the surface it emits.""" + + entry: str + surface: str + negated: bool + predicate: str + quantifier: str | None + label: str + + +def _step_cells(subject: str, obj: str) -> Iterator[WriterCell]: + for move, predicate, negated, quant, tense, aspect in itertools.product( + tuple(RhetoricalMove), + WRITER_PREDICATES, + (False, True), + WRITER_QUANTIFIERS, + WRITER_TENSES, + WRITER_ASPECTS, + ): + surface = render_step( + move, subject, predicate, obj, + negated=negated, quantifier=quant, tense=tense, aspect=aspect, + ) + yield WriterCell( + entry="render_step", + surface=surface, + negated=negated, + predicate=predicate, + quantifier=quant, + label=f"{move.name}/{predicate}/neg={negated}/q={quant}/{tense}/{aspect}", + ) + + +def _semantic_cells(subject: str, obj: str, secondary: str) -> Iterator[WriterCell]: + for intent, predicate, negated in itertools.product( + tuple(IntentTag), WRITER_PREDICATES, (False, True) + ): + surface = render_semantic( + intent, subject, predicate, obj, secondary=secondary, negated=negated, + ) + yield WriterCell( + entry="render_semantic", + surface=surface, + negated=negated, + predicate=predicate, + quantifier=None, + label=f"{intent.name}/{predicate}/neg={negated}", + ) + + +def writer_cells(subject: str, obj: str, secondary: str) -> Iterator[WriterCell]: + """Every surface the writer emits for one lexical filler triple. + + Both public entry points are swept. ``render_semantic`` is the **serving** + writer (``core/cognition/pipeline.py``); ``render_step`` is the clause owner + it delegates to and is also reached directly by the eval-only + ``realize_target``. Measuring only the serving entry would understate the + grammar's reach; measuring only the clause owner would overstate what ships. + The lane reports both and keeps ``entry`` on every cell. + """ + yield from _step_cells(subject, obj) + yield from _semantic_cells(subject, obj, secondary) + + +__all__ = ( + "SINGULAR_DETERMINER_PROBES", + "WRITER_ASPECTS", + "WRITER_PREDICATES", + "WRITER_QUANTIFIERS", + "WRITER_TENSES", + "WriterCell", + "writer_cells", +) diff --git a/evals/grammar_roundtrip/contract.md b/evals/grammar_roundtrip/contract.md index 4b4e6ff9..73055d49 100644 --- a/evals/grammar_roundtrip/contract.md +++ b/evals/grammar_roundtrip/contract.md @@ -69,6 +69,17 @@ only the *vocabulary* is split — a materially different remedy from both being low. Collapsing them into one boolean would hide the distinction that decides this arc's direction (plan §6). +> **Never quote `g_read_rate` without `g_args_rate`.** They currently disagree: +> `g_read_rate = 0.003413` with `g_args_rate = 0.0`. The single surface that +> "reads" is `all molecules are defined as compounds`, which the reader accepts +> as `subset(molecule, defined_as_compound)` — it chunked the writer's verb +> phrase into a class name. **Acceptance is not comprehension**, and a reader +> scored on whether it returned a graph rather than on whether the graph is the +> one it was given is rewarded for guessing. Plan §6's RESULT drew a conclusion +> from `g_read_rate` alone and had to be corrected; see +> `evals/construction_inventory/contract.md`, which measures the same question +> against the writer's whole output space instead of a 293-case corpus. + ## Corpora | corpus | source | size | diff --git a/tests/test_construction_inventory.py b/tests/test_construction_inventory.py new file mode 100644 index 00000000..779b324d --- /dev/null +++ b/tests/test_construction_inventory.py @@ -0,0 +1,316 @@ +"""Pins for the construction-inventory lane (grammar-unification Phase 5, item 1). + +Three kinds of test live here, and the difference matters: + +**Measurements** record what the two inventories currently share. ``overlap`` is +ratcheted *upward* — that is the number Phase 5 exists to raise. + +**Defect pins** record something that is wrong. Each says so in its docstring and +states which direction it may be revised. ``w2r_fabricating`` may only be +revised *downward*; raising it to accommodate a new fabrication would be +recording a regression as progress. + +**Structural guards** are AST-keyed so the tables cannot rot. A table of "what +the reader reads" that someone must remember to update is worth nothing — the +guards make forgetting fail loudly. +""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path + +from generate.graph_planner import RhetoricalMove +from generate.meaning_graph.reader import Refusal, comprehend +from generate.templates import render_step + +from evals.construction_inventory.corpora import COUNT_FILLERS, MASS_FILLER, SECONDARY +from evals.construction_inventory.reader_space import ( + READER_CONSTRUCTIONS, + READER_MINT_SITES, + render_instance, +) +from evals.construction_inventory.runner import run_lane +from evals.construction_inventory.skeleton import skeleton +from evals.construction_inventory.writer_space import WRITER_ASPECTS, WRITER_TENSES + +_REPO = Path(__file__).resolve().parents[1] + + +# --------------------------------------------------------------------------- # +# Measurements +# --------------------------------------------------------------------------- # + +def test_overlap_is_six_constructions() -> None: + """The headline. Phase 5 item 1 raises this; it may be revised UPWARD only. + + Six constructions are shared: ``X is a Y``, ``the X is a Y``, and the four + categoricals (``all`` / ``no`` / ``some`` / ``some ... not``). All six are + set-theoretic. The writer cannot express a single ordering, propositional or + interrogative construction, and the reader has no template for the bare + transitive clause that is the writer's entire default output. + """ + metrics = run_lane().metrics + assert metrics["overlap_constructions"] == 6 + assert metrics["w2r_faithful"] == 6 + assert metrics["r2w_expressible"] == 6 + + +def test_the_overlap_is_the_same_six_seen_from_both_sides() -> None: + """Internal consistency: the two directions must name the same set. + + ``w2r_faithful`` counts writer constructions the reader reads truly; + ``r2w_expressible`` counts reader constructions the writer can emit. These + are computed by different code over different corpora, and if they ever + disagree, one of the two inventories is measuring something else. + """ + report = run_lane() + faithful = { + d["skeleton"] for d in report.case_details + if d.get("kind") == "writer_construction" and d["bucket"] == "faithful" + } + expressible = { + d["skeleton"] for d in report.case_details + if d.get("kind") == "reader_construction" and d["writer_can_emit"] + } + assert faithful == expressible + + +def test_reader_side_rate_is_pinned() -> None: + """The reader's own inventory is 19 constructions; the writer reaches 6.""" + metrics = run_lane().metrics + assert metrics["reader_constructions"] == 19 + assert metrics["r2w_rate"] == 0.315789 + + +def test_the_shared_six_do_not_depend_on_vocabulary() -> None: + """Morphology control: the overlap must be the same under every filler. + + If a construction reads for ``dogs`` but not for ``wolves`` or ``children``, + the lane would be reporting a morphology gap as a construction gap. Zero + filler-dependence is what licenses calling this a *construction* count. + """ + assert run_lane().metrics["w2r_filler_dependent"] == 0 + + +# --------------------------------------------------------------------------- # +# Defect pins +# --------------------------------------------------------------------------- # + +def test_reader_fabricates_on_twenty_two_writer_constructions() -> None: + """DEFECT PIN. Revise DOWNWARD only — never up to admit a new fabrication. + + On these the reader neither reads nor refuses: it accepts the surface and + mints an argument the writer never wrote. That breaks the reader's own + stated contract ("a fabricated reading is the only failure") and it is on a + serving path, so it is a ``wrong=0`` hazard, not a cosmetic one. + """ + metrics = run_lane().metrics + assert metrics["w2r_fabricating"] == 22 + assert metrics["cells_fabricating"] == 150 + + +def test_reader_invents_propositions_from_ordinary_english() -> None: + """DEFECT PIN. These are user-typable sentences, not writer artefacts. + + ``every``/``each`` are not in ``_RESERVED``, so they are chunked into the + entity id and the asserted fact is about an entity that does not exist. A + later query about ``dog`` then answers UNKNOWN even though the user said it. + """ + reading = comprehend("every dog is a mammal") + assert not isinstance(reading, Refusal) + assert [tuple(r.arguments) for r in reading.meaning_graph.relations] == [ + ("every_dog", "mammal") + ] + + reading = comprehend("each metal is a conductor") + assert not isinstance(reading, Refusal) + assert [tuple(r.arguments) for r in reading.meaning_graph.relations] == [ + ("each_metal", "conductor") + ] + + +def test_reader_promotes_a_discourse_connective_to_a_proposition() -> None: + """DEFECT PIN. ``_parse_propositional`` accepts ANY single token as a fact. + + ``_split_clauses`` splits on commas and strips only a leading ``and``/``or``, + so a one-word discourse opener becomes its own clause, and the bare-atom rule + mints it. CORE then recites it back to the user as a premise — see + ``test_fabricated_premise_reaches_a_served_deduction_surface``. + """ + reading = comprehend("furthermore, all dogs are mammals") + assert not isinstance(reading, Refusal) + minted = [(r.predicate, tuple(r.arguments)) for r in reading.meaning_graph.relations] + assert ("asserted", ("furthermore",)) in minted + + +def test_fabricated_premise_reaches_a_served_deduction_surface() -> None: + """DEFECT PIN — this is the one that shows the hazard is not theoretical. + + The verdict is unaffected here, but the *recital* is: CORE tells the user + they said ``furthermore``. Revise only by removing the fabrication. + """ + from chat.deduction_surface import deduction_grounded_surface + + clean = deduction_grounded_surface("if p then q. p. therefore q.") + leaked = deduction_grounded_surface("furthermore, if p then q. p. therefore q.") + + assert clean == "Given: p implies q; p. Your premises entail: q." + assert leaked == "Given: furthermore; p implies q; p. Your premises entail: q." + + +def test_aspect_arms_drop_negation() -> None: + """DEFECT PIN — ADR-0265's defect class, surviving in the module ADR-0265 + named as the single owner of clause grammar. + + The four aspect arms of ``_inflect_predicate`` match on ``(aspect, ...)`` + with ``negated`` bound to a wildcard and never consult it, so a denial is + rendered as its own assertion. ADR-0265's invariant is *structural* — it + checks that ``negated`` is threaded to each call — and cannot see an arm that + receives the flag and ignores it. This pin is behavioural and can. + + Not reachable today: no producer sets ``aspect`` on the serving path (only + ``generate/realizer.py`` forwards ``step.aspect``, which is always None). It + is a loaded gun, not a casualty — the distinction ADR-0265 got wrong once + already, so it is stated explicitly rather than assumed. + + Revise DOWNWARD only; ``0`` is the fixed state. + """ + assert run_lane().metrics["negation_collapsed_points"] == 10530 + + for aspect in ("perfective", "imperfective"): + affirmed = render_step( + RhetoricalMove.ASSERT, "dog", "is_defined_as", "mammal", + negated=False, aspect=aspect, + ) + denied = render_step( + RhetoricalMove.ASSERT, "dog", "is_defined_as", "mammal", + negated=True, aspect=aspect, + ) + assert affirmed == denied, f"aspect={aspect} now distinguishes polarity — revise DOWN" + + +# --------------------------------------------------------------------------- # +# Structural guards +# --------------------------------------------------------------------------- # + +def _mint_sites(path: Path) -> list[tuple[str, int]]: + """Every ``relations.append`` / ``queries.append`` in *path*.""" + tree = ast.parse(path.read_text(encoding="utf-8")) + found: list[tuple[str, int]] = [] + for node in ast.walk(tree): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "append" + and isinstance(node.func.value, ast.Name) + and node.func.value.id in ("relations", "queries") + ): + found.append((node.func.value.id, node.lineno)) + return found + + +def test_reader_inventory_covers_every_mint_site() -> None: + """STRUCTURAL GUARD. A new reader template must be entered in the table. + + A ``Comprehension`` can only be populated through a ``relations.append`` or + ``queries.append`` call, so the count of those calls bounds the number of + constructions. When someone adds a template, this goes red until + ``READER_CONSTRUCTIONS`` names its mint site — which is the only thing + keeping the overlap number honest as the reader grows. + """ + sites = _mint_sites(_REPO / "generate" / "meaning_graph" / "reader.py") + assert len(sites) == 10, f"reader mint sites changed: {sites}" + assert len(READER_MINT_SITES) == 10 + + +def test_every_reader_construction_is_read_by_the_reader() -> None: + """STRUCTURAL GUARD. The table must describe the reader, not an aspiration. + + A construction the reader refuses would inflate ``reader_constructions`` and + depress ``r2w_rate`` for a reason that has nothing to do with the writer. + """ + for construction in READER_CONSTRUCTIONS: + for filler in COUNT_FILLERS: + instance = render_instance(construction, filler) + reading = comprehend(instance) + assert not isinstance(reading, Refusal), ( + f"{construction.construction_id} under {filler.name}: " + f"{instance!r} -> {reading}" + ) + + +def _match_arm_literals(path: Path, function: str) -> set[str]: + """String literals appearing in *function*'s ``match`` case PATTERNS.""" + tree = ast.parse(path.read_text(encoding="utf-8")) + literals: set[str] = set() + for node in ast.walk(tree): + if not (isinstance(node, ast.FunctionDef) and node.name == function): + continue + for stmt in ast.walk(node): + if not isinstance(stmt, ast.Match): + continue + for case in stmt.cases: + for pat in ast.walk(case.pattern): + if isinstance(pat, ast.MatchValue) and isinstance(pat.value, ast.Constant): + if isinstance(pat.value.value, str): + literals.add(pat.value.value) + return literals + + +def test_declared_writer_axes_cover_the_inflection_arms() -> None: + """STRUCTURAL GUARD. The sweep must not miss a tense or aspect. + + ``WRITER_TENSES`` / ``WRITER_ASPECTS`` are declared for readability, so they + are pinned against the arms of ``_inflect_predicate``. A new aspect added to + the writer without being added to the sweep would silently shrink the space + this lane claims to measure. + """ + literals = _match_arm_literals(_REPO / "generate" / "templates.py", "_inflect_predicate") + declared = {v for v in (*WRITER_TENSES, *WRITER_ASPECTS) if v is not None} + assert literals - declared == set(), f"undeclared inflection axis values: {literals - declared}" + + +def test_committed_corpus_matches_the_fillers_in_use() -> None: + """STRUCTURAL GUARD. The reviewable corpus must be the one that ran. + + ``corpora.py`` holds the fillers as typed objects and ``public/v1/cases.jsonl`` + is what a reviewer reads. Two copies of one fact is how this codebase got two + grammars; since the runner cannot import a JSON file as a dataclass without + losing the typing, they stay separate and this guard keeps them identical. + """ + path = _REPO / "evals" / "construction_inventory" / "public" / "v1" / "cases.jsonl" + committed = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + + in_use = [ + { + "id": f"filler-{f.name.replace('_', '-')}", + "kind": "count_filler" if f in COUNT_FILLERS else "mass_filler", + "x": f.x, "y": f.y, + "plural_x": f.plural_x, "plural_y": f.plural_y, + "secondary": SECONDARY[f.name], + } + for f in (*COUNT_FILLERS, MASS_FILLER) + ] + assert [{k: v for k, v in row.items() if k != "note"} for row in committed] == in_use + + +def test_writer_emits_no_interrogative_construction() -> None: + """STRUCTURAL GUARD. Three reader query templates are unreachable BY SHAPE. + + ``member_query`` and ``subset_query`` need a ``?``; no writer template + contains one, so no amount of vocabulary work reaches them. Recording this + as a shape fact keeps Phase 5 from trying to close that gap with corpora. + """ + report = run_lane() + writer_skeletons = { + d["skeleton"] for d in report.case_details if d.get("kind") == "writer_construction" + } + assert not any("?" in sk for sk in writer_skeletons) + + interrogative = [c for c in READER_CONSTRUCTIONS if c.interrogative] + assert len(interrogative) == 2 + for construction in interrogative: + instance = render_instance(construction, COUNT_FILLERS[0]) + assert skeleton(instance) not in writer_skeletons