From 7c2b753d1d36d9e4310c02f77db755dbc77daee3 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 27 Jul 2026 11:17:38 -0700 Subject: [PATCH 1/2] fix(generate): inflect the head verb on every branch, not just the plural two Phase 3 fixed two of the twelve branches of `_inflect_predicate` by hand and pinned them with a hand-written oracle. The other ten kept handing whole predicate phrases to single-verb functions, so the same root cause was still live on **57 of 96** (branch x multi-word predicate) pairs, 9 of 12 branches: "belongs to" --perfective--> "has belongs toed" "belongs to" --imperfective--> "is belongs toing" "belongs to" --past--> "belongs toed" "is defined as" --future--> "will is defined a" "contrasts with" --negated--> "does not contrasts with" Every branch now routes through `morphology.inflect_phrase_head`, which applies a single-verb inflection to the finite verb and carries tokens 2..n through byte-identically. Two closed tables were needed because `_base_form` is a suffix stripper and the head of every copular predicate is a form of BE: `base_form("is")` returned "i" and `present_participle("is")` returned "iing". Also fixes predicate-nominal object agreement. `render_step` pluralized the subject and never the object, so it wrote "all dogs are a mammal". The object agrees only for a predicate nominal -- a closed set of two -- because a prepositional object carries its own number and a productive rule would write "all claims are grounded in evidences". Measured -------- tail-mangling pairs 57/96 -> 0/96 g_read_rate (round-trip) 0.0 -> 0.003413 first non-zero in the arc grammatical_coverage v1 49/49 (2 cases corrected, see below) english_fluency_ood 117/117 + 39/39 + 13/13 unchanged discourse_paragraph 12/12 + 6/6 + 5/5 + 1/1 unchanged zero_code_domain_acquisition 18/18 + 30/30 + 21/21 unchanged `gram_C14_p01` and `gram_C14_p10` expected "...defined as compound", which is not English. Corrected, and the old string moved into `reject_surfaces` so the case now actively rejects what it used to accept. The correction is not my judgment: the reader independently READS "all molecules are defined as compounds" and REFUSES the singular form, and that is precisely what took g_read_rate off zero. Why the invariant instead of a bigger oracle -------------------------------------------- A per-branch oracle has to be extended by hand for each new branch, and a branch added without one is invisible -- which is how eight survived Phase 3. The pin is structural instead: English marks tense, number and aspect on the finite verb, so inflection must leave tokens 2..n byte-identical. It needs no oracle and covers branches nobody has written yet. It is necessary, not sufficient: "does not contrasts with" preserves its tail perfectly and is still wrong. `test_do_support_puts_the_head_in_the_bare_ infinitive` is the sufficiency half. Mutation -------- baseline 84 pass inflect_phrase_head applied to the whole phrase 39 FAIL _IRREGULAR_BASE emptied 2 FAIL _IRREGULAR_PARTICIPLE auxiliaries removed 2 FAIL PREDICATIVE_NOMINAL emptied 3 FAIL PREDICATIVE_NOMINAL widened to every copular predicate 2 FAIL The last row is the one that matters: the plausible-but-wrong fix -- "pluralize the object under a plural subject" -- is caught by the mass-noun control. Not fixed here, deliberately: "has not the following steps" is archaic rather than wrong, and modernizing it to do-support is a separate judgment call. [Verification]: smoke 621, deductive 406, lane pins 11/11 unchanged (grammatical_coverage is not a pinned lane; no pin was edited). --- .../public/v1/cases.jsonl | 4 +- generate/lexicon.py | 24 +++ generate/morphology.py | 81 ++++++++-- generate/templates.py | 68 ++++++-- tests/test_grammar_roundtrip.py | 58 ++++++- tests/test_lexicon_single_source.py | 4 + tests/test_realizer_quantifier_agreement.py | 149 ++++++++++++++++++ 7 files changed, 354 insertions(+), 34 deletions(-) diff --git a/evals/grammatical_coverage/public/v1/cases.jsonl b/evals/grammatical_coverage/public/v1/cases.jsonl index 0cea81d1..d46c0ab0 100644 --- a/evals/grammatical_coverage/public/v1/cases.jsonl +++ b/evals/grammatical_coverage/public/v1/cases.jsonl @@ -34,7 +34,7 @@ {"id": "gram_C13_p01", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "follows", "obj": "knowledge", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["wisdom is following knowledge"], "constraints": {"must_contain": ["wisdom", "is", "following", "knowledge"], "word_order": ["wisdom", "is", "following", "knowledge"], "max_words": 8}} {"id": "gram_C13_p02", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "supports", "obj": "truth", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["evidence is supporting truth"], "constraints": {"must_contain": ["evidence", "is", "supporting", "truth"], "word_order": ["evidence", "is", "supporting", "truth"], "max_words": 8}} {"id": "gram_C13_p03", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "dawn", "predicate": "precedes", "obj": "day", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["dawn is preceding day"], "constraints": {"must_contain": ["dawn", "is", "preceding", "day"], "word_order": ["dawn", "is", "preceding", "day"], "max_words": 8}} -{"id": "gram_C14_p01", "construction": "C14", "construction_name": "quantified_copular", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "molecule", "predicate": "is_defined_as", "obj": "compound", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all molecules are defined as compound"], "constraints": {"must_contain": ["all", "molecules", "are", "defined", "as", "compound"], "word_order": ["all", "molecules", "are", "defined", "as", "compound"], "max_words": 8}, "reject_surfaces": ["all molecules is defined a compound"]} +{"id": "gram_C14_p01", "construction": "C14", "construction_name": "quantified_copular", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "molecule", "predicate": "is_defined_as", "obj": "compound", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all molecules are defined as compounds"], "constraints": {"must_contain": ["all", "molecules", "are", "defined", "as", "compounds"], "word_order": ["all", "molecules", "are", "defined", "as", "compounds"], "max_words": 8}, "reject_surfaces": ["all molecules is defined a compound", "all molecules are defined as compound"]} {"id": "gram_C14_p02", "construction": "C14", "construction_name": "quantified_copular", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "theory", "predicate": "is_caused_by", "obj": "observation", "quantifier": "some"}], "edges": []}, "accept_surfaces": ["some theories are caused by observation"], "constraints": {"must_contain": ["some", "theories", "are", "caused", "by", "observation"], "word_order": ["some", "theories", "are", "caused", "by", "observation"], "max_words": 8}, "reject_surfaces": ["some theories is caused by observation"]} {"id": "gram_C14_p03", "construction": "C14", "construction_name": "quantified_copular", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wolf", "predicate": "belongs_to", "obj": "pack", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all wolves belong to pack"], "constraints": {"must_contain": ["all", "wolves", "belong", "to", "pack"], "word_order": ["all", "wolves", "belong", "to", "pack"], "max_words": 7}, "reject_surfaces": ["all wolfs belongs to pack"]} {"id": "gram_C14_p04", "construction": "C14", "construction_name": "quantified_copular", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "claim", "predicate": "is_grounded_in", "obj": "evidence", "quantifier": "most"}], "edges": []}, "accept_surfaces": ["most claims are grounded in evidence"], "constraints": {"must_contain": ["most", "claims", "are", "grounded", "in", "evidence"], "word_order": ["most", "claims", "are", "grounded", "in", "evidence"], "max_words": 8}, "reject_surfaces": ["most claims is grounded in evidence"]} @@ -43,7 +43,7 @@ {"id": "gram_C14_p07", "construction": "C14", "construction_name": "quantified_copular", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "axiom", "predicate": "is_distinguished_from", "obj": "theorem", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all axioms are distinguished from theorem"], "constraints": {"must_contain": ["all", "axioms", "are", "distinguished", "from", "theorem"], "word_order": ["all", "axioms", "are", "distinguished", "from", "theorem"], "max_words": 8}, "reject_surfaces": ["all axioms is distinguished from theorem"]} {"id": "gram_C14_p08", "construction": "C14", "construction_name": "quantified_copular", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "is_grounded_in", "obj": "truth", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all evidence is grounded in truth"], "constraints": {"must_contain": ["all", "evidence", "is", "grounded", "in", "truth"], "word_order": ["all", "evidence", "is", "grounded", "in", "truth"], "max_words": 8}} {"id": "gram_C14_p09", "construction": "C14", "construction_name": "quantified_copular", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "is_defined_as", "obj": "justified", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all knowledge is defined as justified"], "constraints": {"must_contain": ["all", "knowledge", "is", "defined", "as", "justified"], "word_order": ["all", "knowledge", "is", "defined", "as", "justified"], "max_words": 8}} -{"id": "gram_C14_p10", "construction": "C14", "construction_name": "quantified_copular", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "molecule", "predicate": "is_defined_as", "obj": "compound", "quantifier": "all", "negated": true}], "edges": []}, "accept_surfaces": ["all molecules are not defined as compound"], "constraints": {"must_contain": ["all", "molecules", "are", "not", "defined", "as", "compound"], "word_order": ["all", "molecules", "are", "not", "defined", "as", "compound"], "max_words": 9}, "reject_surfaces": ["all molecules do not is defined a compound"]} +{"id": "gram_C14_p10", "construction": "C14", "construction_name": "quantified_copular", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "molecule", "predicate": "is_defined_as", "obj": "compound", "quantifier": "all", "negated": true}], "edges": []}, "accept_surfaces": ["all molecules are not defined as compounds"], "constraints": {"must_contain": ["all", "molecules", "are", "not", "defined", "as", "compounds"], "word_order": ["all", "molecules", "are", "not", "defined", "as", "compounds"], "max_words": 9}, "reject_surfaces": ["all molecules do not is defined a compound", "all molecules are not defined as compound"]} {"id": "gram_C14_p11", "construction": "C14", "construction_name": "quantified_copular", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wolf", "predicate": "belongs_to", "obj": "pack", "quantifier": "all", "negated": true}], "edges": []}, "accept_surfaces": ["all wolves do not belong to pack"], "constraints": {"must_contain": ["all", "wolves", "do", "not", "belong", "to", "pack"], "word_order": ["all", "wolves", "do", "not", "belong", "to", "pack"], "max_words": 9}, "reject_surfaces": ["all wolfs do not belongs to pack"]} {"id": "gram_C14_p12", "construction": "C14", "construction_name": "quantified_copular", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "molecule", "predicate": "causes", "obj": "reaction", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all molecules cause reaction"], "constraints": {"must_contain": ["all", "molecules", "cause", "reaction"], "word_order": ["all", "molecules", "cause", "reaction"], "max_words": 6}, "reject_surfaces": ["all molecules caus reaction"]} {"id": "gram_C14_p13", "construction": "C14", "construction_name": "quantified_copular", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "observation", "predicate": "evidences", "obj": "claim", "quantifier": "many"}], "edges": []}, "accept_surfaces": ["many observations evidence claim"], "constraints": {"must_contain": ["many", "observations", "evidence", "claim"], "word_order": ["many", "observations", "evidence", "claim"], "max_words": 6}} diff --git a/generate/lexicon.py b/generate/lexicon.py index d5a07935..82df6b87 100644 --- a/generate/lexicon.py +++ b/generate/lexicon.py @@ -338,3 +338,27 @@ MASS_NOUNS: Final[frozenset[str]] = frozenset( } ) + +#: Predicates whose object is a **predicate nominal** — a second name for the +#: subject's category rather than an independent noun phrase. English makes the +#: object agree in number with the subject in exactly this construction: +#: +#: all dogs are mammals (not "are a mammal") +#: all molecules are defined as compounds +#: +#: and does **not** elsewhere, because a prepositional object carries its own +#: number, chosen by the speaker and not by the subject: +#: +#: all claims are grounded in evidence (mass — never "evidences") +#: some theories are caused by observation (generic singular is fine) +#: +#: So this is a CLOSED SET, not a productive rule — the same discipline +#: ``VES_PLURAL_SINGULARS`` enforces for f/fe → ves. A rule that pluralized +#: every object under a plural subject would produce "grounded in evidences". +#: +#: Deliberately excluded: ``is_distinguished_from``, ``is_caused_by``, +#: ``is_grounded_in``. Their objects read as generic and the reader refuses the +#: whole construction anyway (``reserved_word_in_np``), so widening the set buys +#: no round-trip and commits to a number English leaves open. +PREDICATIVE_NOMINAL: Final[frozenset[str]] = frozenset({"is_a", "is_defined_as"}) + diff --git a/generate/morphology.py b/generate/morphology.py index 78c399f4..8dacf013 100644 --- a/generate/morphology.py +++ b/generate/morphology.py @@ -15,6 +15,8 @@ mammal``. """ from __future__ import annotations +from collections.abc import Callable + from generate.lexicon import ( INVARIANT_NUMBER, VES_PLURAL_SINGULARS, @@ -152,13 +154,40 @@ _IRREGULAR_FORMS: dict[str, tuple[str, str]] = { _IRREGULAR_PAST: dict[str, str] = {v: forms[0] for v, forms in _IRREGULAR_FORMS.items()} _IRREGULAR_PARTICIPLE: dict[str, str] = { - # Present-participle (-ing) is almost always regular. Only handle - # the truly weird cases (lie→lying handled by the suffix rule; - # be→being is the one English present-participle that needs a - # special entry, but `is` doesn't normally surface as a content - # predicate in our realizer pipeline). + # Present-participle (-ing) is almost always regular (lie→lying is handled + # by the suffix rule). The auxiliaries are the exception, and they DO + # surface as predicate heads: every copular predicate in + # ``PREDICATE_DISPLAY`` begins with "is"/"has" ("is defined as", "has the + # following steps"), so the imperfective branch inflects them constantly. + # Without these entries ``present_participle("is")`` fell through + # ``_base_form("is") == "i"`` and produced **"iing"**. + "is": "being", + "are": "being", + "was": "being", + "were": "being", + "has": "having", + "have": "having", + "does": "doing", + "do": "doing", } +#: 3sg present → bare infinitive, for the verbs whose base is not the stem left +#: behind by stripping ``-s``. ``_base_form`` is a suffix stripper, so without +#: this table ``base_form("is")`` returned **"i"** and the future branch emitted +#: "will i defined as". Same closed-set discipline as the ``ves`` plurals: a +#: table, not a rule, because the rule has no way to know. +_IRREGULAR_BASE: dict[str, str] = { + "is": "be", "are": "be", "was": "be", "were": "be", "am": "be", + "has": "have", "have": "have", "had": "have", + "does": "do", "do": "do", "did": "do", +} + +#: Heads that take a bare ``not`` rather than do-support. "was not defined as", +#: never "did not be defined as"; but "did not belong to", never "belonged not". +_BARE_NOT_HEADS: frozenset[str] = frozenset( + {"is", "are", "was", "were", "has", "have", "had", "does", "do", "did"} +) + _IRREGULAR_PAST_PARTICIPLE: dict[str, str] = {v: forms[1] for v, forms in _IRREGULAR_FORMS.items()} @@ -179,6 +208,8 @@ _ES_STEM_ENDINGS = ("ss", "sh", "ch", "x", "z", "o") def _base_form(verb_3sg: str) -> str: + if verb_3sg in _IRREGULAR_BASE: + return _IRREGULAR_BASE[verb_3sg] if verb_3sg in _IES_KEEP_IE: return verb_3sg[:-1] if verb_3sg.endswith("ies"): @@ -207,24 +238,48 @@ def plural_present(verb_3sg: str) -> str: return _base_form(verb_3sg) +def inflect_phrase_head(phrase: str, inflect: Callable[[str], str]) -> str: + """Apply a SINGLE-VERB inflection to a predicate phrase's finite verb. + + English marks tense, number and aspect on the finite verb, which is the + first token of every humanized predicate ("is defined as", "has the + following steps", "belongs to", "contrasts with"). Only that token + inflects; tokens 2..n are carried through **byte-identical**. + + That tail-preservation is a falsifiable invariant, and it is the one this + module kept violating. Every function in here — ``base_form``, + ``past_tense``, ``present_participle``, ``past_participle`` — is written + for a single verb, and ``_inflect_predicate`` was handing them whole + phrases on nine of its ten branches. Phase 3 fixed the two plural branches + by hand; the other eight still produced "belongs toed", "has belongs toed", + "is belongs toing" and "will is defined a". Routing every branch through + this one function is what makes the invariant checkable in one place + instead of eight. + """ + if not phrase: + return phrase + head, sep, rest = phrase.partition(" ") + return inflect(head) + sep + rest + + def agree_plural_phrase(phrase: str) -> str: """Put a whole predicate PHRASE into plural agreement. - Number is marked on the finite verb, which is the first token of every - humanized predicate ("is defined as", "has the following steps", - "belongs to", "contrasts with"). Only that token inflects; the rest is - carried through untouched. - This exists because :func:`base_form` is a SINGLE-VERB function and was being applied to whole phrases, stripping the last character-class of the final word: "is defined as" -> "is defined a", "has the following steps" -> "has the following step". Nine of the 26 seed predicates were wrong that way, and every multi-word one was. """ + return inflect_phrase_head(phrase, plural_present) + + +def takes_bare_not(phrase: str) -> bool: + """True when negation attaches directly to the head ("was not defined as") + rather than through do-support ("did not belong to").""" if not phrase: - return phrase - head, sep, rest = phrase.partition(" ") - return plural_present(head) + sep + rest + return False + return phrase.partition(" ")[0] in _BARE_NOT_HEADS def past_tense(verb_3sg: str) -> str: diff --git a/generate/templates.py b/generate/templates.py index 588ac1b2..7c753cf3 100644 --- a/generate/templates.py +++ b/generate/templates.py @@ -16,6 +16,7 @@ from generate.lexicon import ( IRREGULAR_PLURALS, PLURAL_QUANTIFIERS, PREDICATE_DISPLAY, + PREDICATIVE_NOMINAL, ) from generate.articulation_legality import ( ArticulationLegality, @@ -25,11 +26,13 @@ from generate.graph_planner import RhetoricalMove from generate.morphology import ( agree_plural_phrase, base_form, + inflect_phrase_head, is_mass_noun, past_participle, past_tense, pluralize, present_participle, + takes_bare_not, ) @@ -54,6 +57,16 @@ def _humanize_predicate(predicate: str) -> str: return _PREDICATE_DISPLAY.get(predicate, predicate.replace("_", " ")) +def _drop_indefinite_article(predicate_h: str) -> str: + """Strip a trailing ``a``/``an`` from an inflected predicate. + + ``is_a`` humanizes to "is a" and pluralizes to "are a"; the article cannot + survive a plural nominal ("all dogs are a mammals" is not English). + """ + head, sep, rest = predicate_h.rpartition(" ") + return head if sep and rest in ("a", "an") else predicate_h + + _MOVE_TEMPLATES: dict[RhetoricalMove, str] = { RhetoricalMove.ASSERT: "{subject} {predicate_h} {obj}", RhetoricalMove.ELABORATE: "furthermore, {subject} {predicate_h} {obj}", @@ -83,25 +96,42 @@ def _inflect_predicate( predicate_h.startswith(prefix) for prefix in ("is ", "are ", "has ", "have ", "belongs ") ) - base = base_form(verb) - match (aspect, tense, negated, plural_subject): + # Every branch below inflects the phrase HEAD and carries tokens 2..n + # through untouched. Phase 3 fixed only the two plural branches, so + # these eight were still handing whole phrases to single-verb + # functions: "belongs to" came back "has belongs toed" (perfective), + # "is belongs toing" (imperfective), "belongs toed" (past), and + # "is defined as" came back "will is defined a" (future). case ("perfective", _, _, True): - return f"have {past_participle(verb)}" + return f"have {inflect_phrase_head(verb, past_participle)}" case ("perfective", _, _, False): - return f"has {past_participle(verb)}" + return f"has {inflect_phrase_head(verb, past_participle)}" case ("imperfective", _, _, True): - return f"are {present_participle(verb)}" + return f"are {inflect_phrase_head(verb, present_participle)}" case ("imperfective", _, _, False): - return f"is {present_participle(verb)}" + return f"is {inflect_phrase_head(verb, present_participle)}" case (_, "past", True, _): - return f"did not {base}" + # A be/have head negates in place and carries its own past tense + # ("was not defined as"); anything else takes do-support in the + # past ("did not belong to"), where the head reverts to the base. + if takes_bare_not(verb): + past = inflect_phrase_head(verb, past_tense) + p_head, sep, rest = past.partition(" ") + if plural_subject: + p_head = {"was": "were", "has": "have", "did": "did"}.get(p_head, p_head) + return f"{p_head} not{sep}{rest}" if rest else f"{p_head} not" + return f"did not {inflect_phrase_head(verb, base_form)}" case (_, "past", False, _): - return past_tense(verb) + past = inflect_phrase_head(verb, past_tense) + if plural_subject: + p_head, sep, rest = past.partition(" ") + return {"was": "were"}.get(p_head, p_head) + sep + rest + return past case (_, "future", True, _): - return f"will not {base}" + return f"will not {inflect_phrase_head(verb, base_form)}" case (_, "future", False, _): - return f"will {base}" + return f"will {inflect_phrase_head(verb, base_form)}" case (_, _, True, True): # Plural + negated. Agree the head first, then negate around it: # a plural copula takes a bare "not" ("are not defined as"), while @@ -125,9 +155,13 @@ def _inflect_predicate( return "have not " + predicate_h[5:] if predicate_h.startswith("belongs "): return "does not belong " + predicate_h[8:] - return f"is not {base}" + return f"is not {inflect_phrase_head(verb, base_form)}" case (_, _, True, False): - return f"does not {base}" + # Do-support puts the head in the bare infinitive. This branch was + # the ninth instance of the same defect: ``base_form`` on the whole + # phrase left "contrasts with" untouched (no -s/-es/-ies suffix to + # strip from "with"), yielding "does not contrasts with". + return f"does not {inflect_phrase_head(verb, base_form)}" case (_, _, False, True): # Plural agreement on the whole phrase, not base_form() of it. # This is the branch the 9-of-26 defect lived in: it returned the @@ -170,6 +204,16 @@ def render_step( plural_subject=plural, ) obj_display = obj if obj != "" else "..." + # A predicate nominal names the subject's category, so it agrees with the + # subject in number and sheds its indefinite article: "all dogs are + # mammals", never "all dogs are a mammal". Restricted to the closed + # PREDICATIVE_NOMINAL set — a prepositional object carries its own number + # ("all claims are grounded in evidence") and pluralizing it produces + # "evidences". The reader accepts the agreeing form and refuses the other, + # which is why this was the last writer-side blocker on G-round-trip. + if plural and predicate in PREDICATIVE_NOMINAL: + obj_display = pluralize(obj_display) + predicate_h = _drop_indefinite_article(predicate_h) subject_form = pluralize(subject) if plural else subject subject_display = f"{quantifier} {subject_form}" if quantifier else subject_form return template.format( diff --git a/tests/test_grammar_roundtrip.py b/tests/test_grammar_roundtrip.py index 2718b934..9665290e 100644 --- a/tests/test_grammar_roundtrip.py +++ b/tests/test_grammar_roundtrip.py @@ -134,12 +134,22 @@ def test_shuffles_are_byte_stable_across_calls(): # --------------------------------------------------------------------------- # -def test_g_roundtrip_baseline_is_zero(report): - """BASELINE PIN (a defect, not a goal): CORE reads 0% of what it writes. +def test_g_roundtrip_baseline_is_one_case_of_293(report): + """RATCHET PIN. Was 0/293 on main @ 9696443a — CORE read *nothing* it wrote. - Measured on main @ 9696443a. When the grammar is unified this must be - revised **upward**; it must never be revised downward to accommodate a - regression. + Phase 4 made it **1**. This must only ever be revised upward; never + downward to accommodate a regression. + + The one case is ``gram_C14_p01``, and what unblocked it is worth recording + because it is the opposite of what §6 of the plan predicted. The writer was + emitting ``all molecules are defined as compound`` — a predicate nominal + that does not agree with its subject. The reader accepts + ``...as compounds`` and refuses ``...as compound``, so the blocker was a + one-line **writer** defect, not the MeaningGraph/PropositionGraph type + mismatch of §1.8. + + The remaining 292 decompose cleanly, and none of them is a model mismatch + either — see ``test_the_remaining_blockers_are_reader_construction_coverage``. """ # 293, not the original 280: Phase 3 added 13 quantified-copular cases # (construction C14) to grammatical_coverage/public/v1, and this lane @@ -148,8 +158,42 @@ def test_g_roundtrip_baseline_is_zero(report): # or shrinks should require a deliberate edit here, not pass silently. assert report.metrics["graph_cases"] == 293 assert report.metrics["g_write_rate"] == 1.0 - assert report.metrics["g_read_rate"] == 0.0 - assert report.metrics["g_exact_rate"] == 0.0 + assert report.metrics["g_read_rate"] >= 0.003413, "the ratchet may not go down" + assert report.metrics["g_read_rate"] == 0.003413 + + +def test_the_remaining_blockers_are_reader_construction_coverage(report): + """WHY g_read_rate is 1/293 and not 293/293 — the measurement §6 turns on. + + The plan pre-committed to reading a near-zero rate as evidence for §1.8: + two incompatible graph models, next step an ADR. The refusal reasons say + otherwise. Every one of them is the reader declining a CONSTRUCTION it has + no template for — not a projection disagreeing about a graph it parsed: + + no_template_match 289 reader has no SUBJ-VERB-OBJ template at all + unknown_morphology 2 prepositional objects (reserved_word_in_np) + unsupported_negation 1 reader has no negated-categorical template + + Where a construction IS in both inventories, the round trip closes exactly + (``s_surface_match_rate == s_renderable_rate``). So the barrier is the + *overlap* of the two construction inventories, which is currently one + construction wide — and that is Phase 5's item 1, not an ADR. + """ + reasons: dict[str, int] = {} + for row in report.case_details: + if "wrote" not in row: + continue + reasons[row.get("refusal_reason") or "READ"] = ( + reasons.get(row.get("refusal_reason") or "READ", 0) + 1 + ) + assert reasons == { + "no_template_match": 289, + "unknown_morphology": 2, + "unsupported_negation": 1, + "READ": 1, + } + # The load-bearing claim: not one refusal is a graph-model disagreement. + assert "projection_mismatch" not in reasons def test_s_roundtrip_closes_for_every_renderable_surface(report): diff --git a/tests/test_lexicon_single_source.py b/tests/test_lexicon_single_source.py index 5ecf4a3a..b040f3be 100644 --- a/tests/test_lexicon_single_source.py +++ b/tests/test_lexicon_single_source.py @@ -62,6 +62,10 @@ RECORDED_CONSUMERS: dict[str, frozenset[str]] = { "QUANTIFIER_LEAD": frozenset({"generate.proof_chain.english"}), "QUANTIFIER_TOKENS": frozenset({"generate.proof_chain.member"}), "PLURAL_QUANTIFIERS": frozenset({"generate.templates"}), + # Phase 4: the closed set of predicates whose object is a predicate nominal + # and therefore agrees in number with the subject. Closed, not productive — + # a rule would pluralize "grounded in evidence" into "evidences". + "PREDICATIVE_NOMINAL": frozenset({"generate.templates"}), "PREDICATE_DISPLAY": frozenset({ "generate.templates", "generate.semantic_templates", diff --git a/tests/test_realizer_quantifier_agreement.py b/tests/test_realizer_quantifier_agreement.py index cd663c30..615b9b6d 100644 --- a/tests/test_realizer_quantifier_agreement.py +++ b/tests/test_realizer_quantifier_agreement.py @@ -217,3 +217,152 @@ def test_f_to_ves_is_a_closed_set_not_a_rule(singular: str, plural: str) -> None from generate.morphology import pluralize as _pluralize assert _pluralize(singular) == plural + + +# --------------------------------------------------------------------------- +# Phase 4: the phrase-vs-single-verb defect on the EIGHT branches Phase 3 left +# --------------------------------------------------------------------------- +# +# Phase 3 fixed the two plural branches of ``_inflect_predicate`` by hand and +# pinned them with a hand-written oracle. That left the other eight branches +# handing whole predicate phrases to single-verb functions, so the same root +# cause was still live: +# +# "belongs to" --perfective--> "has belongs toed" +# "belongs to" --imperfective-> "is belongs toing" +# "belongs to" --past--------> "belongs toed" +# "is defined as" --future----> "will is defined a" +# +# 49 of 80 (branch x multi-word predicate) pairs were wrong. A per-branch +# oracle would have to be extended by hand every time a branch is added, and a +# branch added without one is invisible — which is how eight of them survived +# Phase 3. So the pin here is a STRUCTURAL INVARIANT instead: +# +# English marks tense, number and aspect on the FINITE VERB. Inflecting a +# predicate phrase must leave tokens 2..n byte-identical. +# +# It is falsifiable, it needs no oracle, and it covers branches nobody has +# written yet. + +_INFLECTION_BRANCHES: list[tuple[str, dict[str, object]]] = [ + ("plural", {"plural_subject": True}), + ("plural+negated", {"plural_subject": True, "negated": True}), + ("negated", {"negated": True}), + ("past", {"tense": "past"}), + ("past+plural", {"tense": "past", "plural_subject": True}), + ("past+negated", {"tense": "past", "negated": True}), + ("future", {"tense": "future"}), + ("future+negated", {"tense": "future", "negated": True}), + ("perfective", {"aspect": "perfective"}), + ("perfective+plural", {"aspect": "perfective", "plural_subject": True}), + ("imperfective", {"aspect": "imperfective"}), + ("imperfective+plural", {"aspect": "imperfective", "plural_subject": True}), +] + + +def _multi_word_predicates() -> list[str]: + from generate.lexicon import PREDICATE_DISPLAY + + return sorted({d for d in PREDICATE_DISPLAY.values() if " " in d}) + + +@pytest.mark.parametrize(("branch", "kwargs"), _INFLECTION_BRANCHES, ids=[b for b, _ in _INFLECTION_BRANCHES]) +def test_inflection_only_touches_the_head_verb(branch: str, kwargs: dict[str, object]) -> None: + """Tokens 2..n of a predicate phrase survive inflection byte-identically.""" + from generate.templates import _inflect_predicate + + violations = [] + for display in _multi_word_predicates(): + tail = display.split(" ")[1:] + got = _inflect_predicate(display, **kwargs) # type: ignore[arg-type] + if got.split(" ")[-len(tail):] != tail: + violations.append((display, got)) + assert not violations, f"{branch} mangled the phrase tail: {violations}" + + +@pytest.mark.parametrize( + ("kwargs", "expected"), + [ + ({"tense": "past"}, "was defined as"), + ({"tense": "past", "plural_subject": True}, "were defined as"), + ({"tense": "past", "negated": True}, "was not defined as"), + ({"tense": "future"}, "will be defined as"), + ({"tense": "future", "negated": True}, "will not be defined as"), + ({"aspect": "perfective"}, "has been defined as"), + ({"aspect": "perfective", "plural_subject": True}, "have been defined as"), + ({"aspect": "imperfective"}, "is being defined as"), + ({"aspect": "imperfective", "plural_subject": True}, "are being defined as"), + ], +) +def test_copular_head_inflects_as_be(kwargs: dict[str, object], expected: str) -> None: + """The head of every copular predicate is a form of BE, and BE is irregular + in all four of these paradigms. ``_base_form`` is a suffix stripper, so + before the irregular tables ``base_form("is")`` was **"i"** and + ``present_participle("is")`` was **"iing"**.""" + from generate.templates import _inflect_predicate + + assert _inflect_predicate("is defined as", **kwargs) == expected # type: ignore[arg-type] + + +def test_do_support_is_used_when_the_head_is_not_an_auxiliary() -> None: + """"did not belong to", never "did not belonged to" or "belonged not to".""" + from generate.templates import _inflect_predicate + + assert _inflect_predicate("belongs to", tense="past", negated=True) == "did not belong to" + assert _inflect_predicate("belongs to", tense="past") == "belonged to" + assert _inflect_predicate("belongs to", tense="future") == "will belong to" + + +# --------------------------------------------------------------------------- +# Phase 4: predicate-nominal object agreement +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize( + ("predicate", "obj", "expected"), + [ + # Predicate nominal: the object names the subject's category, so it + # agrees in number and the indefinite article goes. + ("is_a", "mammal", "all dogs are mammals"), + ("is_defined_as", "compound", "all dogs are defined as compounds"), + # Prepositional object: number is the speaker's, not the subject's. + ("is_grounded_in", "evidence", "all dogs are grounded in evidence"), + ("is_caused_by", "observation", "all dogs are caused by observation"), + ("belongs_to", "pack", "all dogs belong to pack"), + ], +) +def test_only_predicate_nominals_agree_in_number(predicate: str, obj: str, expected: str) -> None: + """``all dogs are a mammal`` was the last writer-side blocker on + G-round-trip: the reader accepts "all dogs are mammals" and refuses the + other. But the fix must NOT be "pluralize the object under a plural + subject" — that yields "grounded in evidences". Hence a closed set.""" + assert render_step(RhetoricalMove.ASSERT, "dog", predicate, obj, quantifier="all") == expected + + +def test_singular_subjects_keep_the_article_and_the_singular_object() -> None: + assert render_step(RhetoricalMove.ASSERT, "dog", "is_a", "mammal") == "dog is a mammal" + + +def test_predicative_nominal_is_a_closed_set_not_every_copular_predicate() -> None: + """If this set ever becomes "anything starting with is", the mass-noun + control above starts failing.""" + from generate.lexicon import PREDICATIVE_NOMINAL + + assert PREDICATIVE_NOMINAL == frozenset({"is_a", "is_defined_as"}) + + +def test_do_support_puts_the_head_in_the_bare_infinitive() -> None: + """The ninth instance of the same defect, and the one the tail invariant + CANNOT see: "does not contrasts with" preserves the tail perfectly and is + still wrong, because the error is on the head. + + ``base_form("contrasts with")`` returned the phrase unchanged — "with" has + no -s/-es/-ies suffix to strip — so the 3sg -s survived do-support. A tail + invariant is necessary, not sufficient; this is the sufficiency half. + """ + from generate.templates import _inflect_predicate + + assert _inflect_predicate("contrasts with", negated=True) == "does not contrast with" + assert _inflect_predicate("belongs to", negated=True) == "does not belong to" + assert _inflect_predicate("causes", negated=True) == "does not cause" + # A copular head negates in place and keeps its finite form. + assert _inflect_predicate("is defined as", negated=True) == "is not defined as" From 7d6760ae5bd45cc1aff082ba6eb2b640b876b48b Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 27 Jul 2026 11:57:48 -0700 Subject: [PATCH 2/2] feat(evals): the fluency lanes report the writer that actually ships (Phase 4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 4 of the grammar-unification arc, resolved by option (b) of the plan. The problem: 149 green fluency cases scored `realize_target`, and `core/cognition/pipeline.py` never calls it -- it calls `realize_semantic`. `english_fluency_ood` reported 117/117 + 39/39 for a function that does not speak, for the whole life of the lane. The plan framed this as "which realizer is better". Reading the source says it is not a quality question. `render_semantic`'s signature is `(intent, subject, predicate, obj, secondary, language, root)` -- no `negated`, no `quantifier`, no `tense`, no `aspect` -- and `realize_semantic` never reads them off the step. So the serving writer cannot express content the ArticulationStep is carrying: negated=False -> 'Knowledge is defined as opinion.' negated=True -> 'Knowledge is defined as opinion.' It serves the AFFIRMATIVE of a negated proposition. That is the ADR-0261 §5.1 family, not a fluency defect. Pinned here as a defect; NOT fixed, because fixing it changes served output and belongs to Shay. Measured -- identical contract, all seven scored corpora -------------------------------------------------------- bucket n realize_target realize_semantic feature-bearing, single node 214 207 49 no features, multi-node 100 100 3 no features, single node (CONTROL) 33 33 33 total 347 340 85 The control is what makes the rest mean anything. Every corpus hardcodes IntentTag.UNKNOWN, so "the serving writer scores badly" could have been an artifact of never giving it a real intent. On the 33 cases carrying nothing it cannot express, the two are IDENTICAL -- so the gap on the other 314 is the dropped features and the clause joining, not the intent. Delivered --------- - `grammatical_coverage/runner.py` reports `serving_accuracy` beside `accuracy`; `english_fluency_ood` delegates to that run_lane and gains it for free. The realizer is a parameter now instead of a hardcoded import. - `tests/test_phase4_realizer_resolution.py` -- the control, the decomposition, the negation defect pin, and the §6 evidence. - The claim is restated at both places it was made: the lane docstring and `grammar_roundtrip/contract.md`. Zero served bytes change. No serving authorization needed. §6 -- the plan's pre-commitment was wrong, and is corrected rather than quietly edited ------------------------------------------------------------------------ §6 forks on read_rate: risen => Phase 5 diversity; near-zero => §1.8 graph-model mismatch => ADR. It pre-committed to the second. Neither is what the measurement says. g_read_rate went to 1/293. The unblocked case was blocked by a one-line WRITER defect (predicate-nominal object agreement, #135), not by §1.8. The other 292: no_template_match 289 reader has no SUBJ-VERB-OBJ template at all unknown_morphology 2 prepositional objects (reserved_word_in_np) unsupported_negation 1 reader has no negated-categorical template Every one is the reader declining a CONSTRUCTION, not a projection disagreeing about a graph it parsed. Where a construction is in both inventories the round trip closes exactly. So the barrier is the OVERLAP of the two construction inventories, currently one construction wide -- Phase 5's item 1, tractable, not an ADR-scale model decision. §1.6 read "uniform no_template_match" as evidence FOR the type mismatch. It is not: no_template_match is a coverage fact, and the corpus was 289/293 bare transitives, a construction the reader never claimed to read. The measurement was mostly reporting the corpus's composition. And a finding in my own stack ------------------------------ `tests/test_realizer_quantifier_agreement.py` lived ONLY in the `cognition` suite, which is not on the AGENTS.md pre-push gate. So every pin Phases 3 and 4 added to it -- including the invariant covering all twelve inflection branches -- ran in NO gate. That is why smoke stayed at 621 across two PRs that added 13 tests between them, and it is the same silent-red shape the smoke list already calls out for test_adr_index.py. Registered into `deductive`, which now runs 504 instead of 406. Mutation -------- baseline 12 pass serving metric computed with realize_target 4 FAIL render_semantic GAINS a `negated` parameter 2 FAIL realize_semantic delegated to realize_target 6 FAIL Row 2 matters most: if someone FIXES the negation defect, the pin forces a deliberate revision instead of passing silently. Still open, still Shay's: option (a), promoting realize_target to the serving path. It now has a price tag -- 340/347 over 85/347 and the ability to say "not", against a move in every surface hash and whatever the Shadow Coherence Gate ruling in core/cognition/surface_resolution.py was protecting. [Verification]: in-worktree on CPython 3.12.13 with `uv sync --locked` -- smoke 621, deductive 504 (was 406; +12 new Phase 4 pins, +86 previously ungated), lane pins 11/11 unchanged, no pin edited. --- core/cli_test.py | 15 ++ docs/plans/grammar-unification-2026-07-26.md | 110 +++++++- evals/grammar_roundtrip/contract.md | 8 + evals/grammatical_coverage/runner.py | 49 +++- tests/test_phase4_realizer_resolution.py | 268 +++++++++++++++++++ 5 files changed, 445 insertions(+), 5 deletions(-) create mode 100644 tests/test_phase4_realizer_resolution.py diff --git a/core/cli_test.py b/core/cli_test.py index bc88aba3..c01fe3be 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -246,6 +246,21 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_vocab_trigger_instrument.py", "tests/test_grammar_roundtrip.py", "tests/test_lexicon_single_source.py", + # Phase 4 — which realizer serves, and what the other one's score + # means. Registered here deliberately: this file exists because a lane + # reported 117/117 for a function nothing calls, and a pin that no + # curated suite runs is the same defect one level up. Pure in-process + # lane replay, ~0.4s. + "tests/test_phase4_realizer_resolution.py", + # The grammar arc's agreement oracle and the tail-preservation + # invariant. It lived ONLY in the `cognition` suite, which is not on + # the AGENTS.md pre-push gate (smoke + deductive) — so every pin + # Phases 3 and 4 added to it, including the invariant that covers all + # twelve inflection branches, ran in no gate at all. Exactly the + # silent-red shape called out for test_adr_index.py above, and the + # reason smoke stayed at 621 across two PRs that added 13 tests. + # ~0.4s; it belongs with the other grammar pins. + "tests/test_realizer_quantifier_agreement.py", ), "full": ("tests/",), } diff --git a/docs/plans/grammar-unification-2026-07-26.md b/docs/plans/grammar-unification-2026-07-26.md index 95d2619d..001892e1 100644 --- a/docs/plans/grammar-unification-2026-07-26.md +++ b/docs/plans/grammar-unification-2026-07-26.md @@ -766,7 +766,14 @@ not guessed). Mutation now: | baseline | 13/13 | | phrase-head plural agreement | **4/13** | | `-es` stem rule | **12/13** | -| closed `ves` set | **11/13** | +| closed `ves` set | **12/13** | + +> **Correction (Phase 4).** The `ves` row was first recorded here as 11/13. Re-run +> against merged `main` by reverting the hunk in source — not by monkeypatch — it +> is **12/13**, and the single sensitive case is `gram_C14_p06` (`proofs`). The +> pin is still load-bearing; the recorded number was wrong. Corrected rather than +> left standing, because a mutation matrix nobody can reproduce is worth less +> than no matrix at all. The 4 survivors of the first mutation are the mass-noun and unchanged controls, which is the correct behaviour — `all evidence is grounded in truth` must *not* @@ -792,6 +799,55 @@ is taken is recorded with its reasoning. **Why this is gated:** (a) changes what users see and would move surface hashes. It is a serving change and belongs to Shay, not to this plan. +**RESULT — (b), and the framing of the choice was wrong.** + +The plan posed this as "which realizer is better". Reading the source says it is +not a quality question. `render_semantic`'s signature is +`(intent, subject, predicate, obj, secondary, language, root)` — there is **no +`negated`, `quantifier`, `tense` or `aspect` parameter**, and `realize_semantic` +never reads them off the step. So the serving writer does not merely inflect +worse; it cannot express content the `ArticulationStep` is carrying: + +``` +negated=False -> 'Knowledge is defined as opinion.' +negated=True -> 'Knowledge is defined as opinion.' <-- byte-identical +``` + +**It serves the affirmative of a negated proposition.** That is not a fluency +defect. It is the ADR-0261 §5.1 family — v1b served WRONG by dropping premises +it could not express — and it is pinned here as a defect, not fixed, because +fixing it changes served output. + +Scored on the identical contract across all seven corpora that a lane runs +through `realize_target`: + +| bucket | n | `realize_target` | `realize_semantic` | +|---|---|---|---| +| feature-bearing, single node | 214 | 207 | **49** | +| no features, multi-node | 100 | 100 | **3** | +| no features, single node — **CONTROL** | 33 | 33 | **33** | +| total | 347 | **340** | **85** | + +The control is what makes the rest mean anything. Every corpus hardcodes +`IntentTag.UNKNOWN`, so "the serving writer scores badly" could have been an +artifact of never giving it a real intent. On the 33 cases carrying nothing it +cannot express, the two realizers are **identical** — so the gap on the other +314 is the dropped features (214) and clause joining (100), not the intent. + +**What was delivered.** `grammatical_coverage/runner.py` now reports +`serving_accuracy` beside `accuracy`, computed by re-scoring the same cases +through `realize_semantic`; `english_fluency_ood` delegates to that `run_lane` +and gains it for free. The claim is restated at both places it was made — the +lane docstring and `grammar_roundtrip/contract.md` — as a claim about +**eval-only code**. Zero served bytes change, so this needed no serving +authorization. + +**What was not delivered, deliberately.** (a) is still open and still Shay's. +The difference now is that it can be decided against a measured cost: promoting +`realize_target` buys 340/347 over 85/347 and the ability to say "not", and +costs a move in every surface hash plus whatever the Shadow Coherence Gate +ruling in `core/cognition/surface_resolution.py` was protecting. + ### Phase 5 — Raise round-trip, then earn diversity *Depends on:* Phases 1–3 plus 2B, and on the Phase 1 baseline being non-zero-able. @@ -860,6 +916,58 @@ the ADR rather than at fluid prose. --- +### RESULT — measured after Phase 4. **Neither fork. The pre-commitment above was wrong.** + +`g_read_rate` did not stay at zero, and it did not rise materially either. It +went to **1 of 293** (`0.003413`) — the first non-zero in the arc — and the one +case tells you more than the rate does. + +The unblocked case is `gram_C14_p01`. What was blocking it: the writer emitted +`all molecules are defined as compound`, a predicate nominal that does not agree +with its subject. The reader **accepts** `...as compounds` and **refuses** +`...as compound`. So the blocker was a one-line *writer* defect — not §1.8. + +The other 292 decompose, and not one of them is a model mismatch either: + +| refusal reason | n | whose problem | +|---|---|---| +| `no_template_match` | 289 | reader has no SUBJ-VERB-OBJ template at all | +| `unknown_morphology` | 2 | prepositional objects (`reserved_word_in_np`) | +| `unsupported_negation` | 1 | reader has no negated-categorical template | +| read | 1 | — | + +Every one is the reader declining a **construction it has no template for**, not +a projection disagreeing about a graph it successfully parsed. And where a +construction *is* in both inventories the round trip closes exactly — +`s_surface_match_rate == s_renderable_rate` — while the reader independently +reads `all dogs are mammals`, `all wolves are canines`, and +`all molecules are defined as compounds`. + +**So the barrier is the overlap of the two construction inventories, which is +currently one construction wide.** That is Phase 5's item 1 — grow the +inventory until `read_rate` clears a ratcheted floor — and it is tractable work, +not an ADR-scale model decision. + +**Why the pre-commitment was wrong, kept here rather than quietly edited out:** +§1.6 read "uniform `no_template_match` across all 280" as evidence *for* the +type mismatch. It is not evidence for it. `no_template_match` is the reader +saying it has no template — a coverage fact — and §1.8's type mismatch was +inferred from it rather than measured. The corpus was 289/293 bare transitives, +a construction the reader has never claimed to read, so the measurement was +mostly reporting the corpus's composition. The one construction the corpus *did* +share with the reader (Phase 3's C14) was blocked by a writer bug, and until +that was fixed there was no case in the whole corpus capable of testing the +graph-model question at all. + +**Consequence for Phase 5.** Its scope is now decidable. The inventory question +is the arc's live frontier, and it should be sized by measuring the reader's +construction set against the writer's rather than by growing corpora blindly. +The graph-model ADR is **not** cancelled — §1.8's type mismatch is still real — +but nothing measured here obliges it, and it should not be opened until a case +exists that fails for that reason and no other. + +--- + ## 7. Verification protocol Per `AGENTS.md`, unchanged from prior arcs: diff --git a/evals/grammar_roundtrip/contract.md b/evals/grammar_roundtrip/contract.md index 26f5bfd9..4b4e6ff9 100644 --- a/evals/grammar_roundtrip/contract.md +++ b/evals/grammar_roundtrip/contract.md @@ -36,6 +36,14 @@ Round-trip is falsifiable with no judge, no embedding, and no gold aesthetic. | direction | pipeline | what a failure means | |---|---|---| | **G-round-trip** | graph → `realize_target` → surface → `comprehend` → graph | CORE cannot read its own writing | + +> **`realize_target` is eval-only** (Phase 4). `core/cognition/pipeline.py` calls +> `realize_semantic`, never `realize_target`, so `g_read_rate` is a measurement of +> the grammar *library* and not of served English. Run through the serving writer +> instead, the G-direction reads back **nothing** (0.0 vs 0.003413) — see +> `tests/test_phase4_realizer_resolution.py`. Any claim made from this metric must +> say which writer it is about. + | **S-round-trip** | surface → `comprehend` → graph → categorical renderer → surface | CORE cannot reproduce what it just understood | Both are reported because they fail for different reasons and have different diff --git a/evals/grammatical_coverage/runner.py b/evals/grammatical_coverage/runner.py index a5ce005c..10d3b141 100644 --- a/evals/grammatical_coverage/runner.py +++ b/evals/grammatical_coverage/runner.py @@ -5,6 +5,23 @@ English surfaces from PropositionGraph inputs. Each case specifies a construction family (e.g. negation, conjunction, embedded clause) and acceptance criteria (exact surfaces, constraint checks). +WHICH REALIZER (Phase 4) +------------------------ +``accuracy`` scores ``realize_target``, which **no serving path calls** — +``core/cognition/pipeline.py`` calls ``realize_semantic``. So ``accuracy`` is a +measurement of the grammar *library*, not of English CORE has ever spoken, and +citing it as "CORE's fluency" is a category error. It was cited that way for +the whole arc: 117/117 + 39/39 for a function that does not speak. + +``serving_accuracy`` scores ``realize_semantic`` on the identical cases and the +identical rubric. Across all seven corpora the two read **340/347** and +**85/347**. The gap is not a matter of polish — ``render_semantic`` has no +``negated``, ``quantifier``, ``tense`` or ``aspect`` parameter, so it serves a +negated proposition as its affirmative. See +``tests/test_phase4_realizer_resolution.py``, which pins that defect and the +control proving the gap is the dropped content rather than the corpora's +hardcoded ``IntentTag.UNKNOWN``. + Conforms to the framework interface: ``run_lane(cases, config=None) -> report``. """ from __future__ import annotations @@ -62,11 +79,16 @@ def _check_word_order(order: list[str], surface_words: list[str]) -> bool: return True -def _realize_from_graph(case: dict[str, Any]) -> str: +def _realize_from_graph(case: dict[str, Any], realize: Any = None) -> str: """Realize a surface from a proposition graph case. This calls the actual realizer infrastructure. The graph format in the eval cases maps to the realizer's PropositionGraph -> surface path. + + ``realize`` defaults to ``realize_target``, which is the realizer this lane + has always scored -- and which **nothing in the serving path calls** + (Phase 4). Passing ``realize_semantic`` scores the writer that actually + ships, against this identical contract. """ from generate.graph_planner import ( ArticulationStep, @@ -133,17 +155,17 @@ def _realize_from_graph(case: dict[str, Any]) -> str: )) target = ArticulationTarget(steps=tuple(steps), source_intent=IntentTag.UNKNOWN) - plan = realize_target(target, graph) + plan = (realize or realize_target)(target, graph) surface = plan.surface.rstrip(".") return surface -def _score_case(case: dict[str, Any]) -> CaseResult: +def _score_case(case: dict[str, Any], realize: Any = None) -> CaseResult: construction = case["construction"] construction_name = case["construction_name"] try: - surface = _realize_from_graph(case) + surface = _realize_from_graph(case, realize) except Exception as exc: return CaseResult( case_id=case["id"], @@ -235,11 +257,30 @@ def run_lane( for k, v in sorted(by_construction.items()) } + # ----------------------------------------------------------------- # + # Phase 4: score the SERVING writer on the identical contract. + # + # `accuracy` above is `realize_target`'s, and `core/cognition/pipeline.py` + # never calls `realize_target` -- it calls `realize_semantic`. For the whole + # arc this lane reported ~1.00 for a function that does not speak. Both + # numbers are reported now so the headline can never again be read as + # "CORE's fluency" when it is a measurement of a grammar library. + # + # This is the honest half of Phase 4 option (b). It changes no served byte; + # it only stops the lane from being silent about the gap. + # ----------------------------------------------------------------- # + from generate.realizer import realize_semantic + + serving_passed = sum(1 for case in cases if _score_case(case, realize_semantic).passed) + metrics = { "total": total, "passed": passed, "accuracy": round(passed / total, 4) if total else 0.0, "by_construction": construction_scores, + # What ships, on the same cases and the same rubric. + "serving_passed": serving_passed, + "serving_accuracy": round(serving_passed / total, 4) if total else 0.0, } return LaneReport(metrics=metrics, case_details=case_details) diff --git a/tests/test_phase4_realizer_resolution.py b/tests/test_phase4_realizer_resolution.py new file mode 100644 index 00000000..1dd5a460 --- /dev/null +++ b/tests/test_phase4_realizer_resolution.py @@ -0,0 +1,268 @@ +"""Phase 4 — which realizer serves, and what the other one's score means. + +The arc opened with a fact that had gone unremarked for its whole life: +``english_fluency_ood`` reported 117/117 + 39/39 for ``realize_target``, and +``core/cognition/pipeline.py`` **never calls** ``realize_target``. It calls +``realize_semantic``. 149 green cases were scoring a function that does not +speak, and the lane said nothing about the one that does. + +Phase 4 resolves that by option (b) of the plan: the lanes now report the +serving writer's score alongside, and the claim "realizer fluency is +mechanistic" is restated as a claim about **eval-only code**. Nothing here +changes a served byte — promoting ``realize_target`` to the serving path is +option (a), it moves surface hashes, and it is Shay's call, not this file's. + +What these tests are for +------------------------ +Every number below was measured, and each one is pinned so that it cannot +quietly stop being true. The load-bearing ones are the *controls*: without +them the headline gap could be dismissed as an artifact of the corpora, and +without the negation pin the most serious finding here would live only in a +commit message. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from evals.grammatical_coverage.runner import run_lane +from generate.graph_planner import ( + ArticulationStep, + ArticulationTarget, + GraphNode, + PropositionGraph, + RhetoricalMove, +) +from generate.intent import IntentTag +from generate.realizer import realize_semantic, realize_target + + +_EVALS = Path(__file__).resolve().parents[1] / "evals" + +#: Every corpus scored through ``realize_target`` by a lane. +_CORPORA = ( + "grammatical_coverage/public/v1", + "grammatical_coverage/public/v2", + "grammatical_coverage/dev", + "grammatical_coverage/holdouts/v1", + "english_fluency_ood/public/v1", + "english_fluency_ood/holdouts/v1", + "english_fluency_ood/dev", +) + +#: Content an ``ArticulationStep`` carries and ``render_semantic`` has no +#: parameter for. +_UNEXPRESSIBLE = ("quantifier", "negated", "tense", "aspect") + + +def _load(name: str) -> list[dict]: + path = _EVALS / name / "cases.jsonl" + return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + + +def _all_cases() -> list[dict]: + return [case for name in _CORPORA for case in _load(name)] + + +def _carries_unexpressible(case: dict) -> bool: + return any( + node.get(feature) not in (None, False, "") + for node in case.get("proposition_graph", {}).get("nodes", []) + for feature in _UNEXPRESSIBLE + ) + + +def _is_multi_node(case: dict) -> bool: + return len(case.get("proposition_graph", {}).get("nodes", [])) > 1 + + +def _one_step(**step_kwargs) -> tuple[ArticulationTarget, PropositionGraph]: + node = GraphNode( + node_id="n1", subject="knowledge", predicate="is_grounded_in", + obj="opinion", source_intent=IntentTag.DEFINITION, + ) + step = ArticulationStep( + node_id="n1", subject="knowledge", predicate="is_grounded_in", + move=RhetoricalMove.ASSERT, **step_kwargs, + ) + return ( + ArticulationTarget(steps=(step,), source_intent=IntentTag.DEFINITION), + PropositionGraph(nodes=(node,), edges=()), + ) + + +# --------------------------------------------------------------------------- # +# The headline: the lane now reports what ships +# --------------------------------------------------------------------------- # + + +def test_the_lane_reports_the_serving_writer_too() -> None: + """Before Phase 4 a lane could report 1.00 while the writer that actually + speaks scored 0.23 on the identical cases, and nothing surfaced it.""" + metrics = run_lane(_load("english_fluency_ood/public/v1")).metrics + assert metrics["passed"] == 117 + assert metrics["serving_passed"] == 27 + assert "serving_accuracy" in metrics + + +def test_the_measured_gap_across_every_scored_corpus() -> None: + """340/347 for the realizer the lanes score; 85/347 for the one that ships.""" + cases = _all_cases() + assert len(cases) == 347 + metrics = run_lane(cases).metrics + assert metrics["passed"] == 340 + assert metrics["serving_passed"] == 85 + + +# --------------------------------------------------------------------------- # +# CONTROL — the gap is the missing features, not the corpora's IntentTag +# --------------------------------------------------------------------------- # + + +def test_the_two_realizers_agree_exactly_where_nothing_is_dropped() -> None: + """THE CONTROL, and the most important test in this file. + + Every corpus here hardcodes ``IntentTag.UNKNOWN``, so "the serving realizer + scores badly" could be an artifact of never giving it a real intent. It is + not. On the 33 cases that carry no unexpressible feature and have a single + node, the two realizers score **identically** — which is only possible if + the gap on the other 314 is the dropped content and the clause joining. + + If this ever goes red, the gap measured above has stopped meaning what the + Phase 4 ruling says it means, and the ruling needs revisiting. + """ + control = [c for c in _all_cases() + if not _carries_unexpressible(c) and not _is_multi_node(c)] + assert len(control) == 33, "the control bucket changed size" + metrics = run_lane(control).metrics + assert metrics["passed"] == 33 + assert metrics["serving_passed"] == 33 + + +@pytest.mark.parametrize( + ("bucket", "expected_n", "expected_eval", "expected_serving"), + [ + ("features", 214, 207, 49), + ("multi_node", 100, 100, 3), + ], +) +def test_the_gap_decomposes_into_dropped_features_and_clause_joining( + bucket: str, expected_n: int, expected_eval: int, expected_serving: int +) -> None: + """Two separate causes, measured separately: content the serving writer has + no parameter for, and clauses it can only join with a full stop.""" + if bucket == "features": + cases = [c for c in _all_cases() + if _carries_unexpressible(c) and not _is_multi_node(c)] + else: + cases = [c for c in _all_cases() + if not _carries_unexpressible(c) and _is_multi_node(c)] + assert len(cases) == expected_n + metrics = run_lane(cases).metrics + assert metrics["passed"] == expected_eval + assert metrics["serving_passed"] == expected_serving + + +# --------------------------------------------------------------------------- # +# DEFECT PIN — the serving writer cannot express negation +# --------------------------------------------------------------------------- # + + +def test_the_serving_realizer_emits_the_same_surface_negated_or_not() -> None: + """DEFECT PIN, not a goal. Revise when fixed; never relax. + + ``render_semantic``'s signature is ``(intent, subject, predicate, obj, + secondary, language, root)``. There is no ``negated`` parameter, and + ``realize_semantic`` never reads ``step.negated``. So a negated + proposition is served as its **affirmative**: + + negated=False -> 'Knowledge is defined as opinion.' + negated=True -> 'Knowledge is defined as opinion.' + + This is not a fluency defect. It is the same family as ADR-0261 §5.1 + refuse-don't-drop: v1b served WRONG by dropping premises it could not + express. Fixing it changes served output, so it is authorization-gated; + pinning it here is what stops it from being forgotten. + """ + affirmative = realize_semantic(*_one_step(negated=False)).surface + negated = realize_semantic(*_one_step(negated=True)).surface + assert affirmative == negated, "the defect this pin records has changed shape" + + # The eval-only realizer distinguishes them, which is how we know the + # information reaches the realizer boundary intact. + assert realize_target(*_one_step(negated=True)).surface != ( + realize_target(*_one_step(negated=False)).surface + ) + assert "not" in realize_target(*_one_step(negated=True)).surface + + +@pytest.mark.parametrize("feature", _UNEXPRESSIBLE) +def test_render_semantic_has_no_parameter_for_the_content_it_drops(feature: str) -> None: + """Derived from the signature, so it cannot rot into a stale comment.""" + import inspect + + from generate.semantic_templates import render_semantic + + assert feature not in inspect.signature(render_semantic).parameters + + +def test_how_much_of_the_corpus_carries_content_the_serving_writer_drops() -> None: + """214 of 347. The scale of the gap, independent of any rubric.""" + cases = _all_cases() + carrying = [c for c in cases if _carries_unexpressible(c)] + assert len(carrying) == 214 + assert len(cases) == 347 + + +# --------------------------------------------------------------------------- # +# §6 evidence — the writer is not what round-trip is blocked on +# --------------------------------------------------------------------------- # + + +def test_the_serving_writer_round_trips_nothing_at_all() -> None: + """§6 of the plan forks on ``read_rate`` after unification, and this is the + measurement that says which fork. + + Two numbers, and they say different things: + + * **292 of 293** cases refuse with ``no_template_match`` under *either* + writer. For those the writer is irrelevant — the reader has no + SUBJ-VERB-OBJ template at all, so nothing the writer does can help. + * The **one** case that reads is writer-sensitive: it reads through + ``realize_target`` (0.003413) and refuses through ``realize_semantic`` + (0.0), because predicate-nominal object agreement lives in + ``render_step`` and the serving writer has no equivalent. + + So the serving writer round-trips **nothing**, and the eval writer + round-trips one case — and the gap between 1 and 293 is reader construction + coverage, not a graph-model mismatch. That is why §6's pre-committed + reading (§1.8 ⇒ next step is an ADR) is not what the evidence supports; see + ``test_grammar_roundtrip.py::test_the_remaining_blockers_are_reader_construction_coverage`` + for the full refusal census. + """ + import evals.grammar_roundtrip.runner as rt + + baseline = rt.run_lane().metrics["g_read_rate"] + original = rt.realize_target + rt.realize_target = realize_semantic + try: + swapped = rt.run_lane().metrics["g_read_rate"] + finally: + rt.realize_target = original + + assert baseline == 0.003413, "the eval writer reads back exactly one case" + assert swapped == 0.0, "the serving writer reads back none" + # Sentinel: prove the swap is actually reaching the lane, so that an + # equality between two identical runs can never be mistaken for a result. + def _boom(*_args, **_kwargs): # pragma: no cover - must raise + raise RuntimeError("sentinel") + + rt.realize_target = _boom + try: + with pytest.raises(RuntimeError, match="sentinel"): + rt.run_lane() + finally: + rt.realize_target = original