diff --git a/chat/curriculum_surface.py b/chat/curriculum_surface.py index ea2ec0cc..9904aac8 100644 --- a/chat/curriculum_surface.py +++ b/chat/curriculum_surface.py @@ -20,9 +20,12 @@ knowledge it was never given. The only verdicts reachable from a purely positive curriculum are therefore ENTAILED and UNKNOWN — a real limit of the present corpora, recorded in ADR-0262 §5 rather than papered over. -Fail-closed (INV-34): once `looks_like_curriculum_question` fires, every path -below returns a committed, honest surface — typed refusal or decided verdict — -never a silent fall-through to a different composer. +Fail-closed (INV-34) BEHIND A NARROW GATE: once `is_curriculum_question` fires, +every path below returns a committed, honest surface — typed refusal or decided +verdict — never a silent fall-through. The gate itself is deliberately narrow +(routable, not merely `Does …?`-shaped): shape alone is not a signal of intent +for a polar question, so claiming every one of them would hijack ordinary turns. +See :func:`is_curriculum_question`. """ from __future__ import annotations @@ -80,14 +83,44 @@ class CurriculumRefusal: def looks_like_curriculum_question(text: str) -> bool: - """True iff *text* is a ``Does …?`` polar question. + """True iff *text* has the ``Does …?`` polar-question SHAPE. - A cheap, deterministic COMMIT gate — not a decision. The reader below - remains the sole authority on whether the question is actually readable. + A cheap syntactic pre-filter, NOT the commit gate — see + :func:`is_curriculum_question`, which is what decides whether this + composer claims a turn. """ return bool(_QUESTION_RE.match(text or "")) +def is_curriculum_question(text: str) -> bool: + """True iff *text* is a curriculum question this composer should CLAIM. + + The deduction composer can commit on shape alone because its gate — a + sentence-initial "therefore" — is a genuine signal of intent: text shaped + that way IS an argument. ``Does …?`` carries no such signal; it is one of + the most common ways to open any English question. Committing on it would + take "Does the build pass?" away from the rest of dispatch and answer it + with a remark about curriculum gaps, which is worse than useless. + + So the gate is *routability*, not shape: claim the turn only when the + question parses to three tokens AND its terms are vocabulary some served + subject actually teaches. Everything past that point stays fail-closed + (INV-34) — including ``ambiguous_reading``, where both terms ARE taught + and the only problem is that two subjects claim them, and + ``out_of_curriculum``, where the terms are taught and only the relation is + unknown. Those are real curriculum questions with honest answers. A + question whose vocabulary CORE has never been taught is simply not this + composer's turn. + """ + query = read_curriculum_question(text) + if not isinstance(query, CurriculumQuery): + return False + domain = resolve_domain(query) + if isinstance(domain, CurriculumRefusal): + return domain.reason != "untaught_vocabulary" + return True + + def read_curriculum_question(text: str) -> CurriculumQuery | CurriculumRefusal: """Read *text* as ``Does ?``, or refuse (typed).""" match = _QUESTION_RE.match(text or "") @@ -229,11 +262,6 @@ _REFUSAL_SURFACE = { "That reads as a question about a subject I study, but not in a form I " "can decide yet (I read \"Does ?\")." ), - "untaught_vocabulary": ( - "I haven't been taught {detail} — so I can't decide anything about it " - "from what I know. That's a gap in my curriculum, not a claim about " - "the world." - ), "out_of_curriculum": ( "I haven't been taught the relation \"{detail}\", so I can't decide " "that question from my curriculum." @@ -273,11 +301,11 @@ def curriculum_grounded_surface( ) -> str | None: """Return a deterministic CURRICULUM-tier surface, or ``None``. - ``None`` only when *text* is not a ``Does …?`` question at all — the - caller then falls through to its pre-existing dispatch, byte-identical to - before this composer existed. + ``None`` when *text* is not a curriculum question this composer should + claim (:func:`is_curriculum_question`) — the caller then falls through to + its pre-existing dispatch, byte-identical to before this composer existed. """ - if not looks_like_curriculum_question(text): + if not is_curriculum_question(text): return None decision = decide_curriculum_question(text) query = read_curriculum_question(text) @@ -317,15 +345,6 @@ def _detail_of(text: str, decision: CurriculumDecision) -> str: query = read_curriculum_question(text) if not isinstance(query, CurriculumQuery): return "" - if decision.reason == "untaught_vocabulary": - untaught = [ - term - for term in (query.subject, query.obj) - if not any( - term in load_curriculum(d).vocabulary for d in SERVED_DOMAINS - ) - ] - return " or ".join(untaught) if untaught else f"{query.subject} / {query.obj}" if decision.reason == "out_of_curriculum": return query.verb if decision.reason == "ambiguous_reading": @@ -354,6 +373,7 @@ __all__ = [ "band_for", "curriculum_grounded_surface", "decide_curriculum_question", + "is_curriculum_question", "looks_like_curriculum_question", "read_curriculum_question", "resolve_domain", diff --git a/docs/adr/ADR-0262-curriculum-grounded-serving.md b/docs/adr/ADR-0262-curriculum-grounded-serving.md index 959bad43..955e6e44 100644 --- a/docs/adr/ADR-0262-curriculum-grounded-serving.md +++ b/docs/adr/ADR-0262-curriculum-grounded-serving.md @@ -27,7 +27,9 @@ default-off `curriculum_serving_enabled` flag: 1. **Closed question grammar** — `Does ?`. One shape. Anything else refuses `question_shape_out_of_band` rather than guessing - which token is the relation. + which token is the relation. **The commit gate is routability, not shape** + (§5.4): the composer claims a turn only when the question parses AND its + terms are vocabulary a served subject teaches. 2. **Subject routing by vocabulary** — the question goes to the subject whose ratified pack vocabulary contains BOTH terms. No match is `untaught_vocabulary`; more than one is `ambiguous_reading`. The subject is @@ -123,7 +125,32 @@ curriculum to teach a negative ("no X causes Y") or an exclusion. Until a corpus does, `refuted` is a verdict this path can render but never reach. The serving code handles it; the lane documents it. -### 5.3 There is no biology domain-chain corpus +### 5.3 The commit gate must be routability, not shape + +Shipped first as a shape gate (`Does …?`) with the deduction composer's +fail-closed posture copied verbatim. That was wrong, and flipping the flag +would have exposed it: `Does the build pass?` and `Does anyone know the time?` +would have been taken away from the rest of dispatch and answered with a +remark about curriculum gaps. + +The asymmetry is the lesson. The deduction composer may commit on shape +because a sentence-initial "therefore" IS a signal of intent — text shaped +that way is an argument. `Does …?` is one of the most common ways to open any +English question and signals nothing. So the gate became +`is_curriculum_question`: claim the turn only when the question parses to +three tokens AND its terms are vocabulary some served subject teaches. +Everything past the gate stays fail-closed, including `ambiguous_reading` +(both terms taught, two subjects claim them) and `out_of_curriculum` (terms +taught, relation unknown) — those are real curriculum questions with honest +answers. A question whose vocabulary CORE has never been taught is simply not +this composer's turn, and the DECIDER still records it as +`declined`/`untaught_vocabulary` for the lane. + +**Generalizable rule:** a fail-closed composer is only as safe as its commit +gate is narrow. Before copying a fail-closed posture, check whether the new +gate carries the same evidence of intent as the one it was copied from. + +### 5.4 There is no biology domain-chain corpus The plan names "physics and biology" as Phase 2's two subjects because "OOD lanes, seed packs, and domain chains already exist". For biology only the first diff --git a/evals/curriculum_serve/contract.md b/evals/curriculum_serve/contract.md index 829b7686..2e301f73 100644 --- a/evals/curriculum_serve/contract.md +++ b/evals/curriculum_serve/contract.md @@ -61,6 +61,16 @@ disagrees with gold) MUST stay 0; a decline where gold expected a verdict is a coverage miss, tracked in `counts.declined`, never conflated with a confabulation. The runner requires `correct == n`. +## Lane scope vs composer scope + +The lane scores the DECIDER (`decide_curriculum_question`), which records a +verdict for every case including `untaught_vocabulary` ones. The COMPOSER +(`curriculum_grounded_surface`) is narrower: it claims a turn only when the +question is routable (ADR-0262 §5.3), so the anti-recall probes reach the rest +of dispatch rather than being answered with a curriculum remark. Both +statements hold at once — the curriculum path declines them, and it does not +speak for them. + ## What the lane deliberately does NOT do - **It does not compose chains.** `force causes acceleration` and diff --git a/tests/test_curriculum_serve.py b/tests/test_curriculum_serve.py index a027e6f2..f35b3167 100644 --- a/tests/test_curriculum_serve.py +++ b/tests/test_curriculum_serve.py @@ -18,6 +18,7 @@ from chat.curriculum_surface import ( band_for, curriculum_grounded_surface, decide_curriculum_question, + is_curriculum_question, looks_like_curriculum_question, read_curriculum_question, resolve_domain, @@ -115,14 +116,49 @@ def test_question_may_spell_the_relation_in_its_base_form() -> None: ) def test_untaught_vocabulary_declines_rather_than_recalling(text: str) -> None: """Both are true physics and both are outside every mounted pack. The - system has no curriculum to decide them from, and says so.""" + DECIDER declines — it has no curriculum to decide them from — and the + composer does not claim the turn at all, so the question reaches the rest + of dispatch instead of being answered with a remark about curriculum.""" decision = decide_curriculum_question(text) assert decision.verdict == "declined" assert decision.reason == "untaught_vocabulary" - surface = curriculum_grounded_surface(text) - assert surface is not None - assert "haven't been taught" in surface - assert "not a claim about the world" in surface + assert curriculum_grounded_surface(text) is None + + +@pytest.mark.parametrize( + "text", + [ + "Does the build pass?", # ordinary question, untaught terms + "Does it rain often?", # ordinary question, untaught terms + "Does anyone know the time?", # not even the three-token shape + "Does force accelerate?", # two tokens + "Does the force cause acceleration?", # four tokens + ], +) +def test_ordinary_does_questions_are_not_claimed(text: str) -> None: + """The commit gate is ROUTABILITY, not shape. "Does …?" is one of the most + common ways to open any English question; claiming every one of them would + take ordinary turns away from the rest of dispatch and answer them with a + curriculum remark. Only questions whose vocabulary a served subject + actually teaches are this composer's turn.""" + assert curriculum_grounded_surface(text) is None + assert not is_curriculum_question(text) + + +@pytest.mark.parametrize( + "text", + [ + "Does force cause acceleration?", # taught, decidable + "Does force explain acceleration?", # taught terms, unknown relation + "Does force cause motion?", # taught, unsettled + ], +) +def test_routable_questions_are_claimed_and_answered(text: str) -> None: + """Past the gate the composer stays fail-closed: every routable question + gets a committed, honest surface — including the one whose relation the + curriculum does not teach, since its TERMS plainly are curriculum terms.""" + assert is_curriculum_question(text) + assert curriculum_grounded_surface(text) is not None # --- typed refusals ------------------------------------------------------------