diff --git a/docs/plans/grammar-unification-2026-07-26.md b/docs/plans/grammar-unification-2026-07-26.md index 413cd531..788d16c6 100644 --- a/docs/plans/grammar-unification-2026-07-26.md +++ b/docs/plans/grammar-unification-2026-07-26.md @@ -625,10 +625,43 @@ on a ratified, flag-ON band with `wrong=0` intact for the whole arc. The round-trip lane is not a nice-to-have measurement; it is the only guard that reads what the user reads. -⇒ **Carried forward:** any future phase touching surfaces must gate on -`grammar_roundtrip`, not on the lane pins. Adding surface text to the pinned -lane payloads would be the durable fix and is *not* done here — it would move -every pin at once and deserves its own unit. +**One correction to the sabotage table above.** The runner's own docstring named +`tests/test_deduction_surface.py` as the place wording is covered, and I had not +included that file in the first sabotage run. Tested: **all 41 of its tests pass +under the sabotage too.** So the exclusion's stated justification — "wording is +covered by tests/test_deduction_surface.py" — was false, not merely incomplete. +The claim survived the check that could have refuted it. + +#### Closed — the pins now read what the user reads + +Fixed in the same arc rather than deferred. `evals/deduction_serve/runner.py` +now puts `surface_sha256` **and** the per-case `surfaces` into the pinned +artifact, via `build_combined_report` (the earlier attempt missed because that +function re-projects five named fields per split and silently dropped the new +one — the payload is not the report). + +| | before | after | +|---|---|---| +| pinned artifact | 2,766 bytes, verdict counts only | 37,280 bytes, every served sentence | +| `deduction_serve_v1` under sabotage | ✓ byte-identical (blind) | **✗ `52370b73` vs `c855d55c`** | + +Re-pinned surgically — one line, old hash recorded in a comment beside it, never +`--update`. Verdicts are untouched (166/166 correct, `wrong=0`); the hash moved +because the payload grew. + +`test_surface_hash_moves_when_the_renderer_is_sabotaged` makes this permanent: +it corrupts the renderer, requires a digest to move, **and** asserts the +aggregate verdict counts are unchanged — proving the digest tracks prose rather +than smuggling in a decision change. A pin that cannot fail guards nothing. + +**Accepted cost, stated plainly:** wording-only changes now move this pin. That +is the intent — a wording change *is* a user-visible change and should require a +deliberate re-pin, which is exactly the discipline whose absence let +`all dog are mammal` serve for the whole arc. + +The other 10 lanes are untouched. Whether their payloads should also carry +surfaces is a separate question this unit does not answer; `deduction_serve` was +fixed because it is the one demonstrably serving prose to users. The Phase 1 instrument moved on its own, having been built with no knowledge of this fix. That is the whole argument for building the instrument first. diff --git a/evals/deduction_serve/report.json b/evals/deduction_serve/report.json index ce18245f..b6d7398a 100644 --- a/evals/deduction_serve/report.json +++ b/evals/deduction_serve/report.json @@ -33,7 +33,122 @@ "declined": 0, "wrong": 0 }, - "n": 28 + "n": 28, + "surface_sha256": "0c36317b61e7f6ad6f8ac8a8f99a9e6c70d78cc86ccb90cd7d930fd94fdc1f81", + "surfaces": [ + { + "id": "ds-v1-0001", + "surface": "Given: s implies t; s. Your premises entail: t." + }, + { + "id": "ds-v1-0002", + "surface": "Given: s implies t; not t. Your premises entail: not s." + }, + { + "id": "ds-v1-0003", + "surface": "Given: p implies q; q implies r. Your premises entail: p implies r." + }, + { + "id": "ds-v1-0004", + "surface": "Given: p or q; not p. Your premises entail: q." + }, + { + "id": "ds-v1-0005", + "surface": "Given: p or q; not q. Your premises entail: p." + }, + { + "id": "ds-v1-0006", + "surface": "Given: if p then q. Your premises entail: if not q then not p." + }, + { + "id": "ds-v1-0007", + "surface": "Given: p. Your premises entail: p." + }, + { + "id": "ds-v1-0008", + "surface": "Given: p implies q; r implies q; p or r. Your premises entail: q." + }, + { + "id": "ds-v1-0009", + "surface": "Given: p implies q; not q. Your premises entail the opposite of p \u2014 it cannot hold." + }, + { + "id": "ds-v1-0010", + "surface": "Given: p or q; not p. Your premises entail the opposite of p \u2014 it cannot hold." + }, + { + "id": "ds-v1-0011", + "surface": "Given: p implies q; p. Your premises entail the opposite of not q \u2014 it cannot hold." + }, + { + "id": "ds-v1-0012", + "surface": "Given: not p. Your premises entail the opposite of p \u2014 it cannot hold." + }, + { + "id": "ds-v1-0013", + "surface": "Given: p implies q; q implies r; p. Your premises entail the opposite of not r \u2014 it cannot hold." + }, + { + "id": "ds-v1-0014", + "surface": "Given: p or q. Your premises don't settle whether p \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-v1-0015", + "surface": "Given: p implies q. Your premises don't settle whether p \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-v1-0016", + "surface": "Given: p implies q. Your premises don't settle whether q \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-v1-0017", + "surface": "Given: p or q. Your premises don't settle whether q \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-v1-0018", + "surface": "Given: p implies q; r implies q. Your premises don't settle whether p \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-v1-0019", + "surface": "Given: p; not p. Those premises are inconsistent \u2014 they can't all be true, so I won't assert anything from them." + }, + { + "id": "ds-v1-0020", + "surface": "Given: p implies q; p; not q. Those premises are inconsistent \u2014 they can't all be true, so I won't assert anything from them." + }, + { + "id": "ds-v1-0021", + "surface": "Given: p; not p. Those premises are inconsistent \u2014 they can't all be true, so I won't assert anything from them." + }, + { + "id": "ds-v1-0022", + "surface": "Given: p or q; not p; not q. Those premises are inconsistent \u2014 they can't all be true, so I won't assert anything from them." + }, + { + "id": "ds-v1-0023", + "surface": "Given: all mammals are animals; all whales are mammals. That's valid \u2014 all whales are animals follows." + }, + { + "id": "ds-v1-0024", + "surface": "Given: no reptiles are mammals; all snakes are reptiles. That's valid \u2014 no snakes are mammals follows." + }, + { + "id": "ds-v1-0025", + "surface": "Given: if it rains then the ground is wet; it rains. Your premises entail: the ground is wet." + }, + { + "id": "ds-v1-0026", + "surface": "Given: some students are poets; all poets are artists. That's valid \u2014 some students are artists follows." + }, + { + "id": "ds-v1-0028", + "surface": "Given: all cats are animals; all dogs are animals. That doesn't follow \u2014 all dogs are cats isn't guaranteed by those premises." + }, + { + "id": "ds-v1-0027", + "surface": "Given: p implies q; q implies r; r implies s; p. Your premises entail: s." + } + ] }, "v2_condmem": { "all_cases_correct": true, @@ -54,7 +169,114 @@ "declined": 0, "wrong": 0 }, - "n": 26 + "n": 26, + "surface_sha256": "fb7d86b19ee9286b43a1b4475653d62b819e07fd89113476def13f5f55cfc308", + "surfaces": [ + { + "id": "ds-cm-0001", + "surface": "Given: if socrates is a man then socrates is mortal; socrates is a man. Your premises entail: socrates is mortal." + }, + { + "id": "ds-cm-0002", + "surface": "Given: if the museum is open then the ticket booth is staffed; the ticket booth is not staffed. Your premises entail: the museum is not open." + }, + { + "id": "ds-cm-0003", + "surface": "Given: either the witness is lying or the suspect is guilty; the witness is not lying. Your premises entail: the suspect is guilty." + }, + { + "id": "ds-cm-0004", + "surface": "Given: the key is in the drawer or the key is on the hook; the key is not in the drawer. Your premises entail: the key is on the hook." + }, + { + "id": "ds-cm-0005", + "surface": "Given: if the alarm is triggered then the guard is alerted; if the sensor is triggered then the guard is alerted; the alarm is triggered or the sensor is triggered. Your premises entail: the guard is alerted." + }, + { + "id": "ds-cm-0006", + "surface": "Given: if the pressure is high then the valve is open; if the valve is open then the tank is draining; the pressure is high. Your premises entail: the tank is draining." + }, + { + "id": "ds-cm-0007", + "surface": "Given: if nova is a suspect then nova is a person of interest; if nova is a person of interest then nova is a flight risk; nova is a suspect. Your premises entail: nova is a flight risk." + }, + { + "id": "ds-cm-0008", + "surface": "Given: all engineers are employees; if dax is an employee then dax is insured; dax is an engineer. Your premises entail: dax is insured." + }, + { + "id": "ds-cm-0009", + "surface": "Given: if dax is an employee then dax is insured; all contractors are employees; dax is a contractor. Your premises entail: dax is insured." + }, + { + "id": "ds-cm-0010", + "surface": "Given: all students are learners; priya is a student; if priya is a learner then priya is curious. Your premises entail: priya is curious." + }, + { + "id": "ds-cm-0011", + "surface": "Given: if the reactor is critical then the reactor is unstable; the reactor is critical. Your premises entail the opposite of the reactor is not unstable \u2014 it cannot hold." + }, + { + "id": "ds-cm-0012", + "surface": "Given: if the code compiles then the code is buildable; if the code is buildable then the code is deployable; the code compiles. Your premises entail the opposite of the code is not deployable \u2014 it cannot hold." + }, + { + "id": "ds-cm-0013", + "surface": "Given: all interns are trainees; if kai is a trainee then kai is supervised; kai is an intern. Your premises entail the opposite of kai is not supervised \u2014 it cannot hold." + }, + { + "id": "ds-cm-0014", + "surface": "Given: the switch is on or the switch is off; the switch is not on. Your premises entail the opposite of the switch is not off \u2014 it cannot hold." + }, + { + "id": "ds-cm-0015", + "surface": "Given: if it is raining then the ground is wet; the ground is wet. Reading each clause as one indivisible claim, your premises don't settle whether it is raining \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-cm-0016", + "surface": "Given: if the server is overloaded then the response is slow; the server is not overloaded. Reading each clause as one indivisible claim, your premises don't settle whether the response is not slow \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-cm-0017", + "surface": "Given: if the bridge is closed then the traffic is rerouted. Reading each clause as one indivisible claim, your premises don't settle whether the bridge is closed \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-cm-0018", + "surface": "Given: the door is locked or the door is unlocked. Reading each clause as one indivisible claim, your premises don't settle whether the door is locked \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-cm-0019", + "surface": "Given: all senators are legislators; if elio is a legislator then elio is a committee member; priya is a senator. Reading each name as one individual and each class word at face value, your premises don't settle whether elio is a committee member \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-cm-0020", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (reserved_word_in_np)." + }, + { + "id": "ds-cm-0021", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (reserved_word_in_np)." + }, + { + "id": "ds-cm-0022", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (reserved_word_in_np)." + }, + { + "id": "ds-cm-0023", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (reserved_word_in_np)." + }, + { + "id": "ds-cm-0024", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (reserved_word_in_np)." + }, + { + "id": "ds-cm-0025", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (reserved_word_in_np)." + }, + { + "id": "ds-cm-0026", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (reserved_word_in_np)." + } + ] }, "v2_en": { "all_cases_correct": true, @@ -75,7 +297,114 @@ "declined": 0, "wrong": 0 }, - "n": 26 + "n": 26, + "surface_sha256": "ee1866c501556f8907c447050ca42d3e62c38ec5cef60303d65863a5a7827e0c", + "surfaces": [ + { + "id": "ds-en-0001", + "surface": "Given: if it rains then the ground is wet; it rains. Your premises entail: the ground is wet." + }, + { + "id": "ds-en-0002", + "surface": "Given: if the alarm is armed then the light is on; the light is not on. Your premises entail: the alarm is not armed." + }, + { + "id": "ds-en-0003", + "surface": "Given: either the door is open or the window is open; the door is not open. Your premises entail: the window is open." + }, + { + "id": "ds-en-0004", + "surface": "Given: if the pump runs then the tank fills; if the valve opens then the tank fills; the pump runs or the valve opens. Your premises entail: the tank fills." + }, + { + "id": "ds-en-0005", + "surface": "Given: if it rains then the ground is wet; if the ground is wet then the match is cancelled. Your premises entail: if it rains then the match is cancelled." + }, + { + "id": "ds-en-0006", + "surface": "Given: the oven is hot and the dough is ready. Your premises entail: the dough is ready." + }, + { + "id": "ds-en-0007", + "surface": "Given: the kettle is boiling; the toast is ready. Your premises entail: the kettle is boiling and the toast is ready." + }, + { + "id": "ds-en-0008", + "surface": "Given: the train is late. Your premises entail: the train is late or the bus is early." + }, + { + "id": "ds-en-0009", + "surface": "Given: it is not the case that the store is open. Your premises entail: the store is not open." + }, + { + "id": "ds-en-0010", + "surface": "Given: if the seal is broken then the tank is empty; if the tank is empty then the pump is dry; the pump is not dry. Your premises entail: the seal is not broken." + }, + { + "id": "ds-en-0011", + "surface": "Given: if the bell rings then the class is over; if the class is over then the hall is crowded; the bell rings. Your premises entail: the hall is crowded." + }, + { + "id": "ds-en-0012", + "surface": "Given: if the printer is jammed then the queue is stuck; the queue is not stuck. Your premises entail: the printer is not jammed." + }, + { + "id": "ds-en-0013", + "surface": "Given: the report is filed. Your premises entail: the report is filed." + }, + { + "id": "ds-en-0014", + "surface": "Given: if it rains then the ground is wet; it rains. Your premises entail the opposite of the ground is not wet \u2014 it cannot hold." + }, + { + "id": "ds-en-0015", + "surface": "Given: the gate is locked. Your premises entail the opposite of the gate is not locked \u2014 it cannot hold." + }, + { + "id": "ds-en-0016", + "surface": "Given: the cat is inside or the dog is inside; the cat is not inside. Your premises entail the opposite of the dog is not inside \u2014 it cannot hold." + }, + { + "id": "ds-en-0017", + "surface": "Given: if it rains then the ground is wet; the ground is wet. Reading each clause as one indivisible claim, your premises don't settle whether it rains \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-en-0018", + "surface": "Given: if the oven is on then the kitchen is warm; the oven is not on. Reading each clause as one indivisible claim, your premises don't settle whether the kitchen is not warm \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-en-0019", + "surface": "Given: the key is lost or the lock is broken. Reading each clause as one indivisible claim, your premises don't settle whether the key is lost \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-en-0020", + "surface": "Given: the sky is clear. Reading each clause as one indivisible claim, your premises don't settle whether the sea is calm \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-en-0021", + "surface": "Given: the door is open; the door is not open. Those premises are inconsistent \u2014 they can't all be true, so I won't assert anything from them." + }, + { + "id": "ds-en-0022", + "surface": "Given: socrates is a man. Your premises entail: socrates is a man." + }, + { + "id": "ds-en-0023", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-en-0024", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-en-0025", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (reserved_word_in_np)." + }, + { + "id": "ds-en-0026", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (reserved_word_in_np)." + } + ] }, "v2_exist": { "all_cases_correct": true, @@ -96,7 +425,138 @@ "declined": 0, "wrong": 0 }, - "n": 32 + "n": 32, + "surface_sha256": "1713d4a839fb73c0dc7fc416e11c159cdae6661bebadbddd754978b96f98c3c7", + "surfaces": [ + { + "id": "ds-ex-0001", + "surface": "Given: every mammal is a vertebrate; some whales are mammals. Your premises entail: some whales are vertebrates." + }, + { + "id": "ds-ex-0002", + "surface": "Given: some philosophers are greeks; zeno is a greek. Your premises entail: some greeks are philosophers." + }, + { + "id": "ds-ex-0003", + "surface": "Given: socrates is a philosopher; socrates is wise. Your premises entail: some philosophers are wise." + }, + { + "id": "ds-ex-0004", + "surface": "Given: mars is a planet; no planets are stars; some wanderers are planets. Your premises entail: some wanderers are not stars." + }, + { + "id": "ds-ex-0005", + "surface": "Given: all metals conduct heat; some alloys are metals. Your premises entail: some alloys conduct heat." + }, + { + "id": "ds-ex-0006", + "surface": "Given: no reptiles nurse young; some lizards are reptiles. Your premises entail: some lizards do not nurse young." + }, + { + "id": "ds-ex-0007", + "surface": "Given: some canaries are songbirds; all songbirds are perchers; all perchers are agile. Your premises entail: some canaries are agile." + }, + { + "id": "ds-ex-0008", + "surface": "Given: aristotle is a philosopher; all philosophers are scholars. Your premises entail: some scholars are philosophers." + }, + { + "id": "ds-ex-0009", + "surface": "Given: some tortoises are ancient. Your premises entail: some tortoises are ancient." + }, + { + "id": "ds-ex-0010", + "surface": "Given: some hermits are not sociable. Your premises entail: some hermits are not sociable." + }, + { + "id": "ds-ex-0011", + "surface": "Given: every goose is a bird; some geese are gray. Your premises entail: some birds are gray." + }, + { + "id": "ds-ex-0012", + "surface": "Given: no fish are mammals. Your premises entail the opposite of some fish are mammals \u2014 it cannot hold." + }, + { + "id": "ds-ex-0013", + "surface": "Given: every square is a rectangle. Your premises entail the opposite of some squares are not rectangles \u2014 it cannot hold." + }, + { + "id": "ds-ex-0014", + "surface": "Given: bede is a monk; no monks are talkative; some scribes are monks. Your premises entail the opposite of bede is talkative \u2014 it cannot hold." + }, + { + "id": "ds-ex-0015", + "surface": "Given: nile is a river; all rivers are wet; some rivers are wide. Your premises entail the opposite of nile is not wet \u2014 it cannot hold." + }, + { + "id": "ds-ex-0016", + "surface": "Given: some sailors are brave; magellan is a sailor. Reading each name as one individual, each class word and verb phrase at face value, and \"some\" as claiming a member exists (so \"all\" does not), your premises don't settle whether magellan is brave \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-ex-0017", + "surface": "Given: all unicorns are horned. Reading each name as one individual, each class word and verb phrase at face value, and \"some\" as claiming a member exists (so \"all\" does not), your premises don't settle whether some unicorns are horned \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-ex-0018", + "surface": "Given: some auks are flightless; some auks are colorful. Reading each name as one individual, each class word and verb phrase at face value, and \"some\" as claiming a member exists (so \"all\" does not), your premises don't settle whether some flightless are colorful \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-ex-0019", + "surface": "Given: vega is a star; vega is not visible. Reading each name as one individual, each class word and verb phrase at face value, and \"some\" as claiming a member exists (so \"all\" does not), your premises don't settle whether some stars are visible \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-ex-0020", + "surface": "Given: some poems are sonnets; no plays rhyme. Reading each name as one individual, each class word and verb phrase at face value, and \"some\" as claiming a member exists (so \"all\" does not), your premises don't settle whether some poems do not rhyme \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-ex-0021", + "surface": "Given: some larks sing. Reading each name as one individual, each class word and verb phrase at face value, and \"some\" as claiming a member exists (so \"all\" does not), your premises don't settle whether some larks sing sweetly \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-ex-0022", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (reserved_word_in_np)." + }, + { + "id": "ds-ex-0023", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-ex-0024", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-ex-0025", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (reserved_word_in_np)." + }, + { + "id": "ds-ex-0026", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-ex-0027", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (unknown_morphology)." + }, + { + "id": "ds-ex-0028", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-ex-0029", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-ex-0030", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (unknown_morphology)." + }, + { + "id": "ds-ex-0031", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (unknown_morphology)." + }, + { + "id": "ds-ex-0032", + "surface": "" + } + ] }, "v2_member": { "all_cases_correct": true, @@ -117,7 +577,114 @@ "declined": 0, "wrong": 0 }, - "n": 26 + "n": 26, + "surface_sha256": "e1e16e0f061ac2060d7795a8ec3f13b49d14fab369e64f57be42486d4aeecdbf", + "surfaces": [ + { + "id": "ds-mem-0001", + "surface": "Given: socrates is a man; all men are mortal. Your premises entail: socrates is mortal." + }, + { + "id": "ds-mem-0002", + "surface": "Given: ada is a person; all people are mortal. Your premises entail: ada is mortal." + }, + { + "id": "ds-mem-0003", + "surface": "Given: rex is a dog; all dogs are mammals; all mammals are animals. Your premises entail: rex is an animal." + }, + { + "id": "ds-mem-0004", + "surface": "Given: fido is a dog; no dogs are cats. Your premises entail: fido is not a cat." + }, + { + "id": "ds-mem-0005", + "surface": "Given: every child is curious; milo is a child. Your premises entail: milo is curious." + }, + { + "id": "ds-mem-0006", + "surface": "Given: tweety is a canary; all canaries are birds; no birds are reptiles. Your premises entail: tweety is not a reptile." + }, + { + "id": "ds-mem-0007", + "surface": "Given: dolly is a sheep; all sheep are woolly. Your premises entail: dolly is woolly." + }, + { + "id": "ds-mem-0008", + "surface": "Given: rex is not a cat. Your premises entail: rex is not a cat." + }, + { + "id": "ds-mem-0009", + "surface": "Given: each planet is round; mars is a planet. Your premises entail: mars is round." + }, + { + "id": "ds-mem-0010", + "surface": "Given: rex is a guard dog; all guard dogs are loyal. Your premises entail: rex is loyal." + }, + { + "id": "ds-mem-0011", + "surface": "Given: plato is a man; aristotle is a man; all men are mortal. Your premises entail: aristotle is mortal." + }, + { + "id": "ds-mem-0012", + "surface": "Given: it is not the case that rex is a cat. Your premises entail: rex is not a cat." + }, + { + "id": "ds-mem-0013", + "surface": "Given: socrates is a man; all men are mortal. Your premises entail the opposite of socrates is not mortal \u2014 it cannot hold." + }, + { + "id": "ds-mem-0014", + "surface": "Given: fido is a dog; no dogs are cats. Your premises entail the opposite of fido is a cat \u2014 it cannot hold." + }, + { + "id": "ds-mem-0015", + "surface": "Given: rex is loyal. Your premises entail the opposite of rex is not loyal \u2014 it cannot hold." + }, + { + "id": "ds-mem-0016", + "surface": "Given: socrates is mortal; all men are mortal. Reading each name as one individual and each class word at face value, your premises don't settle whether socrates is a man \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-mem-0017", + "surface": "Given: plato is a man; all men are mortal. Reading each name as one individual and each class word at face value, your premises don't settle whether zeus is mortal \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-mem-0018", + "surface": "Given: rex is a dog; all cats are animals. Reading each name as one individual and each class word at face value, your premises don't settle whether rex is an animal \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-mem-0019", + "surface": "Given: milo is a child; all children are curious. Reading each name as one individual and each class word at face value, your premises don't settle whether milo is happy \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-mem-0020", + "surface": "Given: some men are wise; socrates is a man. Reading each name as one individual, each class word and verb phrase at face value, and \"some\" as claiming a member exists (so \"all\" does not), your premises don't settle whether socrates is wise \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-mem-0021", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-mem-0022", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-mem-0023", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (unknown_morphology)." + }, + { + "id": "ds-mem-0024", + "surface": "Given: if socrates is a man then socrates is mortal. Reading each name as one individual and each class word at face value, your premises don't settle whether socrates is mortal \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-mem-0025", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (unknown_morphology)." + }, + { + "id": "ds-mem-0026", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + } + ] }, "v2_verb": { "all_cases_correct": true, @@ -138,7 +705,122 @@ "declined": 0, "wrong": 0 }, - "n": 28 + "n": 28, + "surface_sha256": "1d7d28645222f137da226ad1a1264e0e661fcffa16c5bea6b5ab94aba101987b", + "surfaces": [ + { + "id": "ds-vb-0001", + "surface": "Given: all poets write; homer is a poet. Your premises entail: homer writes." + }, + { + "id": "ds-vb-0002", + "surface": "Given: every senator debates policy; cicero is a senator. Your premises entail: cicero debates policy." + }, + { + "id": "ds-vb-0003", + "surface": "Given: anselm is a monk; each monk prays. Your premises entail: anselm prays." + }, + { + "id": "ds-vb-0004", + "surface": "Given: aristotle is a philosopher; all philosophers are scholars; all scholars read. Your premises entail: aristotle reads." + }, + { + "id": "ds-vb-0005", + "surface": "Given: all sparrows are birds; jack is a sparrow; all birds build nests. Your premises entail: jack builds nests." + }, + { + "id": "ds-vb-0006", + "surface": "Given: miriam is a nun; no nuns marry. Your premises entail: miriam does not marry." + }, + { + "id": "ds-vb-0007", + "surface": "Given: felix is a tortoise; all tortoises are reptiles; no reptiles purr. Your premises entail: felix does not purr." + }, + { + "id": "ds-vb-0008", + "surface": "Given: clara paints portraits; clara is an artist. Your premises entail: clara paints portraits." + }, + { + "id": "ds-vb-0009", + "surface": "Given: boris does not smoke. Your premises entail: it is not the case that boris smokes." + }, + { + "id": "ds-vb-0010", + "surface": "Given: every baby cries; moses is a baby. Your premises entail: moses cries." + }, + { + "id": "ds-vb-0011", + "surface": "Given: all pilgrims go; dante is a pilgrim. Your premises entail: dante goes." + }, + { + "id": "ds-vb-0012", + "surface": "Given: all owls hunt; hedwig is an owl. Your premises entail the opposite of hedwig does not hunt \u2014 it cannot hold." + }, + { + "id": "ds-vb-0013", + "surface": "Given: petra is a judge; no judges gossip. Your premises entail the opposite of petra gossips \u2014 it cannot hold." + }, + { + "id": "ds-vb-0014", + "surface": "Given: omar swims; omar is an athlete. Your premises entail the opposite of omar does not swim \u2014 it cannot hold." + }, + { + "id": "ds-vb-0015", + "surface": "Given: ida is a farmer; all farmers are workers; all workers vote. Your premises entail the opposite of ida does not vote \u2014 it cannot hold." + }, + { + "id": "ds-vb-0016", + "surface": "Given: all monks chant; basil is a monk. Reading each name as one individual and each class word and verb phrase at face value, your premises don't settle whether gregory chants \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-vb-0017", + "surface": "Given: nina sings; all larks sing. Reading each name as one individual and each class word and verb phrase at face value, your premises don't settle whether nina is a lark \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-vb-0018", + "surface": "Given: viktor cooks; viktor is a chef. Reading each name as one individual and each class word and verb phrase at face value, your premises don't settle whether viktor cooks stew \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-vb-0019", + "surface": "Given: rosa is not a lawyer; no lawyers whistle. Reading each name as one individual and each class word and verb phrase at face value, your premises don't settle whether rosa does not whistle \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-vb-0020", + "surface": "Given: edgar does not dance. Reading each name as one individual and each class word and verb phrase at face value, your premises don't settle whether edgar does not skate \u2014 it holds in some cases and fails in others." + }, + { + "id": "ds-vb-0021", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-vb-0022", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-vb-0023", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-vb-0024", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-vb-0025", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-vb-0026", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-vb-0027", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + }, + { + "id": "ds-vb-0028", + "surface": "That reads as an argument, but I can't parse it precisely enough to decide it yet (no_template_match)." + } + ] } }, "wrong_is_zero": true diff --git a/evals/deduction_serve/runner.py b/evals/deduction_serve/runner.py index 80cfed6b..ba3cc04a 100644 --- a/evals/deduction_serve/runner.py +++ b/evals/deduction_serve/runner.py @@ -9,11 +9,33 @@ pipeline ``chat/deduction_surface.py`` calls in serving — the shape-gate (``looks_like_deductive_argument``), the reader (``comprehend``), the projector (``to_deductive_logic``), and the production ROBDD engine (``evaluate_entailment_with_trace``) — and the resulting outcome is -compared to independently-authored gold. The prose-rendering step -(``generate.proof_chain.render.render_entailment``) is presentation, not -decision, and is intentionally NOT re-derived here so this lane's pinned -bytes stay stable against wording-only changes; wording is covered by -``tests/test_deduction_surface.py``. +compared to independently-authored gold. + +**Surfaces are hashed too (2026-07-27).** This lane previously excluded the +prose-rendering step on the stated grounds that it is "presentation, not +decision", so the pinned bytes would "stay stable against wording-only +changes; wording is covered by ``tests/test_deduction_surface.py``." + +That rationale was measured and does not hold. With +``render._display_noun`` sabotaged to prefix every categorical noun — so +CORE served ``all SABOTAGE_dogs are SABOTAGE_animals`` — the following +were all still green: + +* the 11 lane SHA pins (11/11 byte-identical), +* ``tests/test_deduction_serve_lane`` + ``_license`` (20 passed), +* ``tests/test_deduction_surface.py`` (41 passed) — the very file named + above as the wording guard. + +Only ``evals/grammar_roundtrip`` caught it. So "wording is covered +elsewhere" was false, and the exclusion left CORE's user-visible output +unguarded by its own hash pins. That is how the ratified v1b band served +``all dog are mammal`` for the whole arc with ``wrong=0`` intact. + +``surface_sha256`` now covers every served surface, and ``surfaces`` +records them per case so a moved hash can be diffed rather than guessed at. +The accepted cost is exactly what the old rationale wanted to avoid: a +wording-only change moves this pin. That is the point — a wording change IS +a user-visible change and should require a deliberate re-pin. Counts: * ``correct`` — the pipeline's outcome class matches gold (including a @@ -37,11 +59,15 @@ AND that the pipeline declines honestly, not to inflate a pass rate). from __future__ import annotations import argparse +import hashlib import json from collections import Counter from pathlib import Path -from chat.deduction_surface import looks_like_deductive_argument +from chat.deduction_surface import ( + deduction_grounded_surface, + looks_like_deductive_argument, +) from generate.meaning_graph.projectors import to_deductive_logic, to_syllogism from generate.meaning_graph.reader import Comprehension, comprehend from generate.proof_chain.categorical import CategoricalError, decide_syllogism @@ -177,16 +203,33 @@ def _decide_exist(text: str) -> str: return _OUTCOME_TO_CLASS[outcome] +def _served_surface(text: str) -> str: + """The production prose for *text*, or a stable marker when none is served. + + Uses the real ``deduction_grounded_surface`` — the same call chat serving + makes — so what this lane hashes is what a user would actually read. A + raised exception is recorded rather than swallowed: a renderer that + crashes is a surface change too. + """ + try: + surface = deduction_grounded_surface(text) + except Exception as exc: # noqa: BLE001 - recorded, not handled + return f"" + return surface if surface is not None else "" + + def build_report(cases: list[dict]) -> dict: counts = Counter({"correct": 0, "wrong": 0, "declined": 0}) by_gold: Counter[str] = Counter() correct_by_gold: Counter[str] = Counter() wrong_examples: list[dict] = [] + surfaces: list[dict] = [] for case in cases: gold = case["gold"] by_gold[gold] += 1 got = decide(case["text"]) + surfaces.append({"id": case["id"], "surface": _served_surface(case["text"])}) if got == gold: counts["correct"] += 1 correct_by_gold[gold] += 1 @@ -204,6 +247,7 @@ def build_report(cases: list[dict]) -> dict: ) all_cases_correct = counts["correct"] == len(cases) + surface_blob = "\n".join(f"{s['id']}\t{s['surface']}" for s in surfaces) return { "n": len(cases), "counts": dict(counts), @@ -211,6 +255,10 @@ def build_report(cases: list[dict]) -> dict: "correct_by_gold": dict(correct_by_gold), "all_cases_correct": all_cases_correct, "mismatch_examples": wrong_examples, + # What the user actually reads. See the module docstring for why this + # is hashed rather than treated as "presentation, not decision". + "surface_sha256": hashlib.sha256(surface_blob.encode("utf-8")).hexdigest(), + "surfaces": surfaces, } @@ -243,6 +291,14 @@ def build_combined_report() -> dict: "by_gold": report["by_gold"], "correct_by_gold": report["correct_by_gold"], "all_cases_correct": report["all_cases_correct"], + # The served prose, hashed AND recorded. Without these the pinned + # artifact carried verdict counts only, and a renderer that emitted + # "all SABOTAGE_dogs are SABOTAGE_animals" left every pin + # byte-identical. Keeping the surfaces (not just the digest) means a + # moved pin shows the exact sentence that changed in review, instead + # of an opaque hash someone has to go re-derive. + "surface_sha256": report["surface_sha256"], + "surfaces": report["surfaces"], } aggregate["n"] += report["n"] for key in ("correct", "wrong", "declined"): diff --git a/scripts/verify_lane_shas.py b/scripts/verify_lane_shas.py index 4de0290c..182e9ca1 100644 --- a/scripts/verify_lane_shas.py +++ b/scripts/verify_lane_shas.py @@ -51,7 +51,13 @@ PINNED_SHAS: dict[str, str] = { "public_demo": "da7fad654e77aac4573a6fcf6e9eaaf84540be8e135d2e033d9cfd15119df3fc", "math_teaching_corpus_v1": "eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4", "deductive_logic_v1": "97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f", - "deduction_serve_v1": "0b461a5a49c8f8260ca87d0c9c9f9a17232bd1fdedd982e34649eedf9cca30b5", + # Re-pinned 2026-07-27: the report now carries the SERVED PROSE + # (``surface_sha256`` + ``surfaces`` per split) as well as verdict counts. + # Old: 0b461a5a49c8f8260ca87d0c9c9f9a17232bd1fdedd982e34649eedf9cca30b5 + # The move is the payload growing, not a behaviour change — verdicts are + # identical (166/166 correct, wrong=0). Before this, a renderer emitting + # "all SABOTAGE_dogs are SABOTAGE_animals" left all 11 pins byte-identical. + "deduction_serve_v1": "c855d55cf316471fdfe092aa0d5c954e5ceb6f30c9a8db283e0b9a5d5e8b419a", "curriculum_serve_v1": "d9e7ba500f040b865870413a940ee9a49910ac22e1a89c9feec1a60bdd2513f1", } diff --git a/tests/test_deduction_serve_lane.py b/tests/test_deduction_serve_lane.py index 010e0faf..89cec9f0 100644 --- a/tests/test_deduction_serve_lane.py +++ b/tests/test_deduction_serve_lane.py @@ -9,7 +9,12 @@ pipeline ``chat/deduction_surface.py`` runs), distinct from the bare-engine from __future__ import annotations -from evals.deduction_serve.runner import _ROOT, _load, build_report +from evals.deduction_serve.runner import ( + _ROOT, + _load, + build_combined_report, + build_report, +) _V1 = _ROOT / "v1" / "cases.jsonl" _V2_EN = _ROOT / "v2_en" / "cases.jsonl" @@ -136,3 +141,57 @@ def test_runner_treats_wrong_verdict_as_the_only_real_failure() -> None: ]) assert report["counts"]["wrong"] == 1 assert report["all_cases_correct"] is False + + +def test_the_pinned_report_records_what_the_user_actually_reads() -> None: + """Every committed case contributes its SERVED PROSE to the pinned bytes. + + Before 2026-07-27 the pinned artifact held verdict counts only, on the + stated rationale that prose is "presentation, not decision" and that + wording was covered by ``tests/test_deduction_surface.py``. Measured, it + was not: see the mutation test below. + """ + report = build_combined_report() + for name, split in report["splits"].items(): + assert len(split["surfaces"]) == split["n"], name + assert len(split["surface_sha256"]) == 64, name + for row in split["surfaces"]: + assert row["surface"], f"{name}/{row['id']} served empty prose" + + +def test_surface_hash_moves_when_the_renderer_is_sabotaged(monkeypatch) -> None: + """THE MUTATION PROOF — without this, the field above is decoration. + + A pin that cannot fail guards nothing. This corrupts the categorical noun + renderer exactly as the 2026-07-27 investigation did (every clause becomes + "all SABOTAGE_dogs are SABOTAGE_animals") and requires the pinned digest to + move. + + Historical note, and the reason this test exists: under that same sabotage + the 11 lane SHA pins stayed 11/11 byte-identical, this file's other 20 + tests passed, and all 41 tests in ``tests/test_deduction_surface.py`` + passed. Only ``evals/grammar_roundtrip`` caught it. CORE could have served + word salad indefinitely with every hash gate green. + """ + from generate.proof_chain import render + + clean = build_combined_report() + + original = render._display_noun + monkeypatch.setattr( + render, "_display_noun", lambda term: "SABOTAGE_" + original(term) + ) + sabotaged = build_combined_report() + + moved = [ + name + for name in clean["splits"] + if clean["splits"][name]["surface_sha256"] + != sabotaged["splits"][name]["surface_sha256"] + ] + assert moved, "sabotaging the renderer did not move any surface digest" + + # And the verdicts must be untouched — this proves the digest is tracking + # PROSE, not smuggling in a decision change. + assert clean["aggregate"] == sabotaged["aggregate"] + assert sabotaged["wrong_is_zero"] is True