From 472fc0a83f5f1ca104ca00e06707f43aea426bc9 Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 28 Jul 2026 13:16:57 -0700 Subject: [PATCH] =?UTF-8?q?feat(evals,tests):=20Perception=20Arc=20Phase?= =?UTF-8?q?=200=20=E2=80=94=20the=20read-rate=20instrument=20and=20floor?= =?UTF-8?q?=20ratchet=20(G-21,=20G-24)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 1.28% is now an instrument instead of testimony. evals/perception/read_rate.py measures the logic reader over holdout_dev/v1 deterministically (23/1,798 sentences, refusal taxonomy included, 0.57s); tests/test_read_rate_floor.py pins it in both directions — a comprehension regression fails the gate, and an improvement must move the recorded constants in the same reviewed commit, so capability motion is a decision with a diff, never an accident with a story. Non-vacuity guarded (the matcher must still say no; counts must conserve). Two sabotages observed red. Registered in smoke (~0.8s). This is one of exactly two old-plan items that survive the keel triage (the other is FA-1): it is the baseline number the keel's K4 perception layer is accountable for moving in chunks. PR-8, PR-10, Track B widening and Track C are superseded by keel phases K6/K6/K4/K5 and are not being finished here — finishing them would be polishing the quarry. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Wcw2pnMBwyvmNyQg4uPEt4 --- core/cli_test.py | 7 +++ evals/perception/__init__.py | 1 + evals/perception/read_rate.py | 97 +++++++++++++++++++++++++++++++++++ tests/test_read_rate_floor.py | 81 +++++++++++++++++++++++++++++ 4 files changed, 186 insertions(+) create mode 100644 evals/perception/__init__.py create mode 100644 evals/perception/read_rate.py create mode 100644 tests/test_read_rate_floor.py diff --git a/core/cli_test.py b/core/cli_test.py index 471b4696..7536cc98 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -156,6 +156,13 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { # ON surface being the only part where staleness is immediately dangerous. # Four sabotages observed red. Pure file parsing, <1s. "tests/test_flag_register.py", + # Perception Arc Phase 0 / G-21 / G-24 — the read-rate floor. 23 of 1,798 + # holdout sentences read (1.28%): the binding constraint every substrate + # experiment independently hit, and the baseline the keel's perception + # layer is accountable for moving in chunks. Both directions pinned: a + # regression fails; an improvement must move the recorded constants in + # the same reviewed commit. Deterministic, ~0.8s. + "tests/test_read_rate_floor.py", # PR-11 / G-5 / R-9 — the soak's evidence is committed, pinned, and CURRENT. # Two failure modes, deliberately separated: the digest catches a regression, # and the attested-source hashes catch STALENESS — evidence that no longer diff --git a/evals/perception/__init__.py b/evals/perception/__init__.py new file mode 100644 index 00000000..667be795 --- /dev/null +++ b/evals/perception/__init__.py @@ -0,0 +1 @@ +"""Perception measurement lane — the read-rate instrument (Perception Arc Phase 0, G-21/G-24).""" diff --git a/evals/perception/read_rate.py b/evals/perception/read_rate.py new file mode 100644 index 00000000..72a237b2 --- /dev/null +++ b/evals/perception/read_rate.py @@ -0,0 +1,97 @@ +"""The read-rate instrument — what fraction of real English the reader actually reads. + +Perception Arc Phase 0 (G-21's open ask, G-24's headline number, the keel's baseline). + +The 2026-07-28 diagnosis measured, ad hoc, that ``generate.meaning_graph.reader`` +reads **23 of 1,798** sentences of `holdout_dev/v1` (1.28%), with 93.16% failing as +``no_template_match`` — upstream of vocabulary, upstream of every flag. That number +is the binding constraint every substrate experiment independently hit, and it is +the metric the keel's perception layer (K4) is accountable for moving in chunks. + +An ad-hoc measurement is testimony. This module is the instrument: deterministic, +committed, and pinned by ``tests/test_read_rate_floor.py`` as a floor ratchet — the +recorded counts may only be updated deliberately, and only upward for ``read``. + +Deliberately narrow: the LOGIC reader at sentence granularity. The math reader's +decision rate (5/500 holdout cases, G-21) is recorded where it was measured +(`docs/analysis/relational-operator-ablation-dossier-2026-07-19.md`, split table) +and will get its own lane if it ever moves; folding a slow full-solve sweep into +this fast lane would couple two instruments that fail differently. +""" + +from __future__ import annotations + +import json +from collections import Counter +from dataclasses import dataclass +from pathlib import Path + +from generate.meaning_graph.reader import Refusal, _split_sentences, comprehend + +_ROOT = Path(__file__).resolve().parents[2] +CORPUS = _ROOT / "evals" / "gsm8k_math" / "holdout_dev" / "v1" / "cases.jsonl" + + +@dataclass(frozen=True) +class ReadRateReport: + corpus: str + problems: int + sentences: int + read: int + refusals: tuple[tuple[str, int], ...] # (reason, count), descending + + @property + def read_rate(self) -> float: + return self.read / self.sentences if self.sentences else 0.0 + + def as_dict(self) -> dict: + return { + "corpus": self.corpus, + "problems": self.problems, + "sentences": self.sentences, + "read": self.read, + "read_rate": round(self.read_rate, 6), + "refusals": dict(self.refusals), + } + + +def measure(corpus_path: Path = CORPUS) -> ReadRateReport: + """Run the logic reader over every sentence of the corpus. Deterministic.""" + problems = [json.loads(line) for line in corpus_path.read_text(encoding="utf-8").splitlines() if line.strip()] + reasons: Counter[str] = Counter() + read = 0 + sentences = 0 + for case in problems: + for sentence, *_rest in _split_sentences(case["problem"]): + sentence = sentence.strip() + if not sentence: + continue + sentences += 1 + try: + result = comprehend(sentence) + except Exception as exc: # a crashing reader is a finding, not a skip + reasons[f"EXCEPTION:{type(exc).__name__}"] += 1 + continue + if isinstance(result, Refusal): + reasons[result.reason] += 1 + elif result is None: + reasons["returned_none"] += 1 + else: + read += 1 + return ReadRateReport( + corpus=str(corpus_path.relative_to(_ROOT)), + problems=len(problems), + sentences=sentences, + read=read, + refusals=tuple(sorted(reasons.items(), key=lambda kv: (-kv[1], kv[0]))), + ) + + +def main() -> int: + report = measure() + print(json.dumps(report.as_dict(), indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_read_rate_floor.py b/tests/test_read_rate_floor.py new file mode 100644 index 00000000..375e4d44 --- /dev/null +++ b/tests/test_read_rate_floor.py @@ -0,0 +1,81 @@ +"""Perception Arc Phase 0 — the read-rate floor ratchet (G-21, G-24, keel baseline). + +The reader decides **23 of 1,798** sentences of `holdout_dev/v1` (1.28%). That number +is the capability constraint every substrate experiment independently hit (wedge, +operator ablation, §5 — all starved by the reader), the headline of G-24, and the +baseline the keel's perception layer (K4) is accountable for moving **in chunks**. + +This pin makes the number an instrument instead of testimony, with the same +both-directions discipline as every honest baseline in this repo: + + * ``read`` may not FALL below the recorded floor — a silent comprehension + regression in the serving reader fails the gate; + * the recorded counts may not silently DRIFT from the measurement — if read-rate + rises (a reader change, a keel absorption), the constants here move in the same + reviewed commit, so capability motion is a decision with a diff, never an + accident with a story. + +Corpus and reader are deterministic; the counts are exact, not tolerances. +""" + +from __future__ import annotations + +from evals.perception.read_rate import CORPUS, measure + +#: Measured 2026-07-28 at `64ecad00` — the diagnosis baseline (G-24). +FLOOR_READ = 23 +RECORDED_SENTENCES = 1798 +RECORDED_PROBLEMS = 500 +#: The dominant failure, pinned so the *shape* of the gap cannot drift unnoticed: +#: 93.16% of sentences die before vocabulary — upstream of packs, upstream of flags. +RECORDED_NO_TEMPLATE_MATCH = 1675 + + +def test_corpus_is_the_declared_one() -> None: + """The floor means nothing if the corpus quietly changes underneath it.""" + assert CORPUS.exists(), "holdout_dev/v1 moved — re-anchor the instrument first" + report = measure() + assert report.problems == RECORDED_PROBLEMS, ( + f"corpus changed shape: {report.problems} problems vs {RECORDED_PROBLEMS} recorded — " + "this pin measures the reader, not the corpus; re-baseline deliberately" + ) + assert report.sentences == RECORDED_SENTENCES, ( + f"sentence split moved: {report.sentences} vs {RECORDED_SENTENCES} — either the " + "splitter changed or the corpus did; both are reviewed decisions" + ) + + +def test_read_rate_holds_the_floor_and_the_record() -> None: + report = measure() + assert report.read >= FLOOR_READ, ( + f"READ-RATE REGRESSION: {report.read} sentences read, floor is {FLOOR_READ}. " + "The serving reader lost comprehension it had." + ) + assert report.read == FLOOR_READ, ( + f"read-rate MOVED UP: {report.read} vs recorded {FLOOR_READ}. Good news is still " + "a reviewed decision — update FLOOR_READ in this commit so the record matches " + "reality (the G-22 lesson: two agreeing stale records are not evidence)." + ) + refusals = dict(report.refusals) + assert refusals.get("no_template_match") == RECORDED_NO_TEMPLATE_MATCH, ( + f"the gap's shape moved: no_template_match={refusals.get('no_template_match')} " + f"vs {RECORDED_NO_TEMPLATE_MATCH} recorded — re-baseline deliberately" + ) + + +def test_the_instrument_is_not_vacuous() -> None: + """A measurement that cannot fail is not a measurement (R-11's lesson). + + The matcher must still say *no*: the overwhelming majority of real English must + be refused by the template reader, or the instrument (or reader) changed class + entirely and every number above means something different. + """ + report = measure() + assert report.sentences > 1000, "corpus too small to mean anything" + assert report.read < report.sentences * 0.5, ( + "the template reader reads >50% of GSM8K English — that is not this reader; " + "the instrument is measuring something else" + ) + assert sum(dict(report.refusals).values()) + report.read == report.sentences, ( + "counts don't conserve — the instrument dropped sentences silently" + )