test(register): full 7-pack invariant matrix on cognition lane
Adds tests/test_cognition_eval_register_matrix.py — strict superset
of tests/test_register_invariant_grounding.py (which covered only 4
of the 7 ratified register packs).
Parametrizes over all seven ratified register packs
({default_neutral, terse, precise, convivial, pedagogical, formal,
socratic}_v1) and asserts byte-identity against the unregistered
baseline for every per-case projection the cognition eval reports:
* trace_hash (ADR-0072 truth-path-isolation)
* intent_correct (intent runs upstream of realizer)
* terms_captured (scored off canonical surface)
* surface_contains_pass (scored off canonical surface)
* versor_closure (truth-path field invariant)
* versor_condition (exact float, stronger than closure)
* surface (CognitiveTurnResult.surface is the
pre-decoration canonical the trace
hash consumes; substantive transforms
live on turn_log[-1].surface)
Aggregate metrics on EvalReport are pinned identically: total,
intent_correct, terms_captured, terms_expected, surface_grounded,
versor_closures.
Meta-test test_register_matrix_covers_every_ratified_pack enforces
that _RATIFIED_REGISTERS in this file stays in lockstep with
scripts/ratify_register_packs.py::REGISTER_IDS — so the 93 drafted
register packs in packs/register/_catalog.json cannot ratify into
CI without each one passing the full invariant matrix.
Run: 57 cells (8 projections x 7 registers + 1 meta), 27.7s
sequential across 45 cognition cases per register.
Pre-existing smoke failure (test_chat_response_surface_uses_
articulation_plan in tests/test_runtime_config.py) is the ADR-0086
expected-string test on main; unrelated to this change.
This commit is contained in:
Shay
parent
79f1678923
commit
66db063f0b
1 changed files with 281 additions and 0 deletions
281
tests/test_cognition_eval_register_matrix.py
Normal file
281
tests/test_cognition_eval_register_matrix.py
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
"""Full 7-register invariant matrix on the cognition lane (Option 3 CI gate).
|
||||
|
||||
Pins the ADR-0072 invariant across every ratified register pack
|
||||
simultaneously. Strict superset of
|
||||
``tests/test_register_invariant_grounding.py`` (which only covers
|
||||
4 of the 7 ratified registers): each of the seven ratified packs is
|
||||
treated as a parametrized variant, and every per-case projection the
|
||||
eval reports is asserted byte-identical against the unregistered
|
||||
(``None``) baseline.
|
||||
|
||||
Load-bearing claims pinned:
|
||||
|
||||
cognition_eval_register_matrix:
|
||||
For every cognition case AND every ratified register pack:
|
||||
* ``trace_hash`` byte-identical to baseline
|
||||
* ``intent_correct`` byte-identical to baseline
|
||||
* ``terms_captured`` byte-identical to baseline
|
||||
* ``surface_contains_pass`` byte-identical to baseline
|
||||
* ``versor_closure`` byte-identical to baseline
|
||||
* ``versor_condition`` byte-identical to baseline
|
||||
Aggregate metrics on the eval report:
|
||||
* ``intent_correct`` (count) byte-identical to baseline
|
||||
* ``terms_captured`` (count) byte-identical to baseline
|
||||
* ``terms_expected`` (count) byte-identical to baseline
|
||||
* ``surface_grounded`` (count) byte-identical to baseline
|
||||
* ``versor_closures`` (count) byte-identical to baseline
|
||||
* ``total`` byte-identical to baseline
|
||||
|
||||
Why this gate exists. We are about to materialise 93 drafted register
|
||||
packs from ``packs/register/_catalog.json``. Before any new pack
|
||||
lands, this gate proves the seven existing ratified packs are
|
||||
collectively well-behaved against the cognition eval — i.e. the
|
||||
ADR-0072 invariant is not a per-pack accident but a structural
|
||||
guarantee. Every drafted pack added later must extend this matrix
|
||||
(see ``_RATIFIED_REGISTERS`` below) and the gate must stay green.
|
||||
|
||||
Note on surfaces. ``CognitiveTurnResult.surface`` is the pre-
|
||||
decoration / pre-substantive composer output — the truth-path field
|
||||
``compute_trace_hash`` consumes. Substantive register transforms
|
||||
(seeded markers, semantic-domain-clause appends, gloss compression)
|
||||
are applied on the *post-canonical* surface stored on
|
||||
``ChatRuntime.turn_log[-1].surface``. Therefore ``result.surface``
|
||||
is expected to be byte-identical across registers too — the post-
|
||||
decoration variation is asserted by the existing register-tour gate
|
||||
(``evals/register_tour/run_tour.py`` + ``tests/test_register_tour_demo.py``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from core.config import RuntimeConfig
|
||||
from evals.run_cognition_eval import load_cases, run_eval
|
||||
|
||||
|
||||
# The seven ratified register packs on disk. When a drafted pack
|
||||
# from ``packs/register/_catalog.json`` is ratified, append its
|
||||
# ``register_id`` here. The matrix grows; the invariant doesn't bend.
|
||||
_RATIFIED_REGISTERS: tuple[str, ...] = (
|
||||
"default_neutral_v1",
|
||||
"terse_v1",
|
||||
"precise_v1",
|
||||
"convivial_v1",
|
||||
"pedagogical_v1",
|
||||
"formal_v1",
|
||||
"socratic_v1",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def cases():
|
||||
return load_cases()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def baseline_report(cases):
|
||||
"""Unregistered baseline — every register must match this case-for-case."""
|
||||
return run_eval(cases, config=RuntimeConfig(register_pack_id=None))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def baseline_by_id(baseline_report):
|
||||
return {c.case_id: c for c in baseline_report.cases}
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=_RATIFIED_REGISTERS)
|
||||
def register_report(request, cases):
|
||||
"""One eval report per ratified register pack."""
|
||||
return (request.param, run_eval(
|
||||
cases, config=RuntimeConfig(register_pack_id=request.param),
|
||||
))
|
||||
|
||||
|
||||
def _by_id(report):
|
||||
return {c.case_id: c for c in report.cases}
|
||||
|
||||
|
||||
def _diff_rows(register_id: str, baseline_by_id, register_by_id, field: str):
|
||||
rows: list[str] = []
|
||||
for case_id, base in baseline_by_id.items():
|
||||
var = register_by_id[case_id]
|
||||
a = getattr(base, field)
|
||||
b = getattr(var, field)
|
||||
if a != b:
|
||||
rows.append(
|
||||
f" {case_id}: {field} diverged under {register_id!r}\n"
|
||||
f" baseline : {a!r}\n"
|
||||
f" register : {b!r}"
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def test_register_matrix_trace_hash_invariant(
|
||||
register_report, baseline_by_id,
|
||||
):
|
||||
"""``trace_hash`` byte-identical case-by-case under every ratified register.
|
||||
|
||||
This is the load-bearing ADR-0072 truth-path-isolation claim. A
|
||||
single divergence here means a register pack is leaking into the
|
||||
truth path — the entire register subsystem is invalid until fixed.
|
||||
"""
|
||||
register_id, report = register_report
|
||||
rows = _diff_rows(
|
||||
register_id, baseline_by_id, _by_id(report), "trace_hash",
|
||||
)
|
||||
assert not rows, (
|
||||
f"TRUTH-PATH LEAK under register {register_id!r} — trace_hash "
|
||||
"must not vary across the register axis.\n" + "\n".join(rows)
|
||||
)
|
||||
|
||||
|
||||
def test_register_matrix_intent_correct_invariant(
|
||||
register_report, baseline_by_id,
|
||||
):
|
||||
"""Intent classification runs upstream of the realizer; register
|
||||
cannot move it."""
|
||||
register_id, report = register_report
|
||||
rows = _diff_rows(
|
||||
register_id, baseline_by_id, _by_id(report), "intent_correct",
|
||||
)
|
||||
assert not rows, (
|
||||
f"UPSTREAM LEAK under register {register_id!r} — intent "
|
||||
"classification cannot depend on register.\n" + "\n".join(rows)
|
||||
)
|
||||
|
||||
|
||||
def test_register_matrix_terms_captured_invariant(
|
||||
register_report, baseline_by_id,
|
||||
):
|
||||
"""Eval-scored ``terms_captured`` is computed off the pre-
|
||||
decoration / canonical surface; register must not perturb it."""
|
||||
register_id, report = register_report
|
||||
rows = _diff_rows(
|
||||
register_id, baseline_by_id, _by_id(report), "terms_captured",
|
||||
)
|
||||
assert not rows, (
|
||||
f"GROUNDING METRIC LEAK under register {register_id!r} — "
|
||||
"terms_captured varies across registers.\n" + "\n".join(rows)
|
||||
)
|
||||
|
||||
|
||||
def test_register_matrix_surface_contains_pass_invariant(
|
||||
register_report, baseline_by_id,
|
||||
):
|
||||
"""``surface_contains_pass`` scored off canonical surface — must
|
||||
not vary by register."""
|
||||
register_id, report = register_report
|
||||
rows = _diff_rows(
|
||||
register_id, baseline_by_id, _by_id(report),
|
||||
"surface_contains_pass",
|
||||
)
|
||||
assert not rows, (
|
||||
f"GROUNDING METRIC LEAK under register {register_id!r} — "
|
||||
"surface_contains_pass varies across registers.\n"
|
||||
+ "\n".join(rows)
|
||||
)
|
||||
|
||||
|
||||
def test_register_matrix_versor_closure_invariant(
|
||||
register_report, baseline_by_id,
|
||||
):
|
||||
"""``versor_condition < 1e-6`` is a property of the truth path."""
|
||||
register_id, report = register_report
|
||||
rows = _diff_rows(
|
||||
register_id, baseline_by_id, _by_id(report), "versor_closure",
|
||||
)
|
||||
assert not rows, (
|
||||
f"FIELD INVARIANT DRIFT under register {register_id!r} — "
|
||||
"versor_closure varies across registers.\n" + "\n".join(rows)
|
||||
)
|
||||
|
||||
|
||||
def test_register_matrix_versor_condition_byte_identical(
|
||||
register_report, baseline_by_id,
|
||||
):
|
||||
"""Stronger than closure: the exact ``versor_condition`` float
|
||||
must be byte-identical. Catches drift below the closure
|
||||
threshold that closure-only would miss."""
|
||||
register_id, report = register_report
|
||||
rows = _diff_rows(
|
||||
register_id, baseline_by_id, _by_id(report), "versor_condition",
|
||||
)
|
||||
assert not rows, (
|
||||
f"FIELD STATE LEAK under register {register_id!r} — "
|
||||
"versor_condition is not byte-identical.\n" + "\n".join(rows)
|
||||
)
|
||||
|
||||
|
||||
def test_register_matrix_canonical_surface_byte_identical(
|
||||
register_report, baseline_by_id,
|
||||
):
|
||||
"""``CognitiveTurnResult.surface`` is the pre-decoration /
|
||||
canonical composer output (the truth-path field
|
||||
``compute_trace_hash`` consumes). Substantive register transforms
|
||||
apply downstream on ``turn_log[-1].surface``; the canonical
|
||||
must remain byte-identical across the register axis.
|
||||
|
||||
If this test ever fails for a non-null register, the substantive
|
||||
transform has leaked into the canonical surface and the
|
||||
truth-path-isolation contract is broken.
|
||||
"""
|
||||
register_id, report = register_report
|
||||
rows = _diff_rows(
|
||||
register_id, baseline_by_id, _by_id(report), "surface",
|
||||
)
|
||||
assert not rows, (
|
||||
f"CANONICAL SURFACE LEAK under register {register_id!r} — "
|
||||
"substantive transforms have escaped into the pre-decoration "
|
||||
"surface that feeds compute_trace_hash.\n" + "\n".join(rows)
|
||||
)
|
||||
|
||||
|
||||
def test_register_matrix_aggregate_metrics_byte_identical(
|
||||
register_report, baseline_report,
|
||||
):
|
||||
"""Every aggregate count on the eval report matches baseline."""
|
||||
register_id, report = register_report
|
||||
deltas: list[str] = []
|
||||
for field in (
|
||||
"total",
|
||||
"intent_correct",
|
||||
"terms_captured",
|
||||
"terms_expected",
|
||||
"surface_grounded",
|
||||
"versor_closures",
|
||||
):
|
||||
a = getattr(baseline_report, field)
|
||||
b = getattr(report, field)
|
||||
if a != b:
|
||||
deltas.append(
|
||||
f" {field}: baseline={a} register({register_id})={b}"
|
||||
)
|
||||
assert not deltas, (
|
||||
f"AGGREGATE METRIC DRIFT under register {register_id!r} — "
|
||||
"ADR-0072 aggregate-invariant claim violated.\n"
|
||||
+ "\n".join(deltas)
|
||||
)
|
||||
|
||||
|
||||
def test_register_matrix_covers_every_ratified_pack():
|
||||
"""Meta-test: the parametrized matrix must cover exactly the
|
||||
set of ratified register packs on disk. When a drafted pack is
|
||||
ratified, both ``scripts/ratify_register_packs.py::REGISTER_IDS``
|
||||
AND this file's ``_RATIFIED_REGISTERS`` must be widened together.
|
||||
"""
|
||||
from scripts.ratify_register_packs import REGISTER_IDS
|
||||
|
||||
matrix = set(_RATIFIED_REGISTERS)
|
||||
script = set(REGISTER_IDS)
|
||||
only_in_matrix = matrix - script
|
||||
only_in_script = script - matrix
|
||||
assert not only_in_matrix and not only_in_script, (
|
||||
"Register-matrix drift: the ratified register set and the "
|
||||
"test matrix have diverged.\n"
|
||||
f" only in matrix : {sorted(only_in_matrix)}\n"
|
||||
f" only in script : {sorted(only_in_script)}\n"
|
||||
"When ratifying a new register pack, widen "
|
||||
"scripts/ratify_register_packs.py::REGISTER_IDS AND "
|
||||
"tests/test_cognition_eval_register_matrix.py::_RATIFIED_REGISTERS "
|
||||
"in the same change."
|
||||
)
|
||||
Loading…
Reference in a new issue