feat(adr-0243): Phase 4 decisive falsifier — field decoder vs ROBDD gold (wrong=0)
The propositional slice of ADR-0243's falsifiability benchmark (plan §5 Phase 4), the arc's load-bearing evidence. Two mechanism-disjoint engines decide the SAME enumerated propositional problems: - Field decoder: propositional_entails (Cl(4,1) wave-field ground energy). - Gold: generate.proof_chain.entail.evaluate_entailment (exact ROBDD tautology check, ADR-0201 keystone) — genuinely independent, not the field checking itself. A clause→string bridge (render_clause: `a | ~b | c`) lets both engines decide byte-for-byte the same problem. Result over the enumerated panel (atoms a,b,c; premise sets size ≤2): - 1008 satisfiable-premise ID comparisons: field.entailed ⇔ gold is ENTAILED — wrong == 0. - 18 inconsistent-premise cases: field discloses satisfiable_premises=False exactly where gold returns REFUSED — matched refusal, no ex-falso leak. - OOD (>5 atoms): field decoder refuses by construction while gold still decides — the honest ID/OOD scope boundary in the propositional domain. Deterministic (enumerated, no randomness/wall-clock); off-serving (evals/ only, chat/runtime non-import pinned). [Verification]: tests/test_adr_0243_propositional_falsifier.py — 6 passed; run_propositional_falsifier() → wrong=0 over 1026 field-vs-gold decisions.
This commit is contained in:
parent
24078b1118
commit
bdf8146a2b
2 changed files with 304 additions and 0 deletions
222
evals/adr_0243_cognitive_lifecycle/propositional_falsifier.py
Normal file
222
evals/adr_0243_cognitive_lifecycle/propositional_falsifier.py
Normal file
|
|
@ -0,0 +1,222 @@
|
||||||
|
"""ADR-0243 Phase 4 — decisive falsifier: field relaxation decoder vs ROBDD gold.
|
||||||
|
|
||||||
|
The propositional slice of ADR-0243's falsifiability benchmark (plan §5 Phase 4).
|
||||||
|
Two independent engines decide the SAME propositional entailment problems:
|
||||||
|
|
||||||
|
* **Field decoder** — :func:`core.physics.cognitive_lifecycle.propositional_entails`
|
||||||
|
reads the ground energy of a Cl(4,1) wave field relaxed on a diagonal penalty
|
||||||
|
Hamiltonian (ADR-0243 §2.2). ``entailed`` ⇔ ``premises ∧ ¬conclusion`` has
|
||||||
|
ground energy > 0 (UNSAT).
|
||||||
|
* **Gold** — :func:`generate.proof_chain.entail.evaluate_entailment` decides
|
||||||
|
``(⋀premises) → query`` by an exact ROBDD tautology check (ADR-0201 keystone).
|
||||||
|
A canonical, complete, mechanism-disjoint decision procedure.
|
||||||
|
|
||||||
|
The bridge renders each CNF clause to the gold's formula syntax (``a | ~b | c``;
|
||||||
|
premises are one string per clause) so both engines decide byte-for-byte the same
|
||||||
|
problem, and the gold is genuinely independent — not the field decoder checking
|
||||||
|
itself.
|
||||||
|
|
||||||
|
**What wrong == 0 means (plan's decisive falsifier).** On the satisfiable-premise
|
||||||
|
panel the two verdicts are directly comparable: ``field.entailed`` ⇔
|
||||||
|
``gold.outcome is ENTAILED``. A single divergence confirms *"the field engine is
|
||||||
|
NOT the reasoner today"* on this domain; zero divergence over the panel is the
|
||||||
|
evidence that the decode-not-generate thesis holds on checkable propositional
|
||||||
|
logic. Inconsistent premises are handled honestly, not swept under the rug: the
|
||||||
|
field decoder discloses ``satisfiable_premises=False`` (vacuous ex-falso
|
||||||
|
entailment) exactly where the gold returns ``REFUSED`` — a matched refusal, not a
|
||||||
|
disagreement. Out-of-regime problems (> 5 atoms) are where the field decoder
|
||||||
|
*refuses* by construction while the gold still decides — the honest ID/OOD scope
|
||||||
|
boundary, in the propositional domain.
|
||||||
|
|
||||||
|
Off-serving: lives under ``evals/`` only; never imported by ``chat/runtime.py``.
|
||||||
|
Deterministic: the panel is enumerated (no wall-clock, no unseeded randomness).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import itertools
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any, Iterator, Sequence
|
||||||
|
|
||||||
|
from core.physics.cognitive_lifecycle import (
|
||||||
|
HamiltonianCompileError,
|
||||||
|
PropositionalProblem,
|
||||||
|
propositional_entails,
|
||||||
|
)
|
||||||
|
from generate.proof_chain.entail import Entailment, evaluate_entailment
|
||||||
|
|
||||||
|
# A CNF clause: disjunction of (atom, polarity) literals — the shape
|
||||||
|
# ``PropositionalProblem.clauses`` and ``propositional_entails``'s conclusion
|
||||||
|
# both accept. Defined locally rather than importing the module's non-exported
|
||||||
|
# ``Clause`` alias.
|
||||||
|
Clause = tuple[tuple[str, bool], ...]
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"FalsifierCase",
|
||||||
|
"render_clause",
|
||||||
|
"run_propositional_falsifier",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def render_clause(clause: Clause) -> str:
|
||||||
|
"""Render a CNF clause (disjunction of literals) in the gold's syntax.
|
||||||
|
|
||||||
|
``(("a", True), ("b", False))`` → ``"a | ~b"``. A single literal renders
|
||||||
|
bare (``"a"`` / ``"~a"``); the ROBDD parser treats ``|`` as OR, ``~`` as NOT.
|
||||||
|
"""
|
||||||
|
if not clause:
|
||||||
|
raise ValueError("render_clause: empty clause has no propositional meaning")
|
||||||
|
return " | ".join(atom if polarity else f"~{atom}" for atom, polarity in clause)
|
||||||
|
|
||||||
|
|
||||||
|
def _premise_strings(problem: PropositionalProblem) -> tuple[str, ...]:
|
||||||
|
return tuple(render_clause(c) for c in problem.clauses)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class FalsifierCase:
|
||||||
|
"""One field-vs-gold comparison record (JSON-safe via :meth:`as_dict`)."""
|
||||||
|
|
||||||
|
premises: tuple[str, ...]
|
||||||
|
conclusion: str
|
||||||
|
field_entailed: bool
|
||||||
|
field_premises_satisfiable: bool
|
||||||
|
gold_outcome: str
|
||||||
|
gold_reason: str
|
||||||
|
lane: str # "id" | "refusal_parity"
|
||||||
|
agree: bool
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"premises": list(self.premises),
|
||||||
|
"conclusion": self.conclusion,
|
||||||
|
"field_entailed": self.field_entailed,
|
||||||
|
"field_premises_satisfiable": self.field_premises_satisfiable,
|
||||||
|
"gold_outcome": self.gold_outcome,
|
||||||
|
"gold_reason": self.gold_reason,
|
||||||
|
"lane": self.lane,
|
||||||
|
"agree": self.agree,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _atom_literals(atoms: Sequence[str]) -> list[Clause]:
|
||||||
|
"""All 1- and 2-literal clauses over ``atoms`` (a 2-literal clause never
|
||||||
|
repeats an atom — that would be a tautological or contradictory clause the
|
||||||
|
field compiler rejects as a duplicate literal)."""
|
||||||
|
singles: list[Clause] = [((a, p),) for a in atoms for p in (True, False)]
|
||||||
|
pairs: list[Clause] = []
|
||||||
|
for a, b in itertools.combinations(atoms, 2):
|
||||||
|
for pa in (True, False):
|
||||||
|
for pb in (True, False):
|
||||||
|
pairs.append(((a, pa), (b, pb)))
|
||||||
|
return singles + pairs
|
||||||
|
|
||||||
|
|
||||||
|
def _enumerate_problems(
|
||||||
|
atoms: tuple[str, ...],
|
||||||
|
*,
|
||||||
|
max_clauses: int,
|
||||||
|
) -> Iterator[tuple[PropositionalProblem, Clause]]:
|
||||||
|
"""Deterministically enumerate (problem, conclusion) pairs over ``atoms``.
|
||||||
|
|
||||||
|
Premise sets are size 1..``max_clauses`` combinations of the atom clauses;
|
||||||
|
the conclusion ranges over every single literal. Order is fully determined
|
||||||
|
by ``itertools`` iteration over the sorted atom clauses — no randomness.
|
||||||
|
"""
|
||||||
|
clauses = _atom_literals(atoms)
|
||||||
|
conclusions: list[Clause] = [((a, p),) for a in atoms for p in (True, False)]
|
||||||
|
for size in range(1, max_clauses + 1):
|
||||||
|
for premise_combo in itertools.combinations(clauses, size):
|
||||||
|
problem = PropositionalProblem(atoms=atoms, clauses=tuple(premise_combo))
|
||||||
|
for conclusion in conclusions:
|
||||||
|
yield problem, conclusion
|
||||||
|
|
||||||
|
|
||||||
|
def run_propositional_falsifier(
|
||||||
|
*,
|
||||||
|
atoms: tuple[str, ...] = ("a", "b", "c"),
|
||||||
|
max_clauses: int = 2,
|
||||||
|
ood_atoms: tuple[str, ...] = ("a", "b", "c", "d", "e", "f"),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Score the field decoder against the ROBDD gold; return a JSON-safe artifact.
|
||||||
|
|
||||||
|
``wrong`` counts ID-lane divergences (``field.entailed`` ≠
|
||||||
|
``gold is ENTAILED`` on satisfiable premises) plus refusal-parity mismatches
|
||||||
|
(inconsistent premises where the field failed to disclose unsatisfiable
|
||||||
|
premises, or the gold did not refuse). ``ood_field_refused`` records that a
|
||||||
|
> 5-atom problem is rejected by the field decoder (out of its exact regime)
|
||||||
|
while remaining decidable for the gold.
|
||||||
|
"""
|
||||||
|
id_cases: list[FalsifierCase] = []
|
||||||
|
refusal_cases: list[FalsifierCase] = []
|
||||||
|
wrong = 0
|
||||||
|
|
||||||
|
for problem, conclusion in _enumerate_problems(atoms, max_clauses=max_clauses):
|
||||||
|
field = propositional_entails(problem, conclusion)
|
||||||
|
gold = evaluate_entailment(_premise_strings(problem), render_clause(conclusion))
|
||||||
|
|
||||||
|
if gold.outcome is Entailment.REFUSED:
|
||||||
|
# Gold refuses only on inconsistent premises here (input is always
|
||||||
|
# in-regime propositional). The field decoder must disclose the same
|
||||||
|
# via satisfiable_premises=False rather than assert real entailment.
|
||||||
|
agree = field.satisfiable_premises is False
|
||||||
|
refusal_cases.append(
|
||||||
|
FalsifierCase(
|
||||||
|
premises=_premise_strings(problem),
|
||||||
|
conclusion=render_clause(conclusion),
|
||||||
|
field_entailed=field.entailed,
|
||||||
|
field_premises_satisfiable=field.satisfiable_premises,
|
||||||
|
gold_outcome=gold.outcome.value,
|
||||||
|
gold_reason=gold.reason,
|
||||||
|
lane="refusal_parity",
|
||||||
|
agree=agree,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not agree:
|
||||||
|
wrong += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Satisfiable-premise ID lane: field.entailed ⇔ gold is ENTAILED.
|
||||||
|
gold_entailed = gold.outcome is Entailment.ENTAILED
|
||||||
|
agree = field.entailed == gold_entailed
|
||||||
|
id_cases.append(
|
||||||
|
FalsifierCase(
|
||||||
|
premises=_premise_strings(problem),
|
||||||
|
conclusion=render_clause(conclusion),
|
||||||
|
field_entailed=field.entailed,
|
||||||
|
field_premises_satisfiable=field.satisfiable_premises,
|
||||||
|
gold_outcome=gold.outcome.value,
|
||||||
|
gold_reason=gold.reason,
|
||||||
|
lane="id",
|
||||||
|
agree=agree,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not agree:
|
||||||
|
wrong += 1
|
||||||
|
|
||||||
|
# OOD: > 5 atoms is outside the field decoder's exact ≤ 32-assignment regime.
|
||||||
|
# It must refuse (typed) rather than guess; the gold still decides.
|
||||||
|
ood_field_refused = False
|
||||||
|
ood_gold_decided = False
|
||||||
|
ood_conclusion: Clause = ((ood_atoms[0], True),)
|
||||||
|
ood_clauses: tuple[Clause, ...] = tuple(((a, True),) for a in ood_atoms)
|
||||||
|
try:
|
||||||
|
PropositionalProblem(atoms=ood_atoms, clauses=ood_clauses)
|
||||||
|
except HamiltonianCompileError:
|
||||||
|
ood_field_refused = True
|
||||||
|
ood_gold = evaluate_entailment(
|
||||||
|
tuple(render_clause(c) for c in ood_clauses), render_clause(ood_conclusion)
|
||||||
|
)
|
||||||
|
ood_gold_decided = ood_gold.outcome is not Entailment.REFUSED
|
||||||
|
|
||||||
|
return {
|
||||||
|
"wrong": wrong,
|
||||||
|
"id_case_count": len(id_cases),
|
||||||
|
"id_disagreements": [c.as_dict() for c in id_cases if not c.agree],
|
||||||
|
"refusal_parity_count": len(refusal_cases),
|
||||||
|
"refusal_parity_mismatches": [c.as_dict() for c in refusal_cases if not c.agree],
|
||||||
|
"ood_field_refused": ood_field_refused,
|
||||||
|
"ood_gold_decided": ood_gold_decided,
|
||||||
|
"atoms": list(atoms),
|
||||||
|
"max_clauses": max_clauses,
|
||||||
|
}
|
||||||
82
tests/test_adr_0243_propositional_falsifier.py
Normal file
82
tests/test_adr_0243_propositional_falsifier.py
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
"""ADR-0243 Phase 4 — decisive falsifier test (field decoder vs ROBDD gold).
|
||||||
|
|
||||||
|
The load-bearing evidence of the arc: the Cl(4,1) field relaxation decoder must
|
||||||
|
agree with the independent ROBDD gold on every satisfiable-premise propositional
|
||||||
|
problem in the enumerated panel (``wrong == 0``), match its refusal on
|
||||||
|
inconsistent premises, and honestly refuse out-of-regime (> 5-atom) problems the
|
||||||
|
gold still decides. A single ID divergence would confirm the field engine is not
|
||||||
|
the reasoner on this domain.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from evals.adr_0243_cognitive_lifecycle.propositional_falsifier import (
|
||||||
|
render_clause,
|
||||||
|
run_propositional_falsifier,
|
||||||
|
)
|
||||||
|
from generate.proof_chain.entail import Entailment, evaluate_entailment
|
||||||
|
|
||||||
|
_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_render_clause_matches_gold_syntax():
|
||||||
|
assert render_clause((("a", True),)) == "a"
|
||||||
|
assert render_clause((("a", False),)) == "~a"
|
||||||
|
assert render_clause((("a", True), ("b", False), ("c", True))) == "a | ~b | c"
|
||||||
|
|
||||||
|
|
||||||
|
def test_field_decoder_matches_robdd_gold_wrong_zero():
|
||||||
|
"""Decisive falsifier: zero field-vs-gold divergence over the panel."""
|
||||||
|
artifact = run_propositional_falsifier()
|
||||||
|
# Both comparison lanes are actually exercised (not a vacuous pass).
|
||||||
|
assert artifact["id_case_count"] > 0
|
||||||
|
assert artifact["refusal_parity_count"] > 0
|
||||||
|
# The load-bearing assertion.
|
||||||
|
assert artifact["wrong"] == 0, (
|
||||||
|
f"field decoder diverged from ROBDD gold: "
|
||||||
|
f"id_disagreements={artifact['id_disagreements'][:3]} "
|
||||||
|
f"refusal_mismatches={artifact['refusal_parity_mismatches'][:3]}"
|
||||||
|
)
|
||||||
|
assert artifact["id_disagreements"] == []
|
||||||
|
assert artifact["refusal_parity_mismatches"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_out_of_regime_is_field_refused_but_gold_decidable():
|
||||||
|
"""ID/OOD scope boundary: > 5 atoms → field refuses, gold still decides."""
|
||||||
|
artifact = run_propositional_falsifier()
|
||||||
|
assert artifact["ood_field_refused"] is True
|
||||||
|
assert artifact["ood_gold_decided"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_falsifier_artifact_is_deterministic_and_json_safe():
|
||||||
|
a = json.dumps(run_propositional_falsifier(), sort_keys=True)
|
||||||
|
b = json.dumps(run_propositional_falsifier(), sort_keys=True)
|
||||||
|
assert a == b
|
||||||
|
assert json.loads(a)["wrong"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_gold_is_a_genuinely_independent_mechanism():
|
||||||
|
"""Sanity that the gold decides these canonical patterns as expected —
|
||||||
|
it is the ROBDD tautology check, not the field decoder in disguise."""
|
||||||
|
# modus ponens: (a→b) i.e. (~a | b), and a ⊨ b
|
||||||
|
assert evaluate_entailment(("~a | b", "a"), "b").outcome is Entailment.ENTAILED
|
||||||
|
# p ∨ q ⊭ p
|
||||||
|
assert evaluate_entailment(("a | b",), "a").outcome is Entailment.UNKNOWN
|
||||||
|
# inconsistent premises refuse (no ex-falso)
|
||||||
|
assert evaluate_entailment(("a", "~a"), "b").outcome is Entailment.REFUSED
|
||||||
|
|
||||||
|
|
||||||
|
def test_falsifier_is_not_serve_wired():
|
||||||
|
"""Off-serving: chat/runtime.py must never import this eval package."""
|
||||||
|
runtime_src = (_ROOT / "chat" / "runtime.py").read_text(encoding="utf-8")
|
||||||
|
tree = ast.parse(runtime_src)
|
||||||
|
for node in ast.walk(tree):
|
||||||
|
if isinstance(node, ast.ImportFrom) and node.module:
|
||||||
|
assert "adr_0243_cognitive_lifecycle" not in node.module
|
||||||
|
if isinstance(node, ast.Import):
|
||||||
|
for alias in node.names:
|
||||||
|
assert "adr_0243_cognitive_lifecycle" not in alias.name
|
||||||
Loading…
Reference in a new issue