core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions
core/tests/test_domain_contract_predicates.py
Shay 0390491c93 feat(packs): implement ADR-0097 — Mathematics-Logic Reasoning-Capable 
First concrete domain claim under ADR-0091's Domain Pack Contract v1.
en_mathematics_logic_v1 is now formally ratified as reasoning-capable
in the capability ledger: 9/9 ADR-0091 predicates pass.

ADR-0097 §"No code changes outside pack artifacts and corpus" relaxed
to include two latent bug fixes that ADR-0093's predicate enforcement
just exposed:

1. language_packs/schema.py: LanguageRole enum widened to include
   DOMAIN_SEED. Three in-tree packs (en_mathematics_logic_v1,
   en_physics_v1, en_systems_software_v1) have declared role="domain_seed"
   since landing but the enum was never updated; load_pack() always
   raised on them. ADR-0093's P1 predicate exposed the mismatch.

2. core/capability/domain_contract_predicates.py: P2 (gloss checksum)
   was reading manifest["checksums"]["glosses_sha256"]; the canonical
   in-tree location is manifest["glosses_checksum"] (top-level). Fixed
   to prefer the canonical key and fall back to the nested form for
   forward compatibility.

ADR-0097 manifest additions to en_mathematics_logic_v1:
- domain_contract_version: 1
- domain_id: "mathematics_logic"
- axioms: null  (rules in v1 — pack proves reasoning via chain
  composition, not declarative axioms)
- rules: null
- teaching_chains: ["mathematics_logic_chains_v1"]
- eval_lanes: three lanes with dev/public/holdout (elementary_mathematics_ood,
  inference_closure, fabrication_control)
- reviewers: ["shay-j"] (resolved via ADR-0092 registry)
- known_gaps: [] (all math/logic gaps in docs/gaps.md were [x])
- provenance: "adr-0097:reviewed:2026-05-21"

Verified evidence:
- core capability domain-contract --pack-id en_mathematics_logic_v1
  → all_passed=True (P1-P9 all pass)
- core capability ledger → mathematics_logic row shows
  status=reasoning-capable, predicates.reasoning_capable=True,
  predicates.expert_demo=False, open_gaps=[],
  operator_chain_coverage all ready=True (8 chains each),
  intent_shapes_present=5
- 14 ADR-0097 invariant tests in
  test_adr_0097_mathematics_logic_ratification.py pin
  status/provenance/expert-demo-gate/contract shape

Two pre-existing tests updated for the new CLI default
(predicate-running, non-zero on missing contract):
- test_capability_domain_contract_json_absent_contract_is_noop now
  uses --structural-only to assert legacy parse-only shape
- test_cli_returns_nonzero_on_missing_contract switched its fixture
  pack from en_mathematics_logic_v1 (now has a contract) to
  en_core_cognition_v1 (no contract)

The pre-existing test_flag_report_tracks_default_off_flags failure
(discourse_planner flag default mismatch, seen since ADR-0092) is
unchanged and unrelated.

Smoke 67/67, packs 6/6, capability tests 49/50, cognition eval
byte-identical 100/100/100/100; lanes byte-identical:
reviewer_registry 6/6, miner_loop_closure 6/6,
domain_contract_validation 9/9, fabrication_control dev 12/12 +
public 9/9.
2026-05-21 18:51:58 -07:00

487 lines
17 KiB
Python
Raw Blame History

"""ADR-0093 — Domain Pack Contract v1 predicate-evaluation tests."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from core.capability.domain_contract_predicates import (
DomainContractPredicateReport,
PredicateResult,
_parse_gap_states,
evaluate_domain_contract,
)
from core.capability.reviewers import (
Reviewer,
ReviewerRegistry,
SCHEMA_VERSION,
)
REPO_ROOT = Path(__file__).resolve().parent.parent
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _build_synthetic_pack(
tmp_path: Path,
*,
pack_id: str = "synthetic_math_pack_v1",
domain_id: str = "mathematics_logic",
teaching_chains: tuple[str, ...] = ("mathematics_logic_chains_v1",),
reviewers: tuple[str, ...] = ("shay-j",),
eval_lanes: tuple[dict[str, Any], ...] = (
{
"lane": "elementary_mathematics_ood",
"version": "v1",
"splits": ["dev", "public", "holdout"],
},
),
known_gaps: tuple[str, ...] = (),
include_contract: bool = True,
glosses_text: str | None = None,
) -> Path:
"""Create a synthetic pack directory and return the data_root path."""
data_root = tmp_path / "data"
pack_dir = data_root / pack_id
pack_dir.mkdir(parents=True)
manifest: dict[str, Any] = {
"pack_id": pack_id,
"name": "synthetic pack",
}
if include_contract:
manifest.update(
{
"domain_contract_version": 1,
"domain_id": domain_id,
"axioms": None,
"rules": None,
"teaching_chains": list(teaching_chains),
"eval_lanes": [dict(lane) for lane in eval_lanes],
"reviewers": list(reviewers),
"known_gaps": list(known_gaps),
"provenance": "test:fixture:2026-05-21",
}
)
if glosses_text is not None:
glosses_path = pack_dir / "glosses.jsonl"
glosses_path.write_text(glosses_text, encoding="utf-8")
import hashlib
manifest.setdefault("checksums", {})["glosses_sha256"] = hashlib.sha256(
glosses_path.read_bytes()
).hexdigest()
(pack_dir / "manifest.json").write_text(
json.dumps(manifest, sort_keys=True), encoding="utf-8"
)
return data_root
def _primary_registry() -> ReviewerRegistry:
return ReviewerRegistry(
schema_version=SCHEMA_VERSION,
reviewers=(
Reviewer(
reviewer_id="shay-j",
display_name="Joshua Shay",
role="primary",
domains=("*",),
review_scope=("pack", "proposal", "chain", "eval"),
provenance="adr-0092:bootstrap:test",
),
),
)
def _stub_chain_inventory() -> dict[str, Any]:
"""A chain inventory that satisfies P5 and P6 for mathematics_logic."""
return {
"by_domain_operator_family": {
"mathematics_logic": {
"transitive": 10,
"proof_chain": 9,
"contradiction": 8,
},
},
"by_domain_intent_shape": {
"mathematics_logic": {
"cause": 4,
"verification": 3,
"comparison": 5,
},
},
}
# ---------------------------------------------------------------------------
# Contract presence / structural failures
# ---------------------------------------------------------------------------
class TestContractPresence:
def test_pack_without_contract_reports_absent(self, tmp_path: Path) -> None:
data_root = _build_synthetic_pack(tmp_path, include_contract=False)
report = evaluate_domain_contract(
"synthetic_math_pack_v1",
data_root=data_root,
chain_inventory=_stub_chain_inventory(),
reviewer_registry=_primary_registry(),
)
assert isinstance(report, DomainContractPredicateReport)
assert report.contract_present is False
assert report.all_passed is False
assert report.predicates == ()
def test_pack_with_invalid_contract_reports_errors(self, tmp_path: Path) -> None:
data_root = _build_synthetic_pack(
tmp_path,
eval_lanes=(
{"lane": "x", "version": "v1", "splits": ["alien_split"]},
),
)
report = evaluate_domain_contract(
"synthetic_math_pack_v1",
data_root=data_root,
chain_inventory=_stub_chain_inventory(),
reviewer_registry=_primary_registry(),
)
assert report.contract_present is True
assert report.contract_valid is False
assert any("splits" in e for e in report.contract_errors)
# ---------------------------------------------------------------------------
# Per-predicate verification (P3-P9)
#
# P1 / P2 require fully compiled packs and so are exercised against the
# in-tree packs in `TestProductionPacks` below rather than against synthetic
# fixtures (the language_packs.compiler.load_pack call cannot resolve a
# synthetic data root in the same way).
# ---------------------------------------------------------------------------
def _find(predicates: tuple[PredicateResult, ...], predicate_id: str) -> PredicateResult:
matches = [p for p in predicates if p.predicate_id == predicate_id]
assert matches, f"missing predicate {predicate_id}"
return matches[0]
class TestPredicatesAgainstSyntheticPack:
def _report(self, tmp_path: Path, **overrides: Any) -> DomainContractPredicateReport:
data_root = _build_synthetic_pack(tmp_path, **overrides)
return evaluate_domain_contract(
overrides.get("pack_id", "synthetic_math_pack_v1"),
data_root=data_root,
chain_inventory=_stub_chain_inventory(),
reviewer_registry=_primary_registry(),
)
def test_p3_known_domain_passes(self, tmp_path: Path) -> None:
report = self._report(tmp_path)
assert _find(report.predicates, "P3").passed is True
def test_p3_unknown_domain_rejected_by_parser(self, tmp_path: Path) -> None:
report = self._report(tmp_path, domain_id="alien_domain")
assert report.contract_valid is False
assert any("domain_id:unknown" in e for e in report.contract_errors)
def test_p4_registered_corpora_pass(self, tmp_path: Path) -> None:
report = self._report(tmp_path)
assert _find(report.predicates, "P4").passed is True
def test_p4_unregistered_corpus_fails(self, tmp_path: Path) -> None:
report = self._report(
tmp_path, teaching_chains=("ghost_chains_v999",)
)
p4 = _find(report.predicates, "P4")
assert p4.passed is False
assert "ghost_chains_v999" in p4.notes
def test_p5_sufficient_chain_coverage_passes(self, tmp_path: Path) -> None:
report = self._report(tmp_path)
assert _find(report.predicates, "P5").passed is True
def test_p5_shortfall_fails(self, tmp_path: Path) -> None:
sparse_inventory: dict[str, Any] = {
"by_domain_operator_family": {
"mathematics_logic": {
"transitive": 2,
"proof_chain": 9,
"contradiction": 8,
},
},
"by_domain_intent_shape": {
"mathematics_logic": {"cause": 4, "verification": 3, "comparison": 5},
},
}
data_root = _build_synthetic_pack(tmp_path)
report = evaluate_domain_contract(
"synthetic_math_pack_v1",
data_root=data_root,
chain_inventory=sparse_inventory,
reviewer_registry=_primary_registry(),
)
p5 = _find(report.predicates, "P5")
assert p5.passed is False
assert "transitive=2" in p5.notes
def test_p6_three_intent_shapes_passes(self, tmp_path: Path) -> None:
report = self._report(tmp_path)
assert _find(report.predicates, "P6").passed is True
def test_p6_too_few_intents_fails(self, tmp_path: Path) -> None:
sparse_inventory: dict[str, Any] = {
"by_domain_operator_family": {
"mathematics_logic": {
"transitive": 10,
"proof_chain": 9,
"contradiction": 8,
},
},
"by_domain_intent_shape": {
"mathematics_logic": {"cause": 1, "verification": 0, "comparison": 0},
},
}
data_root = _build_synthetic_pack(tmp_path)
report = evaluate_domain_contract(
"synthetic_math_pack_v1",
data_root=data_root,
chain_inventory=sparse_inventory,
reviewer_registry=_primary_registry(),
)
p6 = _find(report.predicates, "P6")
assert p6.passed is False
assert "1 intent" in p6.notes
def test_p7_complete_splits_passes(self, tmp_path: Path) -> None:
report = self._report(tmp_path)
assert _find(report.predicates, "P7").passed is True
def test_p7_missing_holdout_fails(self, tmp_path: Path) -> None:
report = self._report(
tmp_path,
eval_lanes=(
{
"lane": "elementary_mathematics_ood",
"version": "v1",
"splits": ["dev", "public"],
},
),
)
p7 = _find(report.predicates, "P7")
assert p7.passed is False
assert "elementary_mathematics_ood" in p7.notes
def test_p8_known_reviewer_passes(self, tmp_path: Path) -> None:
report = self._report(tmp_path)
assert _find(report.predicates, "P8").passed is True
def test_p8_unknown_reviewer_fails(self, tmp_path: Path) -> None:
report = self._report(tmp_path, reviewers=("ghost-reviewer",))
p8 = _find(report.predicates, "P8")
assert p8.passed is False
assert "ghost-reviewer" in p8.notes
def test_p8_domain_scope_mismatch_fails(self, tmp_path: Path) -> None:
registry = ReviewerRegistry(
schema_version=SCHEMA_VERSION,
reviewers=(
Reviewer(
reviewer_id="physics-only",
display_name="Physics Reviewer",
role="domain",
domains=("physics",),
review_scope=("pack",),
provenance="adr-0092:test",
),
),
)
data_root = _build_synthetic_pack(tmp_path, reviewers=("physics-only",))
report = evaluate_domain_contract(
"synthetic_math_pack_v1",
data_root=data_root,
chain_inventory=_stub_chain_inventory(),
reviewer_registry=registry,
)
p8 = _find(report.predicates, "P8")
assert p8.passed is False
assert "out_of_scope" in p8.notes
def test_p9_no_gaps_passes(self, tmp_path: Path) -> None:
report = self._report(tmp_path)
assert _find(report.predicates, "P9").passed is True
def test_p9_with_closed_gaps_passes(self, tmp_path: Path) -> None:
# All math/logic gaps in docs/gaps.md are closed (per current state).
report = self._report(
tmp_path,
known_gaps=("gap:mathematics_logic_pack_absent",),
)
assert _find(report.predicates, "P9").passed is True
def test_p9_with_unknown_gap_fails(self, tmp_path: Path) -> None:
report = self._report(
tmp_path,
known_gaps=("gap:absolutely_imaginary_blocker",),
)
p9 = _find(report.predicates, "P9")
assert p9.passed is False
# ---------------------------------------------------------------------------
# Gap parser
# ---------------------------------------------------------------------------
class TestGapStateParser:
def test_parses_open_and_closed(self) -> None:
text = (
"# Header\n"
"- [x] `gap:foo_closed`\n"
"- [ ] `gap:bar_open`\n"
"irrelevant line\n"
"- [x] `gap:baz_closed`\n"
)
states = _parse_gap_states(text)
assert states == {
"gap:foo_closed": True,
"gap:bar_open": False,
"gap:baz_closed": True,
}
def test_skips_malformed_lines(self) -> None:
text = "- [?] `gap:typo`\nfree text\n- [x] `gap:ok`\n"
states = _parse_gap_states(text)
assert states == {"gap:ok": True}
# ---------------------------------------------------------------------------
# Eval lane artifact resolution
# ---------------------------------------------------------------------------
class TestEvalLaneArtifacts:
def test_artifacts_surface_existing_reports(self, tmp_path: Path) -> None:
# Use a real in-tree lane (reviewer_registry) that we know has a
# v1_dev.json under results/ from ADR-0092.
data_root = _build_synthetic_pack(
tmp_path,
eval_lanes=(
{
"lane": "reviewer_registry",
"version": "v1",
"splits": ["dev", "public", "holdout"],
},
),
)
report = evaluate_domain_contract(
"synthetic_math_pack_v1",
data_root=data_root,
chain_inventory=_stub_chain_inventory(),
reviewer_registry=_primary_registry(),
)
assert len(report.eval_lane_artifacts) == 1
artifact = report.eval_lane_artifacts[0]
assert artifact["lane"] == "reviewer_registry"
assert artifact["splits"]["dev"]["exists"] is True
assert artifact["splits"]["dev"]["sha256"] is not None
# public/holdout don't exist for that lane yet
assert artifact["splits"]["public"]["exists"] is False
# ---------------------------------------------------------------------------
# CLI smoke
# ---------------------------------------------------------------------------
class TestCli:
def test_cli_returns_nonzero_on_missing_contract(self) -> None:
"""A pack without a domain contract exits 1 under default mode."""
import os
import subprocess
import sys
env = os.environ.copy()
env["PYTHONPATH"] = str(REPO_ROOT)
result = subprocess.run(
[
sys.executable,
"-m",
"core.cli",
"capability",
"domain-contract",
"--pack-id",
"en_core_cognition_v1",
"--json",
],
cwd=REPO_ROOT,
env=env,
capture_output=True,
text=True,
timeout=30,
)
assert result.returncode == 1
payload = json.loads(result.stdout)
assert payload["contract_present"] is False
assert payload["all_passed"] is False
def test_cli_structural_only_skips_predicates(self) -> None:
import os
import subprocess
import sys
env = os.environ.copy()
env["PYTHONPATH"] = str(REPO_ROOT)
result = subprocess.run(
[
sys.executable,
"-m",
"core.cli",
"capability",
"domain-contract",
"--pack-id",
"en_core_cognition_v1",
"--json",
"--structural-only",
],
cwd=REPO_ROOT,
env=env,
capture_output=True,
text=True,
timeout=30,
)
# structural-only returns 0 because parse alone passes (no contract).
assert result.returncode == 0
payload = json.loads(result.stdout)
assert "predicates" not in payload # legacy shape
assert payload["valid"] is True
# ---------------------------------------------------------------------------
# Determinism
# ---------------------------------------------------------------------------
class TestDeterminism:
def test_same_inputs_same_report(self, tmp_path: Path) -> None:
data_root = _build_synthetic_pack(tmp_path)
a = evaluate_domain_contract(
"synthetic_math_pack_v1",
data_root=data_root,
chain_inventory=_stub_chain_inventory(),
reviewer_registry=_primary_registry(),
)
b = evaluate_domain_contract(
"synthetic_math_pack_v1",
data_root=data_root,
chain_inventory=_stub_chain_inventory(),
reviewer_registry=_primary_registry(),
)
assert a.as_dict() == b.as_dict()
Reference in a new issue View git blame Copy permalink