diff --git a/evals/industry_demos/__init__.py b/evals/industry_demos/__init__.py index e813e994..6226b5e2 100644 --- a/evals/industry_demos/__init__.py +++ b/evals/industry_demos/__init__.py @@ -7,11 +7,40 @@ no transformer-LLM wrapper can reproduce. Run individually: python -m evals.industry_demos.demo_02_geometry_drives_identity python -m evals.industry_demos.demo_03_deterministic_audit -Each exits 0 on pass, 1 on fail, and prints structured JSON evidence -to stdout. +Or run the full suite via the suite runner: + + python -m evals.industry_demos.run_all + +Each individual demo exits 0 on pass, 1 on fail, and prints structured +JSON evidence to stdout. The suite runner exits 0 iff all three pass +and emits a final machine-readable JSON summary for CI consumption. + +Demos +----- +demo_01_forward_constraint + Claim: PropositionGraph constrains the generation walk via CGA + geometry (not a prompt filter or keyword list) BEFORE any tokens + are produced. Evidence: allowed_indices < vocab_size; every index + scores positive cga_inner against at least one named node versor; + region label encodes the graph root node ID. + +demo_02_geometry_drives_identity + Claim: Swapping identity pack (precision_first_v1 vs + generosity_first_v1) changes the manifold's alignment_threshold + geometrically — not via a system prompt, different model, or + temperature setting. Evidence: threshold_p (0.55) ≠ threshold_g + (0.40); identity_score.alignment ordered precision ≤ generosity + on the same input. + +demo_03_deterministic_audit + Claim: Three independent ChatRuntime instances on the same input + produce byte-identical JSONL audit records for the seven + architecturally-determined fields (versor_condition, vault_hits, + dialogue_role, stub_path, safety_upheld, ethics_upheld, flagged). + This is structural determinism, not seeded randomness. The exact-recall-at-scale claim (CGA vault recall at N up to 100k) is -covered by ADR-0045 — measured on the actual vault path, with properly +covered by ADR-0045 — measured on the actual vault path with properly constructed versors — and is not duplicated here under a weaker construction. See ADR-0046, "Industry Demo Suite", for the rationale. """ diff --git a/evals/industry_demos/run_all.py b/evals/industry_demos/run_all.py new file mode 100644 index 00000000..b2a2033f --- /dev/null +++ b/evals/industry_demos/run_all.py @@ -0,0 +1,101 @@ +""" +evals/industry_demos/run_all.py — ADR-0046 suite runner. + +Runs all three falsifiable industry demos in sequence. Each demo makes +exactly one claim that a transformer-LLM wrapper cannot reproduce. This +runner collects structured JSON evidence from all three, prints a human- +readable report, and exits 0 iff every demo passes. + +Usage:: + + python -m evals.industry_demos.run_all + +Output (stdout): + - Per-demo banner + JSON evidence block. + - Final structured JSON: {"all_passed": bool, "results": [...]} + +Exit code: + 0 — all three demos passed. + 1 — one or more demos failed or raised an exception. +""" + +from __future__ import annotations + +import json +import sys +import traceback +from typing import Any + + +# ---------- demo registry ---------- + +_DEMOS = [ + ("demo_01_forward_constraint", "evals.industry_demos.demo_01_forward_constraint"), + ("demo_02_geometry_drives_identity", "evals.industry_demos.demo_02_geometry_drives_identity"), + ("demo_03_deterministic_audit", "evals.industry_demos.demo_03_deterministic_audit"), +] + +_SEP = "=" * 72 + + +# ---------- runner ---------- + + +def _run_demo(name: str, module_path: str) -> dict[str, Any]: + """Import and run a single demo module. Returns its result dict. + + Wraps the run() call in a broad try/except so that a crash in one + demo does not prevent the others from executing. A crashed demo + is recorded as failed with the traceback embedded in the evidence. + """ + import importlib + try: + mod = importlib.import_module(module_path) + result: dict[str, Any] = mod.run() + except Exception: # noqa: BLE001 + tb = traceback.format_exc() + result = { + "demo": name, + "claim": "(exception — see evidence.traceback)", + "evidence": {"traceback": tb}, + "passed": False, + } + return result + + +def run_all() -> dict[str, Any]: + """Execute all registered demos and return a summary dict.""" + results: list[dict[str, Any]] = [] + + for name, module_path in _DEMOS: + print(_SEP) + print(f"DEMO: {name}") + print(_SEP) + + result = _run_demo(name, module_path) + results.append(result) + + status = "PASS" if result.get("passed") else "FAIL" + print(f"Status : {status}") + print(f"Claim : {result.get('claim', '')}") + print("Evidence:") + print(json.dumps(result.get("evidence", {}), indent=2)) + print() + + all_passed = all(r.get("passed") for r in results) + + print(_SEP) + print(f"SUITE RESULT: {'ALL PASSED' if all_passed else 'ONE OR MORE FAILED'}") + print(_SEP) + + summary = { + "all_passed": all_passed, + "results": results, + } + print(json.dumps(summary, indent=2)) + return summary + + +if __name__ == "__main__": + summary = run_all() + sys.exit(0 if summary["all_passed"] else 1)