evals/industry_demos: add run_all.py suite runner
ADR-0046 — Industry Demo Suite runner.
Adds `evals/industry_demos/run_all.py`, the single-entry-point script
that executes all three falsifiable demos in sequence, collects their
structured JSON evidence, and exits 0 iff every demo passes.
Design choices:
- Runs each demo in an isolated try/except so a crash in demo_01 does
not suppress evidence from demo_02 and demo_03 (fail-open evidence
collection, fail-closed exit code).
- Prints a human-readable banner + structured JSON evidence per demo.
- Prints a final machine-readable JSON summary `{"all_passed": bool,
"results": [...]}` on stdout for CI consumption.
- Exits 0 when all_passed, 1 otherwise.
- Zero new dependencies: only stdlib + the same imports each individual
demo already uses.
Also updates `evals/industry_demos/__init__.py` to document the new
runner in the module docstring.
Verification path:
python -m evals.industry_demos.run_all
echo $? # 0 on full pass
This commit is contained in:
parent
f6f8ee603f
commit
cc3beede53
2 changed files with 133 additions and 3 deletions
|
|
@ -7,11 +7,40 @@ no transformer-LLM wrapper can reproduce. Run individually:
|
|||
python -m evals.industry_demos.demo_02_geometry_drives_identity
|
||||
python -m evals.industry_demos.demo_03_deterministic_audit
|
||||
|
||||
Each exits 0 on pass, 1 on fail, and prints structured JSON evidence
|
||||
to stdout.
|
||||
Or run the full suite via the suite runner:
|
||||
|
||||
python -m evals.industry_demos.run_all
|
||||
|
||||
Each individual demo exits 0 on pass, 1 on fail, and prints structured
|
||||
JSON evidence to stdout. The suite runner exits 0 iff all three pass
|
||||
and emits a final machine-readable JSON summary for CI consumption.
|
||||
|
||||
Demos
|
||||
-----
|
||||
demo_01_forward_constraint
|
||||
Claim: PropositionGraph constrains the generation walk via CGA
|
||||
geometry (not a prompt filter or keyword list) BEFORE any tokens
|
||||
are produced. Evidence: allowed_indices < vocab_size; every index
|
||||
scores positive cga_inner against at least one named node versor;
|
||||
region label encodes the graph root node ID.
|
||||
|
||||
demo_02_geometry_drives_identity
|
||||
Claim: Swapping identity pack (precision_first_v1 vs
|
||||
generosity_first_v1) changes the manifold's alignment_threshold
|
||||
geometrically — not via a system prompt, different model, or
|
||||
temperature setting. Evidence: threshold_p (0.55) ≠ threshold_g
|
||||
(0.40); identity_score.alignment ordered precision ≤ generosity
|
||||
on the same input.
|
||||
|
||||
demo_03_deterministic_audit
|
||||
Claim: Three independent ChatRuntime instances on the same input
|
||||
produce byte-identical JSONL audit records for the seven
|
||||
architecturally-determined fields (versor_condition, vault_hits,
|
||||
dialogue_role, stub_path, safety_upheld, ethics_upheld, flagged).
|
||||
This is structural determinism, not seeded randomness.
|
||||
|
||||
The exact-recall-at-scale claim (CGA vault recall at N up to 100k) is
|
||||
covered by ADR-0045 — measured on the actual vault path, with properly
|
||||
covered by ADR-0045 — measured on the actual vault path with properly
|
||||
constructed versors — and is not duplicated here under a weaker
|
||||
construction. See ADR-0046, "Industry Demo Suite", for the rationale.
|
||||
"""
|
||||
|
|
|
|||
101
evals/industry_demos/run_all.py
Normal file
101
evals/industry_demos/run_all.py
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
"""
|
||||
evals/industry_demos/run_all.py — ADR-0046 suite runner.
|
||||
|
||||
Runs all three falsifiable industry demos in sequence. Each demo makes
|
||||
exactly one claim that a transformer-LLM wrapper cannot reproduce. This
|
||||
runner collects structured JSON evidence from all three, prints a human-
|
||||
readable report, and exits 0 iff every demo passes.
|
||||
|
||||
Usage::
|
||||
|
||||
python -m evals.industry_demos.run_all
|
||||
|
||||
Output (stdout):
|
||||
- Per-demo banner + JSON evidence block.
|
||||
- Final structured JSON: {"all_passed": bool, "results": [...]}
|
||||
|
||||
Exit code:
|
||||
0 — all three demos passed.
|
||||
1 — one or more demos failed or raised an exception.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import traceback
|
||||
from typing import Any
|
||||
|
||||
|
||||
# ---------- demo registry ----------
|
||||
|
||||
_DEMOS = [
|
||||
("demo_01_forward_constraint", "evals.industry_demos.demo_01_forward_constraint"),
|
||||
("demo_02_geometry_drives_identity", "evals.industry_demos.demo_02_geometry_drives_identity"),
|
||||
("demo_03_deterministic_audit", "evals.industry_demos.demo_03_deterministic_audit"),
|
||||
]
|
||||
|
||||
_SEP = "=" * 72
|
||||
|
||||
|
||||
# ---------- runner ----------
|
||||
|
||||
|
||||
def _run_demo(name: str, module_path: str) -> dict[str, Any]:
|
||||
"""Import and run a single demo module. Returns its result dict.
|
||||
|
||||
Wraps the run() call in a broad try/except so that a crash in one
|
||||
demo does not prevent the others from executing. A crashed demo
|
||||
is recorded as failed with the traceback embedded in the evidence.
|
||||
"""
|
||||
import importlib
|
||||
try:
|
||||
mod = importlib.import_module(module_path)
|
||||
result: dict[str, Any] = mod.run()
|
||||
except Exception: # noqa: BLE001
|
||||
tb = traceback.format_exc()
|
||||
result = {
|
||||
"demo": name,
|
||||
"claim": "(exception — see evidence.traceback)",
|
||||
"evidence": {"traceback": tb},
|
||||
"passed": False,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def run_all() -> dict[str, Any]:
|
||||
"""Execute all registered demos and return a summary dict."""
|
||||
results: list[dict[str, Any]] = []
|
||||
|
||||
for name, module_path in _DEMOS:
|
||||
print(_SEP)
|
||||
print(f"DEMO: {name}")
|
||||
print(_SEP)
|
||||
|
||||
result = _run_demo(name, module_path)
|
||||
results.append(result)
|
||||
|
||||
status = "PASS" if result.get("passed") else "FAIL"
|
||||
print(f"Status : {status}")
|
||||
print(f"Claim : {result.get('claim', '')}")
|
||||
print("Evidence:")
|
||||
print(json.dumps(result.get("evidence", {}), indent=2))
|
||||
print()
|
||||
|
||||
all_passed = all(r.get("passed") for r in results)
|
||||
|
||||
print(_SEP)
|
||||
print(f"SUITE RESULT: {'ALL PASSED' if all_passed else 'ONE OR MORE FAILED'}")
|
||||
print(_SEP)
|
||||
|
||||
summary = {
|
||||
"all_passed": all_passed,
|
||||
"results": results,
|
||||
}
|
||||
print(json.dumps(summary, indent=2))
|
||||
return summary
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
summary = run_all()
|
||||
sys.exit(0 if summary["all_passed"] else 1)
|
||||
Loading…
Reference in a new issue