diff --git a/demos/epistemic_truth_state/.gitignore b/demos/epistemic_truth_state/.gitignore new file mode 100644 index 00000000..89f9ac04 --- /dev/null +++ b/demos/epistemic_truth_state/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/demos/epistemic_truth_state/README.md b/demos/epistemic_truth_state/README.md new file mode 100644 index 00000000..69700e21 --- /dev/null +++ b/demos/epistemic_truth_state/README.md @@ -0,0 +1,116 @@ +# Epistemic Truth-State Authority Demo + +This demo proves one narrow boundary: + +```text +A model-style proposer submits a claim, an evidence bundle, and an optional +bounded-inference block. +CORE alone assigns the typed epistemic state. +The output is a deterministic, replayable evidence artifact. +The proposer never controls its own truth-state. +``` + +## Public proof spine + +```text +model proposes +substrate decides +trace proves +state is typed +``` + +## What this proves + +* A model-style proposer can submit a claim without gaining authority over its + epistemic standing. +* CORE alone assigns the typed state, drawn from the canonical taxonomy in + [`core/epistemic_state.py`](../../core/epistemic_state.py) — `verified`, + `evidenced`, `inferred`, `undetermined`, `scope_boundary`. There is no + parallel enum. +* CORE derives `normative_clearance`, the `evidence_ledger`, the + `authority_path`, and a fresh `trace_hash` itself. +* Any proposer-supplied `proposed_state` or `trace_hash` is recorded as ignored + and never read by the decision path. +* Invalid payloads fail at the typed boundary *before* any state evaluation + runs. +* `verified` requires two or more independent evidence records that explicitly + match the claim's subject and predicate — corroboration the proposer cannot + fabricate through the schema. + +## What this does not prove + +* It is not the runtime epistemic-state tagger; it is a local demo over fixed + fixtures. +* It assigns epistemic truth-state only. It runs **no** normative / safety / + ethics clearance pass, so `normative_clearance` is `unassessable` on every + non-invalid output — including `verified`. This demo never positively clears + a claim and makes no safety guarantee. +* It does not call a network, a model API, a subprocess, or any side-effecting + tool. It evaluates JSON and returns JSON. +* It does not claim broader epistemic coverage than the small local envelope and + the deterministic rules encoded in `authority.py`. + +## Why proposer state is not authority + +A model can *say* a claim is verified. Saying so is data, not standing. The +closed schema makes `assigned_state`, `status`, `evidence_ledger`, +`authority_path`, `trace_hash`, and `normative_clearance` impossible to supply +at the root — any attempt is an unexpected property and the payload is rejected. +The only state-bearing fields the proposer may include (`proposed_state`, +`trace_hash`, both inside `proposer`) are accepted by the schema purely so the +demo can prove CORE *ignores* them: they are echoed back as +`proposer_state_ignored` / `proposer_trace_hash_ignored` and never read by +`assign_epistemic_state`. + +## Relation to #687 and #688 + +```text +#687 -> authority over claims (System 1 proposal -> CORE verifies/refuses/asks) +#688 -> authority over proposed tool actions (proposer suggests, CORE licenses) +this -> authority over epistemic state assignment (proposer claims, CORE types it) +``` + +Each layer keeps the same doctrine: a model-style proposer contributes typed +data, CORE alone decides, and the decision is a deterministic trace artifact +with no proposer-held authority and no execution path. + +## The six scenarios + +* `verified-supported-claim` — two independent matching records → `verified`, + `verified_by_matching_evidence` (clearance `unassessable`). +* `evidenced-but-not-verified-claim` — one supporting record → `evidenced`, + `evidence_present_but_not_verifying`. +* `inferred-from-bounded-evidence` — claim follows from resolved premises → + `inferred`, `bounded_inference_from_evidence`, with `inference_basis` IDs. +* `undetermined-insufficient-evidence` — no relevant evidence → `undetermined`, + `insufficient_evidence`, with an explicit `question`. +* `refused-outside-scope` — claim declared outside the local envelope → + status `refused`, `scope_boundary`, `outside_epistemic_envelope`. +* `invalid-state-smuggling-attempt` — root-level `assigned_state` / `status` / + `evidence_ledger` / `authority_path` / `trace_hash` injection → status + `invalid`, `authority_evaluated: false`, every smuggled property listed in + `invalid_reason`. + +## Honesty ledger + +* Real: closed recursive schema validation, canonical-enum state assignment via + `core.epistemic_state`, deterministic trace hashing over the response minus + `trace_hash`, evidence-ledger derivation, expected-artifact pinning, + double-run byte-identical determinism, output-directory hardening. +* Simulated: the proposer side is static fixture data standing in for a + model-style proposer; the evidence bundle is hand-authored, not retrieved from + the live vault. +* Honest non-claim: `normative_clearance` is `unassessable` on every + non-invalid output, including `verified`, because this demo runs no + safety/ethics verdict pass and therefore has no basis to clear anything. +* Not claimed: runtime integration, serving integration, real evidence + retrieval, a safety guarantee, or any coverage beyond this local envelope. + +## Example commands + +```bash +python demos/epistemic_truth_state/run_demo.py +python demos/epistemic_truth_state/run_demo.py --json +python demos/epistemic_truth_state/run_demo.py --update-expected +pytest -q tests/test_epistemic_truth_state_demo.py +``` diff --git a/demos/epistemic_truth_state/__init__.py b/demos/epistemic_truth_state/__init__.py new file mode 100644 index 00000000..b9a0d22a --- /dev/null +++ b/demos/epistemic_truth_state/__init__.py @@ -0,0 +1 @@ +"""Local deterministic epistemic truth-state authority demo package.""" diff --git a/demos/epistemic_truth_state/authority.py b/demos/epistemic_truth_state/authority.py new file mode 100644 index 00000000..4ab5c724 --- /dev/null +++ b/demos/epistemic_truth_state/authority.py @@ -0,0 +1,425 @@ +"""Local deterministic epistemic-state authority substrate for the demo. + +A model-style proposer submits a claim, an evidence bundle, and (optionally) a +bounded-inference block. The proposer contributes *data only*. CORE alone: + +* validates a closed payload, +* assigns the typed epistemic state drawn from the canonical taxonomy in + :mod:`core.epistemic_state` (never a parallel enum), +* derives normative clearance, +* builds the evidence ledger, and +* regenerates a deterministic trace hash. + +The proposer cannot set ``assigned_state``, ``status``, ``trace_hash``, +``authority_path``, the evidence ledger, or ``normative_clearance``. Any +proposer-supplied ``proposed_state`` / ``trace_hash`` is recorded as ignored +and never read by the decision path. Nothing here executes a side effect. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from functools import lru_cache +from pathlib import Path +from typing import Any, Final + +from core.epistemic_state import ( + EpistemicState, + NormativeClearance, + coerce_epistemic_state, + coerce_normative_clearance, +) + +TOOL_NAME: Final[str] = "core.epistemic_truth_state.review" +_HERE: Final[Path] = Path(__file__).resolve().parent +SCHEMA_PATH: Final[Path] = _HERE / "schema.json" + +# The local epistemic authority envelope: only claims declared inside these +# domains are evaluated. Anything else is refused as outside scope rather than +# guessed at. +ENVELOPE_DOMAINS: Final[frozenset[str]] = frozenset({"demo.local_factual"}) + +_ROOT_AUTHORITY: Final[str] = "demos.epistemic_truth_state.authority.validate_payload" +_ASSIGN_AUTHORITY: Final[str] = "demos.epistemic_truth_state.authority.assign_epistemic_state" +_ENVELOPE_AUTHORITY: Final[str] = "demo_epistemic_truth_state_envelope(local-v1)" +_TAXONOMY_AUTHORITY: Final[str] = "core.epistemic_state.coerce_epistemic_state" + +_SUPPORTED_SCHEMA_KEYS: Final[frozenset[str]] = frozenset( + { + "type", + "properties", + "required", + "additionalProperties", + "enum", + "pattern", + "minLength", + "maxLength", + "items", + } +) +_SCALAR_TYPES: Final[dict[str, type]] = {"string": str, "boolean": bool} + + +def _canonical(payload: Any) -> str: + return json.dumps(payload, sort_keys=True, separators=(",", ":")) + + +def _hash_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +@lru_cache(maxsize=1) +def load_schema() -> dict[str, Any]: + return json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + + +def _clip(value: object) -> str: + rendered = repr(value) + return rendered if len(rendered) <= 80 else rendered[:79] + "…" + + +def _ensure_supported_schema(spec: dict[str, Any], *, path: str) -> None: + unsupported = set(spec) - _SUPPORTED_SCHEMA_KEYS + if unsupported: + raise ValueError( + f"{path} uses unsupported schema keywords {sorted(unsupported)}; " + "extend the validator before extending the schema" + ) + schema_type = spec.get("type") + if isinstance(schema_type, list): + return + if schema_type == "object": + for name, child in spec.get("properties", {}).items(): + _ensure_supported_schema(child, path=f"{path}.{name}") + elif schema_type == "array": + _ensure_supported_schema(spec["items"], path=f"{path}[]") + elif schema_type not in _SCALAR_TYPES: + raise ValueError(f"{path} has unsupported schema type {schema_type!r}") + + +@lru_cache(maxsize=1) +def _input_schema() -> dict[str, Any]: + schema = load_schema()["inputSchema"] + _ensure_supported_schema(schema, path="inputSchema") + if schema.get("type") != "object" or schema.get("additionalProperties") is not False: + raise ValueError("inputSchema must be a closed object") + return schema + + +def _validate(spec: dict[str, Any], value: Any, *, path: str, errors: list[str]) -> None: + schema_type = spec["type"] + if isinstance(schema_type, list): + for entry in schema_type: + if entry == "null" and value is None: + return + if value is None: + errors.append(f"{path} must be one of {schema_type}") + return + non_null = [entry for entry in schema_type if entry != "null"] + if len(non_null) != 1: + raise ValueError(f"{path} uses unsupported union type {schema_type!r}") + schema_type = non_null[0] + + if schema_type == "object": + if not isinstance(value, dict): + errors.append(f"{path} must be object") + return + props = spec.get("properties", {}) + if spec.get("additionalProperties") is False: + for unknown in sorted(set(value) - set(props)): + errors.append( + f"{path} unexpected property {_clip(unknown)} " + "(additionalProperties is false)" + ) + for required in spec.get("required", []): + if required not in value: + errors.append(f"{path} missing required property {required!r}") + for name, child in props.items(): + if name in value: + _validate(child, value[name], path=f"{path}.{name}", errors=errors) + return + + if schema_type == "array": + if not isinstance(value, list): + errors.append(f"{path} must be array") + return + for index, item in enumerate(value): + _validate(spec["items"], item, path=f"{path}[{index}]", errors=errors) + return + + expected = _SCALAR_TYPES[schema_type] + if type(value) is not expected: + errors.append(f"{path} must be {schema_type}") + return + if isinstance(value, str): + if "minLength" in spec and len(value) < spec["minLength"]: + errors.append(f"{path} shorter than {spec['minLength']}") + if "maxLength" in spec and len(value) > spec["maxLength"]: + errors.append(f"{path} longer than {spec['maxLength']}") + if "enum" in spec and value not in spec["enum"]: + errors.append(f"{path} must be one of {spec['enum']}") + if "pattern" in spec and re.fullmatch(spec["pattern"], value) is None: + errors.append(f"{path} does not match {spec['pattern']!r}") + elif "enum" in spec and value not in spec["enum"]: + errors.append(f"{path} must be one of {spec['enum']}") + + +def validate_payload(payload: Any) -> tuple[str, ...]: + errors: list[str] = [] + _validate(_input_schema(), payload, path="payload", errors=errors) + return tuple(errors) + + +def _safe_field(payload: Any, field: str) -> str | None: + value = payload.get(field) if isinstance(payload, dict) else None + if not isinstance(value, str): + return None + pattern = _input_schema()["properties"][field]["pattern"] + return value if re.fullmatch(pattern, value) else None + + +def _response_hash(response: dict[str, Any]) -> str: + body = dict(response) + body.pop("trace_hash", None) + return _hash_text(_canonical(body)) + + +def _finalize(response: dict[str, Any]) -> dict[str, Any]: + response["trace_hash"] = _response_hash(response) + return response + + +def _proposer_trace_hash_present(payload: Any) -> bool: + return bool( + isinstance(payload, dict) + and isinstance(payload.get("proposer"), dict) + and "trace_hash" in payload["proposer"] + ) + + +def _proposer_state_present(payload: Any) -> bool: + return bool( + isinstance(payload, dict) + and isinstance(payload.get("proposer"), dict) + and "proposed_state" in payload["proposer"] + ) + + +def _invalid_response(payload: Any, errors: tuple[str, ...]) -> dict[str, Any]: + return _finalize( + { + "tool": TOOL_NAME, + "status": "invalid", + "request_id": _safe_field(payload, "request_id"), + "scenario_id": _safe_field(payload, "scenario_id"), + "authority_path": [_ROOT_AUTHORITY], + "decision_reason": "invalid_payload", + "assigned_state": None, + "normative_clearance": None, + "evidence_ledger": [], + "trace_summary": { + "authority_evaluated": False, + "validation_errors": list(errors), + "proposer_trace_hash_ignored": _proposer_trace_hash_present(payload), + "proposer_state_ignored": _proposer_state_present(payload), + }, + "invalid_reason": "; ".join(errors), + } + ) + + +def _claim_fingerprint(payload: dict[str, Any]) -> str: + proposer = payload["proposer"] + digest_input = { + "claim": payload["claim"], + "proposer": { + "lane": proposer["lane"], + "model_family": proposer["model_family"], + "proposal_id": proposer["proposal_id"], + }, + "request_id": payload["request_id"], + "scenario_id": payload["scenario_id"], + } + return _hash_text(_canonical(digest_input)) + + +def _matching_evidence(claim: dict[str, Any], evidence: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Evidence records that explicitly support the claim subject and predicate.""" + subject = claim["subject"] + predicate = claim["predicate"] + return [ + record + for record in evidence + if bool(record.get("supports")) + and record.get("subject") == subject + and record.get("predicate") == predicate + ] + + +def _resolved_inference_basis(payload: dict[str, Any]) -> list[str] | None: + """Return sorted premise IDs iff an inference block resolves fully, else None.""" + inference = payload.get("inference") + if not isinstance(inference, dict): + return None + premise_ids = inference.get("premise_ids") or [] + known_ids = {record["evidence_id"] for record in payload.get("evidence", [])} + if not premise_ids or any(pid not in known_ids for pid in premise_ids): + return None + return sorted(premise_ids) + + +def _base_trace_summary(payload: dict[str, Any], evidence: list[dict[str, Any]], independent: list[dict[str, Any]]) -> dict[str, Any]: + return { + "authority_evaluated": True, + "envelope_version": "local-v1", + "claim_fingerprint": _claim_fingerprint(payload), + "evidence_considered": len(evidence), + "independent_support_count": len(independent), + "proposer_trace_hash_ignored": "trace_hash" in payload["proposer"], + "proposer_state_ignored": "proposed_state" in payload["proposer"], + } + + +def _assigned( + payload: dict[str, Any], + *, + state: EpistemicState, + clearance: NormativeClearance, + decision_reason: str, + evidence_ledger: list[str], + trace_summary: dict[str, Any], + extra: dict[str, Any] | None = None, +) -> dict[str, Any]: + response = { + "tool": TOOL_NAME, + "status": "assigned", + "request_id": payload["request_id"], + "scenario_id": payload["scenario_id"], + "authority_path": [ + _ROOT_AUTHORITY, + _ASSIGN_AUTHORITY, + _ENVELOPE_AUTHORITY, + _TAXONOMY_AUTHORITY, + ], + "decision_reason": decision_reason, + "assigned_state": coerce_epistemic_state(state).value, + "normative_clearance": coerce_normative_clearance(clearance).value, + "evidence_ledger": evidence_ledger, + "trace_summary": trace_summary, + } + if extra: + response.update(extra) + return _finalize(response) + + +def assign_epistemic_state(payload: dict[str, Any]) -> dict[str, Any]: + """CORE's sole authority over the typed epistemic state. + + The proposer's ``proposed_state`` is never read here. The state is derived + only from the claim, the evidence bundle, and the bounded-inference block. + """ + claim = payload["claim"] + evidence: list[dict[str, Any]] = payload.get("evidence", []) + matching = _matching_evidence(claim, evidence) + independent = [record for record in matching if bool(record.get("independent"))] + trace_summary = _base_trace_summary(payload, evidence, independent) + + # Scope: refuse claims declared outside the local epistemic envelope. + if claim["domain"] not in ENVELOPE_DOMAINS: + return _finalize( + { + "tool": TOOL_NAME, + "status": "refused", + "request_id": payload["request_id"], + "scenario_id": payload["scenario_id"], + "authority_path": [ + _ROOT_AUTHORITY, + _ASSIGN_AUTHORITY, + _ENVELOPE_AUTHORITY, + ], + "decision_reason": "outside_epistemic_envelope", + "assigned_state": coerce_epistemic_state(EpistemicState.SCOPE_BOUNDARY).value, + "normative_clearance": coerce_normative_clearance( + NormativeClearance.UNASSESSABLE + ).value, + "evidence_ledger": [], + "trace_summary": trace_summary, + "refusal_reason": "outside_epistemic_envelope", + } + ) + + # Verified: two or more independent records that match subject and predicate. + # Clearance stays UNASSESSABLE even here: this demo assigns epistemic + # truth-state only and runs no normative/safety/ethics clearance pass, so it + # has no basis to positively clear anything. + if len(independent) >= 2: + return _assigned( + payload, + state=EpistemicState.VERIFIED, + clearance=NormativeClearance.UNASSESSABLE, + decision_reason="verified_by_matching_evidence", + evidence_ledger=sorted(record["evidence_id"] for record in independent), + trace_summary=trace_summary, + ) + + # Inferred: claim follows from bounded premises and is not directly matched. + inference_basis = _resolved_inference_basis(payload) + if inference_basis is not None and not matching: + return _assigned( + payload, + state=EpistemicState.INFERRED, + clearance=NormativeClearance.UNASSESSABLE, + decision_reason="bounded_inference_from_evidence", + evidence_ledger=inference_basis, + trace_summary=trace_summary, + extra={"inference_basis": inference_basis}, + ) + + # Evidenced: at least one supporting record, but not enough to verify. + if matching: + return _assigned( + payload, + state=EpistemicState.EVIDENCED, + clearance=NormativeClearance.UNASSESSABLE, + decision_reason="evidence_present_but_not_verifying", + evidence_ledger=sorted(record["evidence_id"] for record in matching), + trace_summary=trace_summary, + ) + + # Undetermined: nothing grounds the claim. CORE asks rather than guesses. + return _assigned( + payload, + state=EpistemicState.UNDETERMINED, + clearance=NormativeClearance.UNASSESSABLE, + decision_reason="insufficient_evidence", + evidence_ledger=[], + trace_summary=trace_summary, + extra={ + "question": ( + "CORE has insufficient grounded evidence to assign a determined " + "epistemic state to this claim. Provide supporting, refuting, or " + "premise evidence." + ) + }, + ) + + +def run_authority(payload: Any) -> dict[str, Any]: + errors = validate_payload(payload) + if errors: + return _invalid_response(payload, errors) + assert isinstance(payload, dict) + return assign_epistemic_state(payload) + + +__all__ = [ + "ENVELOPE_DOMAINS", + "SCHEMA_PATH", + "TOOL_NAME", + "assign_epistemic_state", + "load_schema", + "run_authority", + "validate_payload", +] diff --git a/demos/epistemic_truth_state/expected/evidenced-but-not-verified-claim.json b/demos/epistemic_truth_state/expected/evidenced-but-not-verified-claim.json new file mode 100644 index 00000000..ead52c96 --- /dev/null +++ b/demos/epistemic_truth_state/expected/evidenced-but-not-verified-claim.json @@ -0,0 +1,28 @@ +{ + "assigned_state": "evidenced", + "authority_path": [ + "demos.epistemic_truth_state.authority.validate_payload", + "demos.epistemic_truth_state.authority.assign_epistemic_state", + "demo_epistemic_truth_state_envelope(local-v1)", + "core.epistemic_state.coerce_epistemic_state" + ], + "decision_reason": "evidence_present_but_not_verifying", + "evidence_ledger": [ + "ev-b1-log" + ], + "normative_clearance": "unassessable", + "request_id": "demo-eps-b1", + "scenario_id": "evidenced-but-not-verified-claim", + "status": "assigned", + "tool": "core.epistemic_truth_state.review", + "trace_hash": "f9f2e153e66aaba9dcb1d7e57bb30947cb7f4dd9b09de3625b12c21b55d80b3c", + "trace_summary": { + "authority_evaluated": true, + "claim_fingerprint": "289e494aa06b8bd26b9cd26d61e08b966f78dcfc0f211db250ccf0ec7539a744", + "envelope_version": "local-v1", + "evidence_considered": 1, + "independent_support_count": 1, + "proposer_state_ignored": true, + "proposer_trace_hash_ignored": true + } +} diff --git a/demos/epistemic_truth_state/expected/inferred-from-bounded-evidence.json b/demos/epistemic_truth_state/expected/inferred-from-bounded-evidence.json new file mode 100644 index 00000000..782fc698 --- /dev/null +++ b/demos/epistemic_truth_state/expected/inferred-from-bounded-evidence.json @@ -0,0 +1,33 @@ +{ + "assigned_state": "inferred", + "authority_path": [ + "demos.epistemic_truth_state.authority.validate_payload", + "demos.epistemic_truth_state.authority.assign_epistemic_state", + "demo_epistemic_truth_state_envelope(local-v1)", + "core.epistemic_state.coerce_epistemic_state" + ], + "decision_reason": "bounded_inference_from_evidence", + "evidence_ledger": [ + "ev-c1-branch-from-main", + "ev-c1-tag-on-main" + ], + "inference_basis": [ + "ev-c1-branch-from-main", + "ev-c1-tag-on-main" + ], + "normative_clearance": "unassessable", + "request_id": "demo-eps-c1", + "scenario_id": "inferred-from-bounded-evidence", + "status": "assigned", + "tool": "core.epistemic_truth_state.review", + "trace_hash": "bc11e858ece140813c2307aa9c6c602650a4cec63972f520f29ba6863f437949", + "trace_summary": { + "authority_evaluated": true, + "claim_fingerprint": "12e38c11f88615b1afb8e035e6e942eedd434ea0c6e3b4b25fbb7ff06695204a", + "envelope_version": "local-v1", + "evidence_considered": 2, + "independent_support_count": 0, + "proposer_state_ignored": true, + "proposer_trace_hash_ignored": true + } +} diff --git a/demos/epistemic_truth_state/expected/invalid-state-smuggling-attempt.json b/demos/epistemic_truth_state/expected/invalid-state-smuggling-attempt.json new file mode 100644 index 00000000..07b916e3 --- /dev/null +++ b/demos/epistemic_truth_state/expected/invalid-state-smuggling-attempt.json @@ -0,0 +1,27 @@ +{ + "assigned_state": null, + "authority_path": [ + "demos.epistemic_truth_state.authority.validate_payload" + ], + "decision_reason": "invalid_payload", + "evidence_ledger": [], + "invalid_reason": "payload unexpected property 'assigned_state' (additionalProperties is false); payload unexpected property 'authority_path' (additionalProperties is false); payload unexpected property 'evidence_ledger' (additionalProperties is false); payload unexpected property 'status' (additionalProperties is false); payload unexpected property 'trace_hash' (additionalProperties is false)", + "normative_clearance": null, + "request_id": "demo-eps-f1", + "scenario_id": "invalid-state-smuggling-attempt", + "status": "invalid", + "tool": "core.epistemic_truth_state.review", + "trace_hash": "18dda5b4017b223be8eb7272eab1b8c6f1dc7a69d0326e3d1869ee95c8cf5352", + "trace_summary": { + "authority_evaluated": false, + "proposer_state_ignored": true, + "proposer_trace_hash_ignored": true, + "validation_errors": [ + "payload unexpected property 'assigned_state' (additionalProperties is false)", + "payload unexpected property 'authority_path' (additionalProperties is false)", + "payload unexpected property 'evidence_ledger' (additionalProperties is false)", + "payload unexpected property 'status' (additionalProperties is false)", + "payload unexpected property 'trace_hash' (additionalProperties is false)" + ] + } +} diff --git a/demos/epistemic_truth_state/expected/refused-outside-scope.json b/demos/epistemic_truth_state/expected/refused-outside-scope.json new file mode 100644 index 00000000..aa5ec03c --- /dev/null +++ b/demos/epistemic_truth_state/expected/refused-outside-scope.json @@ -0,0 +1,26 @@ +{ + "assigned_state": "scope_boundary", + "authority_path": [ + "demos.epistemic_truth_state.authority.validate_payload", + "demos.epistemic_truth_state.authority.assign_epistemic_state", + "demo_epistemic_truth_state_envelope(local-v1)" + ], + "decision_reason": "outside_epistemic_envelope", + "evidence_ledger": [], + "normative_clearance": "unassessable", + "refusal_reason": "outside_epistemic_envelope", + "request_id": "demo-eps-e1", + "scenario_id": "refused-outside-scope", + "status": "refused", + "tool": "core.epistemic_truth_state.review", + "trace_hash": "c9ef9560bcf71052f9e4f323d5f6c49dc23113dcaa6a63048cfdffafec6dd024", + "trace_summary": { + "authority_evaluated": true, + "claim_fingerprint": "ef9317e405906823799755113ccb216ab39dd7b055c9f9883ada6480b80ef98c", + "envelope_version": "local-v1", + "evidence_considered": 1, + "independent_support_count": 1, + "proposer_state_ignored": true, + "proposer_trace_hash_ignored": true + } +} diff --git a/demos/epistemic_truth_state/expected/undetermined-insufficient-evidence.json b/demos/epistemic_truth_state/expected/undetermined-insufficient-evidence.json new file mode 100644 index 00000000..5d6654d1 --- /dev/null +++ b/demos/epistemic_truth_state/expected/undetermined-insufficient-evidence.json @@ -0,0 +1,27 @@ +{ + "assigned_state": "undetermined", + "authority_path": [ + "demos.epistemic_truth_state.authority.validate_payload", + "demos.epistemic_truth_state.authority.assign_epistemic_state", + "demo_epistemic_truth_state_envelope(local-v1)", + "core.epistemic_state.coerce_epistemic_state" + ], + "decision_reason": "insufficient_evidence", + "evidence_ledger": [], + "normative_clearance": "unassessable", + "question": "CORE has insufficient grounded evidence to assign a determined epistemic state to this claim. Provide supporting, refuting, or premise evidence.", + "request_id": "demo-eps-d1", + "scenario_id": "undetermined-insufficient-evidence", + "status": "assigned", + "tool": "core.epistemic_truth_state.review", + "trace_hash": "35b319eb0186be2d9bedf65263d5acb8f397f747e7efe1ea01960655deafc7d2", + "trace_summary": { + "authority_evaluated": true, + "claim_fingerprint": "dc8720472531486a19edbb82c2accd279d291afe2a797c23d2745469c4f91662", + "envelope_version": "local-v1", + "evidence_considered": 1, + "independent_support_count": 0, + "proposer_state_ignored": true, + "proposer_trace_hash_ignored": true + } +} diff --git a/demos/epistemic_truth_state/expected/verified-supported-claim.json b/demos/epistemic_truth_state/expected/verified-supported-claim.json new file mode 100644 index 00000000..2279ab4b --- /dev/null +++ b/demos/epistemic_truth_state/expected/verified-supported-claim.json @@ -0,0 +1,29 @@ +{ + "assigned_state": "verified", + "authority_path": [ + "demos.epistemic_truth_state.authority.validate_payload", + "demos.epistemic_truth_state.authority.assign_epistemic_state", + "demo_epistemic_truth_state_envelope(local-v1)", + "core.epistemic_state.coerce_epistemic_state" + ], + "decision_reason": "verified_by_matching_evidence", + "evidence_ledger": [ + "ev-a1-replay", + "ev-a1-trace" + ], + "normative_clearance": "unassessable", + "request_id": "demo-eps-a1", + "scenario_id": "verified-supported-claim", + "status": "assigned", + "tool": "core.epistemic_truth_state.review", + "trace_hash": "1341c27c5906ae528ecc1f0cc4ebd3f71db68f138f6d8365fe0e93dd2d7f7097", + "trace_summary": { + "authority_evaluated": true, + "claim_fingerprint": "cef9198360d02ce3133c1095e776ae86a714fd1c9718aa3576616f935eccdf98", + "envelope_version": "local-v1", + "evidence_considered": 2, + "independent_support_count": 2, + "proposer_state_ignored": true, + "proposer_trace_hash_ignored": true + } +} diff --git a/demos/epistemic_truth_state/fixtures/evidenced-but-not-verified-claim.json b/demos/epistemic_truth_state/fixtures/evidenced-but-not-verified-claim.json new file mode 100644 index 00000000..5676936e --- /dev/null +++ b/demos/epistemic_truth_state/fixtures/evidenced-but-not-verified-claim.json @@ -0,0 +1,31 @@ +{ + "tool": "core.epistemic_truth_state.review", + "expected_status": "assigned", + "arguments": { + "request_id": "demo-eps-b1", + "scenario_id": "evidenced-but-not-verified-claim", + "proposer": { + "lane": "frontier_fixture", + "model_family": "model_style_proposer", + "proposal_id": "proposal-b1", + "proposed_state": "verified", + "trace_hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "claim": { + "text": "The smoke suite passed on the current branch.", + "domain": "demo.local_factual", + "subject": "smoke_suite", + "predicate": "passed_on_current_branch" + }, + "evidence": [ + { + "evidence_id": "ev-b1-log", + "subject": "smoke_suite", + "predicate": "passed_on_current_branch", + "supports": true, + "independent": true, + "source_kind": "observation" + } + ] + } +} diff --git a/demos/epistemic_truth_state/fixtures/inferred-from-bounded-evidence.json b/demos/epistemic_truth_state/fixtures/inferred-from-bounded-evidence.json new file mode 100644 index 00000000..e5be2f22 --- /dev/null +++ b/demos/epistemic_truth_state/fixtures/inferred-from-bounded-evidence.json @@ -0,0 +1,43 @@ +{ + "tool": "core.epistemic_truth_state.review", + "expected_status": "assigned", + "arguments": { + "request_id": "demo-eps-c1", + "scenario_id": "inferred-from-bounded-evidence", + "proposer": { + "lane": "frontier_fixture", + "model_family": "model_style_proposer", + "proposal_id": "proposal-c1", + "proposed_state": "verified", + "trace_hash": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "claim": { + "text": "The release tag is reachable from the current branch.", + "domain": "demo.local_factual", + "subject": "release_tag", + "predicate": "reachable_from_current_branch" + }, + "evidence": [ + { + "evidence_id": "ev-c1-tag-on-main", + "subject": "release_tag", + "predicate": "points_at_main", + "supports": true, + "independent": true, + "source_kind": "premise" + }, + { + "evidence_id": "ev-c1-branch-from-main", + "subject": "current_branch", + "predicate": "branched_from_main", + "supports": true, + "independent": true, + "source_kind": "premise" + } + ], + "inference": { + "mode": "bounded_deductive", + "premise_ids": ["ev-c1-tag-on-main", "ev-c1-branch-from-main"] + } + } +} diff --git a/demos/epistemic_truth_state/fixtures/invalid-state-smuggling-attempt.json b/demos/epistemic_truth_state/fixtures/invalid-state-smuggling-attempt.json new file mode 100644 index 00000000..2f22706a --- /dev/null +++ b/demos/epistemic_truth_state/fixtures/invalid-state-smuggling-attempt.json @@ -0,0 +1,27 @@ +{ + "tool": "core.epistemic_truth_state.review", + "expected_status": "invalid", + "arguments": { + "request_id": "demo-eps-f1", + "scenario_id": "invalid-state-smuggling-attempt", + "proposer": { + "lane": "frontier_fixture", + "model_family": "model_style_proposer", + "proposal_id": "proposal-f1", + "proposed_state": "verified", + "trace_hash": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + }, + "claim": { + "text": "This claim ships its own verdict.", + "domain": "demo.local_factual", + "subject": "self_asserting_claim", + "predicate": "ships_its_own_verdict" + }, + "evidence": [], + "assigned_state": "verified", + "status": "assigned", + "evidence_ledger": ["forged-ledger-entry"], + "authority_path": ["proposer.self.authority"], + "trace_hash": "0000000000000000000000000000000000000000000000000000000000000000" + } +} diff --git a/demos/epistemic_truth_state/fixtures/refused-outside-scope.json b/demos/epistemic_truth_state/fixtures/refused-outside-scope.json new file mode 100644 index 00000000..e2985a0c --- /dev/null +++ b/demos/epistemic_truth_state/fixtures/refused-outside-scope.json @@ -0,0 +1,31 @@ +{ + "tool": "core.epistemic_truth_state.review", + "expected_status": "refused", + "arguments": { + "request_id": "demo-eps-e1", + "scenario_id": "refused-outside-scope", + "proposer": { + "lane": "frontier_fixture", + "model_family": "model_style_proposer", + "proposal_id": "proposal-e1", + "proposed_state": "verified", + "trace_hash": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + }, + "claim": { + "text": "A named public stock will close higher tomorrow.", + "domain": "external.realtime_market_forecast", + "subject": "named_public_stock", + "predicate": "closes_higher_tomorrow" + }, + "evidence": [ + { + "evidence_id": "ev-e1-rumor", + "subject": "named_public_stock", + "predicate": "closes_higher_tomorrow", + "supports": true, + "independent": true, + "source_kind": "observation" + } + ] + } +} diff --git a/demos/epistemic_truth_state/fixtures/undetermined-insufficient-evidence.json b/demos/epistemic_truth_state/fixtures/undetermined-insufficient-evidence.json new file mode 100644 index 00000000..788ac61b --- /dev/null +++ b/demos/epistemic_truth_state/fixtures/undetermined-insufficient-evidence.json @@ -0,0 +1,31 @@ +{ + "tool": "core.epistemic_truth_state.review", + "expected_status": "assigned", + "arguments": { + "request_id": "demo-eps-d1", + "scenario_id": "undetermined-insufficient-evidence", + "proposer": { + "lane": "frontier_fixture", + "model_family": "model_style_proposer", + "proposal_id": "proposal-d1", + "proposed_state": "verified", + "trace_hash": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + }, + "claim": { + "text": "The nightly benchmark improved over last week.", + "domain": "demo.local_factual", + "subject": "nightly_benchmark", + "predicate": "improved_over_last_week" + }, + "evidence": [ + { + "evidence_id": "ev-d1-unrelated", + "subject": "documentation_pass", + "predicate": "merged_to_main", + "supports": true, + "independent": true, + "source_kind": "observation" + } + ] + } +} diff --git a/demos/epistemic_truth_state/fixtures/verified-supported-claim.json b/demos/epistemic_truth_state/fixtures/verified-supported-claim.json new file mode 100644 index 00000000..ea8b9893 --- /dev/null +++ b/demos/epistemic_truth_state/fixtures/verified-supported-claim.json @@ -0,0 +1,39 @@ +{ + "tool": "core.epistemic_truth_state.review", + "expected_status": "assigned", + "arguments": { + "request_id": "demo-eps-a1", + "scenario_id": "verified-supported-claim", + "proposer": { + "lane": "frontier_fixture", + "model_family": "model_style_proposer", + "proposal_id": "proposal-a1", + "proposed_state": "verified", + "trace_hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "claim": { + "text": "The reviewed teaching loop ran on the pinned cognition pack.", + "domain": "demo.local_factual", + "subject": "reviewed_teaching_loop", + "predicate": "ran_on_pinned_cognition_pack" + }, + "evidence": [ + { + "evidence_id": "ev-a1-trace", + "subject": "reviewed_teaching_loop", + "predicate": "ran_on_pinned_cognition_pack", + "supports": true, + "independent": true, + "source_kind": "vault" + }, + { + "evidence_id": "ev-a1-replay", + "subject": "reviewed_teaching_loop", + "predicate": "ran_on_pinned_cognition_pack", + "supports": true, + "independent": true, + "source_kind": "teaching" + } + ] + } +} diff --git a/demos/epistemic_truth_state/run_demo.py b/demos/epistemic_truth_state/run_demo.py new file mode 100644 index 00000000..6c6b9f45 --- /dev/null +++ b/demos/epistemic_truth_state/run_demo.py @@ -0,0 +1,179 @@ +"""Run the model-proposer-to-CORE epistemic truth-state authority demo. + +Each fixture is evaluated twice. The run fails if any scenario drifts from its +committed expected artifact or if the two executions differ byte-for-byte. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +import tempfile +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) + +from demos.epistemic_truth_state.authority import TOOL_NAME, load_schema, run_authority # noqa: E402 + +_HERE = Path(__file__).resolve().parent +FIXTURES_DIR = _HERE / "fixtures" +EXPECTED_DIR = _HERE / "expected" +DEFAULT_OUT_DIR = _HERE / "out" +_TMP_ROOT = Path(tempfile.gettempdir()).resolve() + + +def _render(payload: dict[str, Any]) -> str: + return json.dumps(payload, sort_keys=True, indent=2) + "\n" + + +def fixture_paths() -> list[Path]: + return sorted(FIXTURES_DIR.glob("*.json")) + + +def expected_path(scenario_id: str) -> Path: + return EXPECTED_DIR / f"{scenario_id}.json" + + +def run_fixture(path: Path) -> dict[str, Any]: + fixture = json.loads(path.read_text(encoding="utf-8")) + if fixture["tool"] != TOOL_NAME: + raise ValueError(f"{path.name} names {fixture['tool']!r}; expected {TOOL_NAME!r}") + run_a = run_authority(fixture["arguments"]) + run_b = run_authority(fixture["arguments"]) + if _render(run_a) != _render(run_b): + raise AssertionError(f"{path.name} is not deterministic across double-run execution") + return run_a + + +def _is_relative_to(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + return True + except ValueError: + return False + + +def _custom_out_allowed(out_dir: Path) -> bool: + resolved = out_dir.resolve() + if resolved in { + Path("/"), + REPO_ROOT.resolve(), + _HERE.resolve(), + _HERE.parent.resolve(), + Path.home().resolve(), + Path.cwd().resolve(), + }: + return False + return _is_relative_to(resolved, _HERE.resolve()) or _is_relative_to(resolved, _TMP_ROOT) + + +def _clearable_out_dir(out_dir: Path) -> tuple[bool, str | None]: + resolved = out_dir.resolve() + default_resolved = DEFAULT_OUT_DIR.resolve() + if resolved == default_resolved: + return True, None + if not _custom_out_allowed(out_dir): + return False, ( + f"refusing to clear {out_dir}: custom out dir must resolve under " + f"{_HERE} or {_TMP_ROOT}" + ) + if not (out_dir / "summary.json").exists(): + return False, ( + f"refusing to clear {out_dir}: custom out dir requires demo summary.json marker" + ) + return True, None + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT_DIR) + parser.add_argument("--allow-custom-out", action="store_true") + parser.add_argument("--json", action="store_true") + parser.add_argument("--update-expected", action="store_true") + args = parser.parse_args() + + out_dir = args.out + if out_dir.resolve() != DEFAULT_OUT_DIR.resolve() and args.allow_custom_out: + print( + "--allow-custom-out is deprecated; this demo now enforces the output-root policy directly", + file=sys.stderr, + ) + if out_dir.exists(): + allowed, reason = _clearable_out_dir(out_dir) + if not allowed: + assert reason is not None + print(reason, file=sys.stderr) + return 2 + shutil.rmtree(out_dir) + out_dir.mkdir(parents=True) + + schema_doc = load_schema() + results: list[dict[str, Any]] = [] + all_passed = True + + for fixture_path in fixture_paths(): + fixture = json.loads(fixture_path.read_text(encoding="utf-8")) + scenario_id = fixture["arguments"]["scenario_id"] + scenario_dir = out_dir / scenario_id + scenario_dir.mkdir(parents=True, exist_ok=True) + response = run_fixture(fixture_path) + artifact = _render(response) + (scenario_dir / "response.json").write_text(artifact, encoding="utf-8") + + problems: list[str] = [] + if response["status"] != fixture["expected_status"]: + problems.append( + f"status {response['status']!r} != expected {fixture['expected_status']!r}" + ) + ref = expected_path(scenario_id) + if args.update_expected: + EXPECTED_DIR.mkdir(exist_ok=True) + ref.write_text(artifact, encoding="utf-8") + elif not ref.exists(): + problems.append("missing committed expected artifact") + elif ref.read_text(encoding="utf-8") != artifact: + problems.append("response drifted from committed expected artifact") + + all_passed &= not problems + results.append( + { + "scenario_id": scenario_id, + "status": response["status"], + "assigned_state": response["assigned_state"], + "decision_reason": response["decision_reason"], + "trace_hash": response["trace_hash"], + "passed": not problems, + "problems": problems, + } + ) + + summary = { + "tool": schema_doc["name"], + "all_passed": all_passed, + "scenarios": results, + "updated_expected": bool(args.update_expected), + } + summary_text = _render(summary) + (out_dir / "summary.json").write_text(summary_text, encoding="utf-8") + if args.json: + print(summary_text, end="") + else: + for row in results: + mark = "PASS" if row["passed"] else "FAIL" + print( + f"[{mark}] {row['scenario_id']}: {row['status']}" + f" / {row['assigned_state']} ({row['decision_reason']})" + ) + print(f" trace_hash: {row['trace_hash'][:16]}…") + for problem in row["problems"]: + print(f" x {problem}") + print(f"{sum(r['passed'] for r in results)}/{len(results)} scenarios passed") + return 0 if all_passed else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/demos/epistemic_truth_state/schema.json b/demos/epistemic_truth_state/schema.json new file mode 100644 index 00000000..d98a9995 --- /dev/null +++ b/demos/epistemic_truth_state/schema.json @@ -0,0 +1,192 @@ +{ + "name": "core.epistemic_truth_state.review", + "title": "CORE epistemic truth-state authority demo", + "description": "Submit one model-style claim proposal to CORE's local deterministic epistemic-state authority. CORE validates a closed payload, ignores any proposer-supplied proposed_state and trace hash, assigns a typed epistemic state drawn from the canonical core.epistemic_state taxonomy, derives normative clearance, builds an evidence ledger, and returns assigned | refused | invalid plus a deterministic trace artifact. The proposer cannot set the assigned state, status, clearance, evidence ledger, authority path, or trace hash.", + "inputSchema": { + "type": "object", + "properties": { + "request_id": { + "type": "string", + "pattern": "^[A-Za-z0-9._-]{1,64}$" + }, + "scenario_id": { + "type": "string", + "pattern": "^[a-z0-9._-]{1,64}$" + }, + "proposer": { + "type": "object", + "properties": { + "lane": { + "type": "string", + "enum": ["frontier_fixture", "local_fixture"] + }, + "model_family": { + "type": "string", + "enum": ["model_style_proposer"] + }, + "proposal_id": { + "type": "string", + "pattern": "^[A-Za-z0-9._-]{1,64}$" + }, + "proposed_state": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "trace_hash": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": ["lane", "model_family", "proposal_id"], + "additionalProperties": false + }, + "claim": { + "type": "object", + "properties": { + "text": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "domain": { + "type": "string", + "pattern": "^[a-z0-9._-]{1,64}$" + }, + "subject": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "predicate": { + "type": "string", + "minLength": 1, + "maxLength": 120 + } + }, + "required": ["text", "domain", "subject", "predicate"], + "additionalProperties": false + }, + "evidence": { + "type": "array", + "items": { + "type": "object", + "properties": { + "evidence_id": { + "type": "string", + "pattern": "^[A-Za-z0-9._-]{1,64}$" + }, + "subject": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "predicate": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "supports": { + "type": "boolean" + }, + "independent": { + "type": "boolean" + }, + "source_kind": { + "type": "string", + "enum": ["pack", "teaching", "vault", "observation", "premise"] + } + }, + "required": ["evidence_id", "subject", "predicate", "supports"], + "additionalProperties": false + } + }, + "inference": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["bounded_deductive", "bounded_arithmetic"] + }, + "premise_ids": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z0-9._-]{1,64}$" + } + } + }, + "required": ["premise_ids"], + "additionalProperties": false + } + }, + "required": ["request_id", "scenario_id", "proposer", "claim"], + "additionalProperties": false + }, + "outputSchema": { + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": ["assigned", "refused", "invalid"] + }, + "request_id": { + "type": ["string", "null"] + }, + "scenario_id": { + "type": ["string", "null"] + }, + "authority_path": { + "type": "array", + "items": { + "type": "string" + } + }, + "decision_reason": { + "type": "string" + }, + "assigned_state": { + "type": ["string", "null"] + }, + "normative_clearance": { + "type": ["string", "null"] + }, + "evidence_ledger": { + "type": "array", + "items": { + "type": "string" + } + }, + "trace_hash": { + "type": "string" + }, + "trace_summary": { + "type": "object" + }, + "inference_basis": { + "type": ["array", "null"] + }, + "question": { + "type": ["string", "null"] + }, + "refusal_reason": { + "type": ["string", "null"] + }, + "invalid_reason": { + "type": ["string", "null"] + } + }, + "required": [ + "status", + "request_id", + "scenario_id", + "authority_path", + "decision_reason", + "assigned_state", + "normative_clearance", + "evidence_ledger", + "trace_hash", + "trace_summary" + ] + } +} diff --git a/tests/test_epistemic_truth_state_demo.py b/tests/test_epistemic_truth_state_demo.py new file mode 100644 index 00000000..101770a7 --- /dev/null +++ b/tests/test_epistemic_truth_state_demo.py @@ -0,0 +1,283 @@ +from __future__ import annotations + +import ast +import copy +import json +from pathlib import Path + +import pytest + +from core.epistemic_state import EpistemicState, NormativeClearance +from demos.epistemic_truth_state import authority +from demos.epistemic_truth_state import run_demo as demo_runner + +DEMO_DIR = Path(authority.__file__).resolve().parent +FIXTURES_DIR = DEMO_DIR / "fixtures" +EXPECTED_DIR = DEMO_DIR / "expected" + +SCENARIOS = { + "verified-supported-claim": ("assigned", "verified"), + "evidenced-but-not-verified-claim": ("assigned", "evidenced"), + "inferred-from-bounded-evidence": ("assigned", "inferred"), + "undetermined-insufficient-evidence": ("assigned", "undetermined"), + "refused-outside-scope": ("refused", "scope_boundary"), + "invalid-state-smuggling-attempt": ("invalid", None), +} + + +def _fixture(name: str) -> dict[str, object]: + return json.loads((FIXTURES_DIR / name).read_text(encoding="utf-8")) + + +def _arguments(name: str) -> dict[str, object]: + return _fixture(f"{name}.json")["arguments"] + + +def _run(name: str) -> dict[str, object]: + return authority.run_authority(_arguments(name)) + + +def test_schema_is_closed_recursively(): + schema = authority.load_schema()["inputSchema"] + assert schema["type"] == "object" + assert schema["additionalProperties"] is False + assert schema["properties"]["proposer"]["additionalProperties"] is False + assert schema["properties"]["claim"]["additionalProperties"] is False + assert schema["properties"]["evidence"]["items"]["additionalProperties"] is False + assert schema["properties"]["inference"]["additionalProperties"] is False + + # Every nested object in the input schema is closed, recursively. + def _assert_closed(spec: dict[str, object], path: str) -> None: + node_type = spec.get("type") + if node_type == "object": + assert spec.get("additionalProperties") is False, f"{path} is not closed" + for name, child in spec.get("properties", {}).items(): + _assert_closed(child, f"{path}.{name}") + elif node_type == "array": + _assert_closed(spec["items"], f"{path}[]") + + _assert_closed(schema, "inputSchema") + + +def test_all_scenarios_work(): + for name, (status, assigned_state) in SCENARIOS.items(): + response = _run(name) + assert response["status"] == status, name + assert response["assigned_state"] == assigned_state, name + + +def test_uses_canonical_epistemic_state_enum(): + # authority.py imports the canonical taxonomy and defines no parallel enum. + tree = ast.parse((DEMO_DIR / "authority.py").read_text(encoding="utf-8")) + imported_from_core = False + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + base_names = { + base.id for base in node.bases if isinstance(base, ast.Name) + } | { + base.attr for base in node.bases if isinstance(base, ast.Attribute) + } + assert "Enum" not in base_names, "demo must not define a parallel enum" + if isinstance(node, ast.ImportFrom) and node.module == "core.epistemic_state": + names = {alias.name for alias in node.names} + assert {"EpistemicState", "coerce_epistemic_state"} <= names + imported_from_core = True + assert imported_from_core + + # Every emitted state/clearance is a member of the canonical enums. + valid_states = {state.value for state in EpistemicState} + valid_clearances = {clearance.value for clearance in NormativeClearance} + for name in SCENARIOS: + artifact = json.loads((EXPECTED_DIR / f"{name}.json").read_text(encoding="utf-8")) + if artifact["assigned_state"] is not None: + assert artifact["assigned_state"] in valid_states, name + if artifact["normative_clearance"] is not None: + assert artifact["normative_clearance"] in valid_clearances, name + + +def test_verified_requires_matching_evidence(): + response = _run("verified-supported-claim") + assert response["assigned_state"] == "verified" + # This demo runs no normative/safety/ethics clearance pass, so even a + # verified claim stays UNASSESSABLE rather than positively cleared. + assert response["normative_clearance"] == "unassessable" + assert response["evidence_ledger"] == ["ev-a1-replay", "ev-a1-trace"] + + # Drop one independent record: a single supporting record cannot verify. + one_record = _arguments("verified-supported-claim") + one_record["evidence"] = one_record["evidence"][:1] + downgraded = authority.run_authority(one_record) + assert downgraded["assigned_state"] == "evidenced" + assert downgraded["decision_reason"] == "evidence_present_but_not_verifying" + + # Subject/predicate mismatch is not matching evidence at all. + mismatched = _arguments("verified-supported-claim") + for record in mismatched["evidence"]: + record["predicate"] = "something_else" + none_match = authority.run_authority(mismatched) + assert none_match["assigned_state"] == "undetermined" + assert none_match["decision_reason"] == "insufficient_evidence" + + +def test_clearance_is_unassessable_for_all_non_invalid_outputs(): + # The demo assigns epistemic truth-state only; it runs no normative/safety/ + # ethics clearance pass, so it must never positively clear anything. + for name in SCENARIOS: + response = _run(name) + if response["status"] == "invalid": + assert response["normative_clearance"] is None, name + else: + assert response["normative_clearance"] == "unassessable", name + + +def test_evidenced_state_for_single_support(): + response = _run("evidenced-but-not-verified-claim") + assert response["status"] == "assigned" + assert response["assigned_state"] == "evidenced" + assert response["decision_reason"] == "evidence_present_but_not_verifying" + assert response["evidence_ledger"] == ["ev-b1-log"] + + +def test_inferred_includes_inference_basis(): + response = _run("inferred-from-bounded-evidence") + assert response["assigned_state"] == "inferred" + assert response["decision_reason"] == "bounded_inference_from_evidence" + assert response["inference_basis"] == ["ev-c1-branch-from-main", "ev-c1-tag-on-main"] + assert response["evidence_ledger"] == response["inference_basis"] + + +def test_inference_with_unresolved_premise_does_not_infer(): + payload = _arguments("inferred-from-bounded-evidence") + payload["inference"]["premise_ids"] = ["ev-c1-tag-on-main", "does-not-exist"] + response = authority.run_authority(payload) + assert response["assigned_state"] == "undetermined" + + +def test_undetermined_emits_question(): + response = _run("undetermined-insufficient-evidence") + assert response["assigned_state"] == "undetermined" + assert response["evidence_ledger"] == [] + assert "insufficient grounded evidence" in response["question"] + + +def test_scope_boundary_refuses(): + response = _run("refused-outside-scope") + assert response["status"] == "refused" + assert response["assigned_state"] == "scope_boundary" + assert response["refusal_reason"] == "outside_epistemic_envelope" + assert response["decision_reason"] == "outside_epistemic_envelope" + assert response["evidence_ledger"] == [] + + +def test_proposer_cannot_set_assigned_state(): + payload = _arguments("verified-supported-claim") + payload["assigned_state"] = "verified" + response = authority.run_authority(payload) + assert response["status"] == "invalid" + assert "payload unexpected property 'assigned_state'" in response["invalid_reason"] + + +def test_proposer_cannot_smuggle_root_authority_fields(): + response = _run("invalid-state-smuggling-attempt") + assert response["status"] == "invalid" + assert response["assigned_state"] is None + assert response["normative_clearance"] is None + for forged in ("assigned_state", "status", "evidence_ledger", "authority_path", "trace_hash"): + assert f"unexpected property '{forged}'" in response["invalid_reason"] + + +def test_proposer_trace_hash_is_ignored(): + payload = _arguments("verified-supported-claim") + response = authority.run_authority(payload) + assert response["trace_summary"]["proposer_trace_hash_ignored"] is True + assert response["trace_hash"] != payload["proposer"]["trace_hash"] + + +def test_proposer_proposed_state_is_ignored(): + # Proposer claims "verified" but supplies zero evidence: CORE assigns undetermined. + payload = _arguments("undetermined-insufficient-evidence") + payload["proposer"]["proposed_state"] = "verified" + payload["evidence"] = [] + response = authority.run_authority(payload) + assert response["trace_summary"]["proposer_state_ignored"] is True + assert response["assigned_state"] == "undetermined" + + +def test_invalid_payload_fails_before_state_evaluation(monkeypatch): + def _explode(_payload: dict[str, object]) -> dict[str, object]: + raise AssertionError("state evaluation ran for an invalid payload") + + monkeypatch.setattr(authority, "assign_epistemic_state", _explode) + response = _run("invalid-state-smuggling-attempt") + assert response["status"] == "invalid" + assert response["trace_summary"]["authority_evaluated"] is False + + +def test_double_run_outputs_byte_identical(): + for name in SCENARIOS: + payload = _arguments(name) + run_a = authority.run_authority(payload) + run_b = authority.run_authority(payload) + rendered_a = json.dumps(run_a, sort_keys=True, indent=2) + rendered_b = json.dumps(run_b, sort_keys=True, indent=2) + assert rendered_a == rendered_b, name + + +def test_expected_artifact_tampering_fails(tmp_path): + scenario_id = "verified-supported-claim" + ref = EXPECTED_DIR / f"{scenario_id}.json" + original = ref.read_text(encoding="utf-8") + try: + ref.write_text(original.replace("verified", "evidenced", 1), encoding="utf-8") + with pytest.MonkeyPatch.context() as patch: + patch.setattr("sys.argv", ["run_demo.py", "--out", str(tmp_path / "out")]) + assert demo_runner.main() == 1 + finally: + ref.write_text(original, encoding="utf-8") + + +def test_run_demo_all_passes_against_expected(tmp_path, monkeypatch): + out_dir = tmp_path / "demo-out" + monkeypatch.setattr("sys.argv", ["run_demo.py", "--out", str(out_dir)]) + assert demo_runner.main() == 0 + + +def test_run_demo_default_out_still_works(monkeypatch): + monkeypatch.setattr("sys.argv", ["run_demo.py"]) + assert demo_runner.main() == 0 + + +def test_run_demo_refuses_non_default_dir_without_marker(tmp_path, monkeypatch): + out_dir = tmp_path / "missing-marker" + out_dir.mkdir() + monkeypatch.setattr("sys.argv", ["run_demo.py", "--out", str(out_dir)]) + assert demo_runner.main() == 2 + + +@pytest.mark.parametrize("target", [".", ".."]) +def test_run_demo_refuses_parent_like_paths(target: str, monkeypatch): + monkeypatch.setattr("sys.argv", ["run_demo.py", "--out", target]) + assert demo_runner.main() == 2 + + +def test_no_network_subprocess_eval_or_exec_imports_or_calls(): + forbidden_imports = {"subprocess", "socket", "requests", "httpx", "urllib", "urllib.request"} + forbidden_calls = {"eval", "exec", "compile", "open_connection", "create_connection", "Popen", "run"} + scanned = [ + DEMO_DIR / "__init__.py", + DEMO_DIR / "authority.py", + DEMO_DIR / "run_demo.py", + ] + for path in scanned: + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + assert alias.name not in forbidden_imports + elif isinstance(node, ast.ImportFrom): + assert (node.module or "") not in forbidden_imports + elif isinstance(node, ast.Call): + if isinstance(node.func, ast.Name): + assert node.func.id not in forbidden_calls + elif isinstance(node.func, ast.Attribute): + assert node.func.attr not in forbidden_calls