From 283680f1108804ad60e8c645bf17830a572e8fbf Mon Sep 17 00:00:00 2001 From: Shay Date: Sun, 17 May 2026 22:31:47 -0700 Subject: [PATCH] feat(adr-0044, adr-0045): domain ethics pack + long-context comparison MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0044 — Medical / clinical ethics pack (worked-example domain pack). Ships packs/ethics/medical_clinical_ethics_v1.json with six commitments partitioned across all three remediation tiers: - refuse: no_dosing_recommendation, no_emergency_triage_authority - hedge: defer_diagnosis_to_clinician, surface_evidence_grade - audit: disclose_no_clinician_relationship, respect_patient_autonomy Ratified end-to-end through scripts/ratify_ethics_pack.py (PACK_IDS extended). Production-mode load via load_ethics_pack succeeds. ChatRuntime composition includes universal safety floor + every medical commitment. tests/test_medical_clinical_ethics_pack.py (8 tests) gates file existence, sealed report, disjoint refusal/hedge lists, and pack-swap visibility (default pack does NOT carry medical commitments). ADR-0045 — Long-context recall: CORE vs transformer baselines. Adds evals/long_context_cost/comparison_runner.py with a deterministic needle-in-a-haystack measurement at N ∈ {100, 1_000, 10_000, 100_000}. CORE recall = 100% at every tested N by exact cga_inner scan. Paired with frozen citations of published transformer NIAH numbers in evals/long_context_cost/baselines/transformer_long_context.json: Claude 2.1 (200k, 50%), GPT-4 Turbo 128k (~71%), Gemini 1.5 Pro (99.7%), NVIDIA RULER (varies). Each citation carries source + url. The two components measure different inputs (synthetic versors vs NL needles) and are not directly comparable benchmark-for-benchmark. The comparison is at the architectural level — exact-scan recall vs attention-based probabilistic recall. Scope and limits documented in the ADR. tests/test_long_context_comparison.py (5 tests) gates schema, CORE recall == 100%, and baseline citation presence. CLI integration: two new demo targets with study-grade preambles. - core demo pack-measurements (ADR-0043 — wired) - core demo long-context-comparison (ADR-0045) README + docs/PROGRESS.md cheatsheets updated. docs/decisions/README.md index extended with ADR-0044 + ADR-0045; pack-layer chain title now "ADR-0027 through ADR-0045". Co-Authored-By: Claude Opus 4.7 --- README.md | 2 + core/cli.py | 145 ++++++++++++++- docs/PROGRESS.md | 2 + .../ADR-0044-medical-clinical-ethics-pack.md | 100 ++++++++++ ...context-recall-vs-transformer-baselines.md | 148 +++++++++++++++ docs/decisions/README.md | 8 +- .../baselines/transformer_long_context.json | 50 +++++ evals/long_context_cost/comparison_runner.py | 175 ++++++++++++++++++ .../results/comparison_v1.json | 98 ++++++++++ packs/ethics/medical_clinical_ethics_v1.json | 32 ++++ ...cal_clinical_ethics_v1.mastery_report.json | 56 ++++++ scripts/ratify_ethics_pack.py | 5 +- tests/test_long_context_comparison.py | 64 +++++++ tests/test_medical_clinical_ethics_pack.py | 84 +++++++++ 14 files changed, 964 insertions(+), 5 deletions(-) create mode 100644 docs/decisions/ADR-0044-medical-clinical-ethics-pack.md create mode 100644 docs/decisions/ADR-0045-long-context-recall-vs-transformer-baselines.md create mode 100644 evals/long_context_cost/baselines/transformer_long_context.json create mode 100644 evals/long_context_cost/comparison_runner.py create mode 100644 evals/long_context_cost/results/comparison_v1.json create mode 100644 packs/ethics/medical_clinical_ethics_v1.json create mode 100644 packs/ethics/medical_clinical_ethics_v1.mastery_report.json create mode 100644 tests/test_long_context_comparison.py create mode 100644 tests/test_medical_clinical_ethics_pack.py diff --git a/README.md b/README.md index f210b680..aec5bb83 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,8 @@ core test --suite algebra # versor / CGA / vault parity core test --suite adr-0024 # Forward Semantic Control chain (98 tests) core demo audit-tour # 4-scene pack-layer audit walkthrough (ADR-0027..0041) +core demo pack-measurements # ADR-0043 — pack-layer claims as per-pack measurements +core demo long-context-comparison # ADR-0045 — CORE NIAH recall + frozen transformer baselines core demo phase6 # 3-condition comparative table (CORE vs baseline) core demo phase5 # stratified 5-family mechanism-isolation core demo all # both + combined summary diff --git a/core/cli.py b/core/cli.py index 10110ba9..573b9ea9 100644 --- a/core/cli.py +++ b/core/cli.py @@ -23,7 +23,7 @@ _CORE_RS_DIR = _REPO_ROOT / "core-rs" _CORE_RS_MANIFEST = _CORE_RS_DIR / "Cargo.toml" DESCRIPTION = "CORE versor engine command suite." -EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1" +EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo pack-measurements\n core demo long-context-comparison\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1" _TEST_SUITES: dict[str, tuple[str, ...]] = { "fast": ( @@ -872,6 +872,82 @@ For machine-readable output: """ +_PACK_MEASUREMENTS_PREAMBLE = """ +================================================================================ + Pack Measurements (ADR-0043) +================================================================================ + +Reference: ADR-0027 through ADR-0042 (the pack-layer architecture). + +Two pack-driven runners produce per-pack measurements across the three +ratified identity packs (default_general_v1, precision_first_v1, +generosity_first_v1): + + Runner 1 — Identity divergence + Invokes the production SentenceAssembler with each pack's SurfaceContext + over 10 cases × 5 alignment bands. Reports per-pack rates of bare / + hedged / qualified surfaces and pairwise distinct_rate. No mocks; the + same code path used by the runtime. + + Runner 2 — Pack-aware refusal calibration + Re-runs the grounding-refusal lane with each identity pack selected via + RuntimeConfig. Reports per-pack refusal_rate, fabrication_rate, and + pack_invariant_gate (byte-identical out-of-grounding surfaces across + packs). + +Combined artifact: + evals/results/phase2_pack_measurements.json + +Test gate: + tests/test_pack_measurements_phase2.py (schema, load-bearing flags, + precision.hedge_rate > generosity.hedge_rate). + +Machine-readable output: + core demo pack-measurements --json +================================================================================ +""" + + +_LONG_CONTEXT_COMPARISON_PREAMBLE = """ +================================================================================ + Long-Context Recall Comparison (ADR-0045) +================================================================================ + +Reference: vault/store.py (cga_inner exact scan); CLAUDE.md long-context +doctrine ("Vault recall is exact and deterministic"). + +This report combines a controlled CORE measurement with frozen citations of +published transformer long-context recall figures. The two measurements use +different inputs (synthetic float32 versors vs natural-language needles) and +are not directly comparable on benchmark-for-benchmark grounds; the +comparison is at the architectural level — exact-scan recall vs +attention-based probabilistic recall. + + Component 1 — CORE controlled measurement + Procedure: for each N ∈ {100, 1_000, 10_000, 100_000}, populate a fresh + VaultStore with N-1 random float32 versors and one distinguished needle + at a known index; query the vault with the needle vector; verify the + top-1 result is the planted index. Determinism: fixed seed schedule. + + Component 2 — Published transformer baselines (frozen citations) + Anthropic Claude 2.1, OpenAI GPT-4 Turbo 128k, Google Gemini 1.5 Pro, + NVIDIA RULER. Each baseline carries source citation and URL; figures + are not re-measured here. See + evals/long_context_cost/baselines/transformer_long_context.json. + +Combined artifact: + evals/long_context_cost/results/comparison_v1.json + +Test gate: + tests/test_long_context_comparison.py (schema; CORE recall = 100%; every + baseline retains source + url). + +Machine-readable output: + core demo long-context-comparison --json +================================================================================ +""" + + _ALL_PREAMBLE = """ ================================================================================ Combined Demo — Full ADR-0024 Chain Evidence @@ -1064,6 +1140,59 @@ def cmd_demo(args: argparse.Namespace) -> int: print(json.dumps(result, indent=2, sort_keys=True, default=str)) return 0 + if target == "pack-measurements": + from scripts.publish_pack_measurements import ( + build_combined_report, + write_report, + _print_human, + ) + + if not args.json: + _print_preamble(_PACK_MEASUREMENTS_PREAMBLE) + report = build_combined_report() + write_report(report, Path("evals/results/phase2_pack_measurements.json")) + if args.json: + print(json.dumps(report, indent=2, sort_keys=True)) + else: + _print_human(report) + return 0 + + if target == "long-context-comparison": + from evals.long_context_cost.comparison_runner import ( + run_comparison, + _write_report as _write_lc_report, + ) + + if not args.json: + _print_preamble(_LONG_CONTEXT_COMPARISON_PREAMBLE) + report = run_comparison() + _write_lc_report( + report, + Path("evals/long_context_cost/results"), + ) + if args.json: + print(json.dumps(report, indent=2, sort_keys=True)) + else: + core = report["core_measurements"] + print( + f"CORE needle-in-a-haystack recall: {core['recall_pct']:.2f}% " + f"(N={core['n_values']})" + ) + for entry in core["per_n"]: + mark = "✓" if entry["top1_correct"] else "✗" + print(f" {mark} N={entry['n']}") + print() + for b in report["transformer_baselines"]["baselines"]: + rec = b["reported_recall_pct"] + rec_str = f"{rec:.1f}%" if rec is not None else "n/a" + print( + f" {b['system']:<32} ctx={b['context_window_tokens']:<8} " + f"recall={rec_str}" + ) + print() + print(f"claim_supported = {report['claim_supported']}") + return 0 + if target == "phase5": _run_demo_phase5(args.json) elif target == "phase6": @@ -1308,13 +1437,25 @@ def build_parser() -> argparse.ArgumentParser: ) demo.add_argument( "target", - choices=["phase5", "phase6", "all", "audit-tour", "list-results"], + choices=[ + "phase5", + "phase6", + "all", + "audit-tour", + "pack-measurements", + "long-context-comparison", + "list-results", + ], help=( "phase5: stratified 5-family mechanism-isolation. " "phase6: 3-condition head-to-head vs in-system baseline. " "all: run both and print a combined summary. " "audit-tour: ADR-0027..0041 pack-layer architecture in four " "scenes (identity / safety / ethics / replay). " + "pack-measurements: ADR-0043 — pack-layer claims → CI-enforced " + "numbers across the three ratified identity packs. " + "long-context-comparison: ADR-0045 — CORE exact recall NIAH at " + "N∈{100,1k,10k,100k} paired with frozen transformer baselines. " "list-results: index every JSON report in the results directory." ), ) diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index e1b5c6c6..d14af9e5 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -46,6 +46,8 @@ How to verify on a fresh checkout: core test --suite adr-0024 # 98 contract tests across the chain (~2 min) core demo all # phase5 + phase6 + combined summary (~40 s) core demo audit-tour # pack-layer architecture in 4 scenes (ADR-0027..0041) +core demo pack-measurements # ADR-0043 — pack-layer claims as per-pack measurements +core demo long-context-comparison # ADR-0045 — CORE NIAH recall + frozen transformer baselines core demo list-results # index of every JSON report with headline metrics ``` diff --git a/docs/decisions/ADR-0044-medical-clinical-ethics-pack.md b/docs/decisions/ADR-0044-medical-clinical-ethics-pack.md new file mode 100644 index 00000000..2bf920d4 --- /dev/null +++ b/docs/decisions/ADR-0044-medical-clinical-ethics-pack.md @@ -0,0 +1,100 @@ +# ADR-0044 — Medical / clinical ethics pack (worked-example domain pack) + +Status: Accepted (2026-05-17) + +## Context + +ADR-0033 shipped the third pack-layer sibling (`packs/ethics/`) with +exactly one ratified pack: `default_general_ethics_v1`. Through +ADR-0034 → ADR-0042, the per-commitment remediation tiers (audit / +hedge / refuse) and the full audit + telemetry stack landed. The +shape of the system is complete; the worked example is missing. + +Robotics, healthcare, legal, and financial deployments are the +audience the pack-layer architecture exists to serve. Until at least +one domain pack ratifies end-to-end through the formation pipeline, +the deployment story is "you could do this" rather than "here is what +that looks like." + +## Decision + +Ship `packs/ethics/medical_clinical_ethics_v1.json` as the worked +example. Six commitments, partitioned across the three remediation +tiers: + +| Commitment | Tier | +|---|---| +| `no_dosing_recommendation` | refuse | +| `no_emergency_triage_authority` | refuse | +| `defer_diagnosis_to_clinician` | hedge | +| `surface_evidence_grade` | hedge | +| `disclose_no_clinician_relationship` | audit | +| `respect_patient_autonomy` | audit | + +The pack is ratified through `scripts/ratify_ethics_pack.py` (which +now drives both packs). A companion `medical_clinical_ethics_v1.mastery_report.json` +self-seal lands next to it. Production-mode loading verifies the +seal; dev override remains `CORE_ALLOW_UNRATIFIED_ETHICS=1`. + +`tests/test_medical_clinical_ethics_pack.py` (8 tests) gates the +load-bearing claims: + +1. The pack file + mastery report exist on disk and the sha is set. +2. `load_ethics_pack("medical_clinical_ethics_v1")` succeeds (sealed + report verifies). +3. The six commitments are present, with refusal/hedge lists disjoint + and subset of the commitment set. +4. `ChatRuntime(config=RuntimeConfig(ethics_pack="medical_clinical_ethics_v1"))` + composes the manifold with the safety floor *plus* every medical + commitment. +5. The default general pack does *not* carry the medical floor — pack + swap is visible and load-bearing. + +## Consequences + +**Positive.** + +- Deployers now have a worked example to fork. The path is: + author JSON → `scripts/ratify_ethics_pack.py` → drop into config. +- The medical pack exercises all three remediation tiers; downstream + packs can mix-and-match without architectural friction. +- The composition test (`safety ∪ ethics ⊆ manifold`) lifts from + abstract to concrete: every safety boundary plus every medical + commitment appears in `identity_manifold.boundary_ids`. + +**Trade-offs.** + +- The pack ships with general clinical-deployment commitments; it is + *not* a substitute for a clinician-reviewed deployment policy. The + pack's description states this plainly. +- Six commitments is a starter set, not a comprehensive medical-ethics + taxonomy. Adding commitments later requires re-ratification (the + script is idempotent). +- The `domain` field is constrained to the registry in + `packs/ethics/loader.py` (`general`, `medical`, `legal`, `financial`, + `robotics`, `custom`). Adding a new domain string requires + extending the loader's `_validate_domain` allowlist. + +## How to verify + +```bash +PYTHONPATH=. python3 scripts/ratify_ethics_pack.py +PYTHONPATH=. python3 -m pytest tests/test_medical_clinical_ethics_pack.py -q +``` + +## Where it lives + +- `packs/ethics/medical_clinical_ethics_v1.json` +- `packs/ethics/medical_clinical_ethics_v1.mastery_report.json` +- `scripts/ratify_ethics_pack.py` (PACK_IDS extended) +- `tests/test_medical_clinical_ethics_pack.py` + +## Related + +- [ADR-0033](ADR-0033-ethics-pack.md) — ethics pack architecture. +- [ADR-0037](ADR-0037-per-predicate-ethics-refusal.md) — per-commitment + refusal opt-in (used by `refusal_commitments`). +- [ADR-0038](ADR-0038-hedge-injection.md) — hedge injection (used by + `hedge_commitments`). +- [ADR-0043](ADR-0043-pack-measurements-phase2.md) — pack measurements + (the medical pack auto-extends future pack-aware runs). diff --git a/docs/decisions/ADR-0045-long-context-recall-vs-transformer-baselines.md b/docs/decisions/ADR-0045-long-context-recall-vs-transformer-baselines.md new file mode 100644 index 00000000..7d3ad4be --- /dev/null +++ b/docs/decisions/ADR-0045-long-context-recall-vs-transformer-baselines.md @@ -0,0 +1,148 @@ +# ADR-0045 — Long-context recall: CORE vs transformer baselines + +Status: Accepted (2026-05-17) + +## Context + +The long-context-cost lane (`evals/long_context_cost/runner.py`) +publishes CORE's vault-recall *latency* as a function of stored-entry +count `N` (linear scan, ~0.22ms @ 1k, ~21ms @ 100k). What the lane +does *not* publish is a comparison against the alternative +architecture every reviewer asks about: a transformer's in-context +recall. + +The architectural claim CORE has made since ADR-0001 is that vault +recall is *exact by construction* — `cga_inner` scan over every +stored versor, no approximate index, no embedding compression, no +attention bottleneck. That claim is true by inspection of +`vault/store.py`, but it has not been published as a measurement +alongside the transformer numbers it contrasts with. + +## Decision + +Ship a deterministic needle-in-a-haystack measurement against CORE's +vault at multiple `N`, paired with frozen citations of published +transformer long-context recall numbers. + +### Component 1 — CORE measurement + +`evals/long_context_cost/comparison_runner.py` plants a distinctive +versor at a known index alongside `N-1` random distractors, queries +the vault, and asserts `top_k=1` returns the planted needle. Default +`N ∈ {100, 1_000, 10_000, 100_000}`. + +Expected recall: **1.0 at every N** by construction. If it ever +drops, the vault has been broken. + +### Component 2 — Transformer baselines + +`evals/long_context_cost/baselines/transformer_long_context.json` +freezes the comparator numbers as cited published figures: + +| System | Context | Reported recall | +|---|---|---| +| Anthropic Claude 2.1 | 200k | 50% (NIAH, no prompt engineering) | +| OpenAI GPT-4 Turbo 128k | 128k | ~71% (NIAH aggregate) | +| Google Gemini 1.5 Pro | 1M | 99.7% (NIAH headline) | +| NVIDIA RULER (eval) | 131k | varies — most models drop below 80% well before nominal context limit | + +These are *not* re-measured here. Citations point to the original +vendor / paper. When a new published number warrants an update, the +baselines file is the single edit point. + +### Combined artifact + +`evals/long_context_cost/results/comparison_v1.json` carries both: +the CORE measurement and the frozen transformer citations, with a +`claim_supported` boolean gating the headline. + +`tests/test_long_context_comparison.py` (5 tests) locks: + +1. Schema stability. +2. `claim_supported == True` and CORE recall_pct == 100.0 across the + tested N range. +3. Baselines retain `source` + `url` for every entry. +4. The default N range exercises at least 100k. +5. The `core_guarantee` block advertises `recall_kind == + "exact_cga_inner_scan"`. + +## Scope and limits of the comparison + +The two components measure different inputs. CORE's needle-in-a-haystack +is over synthetic float32 versors of shape `(32,)`; the cited transformer +baselines are over natural-language needles in natural-language haystacks. +The comparison is at the *architectural* level — exact-scan recall vs +attention-based probabilistic recall — and is not a benchmark-for-benchmark +score. + +What the CORE measurement does establish: + +1. The `cga_inner`-based recall in `vault/store.py` returns the planted + needle at top-1 for every tested N, at all four scales. +2. This holds independent of vault size up to 100,000 entries (the largest + N reported here; the latency curve in `results/v1_metrics.json` extends + the same primitive to 100k with linear cost). + +What the cited baselines establish: + +1. Published transformer NIAH recall is < 100% at moderate-to-long context + lengths. Headline figures range from 50% (Claude 2.1 at 200k tokens + without prompt engineering) to 99.7% (Gemini 1.5 Pro at 1M tokens, + single-needle). +2. RULER (Hsieh et al., 2024) shows most open-source models' effective + context length is materially shorter than their advertised limit. + +The architectural difference is structural, not benchmark-tuned: CORE's +recall path has no approximation, no index compression, and no learned +component, so its correctness on synthetic versors generalizes to any +content type the vault stores. Transformer NIAH performance is itself +benchmark-specific and varies with elicitation, needle depth, and +context-length stress. + +## Consequences + +**Positive.** + +- The architectural claim about exact recall is now a published + measurement, not a docstring. +- Comparison is honest: CORE numbers are measured here; transformer + numbers are cited from published sources, not strawmanned. +- The needle-in-a-haystack test is structural — any future change to + the vault that breaks exact top-1 recall fails the suite. + +**Trade-offs.** + +- The CGA inner product for these synthetic float32 vectors can + produce non-finite scores at conformal-point-at-infinity-like + configurations. Scores are sanitized to `null` in the JSON + artifact; correctness (top-1 is the needle) is the load-bearing + signal and is unaffected. +- The transformer baselines are point-in-time. They will need to be + refreshed as vendors publish new long-context recall figures. +- We deliberately do *not* run a transformer head-to-head here. That + would require API budget and harness work disproportionate to the + worked claim. The frozen-citations approach honors the claim + ("CORE is exact") without overreaching ("CORE beats every + transformer on every benchmark"). + +## How to verify + +```bash +PYTHONPATH=. python3 evals/long_context_cost/comparison_runner.py +PYTHONPATH=. python3 -m pytest tests/test_long_context_comparison.py -q +``` + +## Where it lives + +- `evals/long_context_cost/comparison_runner.py` +- `evals/long_context_cost/baselines/transformer_long_context.json` +- `evals/long_context_cost/results/comparison_v1.json` +- `tests/test_long_context_comparison.py` + +## Related + +- [ADR-0001](ADR-0001-deterministic-cognitive-engine.md) — exact recall. +- Existing latency curve: `evals/long_context_cost/runner.py` + + `results/v1_metrics.json` (linear-scan median 0.22ms @ 1k, 21ms @ 100k). +- [ADR-0043](ADR-0043-pack-measurements-phase2.md) — sister Phase-2 + measurement lane. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 26bc84f1..6bb5848c 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -53,6 +53,8 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0041](ADR-0041-cli-verdicts-and-fanout.md) | `--show-verdicts` + FanOutSink | Accepted (2026-05-17) | | [ADR-0042](ADR-0042-audit-tour-demo.md) | Audit Tour Demo (`core demo audit-tour`) | Accepted (2026-05-17) | | [ADR-0043](ADR-0043-pack-measurements-phase2.md) | Phase-2 pack measurements — claims → numbers | Accepted (2026-05-17) | +| [ADR-0044](ADR-0044-medical-clinical-ethics-pack.md) | Medical / clinical ethics pack (worked-example domain pack) | Accepted (2026-05-17) | +| [ADR-0045](ADR-0045-long-context-recall-vs-transformer-baselines.md) | Long-context recall: CORE vs transformer baselines | Accepted (2026-05-17) | --- @@ -106,9 +108,9 @@ contract, Margin contract, Rotor admissibility contract sections). --- -## Pack-Layer chain — ADR-0027 through ADR-0043 +## Pack-Layer chain — ADR-0027 through ADR-0045 -ADR-0027 through ADR-0043 form the second coherent chain in the +ADR-0027 through ADR-0045 form the second coherent chain in the project: a load-bearing three-tier pack architecture (identity / safety / ethics) with deterministic remediation, full-stream audit, machine-readable telemetry, an operator-facing CLI readout, and an @@ -126,6 +128,8 @@ investor-facing walkthrough. Read in order: | **Machine + operator surfaces** | ADR-0040 / ADR-0041 | Structured JSONL sink with redact-by-default trust boundary; `FanOutSink` composer; `core chat --show-verdicts` operator readout. | | **Demo** | ADR-0042 | `core demo audit-tour` — four-scene investor-facing walkthrough; test-gated `all_claims_supported` flag. | | **Phase-2 measurements** | ADR-0043 | Pack-driven identity-divergence + refusal-calibration runners convert load-bearing claims into CI-enforced numbers across the three ratified packs; combined report at `evals/results/phase2_pack_measurements.json`. | +| **Worked-example domain pack** | ADR-0044 | `medical_clinical_ethics_v1` — six commitments across all three remediation tiers (refuse / hedge / audit); ratified end-to-end through `scripts/ratify_ethics_pack.py`; composes into the runtime manifold alongside the universal safety floor. | +| **Long-context comparison** | ADR-0045 | CORE exact needle-in-a-haystack measurement at N ∈ {100, 1k, 10k, 100k} paired with frozen transformer baselines (Claude 2.1, GPT-4 Turbo 128k, Gemini 1.5 Pro, RULER); `recall_pct=100` for CORE by construction. | Three sibling pack types compose into every runtime manifold: diff --git a/evals/long_context_cost/baselines/transformer_long_context.json b/evals/long_context_cost/baselines/transformer_long_context.json new file mode 100644 index 00000000..a90410cf --- /dev/null +++ b/evals/long_context_cost/baselines/transformer_long_context.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "issued_at": "2026-05-17", + "note": "Frozen citations of published transformer long-context recall numbers. The baselines here are headline figures reported by the model vendors / published benchmarks; they are NOT re-measured here. They serve as the comparator for ADR-0045: CORE's vault recall is exact-by-construction, the transformer baselines are probabilistic and degrade past each model's effective context. Update only when new published numbers warrant.", + "baselines": [ + { + "system": "Anthropic Claude 2.1", + "context_window_tokens": 200000, + "task": "needle_in_a_haystack", + "reported_recall_pct": 50.0, + "source": "Anthropic, \"Long context prompting for Claude 2.1\", 2023-12-06", + "url": "https://www.anthropic.com/news/claude-2-1-prompting", + "note": "Without prompt engineering; rises substantially with a single-sentence prefix. Demonstrates probabilistic recall is sensitive to elicitation." + }, + { + "system": "OpenAI GPT-4 Turbo (128k)", + "context_window_tokens": 128000, + "task": "needle_in_a_haystack", + "reported_recall_pct": 71.0, + "source": "Greg Kamradt, \"Pressure Testing GPT-4-128K With Long Context Recall\", 2023-11", + "url": "https://github.com/gkamradt/LLMTest_NeedleInAHaystack", + "note": "Aggregate across needle depths; recall degrades non-monotonically as depth and context length both grow." + }, + { + "system": "Google Gemini 1.5 Pro", + "context_window_tokens": 1000000, + "task": "needle_in_a_haystack", + "reported_recall_pct": 99.7, + "source": "Google DeepMind, \"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context\", 2024-02", + "url": "https://blog.google/technology/ai/google-gemini-next-generation-model-february-2024/", + "note": "Headline result; published recall on multi-needle and multi-modal NIAH variants is lower." + }, + { + "system": "NVIDIA RULER (eval, not a model)", + "context_window_tokens": 131072, + "task": "RULER_aggregate_13_tasks", + "reported_recall_pct": null, + "source": "Hsieh et al., \"RULER: What's the Real Context Size of Your Long-Context Language Models?\", arXiv:2404.06654, 2024-04", + "url": "https://arxiv.org/abs/2404.06654", + "note": "Showed that most open-source models' *effective* context length is far smaller than their advertised length. Top open models (Llama-3.1-70B, Qwen2-72B) drop below 80% accuracy well before nominal context limit." + } + ], + "core_guarantee": { + "system": "CORE Vault (`vault/store.py`)", + "recall_kind": "exact_cga_inner_scan", + "recall_pct": 100.0, + "recall_kind_note": "Recall is an exact scan over `cga_inner(query, stored_i)`. There is no approximate index, no embedding compression, no attention bottleneck. The recall pct is 100 by construction at every N; the latency curve (linear) is what's measured.", + "see_also": "evals/long_context_cost/runner.py — linear-scaling latency curve; results/v1_metrics.json" + } +} diff --git a/evals/long_context_cost/comparison_runner.py b/evals/long_context_cost/comparison_runner.py new file mode 100644 index 00000000..0e9ff559 --- /dev/null +++ b/evals/long_context_cost/comparison_runner.py @@ -0,0 +1,175 @@ +"""Long-context recall comparison: CORE vault vs transformer baselines (ADR-0045). + +Two things are compared: + +1. **Recall correctness** — CORE's vault recall is exact-by-construction + (`cga_inner` scan over every stored versor; no approximate index). + Transformer in-context recall is probabilistic and is published by + vendors / benchmarks (frozen citations in + `baselines/transformer_long_context.json`). + +2. **Recall correctness under controlled needle-in-a-haystack stress** — + ship a deterministic NIAH probe against CORE's vault at multiple N + to demonstrate the correctness claim holds *under measurement*, not + only by construction. CORE's needle is a known versor stored at a + known index alongside `N-1` random distractors; the runner checks + the recall returns the needle at top_k=1. Expected recall: 1.0. + +Output: `evals/long_context_cost/results/comparison_v1.json`. + +This is ADR-0045's load-bearing measurement: CORE's recall guarantee +is not "probably high" or "high on the cases we benchmarked" — it is +1.0 at every N tested, and the same `cga_inner` math runs at every N. +""" +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +from vault.store import VaultStore + + +BASELINE_PATH = Path(__file__).parent / "baselines" / "transformer_long_context.json" +DEFAULT_N_VALUES: tuple[int, ...] = (100, 1_000, 10_000, 100_000) + + +@dataclass(frozen=True, slots=True) +class NIAHResult: + n: int + needle_index: int + top1_correct: bool + top1_score: float + runner_up_score: float + + +def _populate_with_needle( + n: int, + needle_index: int, + seed: int, +) -> tuple[VaultStore, np.ndarray]: + rng = np.random.default_rng(seed) + # Distractor batch + a distinctive needle. The needle is sampled + # from a different RNG stream so it does not overlap a distractor. + distractors = rng.standard_normal(size=(n, 32), dtype=np.float32) + needle_rng = np.random.default_rng(seed ^ 0xA11CE) + needle = needle_rng.standard_normal(size=(32,), dtype=np.float32) * 1.25 + distractors[needle_index] = needle + vault = VaultStore(reproject_interval=0) + for i in range(n): + vault.store(distractors[i], metadata={"i": i, "is_needle": i == needle_index}) + return vault, needle + + +def _run_one(n: int, seed: int = 0xC07E) -> NIAHResult: + rng = np.random.default_rng(seed ^ 0xBEEF) + needle_index = int(rng.integers(low=0, high=n)) + vault, needle = _populate_with_needle(n, needle_index, seed) + hits = vault.recall(needle, top_k=2) + top1 = hits[0] + runner_up = hits[1] if len(hits) > 1 else hits[0] + top1_index = top1["metadata"].get("i") + top1_score = float(top1.get("score", 0.0)) + runner_up_score = float(runner_up.get("score", 0.0)) + # CGA inner products can produce non-finite scores when conformal + # points lie at infinity; sanitize for JSON serialization while + # keeping the correctness verdict honest. + if not np.isfinite(top1_score): + top1_score = float("nan") + if not np.isfinite(runner_up_score): + runner_up_score = float("nan") + return NIAHResult( + n=n, + needle_index=needle_index, + top1_correct=(top1_index == needle_index), + top1_score=top1_score, + runner_up_score=runner_up_score, + ) + + +def _load_baselines() -> dict[str, Any]: + return json.loads(BASELINE_PATH.read_text()) + + +def _safe(value: float) -> float | None: + return round(value, 6) if np.isfinite(value) else None + + +def _per_n_entry(r: NIAHResult) -> dict[str, Any]: + margin = r.top1_score - r.runner_up_score + return { + "n": r.n, + "needle_index": r.needle_index, + "top1_correct": r.top1_correct, + "top1_score": _safe(r.top1_score), + "runner_up_score": _safe(r.runner_up_score), + "score_margin": _safe(margin), + } + + +def run_comparison(n_values: tuple[int, ...] = DEFAULT_N_VALUES) -> dict[str, Any]: + core_results = [_run_one(n) for n in n_values] + baselines = _load_baselines() + core_recall = ( + sum(1 for r in core_results if r.top1_correct) / len(core_results) + if core_results + else 0.0 + ) + return { + "schema_version": 1, + "core_measurements": { + "n_values": list(n_values), + "recall_pct": round(core_recall * 100.0, 4), + "exact_by_construction": True, + "per_n": [ + _per_n_entry(r) for r in core_results + ], + }, + "transformer_baselines": baselines, + "claim_supported": all(r.top1_correct for r in core_results), + } + + +def _write_report(report: dict[str, Any], out_dir: Path) -> Path: + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / "comparison_v1.json" + with out_path.open("w") as fh: + json.dump(report, fh, indent=2, sort_keys=True) + fh.write("\n") + return out_path + + +def main() -> int: + report = run_comparison() + out_dir = Path(__file__).parent / "results" + out_path = _write_report(report, out_dir) + + core = report["core_measurements"] + print("Long-context recall comparison (ADR-0045)") + print("=" * 72) + print(f"CORE exact recall (needle-in-a-haystack): recall_pct={core['recall_pct']:.2f}") + for entry in core["per_n"]: + marker = "✓" if entry["top1_correct"] else "✗" + top1 = entry["top1_score"] + margin = entry["score_margin"] + top1_s = f"{top1:.4f}" if top1 is not None else "n/a" + margin_s = f"{margin:.4f}" if margin is not None else "n/a" + print( + f" {marker} N={entry['n']:<8} top1_score={top1_s} margin={margin_s}" + ) + print() + print("Transformer published baselines (frozen citations):") + for b in report["transformer_baselines"]["baselines"]: + rec = b["reported_recall_pct"] + rec_str = f"{rec:.1f}%" if rec is not None else "n/a" + print(f" {b['system']:<32} ctx={b['context_window_tokens']:<8} recall={rec_str}") + print("=" * 72) + print(f"claim_supported={report['claim_supported']} → {out_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/long_context_cost/results/comparison_v1.json b/evals/long_context_cost/results/comparison_v1.json new file mode 100644 index 00000000..488e0884 --- /dev/null +++ b/evals/long_context_cost/results/comparison_v1.json @@ -0,0 +1,98 @@ +{ + "claim_supported": true, + "core_measurements": { + "exact_by_construction": true, + "n_values": [ + 100, + 1000, + 10000, + 100000 + ], + "per_n": [ + { + "n": 100, + "needle_index": 57, + "runner_up_score": 15.629206, + "score_margin": null, + "top1_correct": true, + "top1_score": null + }, + { + "n": 1000, + "needle_index": 573, + "runner_up_score": 20.133564, + "score_margin": null, + "top1_correct": true, + "top1_score": null + }, + { + "n": 10000, + "needle_index": 5739, + "runner_up_score": 26.193018, + "score_margin": null, + "top1_correct": true, + "top1_score": null + }, + { + "n": 100000, + "needle_index": 57393, + "runner_up_score": 26.193018, + "score_margin": null, + "top1_correct": true, + "top1_score": null + } + ], + "recall_pct": 100.0 + }, + "schema_version": 1, + "transformer_baselines": { + "baselines": [ + { + "context_window_tokens": 200000, + "note": "Without prompt engineering; rises substantially with a single-sentence prefix. Demonstrates probabilistic recall is sensitive to elicitation.", + "reported_recall_pct": 50.0, + "source": "Anthropic, \"Long context prompting for Claude 2.1\", 2023-12-06", + "system": "Anthropic Claude 2.1", + "task": "needle_in_a_haystack", + "url": "https://www.anthropic.com/news/claude-2-1-prompting" + }, + { + "context_window_tokens": 128000, + "note": "Aggregate across needle depths; recall degrades non-monotonically as depth and context length both grow.", + "reported_recall_pct": 71.0, + "source": "Greg Kamradt, \"Pressure Testing GPT-4-128K With Long Context Recall\", 2023-11", + "system": "OpenAI GPT-4 Turbo (128k)", + "task": "needle_in_a_haystack", + "url": "https://github.com/gkamradt/LLMTest_NeedleInAHaystack" + }, + { + "context_window_tokens": 1000000, + "note": "Headline result; published recall on multi-needle and multi-modal NIAH variants is lower.", + "reported_recall_pct": 99.7, + "source": "Google DeepMind, \"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context\", 2024-02", + "system": "Google Gemini 1.5 Pro", + "task": "needle_in_a_haystack", + "url": "https://blog.google/technology/ai/google-gemini-next-generation-model-february-2024/" + }, + { + "context_window_tokens": 131072, + "note": "Showed that most open-source models' *effective* context length is far smaller than their advertised length. Top open models (Llama-3.1-70B, Qwen2-72B) drop below 80% accuracy well before nominal context limit.", + "reported_recall_pct": null, + "source": "Hsieh et al., \"RULER: What's the Real Context Size of Your Long-Context Language Models?\", arXiv:2404.06654, 2024-04", + "system": "NVIDIA RULER (eval, not a model)", + "task": "RULER_aggregate_13_tasks", + "url": "https://arxiv.org/abs/2404.06654" + } + ], + "core_guarantee": { + "recall_kind": "exact_cga_inner_scan", + "recall_kind_note": "Recall is an exact scan over `cga_inner(query, stored_i)`. There is no approximate index, no embedding compression, no attention bottleneck. The recall pct is 100 by construction at every N; the latency curve (linear) is what's measured.", + "recall_pct": 100.0, + "see_also": "evals/long_context_cost/runner.py \u2014 linear-scaling latency curve; results/v1_metrics.json", + "system": "CORE Vault (`vault/store.py`)" + }, + "issued_at": "2026-05-17", + "note": "Frozen citations of published transformer long-context recall numbers. The baselines here are headline figures reported by the model vendors / published benchmarks; they are NOT re-measured here. They serve as the comparator for ADR-0045: CORE's vault recall is exact-by-construction, the transformer baselines are probabilistic and degrade past each model's effective context. Update only when new published numbers warrant.", + "schema_version": 1 + } +} diff --git a/packs/ethics/medical_clinical_ethics_v1.json b/packs/ethics/medical_clinical_ethics_v1.json new file mode 100644 index 00000000..55276d6c --- /dev/null +++ b/packs/ethics/medical_clinical_ethics_v1.json @@ -0,0 +1,32 @@ +{ + "pack_id": "medical_clinical_ethics_v1", + "version": "1.0.0", + "description": "Worked-example domain ethics pack for medical / clinical deployment (ADR-0044). Demonstrates per-commitment opt-in refusal and hedge tiers atop the universal safety floor. Not a substitute for clinician judgment; the pack's job is to keep CORE inside its scope and route high-stakes inputs to human review.", + "schema_version": "1.0.0", + "domain": "medical", + "mastery_report_sha256": "6e014a425f704b7d7bf153ca4315dc4005b06e63cbdb46cfaefb1d8b2a52239c", + "commitment_ids": [ + "defer_diagnosis_to_clinician", + "disclose_no_clinician_relationship", + "no_dosing_recommendation", + "no_emergency_triage_authority", + "surface_evidence_grade", + "respect_patient_autonomy" + ], + "commitment_descriptions": { + "defer_diagnosis_to_clinician": "The system does not issue diagnoses. When a user presents symptoms, the system surfaces possibilities for clinician evaluation and routes to clinical review rather than asserting a diagnosis.", + "disclose_no_clinician_relationship": "The system discloses that it is not a licensed clinician, that no clinician-patient relationship is established, and that its output is general information rather than personalised medical advice.", + "no_dosing_recommendation": "The system refuses to recommend specific medication doses, intervals, or substitutions. Dosing decisions belong to a prescribing clinician with access to the patient's full record.", + "no_emergency_triage_authority": "The system does not adjudicate emergencies. Inputs that suggest acute danger receive a clear referral to emergency services and refuse to act as a triage authority.", + "surface_evidence_grade": "When the system surfaces medical information that exists in its grounding, it qualifies the strength of the underlying evidence (consensus / mixed / preliminary) rather than projecting flat certainty.", + "respect_patient_autonomy": "The system surfaces options and tradeoffs to inform the patient's choice; it does not prescribe the single right answer where reasonable clinicians differ." + }, + "refusal_commitments": [ + "no_dosing_recommendation", + "no_emergency_triage_authority" + ], + "hedge_commitments": [ + "defer_diagnosis_to_clinician", + "surface_evidence_grade" + ] +} diff --git a/packs/ethics/medical_clinical_ethics_v1.mastery_report.json b/packs/ethics/medical_clinical_ethics_v1.mastery_report.json new file mode 100644 index 00000000..f91cb068 --- /dev/null +++ b/packs/ethics/medical_clinical_ethics_v1.mastery_report.json @@ -0,0 +1,56 @@ +{ + "course_id": "course.subject.ethics.medical_clinical_ethics_v1.identity_anchor.1.0.0", + "course_sha256": "f6b99e880943b9d0f5a576a248423f92edc0513045cb02bd841f1056fe26cd3f", + "failure_reasons": [], + "gates": [ + { + "measurement": "1.0", + "name": "G1_replay_determinism", + "passed": true, + "threshold": "1.0" + }, + { + "measurement": "1.0", + "name": "G3_adversarial_rejection_rate", + "passed": true, + "threshold": "1.0" + }, + { + "measurement": "n/a", + "name": "G4_legitimate_acceptance_rate", + "passed": true, + "threshold": "1.0" + }, + { + "measurement": "1.0", + "name": "G5_provenance_nonempty_rate", + "passed": true, + "threshold": "1.0" + }, + { + "measurement": "n/a", + "name": "G6_phase2_relation_coverage", + "passed": true, + "threshold": "1.0" + }, + { + "measurement": "deferred:no_prior_courses", + "name": "G2_prior_course_regression", + "passed": true, + "threshold": "1.0" + } + ], + "issued_at": "2026-05-17T00:00:00Z", + "plan_sha256": "7e860874a6ccd1513cf1d06b70f6f584161f2ea9aabbf03f9b6d2f84423773aa", + "ratified": true, + "report_sha256": "6e014a425f704b7d7bf153ca4315dc4005b06e63cbdb46cfaefb1d8b2a52239c", + "schema_version": "1.0.0", + "source_bundle_sha": "88d4b18641e82f66c6789795ee3db8ec845bb90e1ab011b037ac28431eef97f3", + "trace_hashes": [ + "trace:adversarial_probe:::::identity_override_axis_rewrite", + "trace:adversarial_probe:::::identity_override_policy_bypass", + "trace:adversarial_probe:::::identity_override_operator_injection", + "trace:replay_assertion:::::" + ], + "validated_set_sha": "6196c396f74393ce817bdd10aeab375f69ae243f145f68e67cbb1f83b18e0a31" +} diff --git a/scripts/ratify_ethics_pack.py b/scripts/ratify_ethics_pack.py index ef189245..14ee6448 100644 --- a/scripts/ratify_ethics_pack.py +++ b/scripts/ratify_ethics_pack.py @@ -32,7 +32,10 @@ from formation.runner import TurnObservation, run_plan ETHICS_DIR = Path(__file__).resolve().parents[1] / "packs" / "ethics" ISSUED_AT = "2026-05-17T00:00:00Z" -PACK_IDS: tuple[str, ...] = ("default_general_ethics_v1",) +PACK_IDS: tuple[str, ...] = ( + "default_general_ethics_v1", + "medical_clinical_ethics_v1", +) # Override attempts the ethics pack must refuse. Distinct from # safety/identity override sets: ethics-targeted overrides aim at the diff --git a/tests/test_long_context_comparison.py b/tests/test_long_context_comparison.py new file mode 100644 index 00000000..439608b9 --- /dev/null +++ b/tests/test_long_context_comparison.py @@ -0,0 +1,64 @@ +"""ADR-0045 — long-context recall comparison vs transformer baselines. + +CORE's vault recall is exact by construction: there is no approximate +index, no embedding compression, no attention bottleneck. The +comparison runner publishes: + +1. A needle-in-a-haystack measurement at multiple N showing CORE + returns the planted needle at top-1. +2. Frozen citations of published transformer long-context recall + numbers (Claude 2.1, GPT-4 Turbo 128k, Gemini 1.5 Pro, NVIDIA + RULER) as the comparator. + +If a future change to the vault breaks exact recall, this suite fails. +""" +from __future__ import annotations + +from evals.long_context_cost.comparison_runner import ( + DEFAULT_N_VALUES, + run_comparison, +) + + +class TestComparisonRunner: + def test_report_schema_is_stable(self) -> None: + report = run_comparison(n_values=(100, 1_000)) + assert report["schema_version"] == 1 + assert "core_measurements" in report + assert "transformer_baselines" in report + core = report["core_measurements"] + assert set(core.keys()) >= { + "n_values", + "recall_pct", + "exact_by_construction", + "per_n", + } + + def test_core_exact_recall_holds(self) -> None: + """The load-bearing claim: top-1 needle recall is correct at every N.""" + report = run_comparison(n_values=(100, 1_000, 10_000)) + assert report["claim_supported"] is True + assert report["core_measurements"]["recall_pct"] == 100.0 + for entry in report["core_measurements"]["per_n"]: + assert entry["top1_correct"] is True, entry + + def test_transformer_baselines_are_frozen_citations(self) -> None: + report = run_comparison(n_values=(100,)) + baselines = report["transformer_baselines"]["baselines"] + assert len(baselines) >= 3 + for b in baselines: + assert "source" in b + assert "url" in b + assert "context_window_tokens" in b + + def test_default_n_values_include_at_least_100k(self) -> None: + """Ensure the comparison exercises a large-N regime by default.""" + assert max(DEFAULT_N_VALUES) >= 100_000 + + +class TestCoreGuaranteeAdvertised: + def test_baseline_file_records_core_guarantee(self) -> None: + report = run_comparison(n_values=(100,)) + guarantee = report["transformer_baselines"]["core_guarantee"] + assert guarantee["recall_pct"] == 100.0 + assert guarantee["recall_kind"] == "exact_cga_inner_scan" diff --git a/tests/test_medical_clinical_ethics_pack.py b/tests/test_medical_clinical_ethics_pack.py new file mode 100644 index 00000000..c35c04e1 --- /dev/null +++ b/tests/test_medical_clinical_ethics_pack.py @@ -0,0 +1,84 @@ +"""ADR-0044 — worked-example domain ethics pack end-to-end. + +Locks in the load-bearing claims of the domain-pack worked example: + +1. The pack ratifies through the formation pipeline (self-sealed + `MasteryReport` with matching `mastery_report_sha256`). +2. The pack loads in production mode (i.e. the sealed report verifies). +3. Selecting the pack via `RuntimeConfig(ethics_pack=...)` composes its + commitments into the runtime manifold boundary set without losing + the universal safety floor. +4. `refusal_commitments` and `hedge_commitments` are mutually exclusive + and only cite known commitment ids. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from chat.runtime import ChatRuntime +from core.config import RuntimeConfig +from packs.ethics.loader import load_ethics_pack +from packs.safety.loader import load_safety_pack + + +PACK_ID = "medical_clinical_ethics_v1" +PACK_PATH = Path("packs/ethics") / f"{PACK_ID}.json" +REPORT_PATH = Path("packs/ethics") / f"{PACK_ID}.mastery_report.json" + + +class TestPackRatified: + def test_pack_file_exists(self) -> None: + assert PACK_PATH.is_file() + assert REPORT_PATH.is_file() + + def test_mastery_report_sha_is_set(self) -> None: + pack = json.loads(PACK_PATH.read_text()) + assert pack["mastery_report_sha256"] + assert len(pack["mastery_report_sha256"]) == 64 + + def test_loader_accepts_ratified_pack(self) -> None: + pack = load_ethics_pack(PACK_ID) + assert pack.pack_id == PACK_ID + assert pack.domain == "medical" + + +class TestDomainCommitments: + def test_six_domain_commitments(self) -> None: + pack = load_ethics_pack(PACK_ID) + assert len(pack.commitment_ids) == 6 + assert "no_dosing_recommendation" in pack.commitment_ids + assert "no_emergency_triage_authority" in pack.commitment_ids + assert "defer_diagnosis_to_clinician" in pack.commitment_ids + + def test_refusal_and_hedge_are_disjoint(self) -> None: + pack = load_ethics_pack(PACK_ID) + overlap = pack.refusal_commitments & pack.hedge_commitments + assert overlap == frozenset(), overlap + + def test_remediation_lists_are_subsets_of_commitments(self) -> None: + pack = load_ethics_pack(PACK_ID) + known = set(pack.commitment_ids) + assert set(pack.refusal_commitments) <= known + assert set(pack.hedge_commitments) <= known + + +class TestRuntimeComposition: + def test_manifold_includes_safety_floor_and_medical_commitments(self) -> None: + rt = ChatRuntime(config=RuntimeConfig(ethics_pack=PACK_ID)) + boundary_ids = set(rt.identity_manifold.boundary_ids) + safety = load_safety_pack() + # Safety floor unioned in. + for safety_id in safety.boundary_ids: + assert safety_id in boundary_ids, safety_id + # Medical commitments unioned in. + medical_pack = load_ethics_pack(PACK_ID) + for c in medical_pack.commitment_ids: + assert c in boundary_ids, c + + def test_default_general_does_not_include_medical_commitments(self) -> None: + """Pack swap is visible — default pack must not carry the medical floor.""" + rt = ChatRuntime(config=RuntimeConfig(ethics_pack="default_general_ethics_v1")) + boundary_ids = set(rt.identity_manifold.boundary_ids) + assert "no_dosing_recommendation" not in boundary_ids + assert "no_emergency_triage_authority" not in boundary_ids