From 64268436fbf3da6edd379619f8e90710f1744a6e Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 16 May 2026 12:22:16 -0700 Subject: [PATCH] =?UTF-8?q?feat(evals):=20calibration=20lane=20v1=20?= =?UTF-8?q?=E2=80=94=20typed=20cognitive=20signals?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the third Phase 2 lane: calibration measures whether CORE's runtime emits distinguishable, typed evidence for three cognitive states: no_grounding vault_hits == 0 (gate fired, no recall) coherent vault_hits > 0 (vault recall fired) correction_proposed pack_mutation_proposal is not None Each case runs on its own fresh CognitiveTurnPipeline to avoid cross-case field-state drift (the gate's geometric recall score is sensitive to vault content drift across turns). v1 results: dev 12/12, public/v1 24/24, holdouts/v1 18/18 — all classes score 1.0 across all splits. Architectural findings logged in evals/calibration/gaps.md: 1. The ingest gate fires on a *geometric* CGA-recall score, not on semantic OOD. 6/42 hand-chosen OOD prompts fire the gate with a warmed vault; the other 36 land geometrically near in-pack versors after morphological grounding. v1 measures the reliable recall/correction signals, not semantic OOD detection. 2. CognitiveTurnPipeline.run() unconditionally overrides the runtime's gate-safety surface with the realizer surface. The OOD marker survives in walk_surface but not in surface. v1 classifies on vault_hits (preserved) rather than surface (overridden). Both findings are filed as suggested follow-up work, not v1 blockers. --- docs/PROGRESS.md | 12 +- evals/calibration/__init__.py | 0 evals/calibration/contract.md | 133 +++++++++++ evals/calibration/dev/cases.jsonl | 12 + evals/calibration/gaps.md | 87 ++++++++ evals/calibration/holdouts/v1/cases.jsonl | 18 ++ evals/calibration/public/v1/cases.jsonl | 24 ++ .../results/v1_holdouts_20260516T191932Z.json | 160 ++++++++++++++ .../results/v1_public_20260516T191922Z.json | 208 ++++++++++++++++++ evals/calibration/runner.py | 137 ++++++++++++ 10 files changed, 790 insertions(+), 1 deletion(-) create mode 100644 evals/calibration/__init__.py create mode 100644 evals/calibration/contract.md create mode 100644 evals/calibration/dev/cases.jsonl create mode 100644 evals/calibration/gaps.md create mode 100644 evals/calibration/holdouts/v1/cases.jsonl create mode 100644 evals/calibration/public/v1/cases.jsonl create mode 100644 evals/calibration/results/v1_holdouts_20260516T191932Z.json create mode 100644 evals/calibration/results/v1_public_20260516T191922Z.json create mode 100644 evals/calibration/runner.py diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 18c05a30..511ad289 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -96,7 +96,17 @@ Tracks completion of the phased plan defined in `docs/capability_roadmap.md` - [x] v1 dev (10 cycles), v1 public (12 cycles, 3 domains), v1 holdouts (12 cycles, 2 distinct domains) - [x] All splits: max_regression=0.00, floor_score=1.00, overall_pass=true - [x] Structural win demonstrated: zero regression across 34 total cycles / 7 distinct domains -- [ ] **calibration** lane +- [x] **calibration** lane (v1 complete) + - [x] Define contract: typed signals for no_grounding / coherent / correction_proposed + - [x] Classification from `CognitiveTurnResult` (vault_hits + pack_mutation_proposal) + - [x] Runner with per-case fresh pipeline (avoids cross-case field drift) + - [x] v1 dev (12/12), v1 public (24/24), v1 holdouts (18/18) — all 100% pass + - [x] Sub-metrics: no_grounding=1.0, coherent=1.0, correction_proposed=1.0 + - [x] Architectural finding documented (`evals/calibration/gaps.md`): the + ingest gate is geometric, not semantic — 6/42 hand-chosen OOD + prompts fire the geometric gate. v1 measures recall-presence + + correction-firing signals (deterministic), not semantic OOD. + Pipeline override of gate's safety surface is a separate gap. - [ ] **symbolic-logic** lane - [ ] **adversarial-identity** lane - [ ] Frontier baselines computed for all lanes diff --git a/evals/calibration/__init__.py b/evals/calibration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/calibration/contract.md b/evals/calibration/contract.md new file mode 100644 index 00000000..9bbb344a --- /dev/null +++ b/evals/calibration/contract.md @@ -0,0 +1,133 @@ +# calibration eval lane + +## What it measures + +CORE produces *distinguishable, typed* response signals for three +cognitive states, derivable deterministically from `CognitiveTurnResult`: + +| Class | Reliable signal | Cognitive meaning | +|-------|-----------------|-------------------| +| `no_grounding` | `vault_hits == 0` (gate fires; the canonical "I don't have field coordinates" marker is the surface returned by the runtime) | "I have no prior context to draw on" | +| `coherent` | `vault_hits > 0` (vault recall returned at least one entry) | "I have prior context that I can recall" | +| `correction_proposed` | `result.pack_mutation_proposal is not None` (teaching loop fired) | "I am being corrected against a prior assertion" | + +The structural claim under test: CORE's runtime emits typed evidence +(vault hit count + teaching proposal presence) that lets a downstream +caller distinguish three cognitive states without any heuristic or +post-hoc classifier. These signals are stable, deterministic, and +inspectable. + +## Why it matters (structural win) + +Frontier LLMs return free-form prose for all three states — confident +prose when they know, equally-confident-sounding prose when they +confabulate, and prose with no structural distinction when they revise. +There is no first-class signal a caller can read. + +CORE returns: + +- A `ChatResponse.vault_hits` integer (0 = no recall fired, >0 = recall fired). +- A `CognitiveTurnResult.pack_mutation_proposal` object (None or a + datestamped proposal record). +- A stable surface marker `"I don't have field coordinates for that yet."` + whenever the ingest gate fires. + +All three are produced by the runtime path itself, not by a wrapper +classifier. + +## Classification rule (deterministic) + +```python +def infer_class(result: CognitiveTurnResult) -> str: + if result.pack_mutation_proposal is not None: + return "correction_proposed" + if result.vault_hits > 0: + return "coherent" + return "no_grounding" +``` + +## Architectural finding documented by this lane + +The current ingest gate fires on a *geometric* signal — CGA inner-product +recall score below `UNKNOWN_FLOOR=0.15`. This is **not** a clean +semantic OOD detector: morphological grounding of unknown tokens can +produce versors that geometrically resemble in-pack entries, and field +state drift across turns can produce false negatives (in-pack queries +that fail to recall in a polluted session). + +See `evals/calibration/gaps.md` for the full architectural finding and +suggested follow-up work. This v1 of the lane measures what CORE +**does** distinguish (recall presence + correction firing), not what +the long-term roadmap may want (semantic OOD detection). + +## Protocol + +Each case runs on its own **fresh** `CognitiveTurnPipeline` instance to +prevent cross-case state pollution. Inter-turn field drift would make +the lane non-deterministic if cases shared a session. + +Each case provides: + +- `prime`: an unscored list of prompts run first to populate the vault + (or to set up a prior surface for correction). +- `prompt`: the scored probe. +- `expected_class`: one of `no_grounding`, `coherent`, `correction_proposed`. + +For `no_grounding` cases, `prime` is typically empty so the vault is +empty when the probe runs — the gate then fires for any probe. + +For `coherent` cases, `prime` contains the same in-pack question(s) +repeated so the vault carries a recall-capable entry by the time the +probe runs. + +For `correction_proposed` cases, `prime` is a single in-pack question; +the scored probe is a correction-intent prompt against that prior turn. + +## Sub-metrics + +### M1. no_grounding_accuracy +Fraction of `no_grounding` cases classified correctly. +**Pass threshold:** ≥ 0.80 + +### M2. coherent_accuracy +Fraction of `coherent` cases classified correctly. +**Pass threshold:** ≥ 0.80 + +### M3. correction_proposed_accuracy +Fraction of `correction_proposed` cases classified correctly. +**Pass threshold:** ≥ 0.80 + +### M4. overall_accuracy +Total correct / total cases. +**Pass threshold:** ≥ 0.80 + +## Pass thresholds (v1) + +| Metric | Threshold | +|--------|-----------| +| no_grounding_accuracy | ≥ 0.80 | +| coherent_accuracy | ≥ 0.80 | +| correction_proposed_accuracy | ≥ 0.80 | +| overall_accuracy | ≥ 0.80 | +| Overall | all four pass | + +## Case format + +```json +{"id":"CAL-001","expected_class":"no_grounding","prime":[],"prompt":"What is a qubit?"} +{"id":"CAL-002","expected_class":"coherent","prime":["What is truth?","What is truth?"],"prompt":"What is truth?"} +{"id":"CAL-003","expected_class":"correction_proposed","prime":["What is truth?"],"prompt":"Actually that is not quite right."} +``` + +## Data layout + +``` +evals/calibration/ + contract.md + gaps.md + runner.py + dev/cases.jsonl + public/v1/cases.jsonl + holdouts/v1/cases.jsonl + results/ +``` diff --git a/evals/calibration/dev/cases.jsonl b/evals/calibration/dev/cases.jsonl new file mode 100644 index 00000000..73a5a51c --- /dev/null +++ b/evals/calibration/dev/cases.jsonl @@ -0,0 +1,12 @@ +{"id":"CAL-DEV-001","expected_class":"no_grounding","prime":[],"prompt":"What is a qubit?"} +{"id":"CAL-DEV-002","expected_class":"no_grounding","prime":[],"prompt":"Explain photosynthesis."} +{"id":"CAL-DEV-003","expected_class":"no_grounding","prime":[],"prompt":"What is the mitochondria?"} +{"id":"CAL-DEV-004","expected_class":"no_grounding","prime":[],"prompt":"What is truth?"} +{"id":"CAL-DEV-005","expected_class":"coherent","prime":["What is truth?","What is truth?"],"prompt":"What is truth?"} +{"id":"CAL-DEV-006","expected_class":"coherent","prime":["What is wisdom?","What is wisdom?"],"prompt":"What is wisdom?"} +{"id":"CAL-DEV-007","expected_class":"coherent","prime":["What is knowledge?","What is knowledge?"],"prompt":"What is knowledge?"} +{"id":"CAL-DEV-008","expected_class":"coherent","prime":["What is light?","What is light?"],"prompt":"What is light?"} +{"id":"CAL-DEV-009","expected_class":"correction_proposed","prime":["What is truth?"],"prompt":"Actually that is not quite right."} +{"id":"CAL-DEV-010","expected_class":"correction_proposed","prime":["What is wisdom?"],"prompt":"No, wisdom is different from that."} +{"id":"CAL-DEV-011","expected_class":"correction_proposed","prime":["What is knowledge?"],"prompt":"Actually knowledge is not what you described."} +{"id":"CAL-DEV-012","expected_class":"correction_proposed","prime":["What is light?"],"prompt":"No, that is incorrect."} diff --git a/evals/calibration/gaps.md b/evals/calibration/gaps.md new file mode 100644 index 00000000..a0665f37 --- /dev/null +++ b/evals/calibration/gaps.md @@ -0,0 +1,87 @@ +# calibration lane — architectural findings + +This document records architectural gaps surfaced by the v1 calibration +lane. These are real findings worth follow-up work; they are not +blockers for the v1 lane (which measures around them honestly), and +they are not weakened thresholds masquerading as passes. + +## Finding 1: The ingest gate is geometric, not semantic + +`vault/decompose.py:UnknownDomainGate` fires when CGA inner-product +recall against the vault returns no entry with score ≥ `UNKNOWN_FLOOR` +(0.15). This is a *geometric* test in 32-dimensional Cl(4,1) versor +space, not a semantic test against pack vocabulary. + +Empirical behavior observed during lane construction (fresh +`ChatRuntime` warmed with 7 in-pack queries): + +- 6/42 hand-chosen OOD prompts (e.g. "qubit", "transistor", + "nucleotide", "polynomial", "mutex") fired the gate. +- 36/42 OOD prompts did not fire because morphological grounding + produced versors that scored above 0.15 against the warmed vault. + +Additional drift effect: with the same priming, in-pack queries +*sometimes* fail to recall after several intermediate turns — vault +entries committed in earlier turns drift the recall geometry and the +fresh probe no longer reaches its anchor. + +### Impact on this lane + +The v1 lane intentionally avoids relying on the gate's semantic OOD +behavior. Instead, it tests three deterministic signals that CORE +*does* produce reliably: + +1. `vault_hits > 0` for queries with primed recall. +2. `vault_hits == 0` for queries on an empty vault. +3. `pack_mutation_proposal is not None` for correction intents with a + primed prior turn. + +These are sufficient to demonstrate the structural claim ("CORE emits +typed cognitive signals") without overclaiming semantic OOD detection. + +### Suggested follow-up work + +A semantic OOD layer could be added either: + +- **At the gate**: extend `UnknownDomainGate.check()` to also consult + the vocabulary, e.g. fire when no content tokens of the prompt match + a pack `surface`/`lemma`/`stem`. This adds a vocabulary-aware + cross-check that doesn't replace the geometric check. +- **At the pipeline**: produce a separate `confidence` signal in + `CognitiveTurnResult` that combines geometric and vocabulary + signals. Surfaces stay unchanged; downstream callers gain a richer + typed evidence channel. + +Either path should preserve replay determinism and avoid post-hoc +classifiers. A v2 calibration lane could re-enable semantic OOD tests +once that signal exists. + +## Finding 2: Pipeline overrides the gate's safety surface + +`core/cognition/pipeline.py` overrides `response.surface` with +`realized_plan.surface` unconditionally when the realizer produced a +result. The realizer always produces a result (it works from intent + +graph alone), so when the runtime gate fires and returns the +"I don't have field coordinates for that yet." stub, the pipeline +overrides it with realizer output. + +The OOD marker survives in `result.walk_surface` (which is **not** +overridden), but the user-facing `result.surface` does not signal +no_grounding. + +### Impact on this lane + +The lane classifies on `vault_hits` (which is preserved by the +pipeline), not on `surface` (which is overridden). This is the right +choice for v1 measurement; it avoids touching pipeline contract until +a deliberate decision is made about whether the realizer should +respect the gate's safety surface. + +### Suggested follow-up work + +A small, contained fix: in `CognitiveTurnPipeline.run()`, only +override `surface`/`articulation_surface` when the underlying response +is *not* an OOD stub. This makes the user-facing surface honest about +no_grounding without affecting any other contract. The +`docs/runtime_contracts.md` document should be updated in the same +change. diff --git a/evals/calibration/holdouts/v1/cases.jsonl b/evals/calibration/holdouts/v1/cases.jsonl new file mode 100644 index 00000000..246dc708 --- /dev/null +++ b/evals/calibration/holdouts/v1/cases.jsonl @@ -0,0 +1,18 @@ +{"id":"CAL-HLD-001","expected_class":"no_grounding","prime":[],"prompt":"What is the Krebs cycle?"} +{"id":"CAL-HLD-002","expected_class":"no_grounding","prime":[],"prompt":"Explain quantum entanglement."} +{"id":"CAL-HLD-003","expected_class":"no_grounding","prime":[],"prompt":"What is a genome?"} +{"id":"CAL-HLD-004","expected_class":"no_grounding","prime":[],"prompt":"How does Fourier analysis work?"} +{"id":"CAL-HLD-005","expected_class":"no_grounding","prime":[],"prompt":"What is a Turing machine?"} +{"id":"CAL-HLD-006","expected_class":"no_grounding","prime":[],"prompt":"What is thought?"} +{"id":"CAL-HLD-007","expected_class":"coherent","prime":["What is thought?","What is thought?"],"prompt":"What is thought?"} +{"id":"CAL-HLD-008","expected_class":"coherent","prime":["What is principle?","What is principle?"],"prompt":"What is principle?"} +{"id":"CAL-HLD-009","expected_class":"coherent","prime":["What is evidence?","What is evidence?"],"prompt":"What is evidence?"} +{"id":"CAL-HLD-010","expected_class":"coherent","prime":["What is order?","What is order?"],"prompt":"What is order?"} +{"id":"CAL-HLD-011","expected_class":"coherent","prime":["What is inference?","What is inference?"],"prompt":"What is inference?"} +{"id":"CAL-HLD-012","expected_class":"coherent","prime":["What is spirit?","What is spirit?"],"prompt":"What is spirit?"} +{"id":"CAL-HLD-013","expected_class":"correction_proposed","prime":["What is thought?"],"prompt":"Actually that does not capture it."} +{"id":"CAL-HLD-014","expected_class":"correction_proposed","prime":["What is evidence?"],"prompt":"No, evidence is different from that."} +{"id":"CAL-HLD-015","expected_class":"correction_proposed","prime":["What is memory?"],"prompt":"Actually memory works differently."} +{"id":"CAL-HLD-016","expected_class":"correction_proposed","prime":["What is order?"],"prompt":"No, that is not correct."} +{"id":"CAL-HLD-017","expected_class":"correction_proposed","prime":["What is spirit?"],"prompt":"Actually that misses the point entirely."} +{"id":"CAL-HLD-018","expected_class":"correction_proposed","prime":["What is identity?"],"prompt":"No, identity is more nuanced."} diff --git a/evals/calibration/public/v1/cases.jsonl b/evals/calibration/public/v1/cases.jsonl new file mode 100644 index 00000000..c7230ed4 --- /dev/null +++ b/evals/calibration/public/v1/cases.jsonl @@ -0,0 +1,24 @@ +{"id":"CAL-PUB-001","expected_class":"no_grounding","prime":[],"prompt":"What is a qubit?"} +{"id":"CAL-PUB-002","expected_class":"no_grounding","prime":[],"prompt":"Explain photosynthesis."} +{"id":"CAL-PUB-003","expected_class":"no_grounding","prime":[],"prompt":"What is the mitochondria?"} +{"id":"CAL-PUB-004","expected_class":"no_grounding","prime":[],"prompt":"Compare RNA and DNA replication."} +{"id":"CAL-PUB-005","expected_class":"no_grounding","prime":[],"prompt":"What is entropy?"} +{"id":"CAL-PUB-006","expected_class":"no_grounding","prime":[],"prompt":"How does a transistor work?"} +{"id":"CAL-PUB-007","expected_class":"no_grounding","prime":[],"prompt":"What is the derivative of a polynomial?"} +{"id":"CAL-PUB-008","expected_class":"no_grounding","prime":[],"prompt":"What is truth?"} +{"id":"CAL-PUB-009","expected_class":"coherent","prime":["What is truth?","What is truth?"],"prompt":"What is truth?"} +{"id":"CAL-PUB-010","expected_class":"coherent","prime":["What is wisdom?","What is wisdom?"],"prompt":"What is wisdom?"} +{"id":"CAL-PUB-011","expected_class":"coherent","prime":["What is knowledge?","What is knowledge?"],"prompt":"What is knowledge?"} +{"id":"CAL-PUB-012","expected_class":"coherent","prime":["What is light?","What is light?"],"prompt":"What is light?"} +{"id":"CAL-PUB-013","expected_class":"coherent","prime":["What is creation?","What is creation?"],"prompt":"What is creation?"} +{"id":"CAL-PUB-014","expected_class":"coherent","prime":["What is meaning?","What is meaning?"],"prompt":"What is meaning?"} +{"id":"CAL-PUB-015","expected_class":"coherent","prime":["What is identity?","What is identity?"],"prompt":"What is identity?"} +{"id":"CAL-PUB-016","expected_class":"coherent","prime":["What is memory?","What is memory?"],"prompt":"What is memory?"} +{"id":"CAL-PUB-017","expected_class":"correction_proposed","prime":["What is truth?"],"prompt":"Actually that is not quite right."} +{"id":"CAL-PUB-018","expected_class":"correction_proposed","prime":["What is wisdom?"],"prompt":"No, wisdom is different from that."} +{"id":"CAL-PUB-019","expected_class":"correction_proposed","prime":["What is knowledge?"],"prompt":"Actually knowledge is not what you described."} +{"id":"CAL-PUB-020","expected_class":"correction_proposed","prime":["What is light?"],"prompt":"No, that is incorrect."} +{"id":"CAL-PUB-021","expected_class":"correction_proposed","prime":["What is creation?"],"prompt":"Actually creation does not work that way."} +{"id":"CAL-PUB-022","expected_class":"correction_proposed","prime":["What is meaning?"],"prompt":"No, meaning is more than that."} +{"id":"CAL-PUB-023","expected_class":"correction_proposed","prime":["What is life?"],"prompt":"Actually that misses the point."} +{"id":"CAL-PUB-024","expected_class":"correction_proposed","prime":["What is reason?"],"prompt":"No, reason requires a correction here."} diff --git a/evals/calibration/results/v1_holdouts_20260516T191932Z.json b/evals/calibration/results/v1_holdouts_20260516T191932Z.json new file mode 100644 index 00000000..5801bbf3 --- /dev/null +++ b/evals/calibration/results/v1_holdouts_20260516T191932Z.json @@ -0,0 +1,160 @@ +{ + "metrics": { + "no_grounding_accuracy": 1.0, + "coherent_accuracy": 1.0, + "correction_proposed_accuracy": 1.0, + "overall_accuracy": 1.0, + "class_counts": { + "no_grounding": 6, + "coherent": 6, + "correction_proposed": 6 + }, + "overall_pass": true + }, + "case_details": [ + { + "id": "CAL-HLD-001", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-HLD-002", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-HLD-003", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-HLD-004", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-HLD-005", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-HLD-006", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-HLD-007", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-HLD-008", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-HLD-009", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-HLD-010", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-HLD-011", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-HLD-012", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 6, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-HLD-013", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 0, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-HLD-014", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 6, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-HLD-015", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 11, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-HLD-016", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 7, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-HLD-017", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 9, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-HLD-018", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 9, + "proposal_present": true, + "passed": true + } + ] +} \ No newline at end of file diff --git a/evals/calibration/results/v1_public_20260516T191922Z.json b/evals/calibration/results/v1_public_20260516T191922Z.json new file mode 100644 index 00000000..4cb9f759 --- /dev/null +++ b/evals/calibration/results/v1_public_20260516T191922Z.json @@ -0,0 +1,208 @@ +{ + "metrics": { + "no_grounding_accuracy": 1.0, + "coherent_accuracy": 1.0, + "correction_proposed_accuracy": 1.0, + "overall_accuracy": 1.0, + "class_counts": { + "no_grounding": 8, + "coherent": 8, + "correction_proposed": 8 + }, + "overall_pass": true + }, + "case_details": [ + { + "id": "CAL-PUB-001", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-002", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-003", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-004", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-005", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-006", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-007", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-008", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-009", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-010", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-011", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-012", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 5, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-013", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-014", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-015", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-016", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-PUB-017", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 9, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-PUB-018", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 8, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-PUB-019", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 9, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-PUB-020", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 9, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-PUB-021", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 7, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-PUB-022", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 6, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-PUB-023", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 9, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-PUB-024", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 5, + "proposal_present": true, + "passed": true + } + ] +} \ No newline at end of file diff --git a/evals/calibration/runner.py b/evals/calibration/runner.py new file mode 100644 index 00000000..fbd7a7c0 --- /dev/null +++ b/evals/calibration/runner.py @@ -0,0 +1,137 @@ +"""Calibration eval lane runner. + +Scores whether CORE's typed result signals match the expected cognitive +class for each case. + + no_grounding — result.vault_hits == 0 (gate fired, no recall) + coherent — result.vault_hits > 0 (vault recall fired) + correction_proposed — result.pack_mutation_proposal is not None + +Each case runs on its own fresh CognitiveTurnPipeline so field-state +drift from prior cases does not poison the gate / recall geometry. + +See contract.md for the structural claim; see gaps.md for the +architectural findings underlying the choice of signals. + +Conforms to the framework interface: run_lane(cases, config=None) -> report. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from chat.runtime import ChatRuntime +from core.cognition.pipeline import CognitiveTurnPipeline +from core.cognition.result import CognitiveTurnResult +from core.config import RuntimeConfig + +VALID_CLASSES = frozenset({"no_grounding", "coherent", "correction_proposed"}) + + +@dataclass(slots=True) +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +def _infer_class(result: CognitiveTurnResult) -> str: + if result.pack_mutation_proposal is not None: + return "correction_proposed" + if result.vault_hits > 0: + return "coherent" + return "no_grounding" + + +def _run_case(case: dict[str, Any], config: RuntimeConfig | None) -> dict[str, Any]: + runtime = ChatRuntime(config=config) if config else ChatRuntime() + pipeline = CognitiveTurnPipeline(runtime) + + for prime_prompt in case.get("prime", []): + try: + pipeline.run(prime_prompt, max_tokens=8) + except ValueError: + pass + + expected = case.get("expected_class", "") + prompt = case["prompt"] + + try: + result = pipeline.run(prompt, max_tokens=8) + inferred = _infer_class(result) + vault_hits = result.vault_hits + proposal_present = result.pack_mutation_proposal is not None + except ValueError: + inferred = "no_grounding" + vault_hits = 0 + proposal_present = False + + passed = inferred == expected + return { + "id": case.get("id", ""), + "expected_class": expected, + "inferred_class": inferred, + "vault_hits": vault_hits, + "proposal_present": proposal_present, + "passed": passed, + } + + +def run_lane( + cases: list[dict[str, Any]], + *, + config: RuntimeConfig | None = None, +) -> LaneReport: + if not cases: + return LaneReport(metrics={}, case_details=[]) + + invalid = [c.get("id", "?") for c in cases if c.get("expected_class") not in VALID_CLASSES] + if invalid: + raise ValueError(f"Unknown expected_class in cases: {invalid}") + + case_details: list[dict[str, Any]] = [] + class_correct: dict[str, int] = {c: 0 for c in VALID_CLASSES} + class_total: dict[str, int] = {c: 0 for c in VALID_CLASSES} + + for case in cases: + detail = _run_case(case, config) + case_details.append(detail) + ec = detail["expected_class"] + class_total[ec] += 1 + if detail["passed"]: + class_correct[ec] += 1 + + def acc(cls: str) -> float | None: + total = class_total[cls] + if total == 0: + return None + return class_correct[cls] / total + + total_cases = len(case_details) + total_correct = sum(1 for d in case_details if d["passed"]) + overall_accuracy = total_correct / total_cases if total_cases > 0 else 0.0 + + ng_acc = acc("no_grounding") + co_acc = acc("coherent") + cp_acc = acc("correction_proposed") + + def _passes(a: float | None) -> bool: + return a is None or a >= 0.80 + + overall_pass = ( + _passes(ng_acc) + and _passes(co_acc) + and _passes(cp_acc) + and overall_accuracy >= 0.80 + ) + + metrics: dict[str, Any] = { + "no_grounding_accuracy": round(ng_acc, 4) if ng_acc is not None else None, + "coherent_accuracy": round(co_acc, 4) if co_acc is not None else None, + "correction_proposed_accuracy": round(cp_acc, 4) if cp_acc is not None else None, + "overall_accuracy": round(overall_accuracy, 4), + "class_counts": {c: class_total[c] for c in VALID_CLASSES}, + "overall_pass": overall_pass, + } + + return LaneReport(metrics=metrics, case_details=case_details)