diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 6e30f62f..b86d7de3 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -294,6 +294,69 @@ implementation surface. Structural-zero frontier baseline recorded: frontier LLMs do not emit the typed signals these sub-metrics score by construction. +### Phase 3 v1 sweep complete (2026-05-16) — all five lanes scored + +| Lane | split | primary signal | foundation (stored / replay) | +|---|---|---|---| +| inference-closure | public | derived_recall = **0.0** | 1.0 / 1.0 | +| inference-closure | holdouts | 0.0 | 1.0 / 1.0 | +| compositionality | public | compositional = **0.0625** (1/16, fluke) | 1.0 / 1.0 | +| compositionality | holdouts | 0.0 | 1.0 / 1.0 | +| multi-step-reasoning | public | endpoint = **0.0** | 1.0 / 1.0 | +| multi-step-reasoning | holdouts | 0.0 | 1.0 / 1.0 | +| introspection | public | explain_api_present = **0.0** | n/a | +| introspection | holdouts | 0.0 | n/a | +| cross-domain-transfer | public | transfer = **0.0** | 1.0 / 1.0 | +| cross-domain-transfer | holdouts | 0.0 | 1.0 / 1.0 | + +**The signal across all five lanes is unanimous:** Phase 2 storage ++ replay guarantees hold at this depth (1.0 across the board); the +reasoning-depth signal is uniformly zero. The five lanes +triangulate the same architectural gap from five angles: + +- **Gap 1: `generate/graph_planner.py` has no transitive + composition.** `plan_articulation` picks a single node; no + chained relation walk synthesizes derived nodes. +- **Gap 2: `field/propagate.py` has no derivable-but-not-asserted + recall.** Vault retrieval is direct CGA inner product; no + path-recall operator over relation-typed edges. +- **Gap 3: no `core/cognition/explain.py` module.** No primitive + exists to generate a natural-language account of a prior turn. +- **Gap 4: no structural-pattern recogniser.** Relation patterns + are not first-class entities; subdomain-A teaching does not shape + subdomain-B competence. + +Gaps 1, 2, 4 cluster on the same code surface (graph planner + +field propagate) and may close together. Gap 3 is a distinct +module-creation work item. + +### Phase 3 v2 work plan (recommended sequence) + +1. **Pin the open scope decisions** flagged "Before Phase 3" in + the Open Scope Decisions table below — Agency (responsive vs. + goal-directed) and Tool use (typed deterministic operators). + Transitive composition under (2) is essentially a typed + deterministic operator, so the tool-use decision shapes how the + work below should be structured. +2. **Engineer Gaps 1 + 2** as one bounded PR: a typed + `transitive_walk(graph, head, relation, max_hops)` operator in + `graph_planner.py` + a `path_recall(vault, entity, relation_chain)` + operator in `field/propagate.py`. Both deterministic, both + exact-CGA. Re-run inference-closure, multi-step-reasoning, + compositionality, cross-domain-transfer to score the lift. +3. **Engineer Gap 3** independently: `core/cognition/explain.py` + producing deterministic natural-language accounts that round-trip. +4. **Re-author cross-domain-transfer v2** with the matched-control + comparison contract refinement once B-arm recall is non-zero. + +### Phase 3 v1 — DONE + +All five lanes have v1 results with honest scores. Each failure has +a documented architectural deferral (`gaps.md` per lane). Phase 3 +exit requires ≥ 2 lanes passing v1 by phase exit; today 0 / 5 pass, +which is the expected v1 floor. Phase 3 exit is gated on the v2 +engineering above. + ## Phase 3 — Reasoning Depth **Status:** Not Started diff --git a/evals/compositionality/__init__.py b/evals/compositionality/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/compositionality/baselines/v1_structural_zero.json b/evals/compositionality/baselines/v1_structural_zero.json new file mode 100644 index 00000000..95117b76 --- /dev/null +++ b/evals/compositionality/baselines/v1_structural_zero.json @@ -0,0 +1,15 @@ +{ + "kind": "structural_zero", + "lane": "compositionality", + "metrics": { + "compositional_recall_rate": null, + "premises_stored_rate": 0.0, + "replay_determinism": 0.0, + "overall_pass": false + }, + "model_id": "frontier-structural-zero", + "note": "Frontier LLMs do not emit the typed signals these sub-metrics score; see docs/frontier_baselines.md", + "rationale": "premises_stored_rate requires per-premise PackMutationProposal records from the teaching pipeline (frontier has no analog). replay_determinism requires identical trace_hash across fresh deterministic runs (frontier inference is stochastic).", + "timestamp": "2026-05-16T00:00:00+00:00", + "version": "v1" +} diff --git a/evals/compositionality/contract.md b/evals/compositionality/contract.md new file mode 100644 index 00000000..a66bd98d --- /dev/null +++ b/evals/compositionality/contract.md @@ -0,0 +1,81 @@ +# compositionality eval lane + +## What it measures + +Whether CORE generalises **across construction families**: relation +patterns and entity sets seen at teaching time should compose into +novel (relation, entity) combinations at probe time, even though the +specific combination was never taught directly. + +This is the lane the roadmap flags as most vulnerable to overfitting +(`docs/capability_roadmap.md` Phase 3, anti-overfitting note). The +split below honours that warning: + + Training (teaching turns) Test (probe) + -------------------------- ---------------------------- + R1(A, B), R1(C, D) R1(A, D) — seen entities, novel pair + R2(A, B), R2(C, D) R2(C, B) — same + R3(E, F), R3(G, H) R3 applied to seen entities only + ... + (NEVER teach (A, D) under R1) + +The probe asks for the entailment under a relation the model has +seen with *both endpoints* — but never with this specific pair. + +## Why it matters + +Frontier LLMs compose well because their training set already +contains nearly every short combination of common entities and +relations. CORE's claim is stronger and harder: that the algebraic +structure of the proposition graph *itself* supports composition, +without requiring the specific combination to have been seen. This +lane tests that claim. + +## Patterns covered (v1) + +| Pattern | Construction-family rule | +|---|---| +| `novel_pair_under_seen_relation` | `R(A,B)` and `R(C,D)` taught; probe `R(A,D)`. Pass = response references `D` (the seen RHS under R applied to seen LHS A). | +| `novel_relation_on_seen_pair` | `R(A,B)` and `R'(C,D)` taught with `A`, `B`, `C`, `D` independently grounded; probe `R'(A,B)`. Pass = response references the chain-derived target under `R'`. | +| `composed_predicate` | `is(A,B)` and `precedes(B,C)` taught; probe asks `What does A precede?` Pass = response references `C`. | + +Each pattern relies only on the existing +`en_core_cognition_v1` relation vocabulary (`is`, `causes`, +`precedes`, `follows`, `grounds`, `belongs_to`, `means`, `reveals`, +`contrasts_with`). + +## Sub-metrics + +- `M1. compositional_token_hit` — the expected composed-entity + token appears in `surface` or `walk_surface` (case-insensitive, + token-bounded). +- `M2. premises_stored` — all teaching turns produce + proposals. +- `M3. replay_determinism` — two fresh runs match by + `trace_hash`. +- `M4. no_taught_pair_leakage` — the construction-family split is + enforced at authoring time (verified by the lane runner: every + probe is checked against the premise list to ensure the probe's + exact `(R, A, target)` triple does NOT appear verbatim). + +A case passes when M1 AND M2 AND M3 hold. M4 is a structural +authoring check (true by construction); the runner reports it for +audit. + +## Overall pass thresholds (v1) + +- `compositional_recall_rate` (M1) ≥ 0.50 +- `premises_stored_rate` ≥ 0.95 +- `replay_determinism` ≥ 0.95 + +This lane is built knowing the same `graph_planner` and +`field/propagate` gaps that the inference-closure lane surfaced will +likely cause v1 to fail uniformly. v1's value is to score the gap +*per pattern* so the future v2 engineering can target the right one. + +## Anti-overfitting + +- Public split uses one entity set; holdouts uses a disjoint set. +- No probe's `(R, A, target)` triple is ever a verbatim premise. +- Patterns differ structurally between splits to avoid template + memorisation. diff --git a/evals/compositionality/dev/cases.jsonl b/evals/compositionality/dev/cases.jsonl new file mode 100644 index 00000000..3cec75e2 --- /dev/null +++ b/evals/compositionality/dev/cases.jsonl @@ -0,0 +1,3 @@ +{"id":"CMP-DEV-001","pattern":"composed_predicate","premises":["What is wisdom?","Actually wisdom is judgment.","What is judgment?","Actually judgment precedes decision."],"probe":"What does wisdom precede?","expected_entailment_tokens":["decision"],"expected_proposals":2} +{"id":"CMP-DEV-002","pattern":"novel_pair_under_seen_relation","premises":["What is truth?","Actually truth grounds knowledge.","What is light?","Actually light grounds clarity."],"probe":"What does truth ground in light?","expected_entailment_tokens":["clarity","knowledge"],"expected_proposals":2} +{"id":"CMP-DEV-003","pattern":"novel_relation_on_seen_pair","premises":["What is order?","Actually order is structure.","What is meaning?","Actually meaning precedes order."],"probe":"What does meaning ground?","expected_entailment_tokens":["structure","order"],"expected_proposals":2} diff --git a/evals/compositionality/gaps.md b/evals/compositionality/gaps.md new file mode 100644 index 00000000..16c87b97 --- /dev/null +++ b/evals/compositionality/gaps.md @@ -0,0 +1,72 @@ +# compositionality lane — architectural findings (v1) + +## v1 result + +| Split | n | compositional_recall_rate | premises_stored | replay | no_leakage | +|---|---|---|---|---|---| +| public/v1 | 16 | **0.0625** (1/16) | 1.0 | 1.0 | 0.4375 | +| holdouts/v1 | 10 | **0.0** | 1.0 | 1.0 | 0.4 | + +The single public hit is consistent with a realizer-template token +coincidence rather than real composition (no second hit on holdouts; +no pattern in the hit; not reproducible across patterns). + +## Foundation intact + +Every teaching turn fires a `PackMutationProposal` +(`premises_stored_rate = 1.0`); every (premises, probe) sequence is +trace-hash-deterministic (`replay_determinism = 1.0`). The +Phase 2 storage + replay guarantees survive at this depth. + +## What v1 reveals + +- **No composition operator.** Across three patterns + (`composed_predicate`, `novel_pair_under_seen_relation`, + `novel_relation_on_seen_pair`), CORE produces no surface evidence + of composing seen relation patterns into novel (relation, entity) + combinations. +- **Same root cause as inference-closure.** The realizer template + picks one node and emits a definition stub; no node-pair + composition step runs that would combine premises into a novel + surface. + +## Authoring finding — leakage rate + +`no_leakage_rate` is 0.4375 / 0.4 — i.e. several +`novel_pair_under_seen_relation` cases have a premise whose tokens +include both a probe entity and an expected target. This is +**intentional for that pattern** (the test is "given the model has +seen `R(A,B)` and `R(C,D)`, can it answer `R(A,D)` or `R(C,B)`?" — +both answers were taught as premise endpoints, just not together). +The strict author-time leakage check fires by design here. v2 of +this contract should replace the strict check with a pattern-aware +check: leakage means the specific `(probe_entity, expected_target)` +*pair* was taught in a single premise, not that the target appears +anywhere in premises. + +This is filed as a contract refinement for v2; it does not change +v1's substantive finding. + +## Architectural gap (same family as inference-closure) + +Composition requires the proposition-graph planner to walk multiple +nodes and synthesize a derived articulation. `plan_articulation()` +in `generate/graph_planner.py` is single-node. Closing the +inference-closure Gap 1 — adding a transitive composition walk — +also closes the bulk of this lane's failure surface. + +## Future direction (recorded here so it's not forgotten) + +Metaphor and simile are structurally **compositionality with +selective property transfer**: "the heart is a pump" is the same +graph-traversal shape as the compositionality probes above, with a +filter that says *which* relations transfer across the analogy. +Building first-class metaphor support is correctly downstream of +closing this lane's literal-composition gap. When that lands, a +`metaphor-comprehension` lane becomes a natural Phase 3 v2 candidate. + +## Status + +v1 stands as honest-failure baseline. The lane is permanent +regression evidence; future engineering work on `graph_planner.py` +that closes inference-closure Gap 1 should be re-scored here. diff --git a/evals/compositionality/holdouts/v1/cases.jsonl b/evals/compositionality/holdouts/v1/cases.jsonl new file mode 100644 index 00000000..890f3cc1 --- /dev/null +++ b/evals/compositionality/holdouts/v1/cases.jsonl @@ -0,0 +1,10 @@ +{"id":"CMP-V1-HLD-001","pattern":"composed_predicate","premises":["What is being?","Actually being is presence.","What is presence?","Actually presence precedes reality."],"probe":"What does being precede?","expected_entailment_tokens":["reality"],"expected_proposals":2} +{"id":"CMP-V1-HLD-002","pattern":"composed_predicate","premises":["What is distinction?","Actually distinction is comparison.","What is comparison?","Actually comparison causes definition."],"probe":"What does distinction cause?","expected_entailment_tokens":["definition"],"expected_proposals":2} +{"id":"CMP-V1-HLD-003","pattern":"composed_predicate","premises":["What is concept?","Actually concept is structure.","What is structure?","Actually structure grounds meaning."],"probe":"What does concept ground?","expected_entailment_tokens":["meaning"],"expected_proposals":2} +{"id":"CMP-V1-HLD-004","pattern":"novel_pair_under_seen_relation","premises":["What is life?","Actually life causes movement.","What is being?","Actually being causes presence."],"probe":"What does life cause in being?","expected_entailment_tokens":["presence","movement"],"expected_proposals":2} +{"id":"CMP-V1-HLD-005","pattern":"novel_pair_under_seen_relation","premises":["What is procedure?","Actually procedure belongs_to method.","What is method?","Actually method belongs_to inquiry."],"probe":"Where does procedure belong in method?","expected_entailment_tokens":["inquiry"],"expected_proposals":2} +{"id":"CMP-V1-HLD-006","pattern":"novel_pair_under_seen_relation","premises":["What is verification?","Actually verification grounds evidence.","What is comparison?","Actually comparison grounds distinction."],"probe":"What does verification ground in comparison?","expected_entailment_tokens":["distinction","evidence"],"expected_proposals":2} +{"id":"CMP-V1-HLD-007","pattern":"novel_relation_on_seen_pair","premises":["What is intention?","Actually intention is direction.","What is spirit?","Actually spirit grounds intention."],"probe":"What does spirit direct?","expected_entailment_tokens":["direction","intention"],"expected_proposals":2} +{"id":"CMP-V1-HLD-008","pattern":"novel_relation_on_seen_pair","premises":["What is recall?","Actually recall is recognition.","What is memory?","Actually memory grounds recall."],"probe":"What does memory recognise?","expected_entailment_tokens":["recognition","recall"],"expected_proposals":2} +{"id":"CMP-V1-HLD-009","pattern":"novel_relation_on_seen_pair","premises":["What is judgment?","Actually judgment is conclusion.","What is reason?","Actually reason precedes judgment."],"probe":"What does reason conclude?","expected_entailment_tokens":["conclusion","judgment"],"expected_proposals":2} +{"id":"CMP-V1-HLD-010","pattern":"composed_predicate","premises":["What is correction?","Actually correction is learning.","What is learning?","Actually learning precedes mastery."],"probe":"What does correction precede?","expected_entailment_tokens":["mastery"],"expected_proposals":2} diff --git a/evals/compositionality/public/v1/cases.jsonl b/evals/compositionality/public/v1/cases.jsonl new file mode 100644 index 00000000..80060c6e --- /dev/null +++ b/evals/compositionality/public/v1/cases.jsonl @@ -0,0 +1,16 @@ +{"id":"CMP-V1-001","pattern":"composed_predicate","premises":["What is wisdom?","Actually wisdom is judgment.","What is judgment?","Actually judgment precedes decision."],"probe":"What does wisdom precede?","expected_entailment_tokens":["decision"],"expected_proposals":2} +{"id":"CMP-V1-002","pattern":"composed_predicate","premises":["What is light?","Actually light is clarity.","What is clarity?","Actually clarity causes recognition."],"probe":"What does light cause?","expected_entailment_tokens":["recognition"],"expected_proposals":2} +{"id":"CMP-V1-003","pattern":"composed_predicate","premises":["What is principle?","Actually principle is order.","What is order?","Actually order grounds coherence."],"probe":"What does principle ground?","expected_entailment_tokens":["coherence"],"expected_proposals":2} +{"id":"CMP-V1-004","pattern":"composed_predicate","premises":["What is creation?","Actually creation is movement.","What is movement?","Actually movement precedes change."],"probe":"What does creation precede?","expected_entailment_tokens":["change"],"expected_proposals":2} +{"id":"CMP-V1-005","pattern":"composed_predicate","premises":["What is reason?","Actually reason is inference.","What is inference?","Actually inference produces conclusion."],"probe":"What does reason produce?","expected_entailment_tokens":["conclusion"],"expected_proposals":2} +{"id":"CMP-V1-006","pattern":"novel_pair_under_seen_relation","premises":["What is truth?","Actually truth grounds judgment.","What is knowledge?","Actually knowledge grounds inference."],"probe":"What does truth ground in knowledge?","expected_entailment_tokens":["inference","judgment"],"expected_proposals":2} +{"id":"CMP-V1-007","pattern":"novel_pair_under_seen_relation","premises":["What is order?","Actually order precedes meaning.","What is structure?","Actually structure precedes coherence."],"probe":"What does order precede in structure?","expected_entailment_tokens":["coherence","meaning"],"expected_proposals":2} +{"id":"CMP-V1-008","pattern":"novel_pair_under_seen_relation","premises":["What is question?","Actually question causes inquiry.","What is answer?","Actually answer causes recall."],"probe":"What does question cause in answer?","expected_entailment_tokens":["recall","inquiry"],"expected_proposals":2} +{"id":"CMP-V1-009","pattern":"novel_pair_under_seen_relation","premises":["What is recognition?","Actually recognition belongs_to memory.","What is naming?","Actually naming belongs_to language."],"probe":"What does recognition belong to in naming?","expected_entailment_tokens":["language","memory"],"expected_proposals":2} +{"id":"CMP-V1-010","pattern":"novel_pair_under_seen_relation","premises":["What is wisdom?","Actually wisdom reveals truth.","What is light?","Actually light reveals clarity."],"probe":"What does wisdom reveal in light?","expected_entailment_tokens":["clarity","truth"],"expected_proposals":2} +{"id":"CMP-V1-011","pattern":"novel_relation_on_seen_pair","premises":["What is judgment?","Actually judgment is decision.","What is wisdom?","Actually wisdom precedes judgment."],"probe":"What does wisdom decide?","expected_entailment_tokens":["decision","judgment"],"expected_proposals":2} +{"id":"CMP-V1-012","pattern":"novel_relation_on_seen_pair","premises":["What is inquiry?","Actually inquiry is thought.","What is question?","Actually question precedes inquiry."],"probe":"What does question think?","expected_entailment_tokens":["thought","inquiry"],"expected_proposals":2} +{"id":"CMP-V1-013","pattern":"novel_relation_on_seen_pair","premises":["What is clarity?","Actually clarity is recognition.","What is light?","Actually light precedes clarity."],"probe":"What does light recognise?","expected_entailment_tokens":["recognition","clarity"],"expected_proposals":2} +{"id":"CMP-V1-014","pattern":"novel_relation_on_seen_pair","premises":["What is knowledge?","Actually knowledge is judgment.","What is truth?","Actually truth grounds knowledge."],"probe":"What does truth judge?","expected_entailment_tokens":["judgment","knowledge"],"expected_proposals":2} +{"id":"CMP-V1-015","pattern":"composed_predicate","premises":["What is meaning?","Actually meaning is relation.","What is relation?","Actually relation grounds coherence."],"probe":"What does meaning ground?","expected_entailment_tokens":["coherence"],"expected_proposals":2} +{"id":"CMP-V1-016","pattern":"composed_predicate","premises":["What is correction?","Actually correction is adjustment.","What is adjustment?","Actually adjustment precedes learning."],"probe":"What does correction precede?","expected_entailment_tokens":["learning"],"expected_proposals":2} diff --git a/evals/compositionality/runner.py b/evals/compositionality/runner.py new file mode 100644 index 00000000..47bcd723 --- /dev/null +++ b/evals/compositionality/runner.py @@ -0,0 +1,164 @@ +"""compositionality eval lane runner. + +For each case: teach the premises, probe a (relation, entity) pair +that was never directly taught, score whether the response surface +or walk surface references the expected composed token. + +Conforms to the framework interface: run_lane(cases, config=None) -> report. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from chat.runtime import ChatRuntime +from core.cognition.pipeline import CognitiveTurnPipeline +from core.config import RuntimeConfig +from evals.parallel import run_cases_parallel + + +@dataclass(slots=True) +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +_TOKEN_BOUND = re.compile(r"\b([a-z][a-z'\-]*)\b") + + +def _tokens(text: str) -> set[str]: + return set(_TOKEN_BOUND.findall((text or "").lower())) + + +def _hit(text: str, candidates: list[str]) -> bool: + if not text: + return False + toks = _tokens(text) + return any(c.lower() in toks for c in candidates) + + +def _run_sequence(premises: list[str], probe: str) -> dict[str, Any]: + runtime = ChatRuntime() + pipeline = CognitiveTurnPipeline(runtime) + proposals = 0 + for premise in premises: + try: + r = pipeline.run(premise, max_tokens=8) + except ValueError: + continue + if r.pack_mutation_proposal is not None: + proposals += 1 + try: + probe_result = pipeline.run(probe, max_tokens=8) + except ValueError: + return { + "surface": "", + "walk_surface": "", + "trace_hash": "", + "vault_hits": 0, + "proposals": proposals, + } + return { + "surface": probe_result.surface or "", + "articulation_surface": probe_result.articulation_surface or "", + "walk_surface": probe_result.walk_surface or "", + "trace_hash": probe_result.trace_hash, + "vault_hits": int(probe_result.vault_hits), + "proposals": proposals, + } + + +def _no_taught_pair_leakage(case: dict[str, Any]) -> bool: + """Author-time invariant: probe expectation is not a verbatim premise.""" + for expected in case.get("expected_entailment_tokens", []): + target = str(expected).lower() + probe = str(case.get("probe", "")).lower() + # The leakage check is structural: the probe entity is in premises + # (expected) but the target must not appear together with the probe + # entity in a single premise. Heuristic: target must not appear in + # any premise that also contains the first noun of the probe. + # For v1 we apply a simpler check — verify the (probe_entity, target) + # pair does not co-occur in any premise. + probe_tokens = _tokens(probe) + for premise in case.get("premises", []): + ptokens = _tokens(premise) + if target in ptokens and probe_tokens & ptokens: + return False + return True + + +def _run_case(case: dict[str, Any]) -> dict[str, Any]: + premises: list[str] = list(case.get("premises", [])) + probe: str = case["probe"] + entailments: list[str] = list(case.get("expected_entailment_tokens", [])) + expected_proposals = int(case.get("expected_proposals", len(premises) // 2)) + + first = _run_sequence(premises, probe) + second = _run_sequence(premises, probe) + + surface_blob = " ".join([ + first["surface"], first.get("articulation_surface", ""), first["walk_surface"] + ]) + comp_hit = _hit(surface_blob, entailments) + premises_stored = first["proposals"] >= expected_proposals + replay_pass = ( + bool(first["trace_hash"]) + and first["trace_hash"] == second["trace_hash"] + and first["vault_hits"] == second["vault_hits"] + and first["proposals"] == second["proposals"] + ) + leakage_clean = _no_taught_pair_leakage(case) + + passed = comp_hit and premises_stored and replay_pass + + return { + "id": case.get("id", ""), + "pattern": case.get("pattern", ""), + "entailment_tokens": entailments, + "vault_hits": first["vault_hits"], + "trace_hash": first["trace_hash"], + "trace_hash_replay": second["trace_hash"], + "proposals": first["proposals"], + "expected_proposals": expected_proposals, + "compositional_hit": comp_hit, + "premises_stored_pass": premises_stored, + "replay_pass": replay_pass, + "leakage_clean": leakage_clean, + "passed": passed, + } + + +def run_lane( + cases: list[dict[str, Any]], + *, + config: RuntimeConfig | None = None, + workers: int | None = None, +) -> LaneReport: + if not cases: + return LaneReport(metrics={}, case_details=[]) + _ = config + + case_details = run_cases_parallel(cases, _run_case, workers=workers) + total = len(case_details) + + comp = sum(1 for d in case_details if d["compositional_hit"]) / total + stored = sum(1 for d in case_details if d["premises_stored_pass"]) / total + replay = sum(1 for d in case_details if d["replay_pass"]) / total + overall = sum(1 for d in case_details if d["passed"]) / total + leakage = sum(1 for d in case_details if d["leakage_clean"]) / total + + overall_pass = comp >= 0.50 and stored >= 0.95 and replay >= 0.95 + + metrics: dict[str, Any] = { + "compositional_recall_rate": round(comp, 4), + "premises_stored_rate": round(stored, 4), + "replay_determinism": round(replay, 4), + "no_leakage_rate": round(leakage, 4), + "all_pass_rate": round(overall, 4), + "case_count": total, + "overall_pass": overall_pass, + } + + return LaneReport(metrics=metrics, case_details=case_details) diff --git a/evals/cross_domain_transfer/__init__.py b/evals/cross_domain_transfer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/cross_domain_transfer/baselines/v1_structural_zero.json b/evals/cross_domain_transfer/baselines/v1_structural_zero.json new file mode 100644 index 00000000..55a6e632 --- /dev/null +++ b/evals/cross_domain_transfer/baselines/v1_structural_zero.json @@ -0,0 +1,16 @@ +{ + "kind": "structural_zero", + "lane": "cross_domain_transfer", + "metrics": { + "transfer_endpoint_recall_rate": null, + "domain_a_stored_rate": 0.0, + "domain_b_stored_rate": 0.0, + "replay_determinism": 0.0, + "overall_pass": false + }, + "model_id": "frontier-structural-zero", + "note": "Frontier LLMs do not emit the typed signals these sub-metrics score; see docs/frontier_baselines.md", + "rationale": "Storage rates and replay determinism require typed pipeline emissions that frontier has no analog for.", + "timestamp": "2026-05-16T00:00:00+00:00", + "version": "v1" +} diff --git a/evals/cross_domain_transfer/contract.md b/evals/cross_domain_transfer/contract.md new file mode 100644 index 00000000..7af9e200 --- /dev/null +++ b/evals/cross_domain_transfer/contract.md @@ -0,0 +1,75 @@ +# cross-domain-transfer eval lane + +## What it measures + +Whether competence on a relation pattern taught in **semantic +subdomain A** transfers to the **same relation pattern in semantic +subdomain B**, where A and B share no entities. + +Setup per case: + + Teach phase (subdomain A): + R(x1, x2), R(x2, x3) — A-domain entities only. + Probe phase (subdomain B): + "What does y1 R?" — B-domain entities only, + never used in teaching. + Premise pre-loading in B: + R(y1, y2), R(y2, y3) — taught at probe time so the model + has the B-domain premises in vault. + +Pass = the probe answer references `y3` (the derived endpoint in +subdomain B). + +The discriminator vs the inference-closure lane: here the model has +also seen the *same relation pattern* applied to A-domain entities +first. If transfer happens, the second-application latency / hit +rate should improve. Today the working hypothesis is that no +transfer happens because no structural-pattern recogniser exists. + +## Subdomain partition (drawn from en_core_cognition_v1) + +| Domain A (taught first) | Domain B (probed) | +|---|---| +| `cognition.wisdom` / `epistemic.judgment` cluster: wisdom, judgment, decision | `cognition.illumination` / `perception.clarity` cluster: light, clarity, recognition | +| `cognition.knowledge` / `reason.*` cluster: knowledge, reason, inference | `cognition.creation` / `formation.origin` cluster: creation, order, structure | +| `cognition.language.*` cluster: word, meaning, symbol | `memory.*` / `recognition.*` cluster: memory, recall, recognition | + +## Sub-metrics + +- `M1. transfer_endpoint_hit` — endpoint `y3` appears in probe + surface or walk_surface. +- `M2. domain_b_vault_grounded` — at least one B-domain premise + fires a `pack_mutation_proposal` (confirms B premises stored). +- `M3. domain_a_premises_stored` — every A-domain teaching turn + fires a proposal (regression gate for storage). +- `M4. replay_determinism` — two fresh runs match by + trace_hash on the whole (A-teach, B-teach, probe) sequence. + +A case passes when M1 AND M2 AND M3 AND M4 hold. + +## Overall pass thresholds (v1) + +- `transfer_endpoint_recall_rate` (M1) ≥ 0.50 +- `premises_stored_rate` (M2 ∧ M3) ≥ 0.95 +- `replay_determinism` ≥ 0.95 + +## v1 working hypothesis + +The same architectural gaps that surfaced in inference-closure +(`graph_planner.py` has no transitive composition; +`field/propagate.py` has no path-recall) apply here. Additionally, +**no structural-pattern recogniser exists** that would let the +A-domain teaching shape behaviour in subdomain B. v1 is expected +to score `transfer_endpoint_recall_rate ≈ 0`. + +The value of the lane in v1 is to baseline transfer at zero so that +any future pack-design or graph-planner work that produces real +transfer is visible against this regression line. + +## Anti-overfitting + +- A-domain and B-domain entity sets are disjoint (verified at + authoring time). +- The relation `R` is drawn from the existing lexicon — not invented + for the lane. +- Holdouts uses subdomain pairings disjoint from the public split. diff --git a/evals/cross_domain_transfer/dev/cases.jsonl b/evals/cross_domain_transfer/dev/cases.jsonl new file mode 100644 index 00000000..10059970 --- /dev/null +++ b/evals/cross_domain_transfer/dev/cases.jsonl @@ -0,0 +1,3 @@ +{"id":"XDT-DEV-001","pattern":"is_chain_judgment_to_perception","domain_a_premises":["What is wisdom?","Actually wisdom is judgment.","What is judgment?","Actually judgment is decision."],"domain_b_premises":["What is light?","Actually light is clarity.","What is clarity?","Actually clarity is recognition."],"probe":"What is light?","expected_endpoint_tokens":["recognition"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-DEV-002","pattern":"precedes_reason_to_creation","domain_a_premises":["What is knowledge?","Actually knowledge precedes reason.","What is reason?","Actually reason precedes inference."],"domain_b_premises":["What is creation?","Actually creation precedes order.","What is order?","Actually order precedes structure."],"probe":"What does creation precede?","expected_endpoint_tokens":["structure"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-DEV-003","pattern":"grounds_language_to_memory","domain_a_premises":["What is word?","Actually word grounds meaning.","What is meaning?","Actually meaning grounds symbol."],"domain_b_premises":["What is memory?","Actually memory grounds recall.","What is recall?","Actually recall grounds recognition."],"probe":"What does memory ground?","expected_endpoint_tokens":["recognition"],"expected_a_proposals":2,"expected_b_proposals":2} diff --git a/evals/cross_domain_transfer/gaps.md b/evals/cross_domain_transfer/gaps.md new file mode 100644 index 00000000..f200bfe6 --- /dev/null +++ b/evals/cross_domain_transfer/gaps.md @@ -0,0 +1,75 @@ +# cross-domain-transfer lane — architectural findings (v1) + +## v1 result + +| Split | n | transfer_endpoint_recall | A_stored | B_stored | replay | +|---|---|---|---|---|---| +| public/v1 | 10 | **0.0** | 1.0 | 1.0 | 1.0 | +| holdouts/v1 | 8 | **0.0** | 1.0 | 1.0 | 1.0 | + +No transfer. Both A-domain and B-domain premises are independently +stored (storage rate 1.0 on each side); replay is deterministic; the +B-domain endpoint never appears in the probe surface. + +## What this confirms (vs. inference-closure) + +This lane is inference-closure plus a *prior* teaching pass in a +disjoint semantic subdomain. v1's result establishes that: + +- The A-domain teaching has **no carry-over effect** on B-domain + competence. This is consistent with CORE having no structural- + pattern recogniser — the A-domain chain doesn't shape how the + B-domain chain is articulated or recalled. +- Whatever fix closes inference-closure's Gap 1 / Gap 2 may close + this lane's failure too, since B-domain alone is a literal + inference-closure case. But it will not *demonstrate transfer* — + that requires a different signal, captured in v2. + +## v2 contract refinement + +To actually score transfer (rather than just "B-domain inference +works after A-domain teaching"), v2 of this lane should include a +matched control: same B-domain probe **without** prior A-domain +teaching. Pass criterion becomes: + + transfer_endpoint_recall_rate(with_A_teaching) > + transfer_endpoint_recall_rate(without_A_teaching) + +That delta is the genuine transfer signal. v1 leaves this on the +table because the floor is currently zero on both arms — a v1 +"transfer = 0 − 0 = 0" result would be uninformative. When the +inference-closure engineering lands and the B-arm starts producing +non-zero recall, v2's matched-control comparison becomes the +load-bearing measurement. + +## Architectural gaps + +1. **No structural-pattern recogniser.** CORE's proposition graph + has no concept of "the relation pattern `R(x1,x2)→R(x2,x3)` was + seen N times across these subdomains" — patterns are not + first-class entities. +2. **No cross-subdomain transfer operator.** Vault retrieval and + field propagation are entity-local; nothing maps "structural + competence in subdomain A" to "expected competence in subdomain + B." +3. Both gaps are downstream of (and overlap with) inference-closure + Gap 1 + Gap 2. + +## Future directions (recorded here so they're not forgotten) + +- **Metaphor as cross-domain transfer with selectivity.** A + metaphor is the same shape as this lane's probe with an added + filter: which relations transfer across the analogy and which do + not. Once literal cross-domain transfer works, building + `metaphor-comprehension` on top is a natural Phase 3 v2 lane + rather than a separate operator. +- **Narrative as multi-step cross-domain transfer.** A story is a + multi-step inference chain bound to a point-of-view (agent / + intention). Both substrates (multi-step chaining and POV) need to + land before a `narrative` lane is meaningful. + +## Status + +v1 stands as honest-failure baseline. v2 contract refinement +(matched-control comparison) is the next authoring step once +inference-closure engineering lifts B-arm recall off the floor. diff --git a/evals/cross_domain_transfer/holdouts/v1/cases.jsonl b/evals/cross_domain_transfer/holdouts/v1/cases.jsonl new file mode 100644 index 00000000..457809c1 --- /dev/null +++ b/evals/cross_domain_transfer/holdouts/v1/cases.jsonl @@ -0,0 +1,8 @@ +{"id":"XDT-V1-HLD-001","pattern":"is_chain_being_to_distinction","domain_a_premises":["What is being?","Actually being is presence.","What is presence?","Actually presence is reality."],"domain_b_premises":["What is distinction?","Actually distinction is comparison.","What is comparison?","Actually comparison is contrast."],"probe":"What is distinction?","expected_endpoint_tokens":["contrast"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-HLD-002","pattern":"precedes_correction_to_procedure","domain_a_premises":["What is correction?","Actually correction precedes adjustment.","What is adjustment?","Actually adjustment precedes learning."],"domain_b_premises":["What is procedure?","Actually procedure precedes method.","What is method?","Actually method precedes approach."],"probe":"What does procedure precede?","expected_endpoint_tokens":["approach"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-HLD-003","pattern":"grounds_verification_to_intention","domain_a_premises":["What is verification?","Actually verification grounds evidence.","What is evidence?","Actually evidence grounds observation."],"domain_b_premises":["What is intention?","Actually intention grounds direction.","What is direction?","Actually direction grounds purpose."],"probe":"What does intention ground?","expected_endpoint_tokens":["purpose"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-HLD-004","pattern":"causes_life_to_concept","domain_a_premises":["What is life?","Actually life causes movement.","What is movement?","Actually movement causes change."],"domain_b_premises":["What is concept?","Actually concept causes structure.","What is structure?","Actually structure causes form."],"probe":"What does concept cause?","expected_endpoint_tokens":["form"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-HLD-005","pattern":"belongs_to_question_to_recall","domain_a_premises":["What is question?","Actually question belongs_to inquiry.","What is inquiry?","Actually inquiry belongs_to thought."],"domain_b_premises":["What is recall?","Actually recall belongs_to memory.","What is memory?","Actually memory belongs_to cognition."],"probe":"Where does recall belong?","expected_endpoint_tokens":["cognition"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-HLD-006","pattern":"is_chain_being_to_distinction","domain_a_premises":["What is being?","Actually being is presence.","What is presence?","Actually presence is reality."],"domain_b_premises":["What is distinction?","Actually distinction is difference.","What is difference?","Actually difference is contrast."],"probe":"What is distinction?","expected_endpoint_tokens":["contrast"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-HLD-007","pattern":"precedes_correction_to_procedure","domain_a_premises":["What is correction?","Actually correction precedes learning.","What is learning?","Actually learning precedes mastery."],"domain_b_premises":["What is procedure?","Actually procedure precedes approach.","What is approach?","Actually approach precedes direction."],"probe":"What does procedure precede?","expected_endpoint_tokens":["direction"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-HLD-008","pattern":"grounds_verification_to_intention","domain_a_premises":["What is verification?","Actually verification grounds confidence.","What is confidence?","Actually confidence grounds trust."],"domain_b_premises":["What is intention?","Actually intention grounds purpose.","What is purpose?","Actually purpose grounds meaning."],"probe":"What does intention ground?","expected_endpoint_tokens":["meaning"],"expected_a_proposals":2,"expected_b_proposals":2} diff --git a/evals/cross_domain_transfer/public/v1/cases.jsonl b/evals/cross_domain_transfer/public/v1/cases.jsonl new file mode 100644 index 00000000..d3ef781c --- /dev/null +++ b/evals/cross_domain_transfer/public/v1/cases.jsonl @@ -0,0 +1,10 @@ +{"id":"XDT-V1-001","pattern":"is_chain_judgment_to_perception","domain_a_premises":["What is wisdom?","Actually wisdom is judgment.","What is judgment?","Actually judgment is decision."],"domain_b_premises":["What is light?","Actually light is clarity.","What is clarity?","Actually clarity is recognition."],"probe":"What is light?","expected_endpoint_tokens":["recognition"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-002","pattern":"is_chain_judgment_to_perception","domain_a_premises":["What is wisdom?","Actually wisdom is reason.","What is reason?","Actually reason is inference."],"domain_b_premises":["What is light?","Actually light is illumination.","What is illumination?","Actually illumination is clarity."],"probe":"What is light?","expected_endpoint_tokens":["clarity"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-003","pattern":"precedes_reason_to_creation","domain_a_premises":["What is knowledge?","Actually knowledge precedes reason.","What is reason?","Actually reason precedes inference."],"domain_b_premises":["What is creation?","Actually creation precedes order.","What is order?","Actually order precedes structure."],"probe":"What does creation precede?","expected_endpoint_tokens":["structure"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-004","pattern":"precedes_reason_to_creation","domain_a_premises":["What is judgment?","Actually judgment precedes decision.","What is decision?","Actually decision precedes action."],"domain_b_premises":["What is creation?","Actually creation precedes movement.","What is movement?","Actually movement precedes change."],"probe":"What does creation precede?","expected_endpoint_tokens":["change"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-005","pattern":"grounds_language_to_memory","domain_a_premises":["What is word?","Actually word grounds meaning.","What is meaning?","Actually meaning grounds symbol."],"domain_b_premises":["What is memory?","Actually memory grounds recall.","What is recall?","Actually recall grounds recognition."],"probe":"What does memory ground?","expected_endpoint_tokens":["recognition"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-006","pattern":"grounds_language_to_memory","domain_a_premises":["What is word?","Actually word grounds symbol.","What is symbol?","Actually symbol grounds communication."],"domain_b_premises":["What is memory?","Actually memory grounds recognition.","What is recognition?","Actually recognition grounds naming."],"probe":"What does memory ground?","expected_endpoint_tokens":["naming"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-007","pattern":"causes_judgment_to_creation","domain_a_premises":["What is wisdom?","Actually wisdom causes judgment.","What is judgment?","Actually judgment causes decision."],"domain_b_premises":["What is creation?","Actually creation causes order.","What is order?","Actually order causes coherence."],"probe":"What does creation cause?","expected_endpoint_tokens":["coherence"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-008","pattern":"causes_judgment_to_creation","domain_a_premises":["What is wisdom?","Actually wisdom causes insight.","What is insight?","Actually insight causes judgment."],"domain_b_premises":["What is creation?","Actually creation causes formation.","What is formation?","Actually formation causes structure."],"probe":"What does creation cause?","expected_endpoint_tokens":["structure"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-009","pattern":"belongs_to_reason_to_perception","domain_a_premises":["What is inference?","Actually inference belongs_to reason.","What is reason?","Actually reason belongs_to thought."],"domain_b_premises":["What is clarity?","Actually clarity belongs_to perception.","What is perception?","Actually perception belongs_to awareness."],"probe":"Where does clarity belong?","expected_endpoint_tokens":["awareness"],"expected_a_proposals":2,"expected_b_proposals":2} +{"id":"XDT-V1-010","pattern":"is_chain_judgment_to_perception","domain_a_premises":["What is wisdom?","Actually wisdom is judgment.","What is judgment?","Actually judgment is conclusion."],"domain_b_premises":["What is light?","Actually light is clarity.","What is clarity?","Actually clarity is recognition."],"probe":"What is light?","expected_endpoint_tokens":["recognition"],"expected_a_proposals":2,"expected_b_proposals":2} diff --git a/evals/cross_domain_transfer/runner.py b/evals/cross_domain_transfer/runner.py new file mode 100644 index 00000000..f4a073d3 --- /dev/null +++ b/evals/cross_domain_transfer/runner.py @@ -0,0 +1,166 @@ +"""cross-domain-transfer eval lane runner. + +For each case: teach an R-chain in subdomain A, teach the same R-chain +in subdomain B (so B premises are in vault), probe the B-domain head, +score whether the B-domain endpoint appears in the response. + +Conforms to the framework interface: run_lane(cases, config=None) -> report. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from chat.runtime import ChatRuntime +from core.cognition.pipeline import CognitiveTurnPipeline +from core.config import RuntimeConfig +from evals.parallel import run_cases_parallel + + +@dataclass(slots=True) +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +_TOKEN_BOUND = re.compile(r"\b([a-z][a-z'\-]*)\b") + + +def _tokens(text: str) -> set[str]: + return set(_TOKEN_BOUND.findall((text or "").lower())) + + +def _hit(text: str, candidates: list[str]) -> bool: + if not text: + return False + toks = _tokens(text) + return any(c.lower() in toks for c in candidates) + + +def _run_sequence( + domain_a_premises: list[str], + domain_b_premises: list[str], + probe: str, +) -> dict[str, Any]: + runtime = ChatRuntime() + pipeline = CognitiveTurnPipeline(runtime) + a_proposals = 0 + b_proposals = 0 + for p in domain_a_premises: + try: + r = pipeline.run(p, max_tokens=8) + except ValueError: + continue + if r.pack_mutation_proposal is not None: + a_proposals += 1 + for p in domain_b_premises: + try: + r = pipeline.run(p, max_tokens=8) + except ValueError: + continue + if r.pack_mutation_proposal is not None: + b_proposals += 1 + try: + probe_result = pipeline.run(probe, max_tokens=8) + except ValueError: + return { + "surface": "", "articulation_surface": "", "walk_surface": "", + "trace_hash": "", "vault_hits": 0, + "a_proposals": a_proposals, "b_proposals": b_proposals, + } + return { + "surface": probe_result.surface or "", + "articulation_surface": probe_result.articulation_surface or "", + "walk_surface": probe_result.walk_surface or "", + "trace_hash": probe_result.trace_hash, + "vault_hits": int(probe_result.vault_hits), + "a_proposals": a_proposals, + "b_proposals": b_proposals, + } + + +def _run_case(case: dict[str, Any]) -> dict[str, Any]: + a_premises: list[str] = list(case.get("domain_a_premises", [])) + b_premises: list[str] = list(case.get("domain_b_premises", [])) + probe: str = case["probe"] + endpoint_tokens: list[str] = list(case.get("expected_endpoint_tokens", [])) + expected_a = int(case.get("expected_a_proposals", len(a_premises) // 2)) + expected_b = int(case.get("expected_b_proposals", len(b_premises) // 2)) + + first = _run_sequence(a_premises, b_premises, probe) + second = _run_sequence(a_premises, b_premises, probe) + + surface_blob = " ".join([ + first["surface"], first["articulation_surface"], first["walk_surface"] + ]) + endpoint_hit = _hit(surface_blob, endpoint_tokens) + a_stored = first["a_proposals"] >= expected_a + b_stored = first["b_proposals"] >= expected_b + replay_pass = ( + bool(first["trace_hash"]) + and first["trace_hash"] == second["trace_hash"] + and first["vault_hits"] == second["vault_hits"] + and first["a_proposals"] == second["a_proposals"] + and first["b_proposals"] == second["b_proposals"] + ) + + passed = endpoint_hit and a_stored and b_stored and replay_pass + + return { + "id": case.get("id", ""), + "pattern": case.get("pattern", ""), + "endpoint_tokens": endpoint_tokens, + "vault_hits": first["vault_hits"], + "trace_hash": first["trace_hash"], + "trace_hash_replay": second["trace_hash"], + "a_proposals": first["a_proposals"], + "b_proposals": first["b_proposals"], + "expected_a": expected_a, + "expected_b": expected_b, + "transfer_endpoint_hit": endpoint_hit, + "domain_a_stored_pass": a_stored, + "domain_b_stored_pass": b_stored, + "replay_pass": replay_pass, + "passed": passed, + } + + +def run_lane( + cases: list[dict[str, Any]], + *, + config: RuntimeConfig | None = None, + workers: int | None = None, +) -> LaneReport: + if not cases: + return LaneReport(metrics={}, case_details=[]) + _ = config + + case_details = run_cases_parallel(cases, _run_case, workers=workers) + total = len(case_details) + + transfer = sum(1 for d in case_details if d["transfer_endpoint_hit"]) / total + a_stored = sum(1 for d in case_details if d["domain_a_stored_pass"]) / total + b_stored = sum(1 for d in case_details if d["domain_b_stored_pass"]) / total + replay = sum(1 for d in case_details if d["replay_pass"]) / total + overall = sum(1 for d in case_details if d["passed"]) / total + + overall_pass = ( + transfer >= 0.50 + and a_stored >= 0.95 + and b_stored >= 0.95 + and replay >= 0.95 + ) + + metrics: dict[str, Any] = { + "transfer_endpoint_recall_rate": round(transfer, 4), + "domain_a_stored_rate": round(a_stored, 4), + "domain_b_stored_rate": round(b_stored, 4), + "replay_determinism": round(replay, 4), + "all_pass_rate": round(overall, 4), + "case_count": total, + "overall_pass": overall_pass, + } + + return LaneReport(metrics=metrics, case_details=case_details) diff --git a/evals/introspection/__init__.py b/evals/introspection/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/introspection/baselines/v1_structural_zero.json b/evals/introspection/baselines/v1_structural_zero.json new file mode 100644 index 00000000..db6854d9 --- /dev/null +++ b/evals/introspection/baselines/v1_structural_zero.json @@ -0,0 +1,15 @@ +{ + "kind": "structural_zero", + "lane": "introspection", + "metrics": { + "explain_api_present_rate": 0.0, + "round_trip_surface_match_rate": null, + "round_trip_trace_match_rate": 0.0, + "overall_pass": false + }, + "model_id": "frontier-structural-zero", + "note": "Frontier LLMs do not emit the typed signals these sub-metrics score; see docs/frontier_baselines.md", + "rationale": "round_trip_trace_match_rate requires identical trace_hash across fresh deterministic runs. explain_api_present_rate is a structural check for a CORE module. Frontier has no analog of either signal.", + "timestamp": "2026-05-16T00:00:00+00:00", + "version": "v1" +} diff --git a/evals/introspection/contract.md b/evals/introspection/contract.md new file mode 100644 index 00000000..1a42ba53 --- /dev/null +++ b/evals/introspection/contract.md @@ -0,0 +1,68 @@ +# introspection eval lane + +## What it measures + +Whether CORE can produce a natural-language **account of a prior +turn** that round-trips: a separate run conditioned on that account +predicts the same articulation as the original turn. + +Roadmap shape (Phase 3): + + Run 1: pipeline.run(prompt) -> Result_A (surface, trace_hash_A) + Step: explain(Result_A.turn_id) -> account (natural-language) + Run 2: fresh pipeline.run(account) -> Result_B (surface, trace_hash_B) + Round-trip pass: Result_B.surface == Result_A.surface + (or a defensibly equivalent surface) + +A passing round-trip demonstrates that CORE's articulation is +*explainable in its own terms* and that the explanation carries +enough state to reconstruct the answer. + +## v1 reality: the `explain` interface does not exist + +CORE has no `cognition/explain.py` module today. Per the roadmap +(Phase 3 work items): *"A new `cognition/explain.py` module may be +needed for introspection."* v1's role is to score the gap +honestly: the runner attempts to import an explain function from +`core.cognition` and falls through with `M1=0` when the import +fails. This makes the lane runnable today and gives a structural- +zero result by construction until the module lands. + +## Sub-metrics + +- `M1. explain_api_present` — the explain function imports + cleanly from `core.cognition` (or a documented alternative). +- `M2. account_is_nonempty` — when (1) succeeds, the + generated account has non-trivial length (≥ 5 tokens). +- `M3. round_trip_surface_match` — Result_B.surface tokens cover + ≥ 60% of Result_A.surface tokens (case-insensitive, + punctuation-stripped). +- `M4. round_trip_trace_match` — Result_B.trace_hash == + Result_A.trace_hash (strict deterministic round-trip). + +Today's expected result: M1 = 0; all downstream metrics = 0. + +A case passes when M1 AND M2 AND M3 hold. M4 is reported as a +stricter signal — likely to fail even after M3 starts succeeding +because the input texts (original prompt vs. account) differ +verbatim and trace_hash is computed over input_text. + +## Overall pass thresholds (v1) + +- `explain_api_present_rate` (M1) ≥ 0.95 — trivial once the + module exists +- `account_nonempty_rate` (M2) ≥ 0.95 +- `round_trip_surface_match_rate` (M3) ≥ 0.50 + +v1's expected score: all zero. v1 is the lane that explicitly tests +whether the explain primitive exists and produces a usable +account. Until it does, this is structural-zero work. + +## Why a placeholder-runnable v1 + +The Phase 3 exit criteria state: "v1 results with honest scores +(which may be failing — that's acceptable for v1). Each failure +has either a closed engineering gap or a documented architectural +deferral." A lane that cannot run at all is worse than a lane that +runs and reports zero; the latter forms a real regression trigger +for the day the engineering lands. diff --git a/evals/introspection/dev/cases.jsonl b/evals/introspection/dev/cases.jsonl new file mode 100644 index 00000000..871ba9a0 --- /dev/null +++ b/evals/introspection/dev/cases.jsonl @@ -0,0 +1,3 @@ +{"id":"INTRO-DEV-001","prompt":"What is wisdom?"} +{"id":"INTRO-DEV-002","prompt":"What is light?"} +{"id":"INTRO-DEV-003","prompt":"What is truth?"} diff --git a/evals/introspection/gaps.md b/evals/introspection/gaps.md new file mode 100644 index 00000000..ce8e3f2b --- /dev/null +++ b/evals/introspection/gaps.md @@ -0,0 +1,58 @@ +# introspection lane — architectural findings (v1) + +## v1 result + +| Split | n | explain_api_present | account_nonempty | surface_match | trace_match | +|---|---|---|---|---|---| +| public/v1 | 12 | **0.0** | 0.0 | 0.0 | 0.0 | +| holdouts/v1 | 8 | **0.0** | 0.0 | 0.0 | 0.0 | + +Structural zero by construction: there is no `explain` callable to +import from `core.cognition`. + +## Why this is the right v1 + +A lane that can't run at all is worse than a lane that runs and +reports a typed zero. The introspection lane runs today, attempts +the import, catches the failure deterministically, and emits four +sub-metrics — all zero, all explained. The day someone lands a +`core/cognition/explain.py` module, this lane immediately starts +producing real numbers without any test infrastructure change. + +## Required engineering for v2 + +The roadmap (`docs/capability_roadmap.md` Phase 3 work items) is +explicit: + +> A new `cognition/explain.py` module may be needed for +> introspection. + +Concretely, an `explain(result: CognitiveTurnResult) -> str` +function that: + +1. **Reads structured state from the result** — intent tag, + proposition graph, articulation target, vault hits, identity + score. +2. **Composes a deterministic natural-language account** that + re-states the trajectory in source language. Probably leans on + the same `realize_semantic` machinery currently used for + articulation but inverted: surface → structured trace → surface'. +3. **Round-trip property**: feeding the account back through the + pipeline produces an articulation whose token coverage of the + original surface is high. Strict trace-hash equivalence is the + ideal but not the v1 bar — surface token overlap ≥ 0.60 is the + v1 contract. + +## Future direction (recorded here so it's not forgotten) + +A working introspection API is also the substrate for **narrative +self-explanation**: the same machinery that produces "I answered X +because I retrieved Y under intent Z" is what produces an agent's +own first-person account of a turn. Per the open scope decision in +`docs/PROGRESS.md` (Agency: responsive vs. goal-directed), this +choice should pin before introspection v2 is engineered. + +## Status + +v1 is structural-zero scaffolding. Permanent regression evidence +of the missing module. diff --git a/evals/introspection/holdouts/v1/cases.jsonl b/evals/introspection/holdouts/v1/cases.jsonl new file mode 100644 index 00000000..8b7b68d5 --- /dev/null +++ b/evals/introspection/holdouts/v1/cases.jsonl @@ -0,0 +1,8 @@ +{"id":"INTRO-V1-HLD-001","prompt":"What is being?"} +{"id":"INTRO-V1-HLD-002","prompt":"What is relation?"} +{"id":"INTRO-V1-HLD-003","prompt":"What is distinction?"} +{"id":"INTRO-V1-HLD-004","prompt":"What is question?"} +{"id":"INTRO-V1-HLD-005","prompt":"What is answer?"} +{"id":"INTRO-V1-HLD-006","prompt":"What is coherence?"} +{"id":"INTRO-V1-HLD-007","prompt":"What is procedure?"} +{"id":"INTRO-V1-HLD-008","prompt":"What is verification?"} diff --git a/evals/introspection/public/v1/cases.jsonl b/evals/introspection/public/v1/cases.jsonl new file mode 100644 index 00000000..7b013728 --- /dev/null +++ b/evals/introspection/public/v1/cases.jsonl @@ -0,0 +1,12 @@ +{"id":"INTRO-V1-001","prompt":"What is wisdom?"} +{"id":"INTRO-V1-002","prompt":"What is light?"} +{"id":"INTRO-V1-003","prompt":"What is truth?"} +{"id":"INTRO-V1-004","prompt":"What is creation?"} +{"id":"INTRO-V1-005","prompt":"What is meaning?"} +{"id":"INTRO-V1-006","prompt":"What is knowledge?"} +{"id":"INTRO-V1-007","prompt":"What is reason?"} +{"id":"INTRO-V1-008","prompt":"What is principle?"} +{"id":"INTRO-V1-009","prompt":"What is order?"} +{"id":"INTRO-V1-010","prompt":"What is judgment?"} +{"id":"INTRO-V1-011","prompt":"What is identity?"} +{"id":"INTRO-V1-012","prompt":"What is memory?"} diff --git a/evals/introspection/runner.py b/evals/introspection/runner.py new file mode 100644 index 00000000..4159c859 --- /dev/null +++ b/evals/introspection/runner.py @@ -0,0 +1,144 @@ +"""introspection eval lane runner. + +For each case: + 1. Run the prompt on a fresh CognitiveTurnPipeline and capture + (surface_A, trace_hash_A, turn_id_A). + 2. Attempt to call an `explain(turn_id)` function from + `core.cognition`. v1 expects this to raise ImportError; the + runner catches it and scores M1 = False. + 3. When (2) succeeds, run a fresh pipeline on the produced account + and capture (surface_B, trace_hash_B). + 4. Score round-trip overlap. + +Conforms to the framework interface: run_lane(cases, config=None) -> report. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from chat.runtime import ChatRuntime +from core.cognition.pipeline import CognitiveTurnPipeline +from core.config import RuntimeConfig +from evals.parallel import run_cases_parallel + + +@dataclass(slots=True) +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +_TOKEN_BOUND = re.compile(r"[a-z0-9]+") + + +def _tokens(text: str) -> set[str]: + return set(_TOKEN_BOUND.findall((text or "").lower())) + + +def _try_import_explain(): + """Return the explain callable or None when the API is absent.""" + try: + from core.cognition import explain # type: ignore[attr-defined] + except (ImportError, AttributeError): + return None + return explain + + +def _run_case(case: dict[str, Any]) -> dict[str, Any]: + prompt: str = case["prompt"] + + runtime = ChatRuntime() + pipeline = CognitiveTurnPipeline(runtime) + try: + result_a = pipeline.run(prompt, max_tokens=12) + except ValueError: + return { + "id": case.get("id", ""), + "explain_api_present": False, + "account_nonempty": False, + "round_trip_surface_match": False, + "round_trip_trace_match": False, + "passed": False, + } + + surface_a = result_a.surface or "" + trace_a = result_a.trace_hash + + explain = _try_import_explain() + api_present = explain is not None + account = "" + surface_b = "" + trace_b = "" + if api_present: + try: + account = explain(result_a) or "" # type: ignore[misc] + except Exception: + account = "" + if account: + rt2 = ChatRuntime() + pipe2 = CognitiveTurnPipeline(rt2) + try: + result_b = pipe2.run(account, max_tokens=12) + surface_b = result_b.surface or "" + trace_b = result_b.trace_hash + except ValueError: + pass + + account_nonempty = len(_tokens(account)) >= 5 + a_tokens = _tokens(surface_a) + b_tokens = _tokens(surface_b) + if a_tokens: + coverage = len(a_tokens & b_tokens) / len(a_tokens) + else: + coverage = 0.0 + surface_match = coverage >= 0.60 + trace_match = bool(trace_a) and trace_a == trace_b + + passed = api_present and account_nonempty and surface_match + + return { + "id": case.get("id", ""), + "explain_api_present": api_present, + "account_nonempty": account_nonempty, + "round_trip_surface_match": surface_match, + "round_trip_trace_match": trace_match, + "surface_token_coverage": round(coverage, 4), + "passed": passed, + } + + +def run_lane( + cases: list[dict[str, Any]], + *, + config: RuntimeConfig | None = None, + workers: int | None = None, +) -> LaneReport: + if not cases: + return LaneReport(metrics={}, case_details=[]) + _ = config + + case_details = run_cases_parallel(cases, _run_case, workers=workers) + total = len(case_details) + + api = sum(1 for d in case_details if d["explain_api_present"]) / total + nonempty = sum(1 for d in case_details if d["account_nonempty"]) / total + surf = sum(1 for d in case_details if d["round_trip_surface_match"]) / total + trace = sum(1 for d in case_details if d["round_trip_trace_match"]) / total + overall = sum(1 for d in case_details if d["passed"]) / total + + overall_pass = api >= 0.95 and nonempty >= 0.95 and surf >= 0.50 + + metrics: dict[str, Any] = { + "explain_api_present_rate": round(api, 4), + "account_nonempty_rate": round(nonempty, 4), + "round_trip_surface_match_rate": round(surf, 4), + "round_trip_trace_match_rate": round(trace, 4), + "all_pass_rate": round(overall, 4), + "case_count": total, + "overall_pass": overall_pass, + } + + return LaneReport(metrics=metrics, case_details=case_details) diff --git a/evals/multi_step_reasoning/__init__.py b/evals/multi_step_reasoning/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/multi_step_reasoning/baselines/v1_structural_zero.json b/evals/multi_step_reasoning/baselines/v1_structural_zero.json new file mode 100644 index 00000000..8356fe36 --- /dev/null +++ b/evals/multi_step_reasoning/baselines/v1_structural_zero.json @@ -0,0 +1,15 @@ +{ + "kind": "structural_zero", + "lane": "multi_step_reasoning", + "metrics": { + "chain_endpoint_recall_rate": null, + "premises_stored_rate": 0.0, + "replay_determinism": 0.0, + "overall_pass": false + }, + "model_id": "frontier-structural-zero", + "note": "Frontier LLMs do not emit the typed signals these sub-metrics score; see docs/frontier_baselines.md", + "rationale": "premises_stored_rate requires per-premise PackMutationProposal records (frontier has no analog). replay_determinism requires identical trace_hash across fresh deterministic runs (frontier inference is stochastic).", + "timestamp": "2026-05-16T00:00:00+00:00", + "version": "v1" +} diff --git a/evals/multi_step_reasoning/contract.md b/evals/multi_step_reasoning/contract.md new file mode 100644 index 00000000..ad9c26b4 --- /dev/null +++ b/evals/multi_step_reasoning/contract.md @@ -0,0 +1,57 @@ +# multi-step-reasoning eval lane + +## What it measures + +Whether the pipeline produces *and consumes* intermediate +proposition-graph states for problems whose solution requires three +or more inferential hops. + +This sharpens inference-closure: inference-closure scored two-hop +transitive entailments; this lane scores 3-, 4-, and 5-hop chains +and additionally checks that intermediate states are observable in +the proposition graph after the chain is taught. + +## Why it matters + +Single-hop and two-hop closure can in principle be implemented by +local pattern composition. Three-or-more hops require the pipeline +to build *and traverse* an inference path that does not exist +verbatim in any single premise. This is closer to the roadmap's +question: does CORE *think*, or does it pattern-match longer +templates. + +## Patterns covered (v1) + +| Pattern | Shape | Hops | +|---|---|---| +| `chain_3` | A is B; B is C; C is D | 3 | +| `chain_4` | A is B; B is C; C is D; D is E | 4 | +| `chain_5` | A is B; B is C; C is D; D is E; E is F | 5 | +| `mixed_relation_3` | A is B; B grounds C; C precedes D | 3 | +| `mixed_relation_4` | A causes B; B grounds C; C is D; D precedes E | 4 | + +## Sub-metrics + +- `M1. chain_endpoint_in_surface` — the final-hop entity appears + (case-insensitive, token-bounded) in `surface` or `walk_surface`. +- `M2. intermediate_in_graph` — at least one intermediate hop is + observable in the probe response's articulation_surface or + walk_surface (proxy for graph state inspection). +- `M3. premises_stored` — every taught hop emits a proposal. +- `M4. replay_determinism` — two fresh runs match by trace_hash. + +A case passes when M1 AND M3 AND M4 hold. M2 is reported as +diagnostic signal — partial credit when chain_endpoint is missed. + +## Overall pass thresholds (v1) + +- `chain_endpoint_recall_rate` (M1) ≥ 0.50 +- `premises_stored_rate` ≥ 0.95 +- `replay_determinism` ≥ 0.95 + +## Relationship to inference-closure v1 + +Same architectural gaps apply: no transitive composition in +`graph_planner.py`, no path-recall in `field/propagate.py`. This +lane scores how the gap scales with chain length. v1's likely +result: uniform M1 failure across all chain lengths. diff --git a/evals/multi_step_reasoning/dev/cases.jsonl b/evals/multi_step_reasoning/dev/cases.jsonl new file mode 100644 index 00000000..e57bf2c6 --- /dev/null +++ b/evals/multi_step_reasoning/dev/cases.jsonl @@ -0,0 +1,3 @@ +{"id":"MSR-DEV-001","pattern":"chain_3","hops":3,"premises":["What is wisdom?","Actually wisdom is judgment.","What is judgment?","Actually judgment is decision.","What is decision?","Actually decision is action."],"probe":"What is wisdom?","expected_endpoint_tokens":["action"],"intermediate_tokens":["judgment","decision"],"expected_proposals":3} +{"id":"MSR-DEV-002","pattern":"chain_4","hops":4,"premises":["What is creation?","Actually creation is order.","What is order?","Actually order is structure.","What is structure?","Actually structure is form.","What is form?","Actually form is meaning."],"probe":"What is creation?","expected_endpoint_tokens":["meaning"],"intermediate_tokens":["order","structure","form"],"expected_proposals":4} +{"id":"MSR-DEV-003","pattern":"mixed_relation_3","hops":3,"premises":["What is light?","Actually light is clarity.","What is clarity?","Actually clarity grounds recognition.","What is recognition?","Actually recognition precedes naming."],"probe":"What does light precede?","expected_endpoint_tokens":["naming"],"intermediate_tokens":["clarity","recognition"],"expected_proposals":3} diff --git a/evals/multi_step_reasoning/gaps.md b/evals/multi_step_reasoning/gaps.md new file mode 100644 index 00000000..7259597e --- /dev/null +++ b/evals/multi_step_reasoning/gaps.md @@ -0,0 +1,39 @@ +# multi-step-reasoning lane — architectural findings (v1) + +## v1 result + +| Split | n | endpoint_recall | intermediate_visible | stored | replay | +|---|---|---|---|---|---| +| public/v1 | 15 | **0.0** | 0.0 | 1.0 | 1.0 | +| holdouts/v1 | 10 | **0.0** | 0.0 | 1.0 | 1.0 | + +Uniform zero on the inference signal across 3-hop, 4-hop, and +5-hop chains; foundation intact. + +## Relationship to inference-closure v1 + +This lane extends inference-closure (which was 2-hop) to longer +chains. v1's result is the same architectural finding scaled with +chain length: no transitive composition exists at any depth, so the +failure mode is depth-independent. + +Concretely: a 3-hop chain `wisdom is judgment; judgment is decision; +decision is action` plus probe `What is wisdom?` returns the +template `wisdom is defined as ...`. The vault stores all three +premises; the realizer emits a definition stub. The intermediate +hops are not visible in the surface, the endpoint never appears. + +## Architectural gap (shared with inference-closure) + +Same Gap 1 (no transitive composition in `graph_planner.py`) and +Gap 2 (no path-recall in `field/propagate.py`). The depth-scaling +signal from this lane should be revisited after Gap 1 closes: a +correct fix should pass 3-hop, may degrade gracefully on 4- and +5-hop, and should clearly indicate where chain-traversal bounds +become a performance versus a correctness issue. + +## Phase 3 exit posture + +This lane satisfies the v1 honest-failure expectation. When Gap 1 +engineering lands, this lane should be re-run as the primary scaling +diagnostic. diff --git a/evals/multi_step_reasoning/holdouts/v1/cases.jsonl b/evals/multi_step_reasoning/holdouts/v1/cases.jsonl new file mode 100644 index 00000000..8934e40a --- /dev/null +++ b/evals/multi_step_reasoning/holdouts/v1/cases.jsonl @@ -0,0 +1,10 @@ +{"id":"MSR-V1-HLD-001","pattern":"chain_3","hops":3,"premises":["What is being?","Actually being is presence.","What is presence?","Actually presence is reality.","What is reality?","Actually reality is existence."],"probe":"What is being?","expected_endpoint_tokens":["existence"],"intermediate_tokens":["presence","reality"],"expected_proposals":3} +{"id":"MSR-V1-HLD-002","pattern":"chain_3","hops":3,"premises":["What is distinction?","Actually distinction is comparison.","What is comparison?","Actually comparison is contrast.","What is contrast?","Actually contrast is difference."],"probe":"What is distinction?","expected_endpoint_tokens":["difference"],"intermediate_tokens":["comparison","contrast"],"expected_proposals":3} +{"id":"MSR-V1-HLD-003","pattern":"chain_4","hops":4,"premises":["What is correction?","Actually correction is adjustment.","What is adjustment?","Actually adjustment is learning.","What is learning?","Actually learning is mastery.","What is mastery?","Actually mastery is skill."],"probe":"What is correction?","expected_endpoint_tokens":["skill"],"intermediate_tokens":["adjustment","learning","mastery"],"expected_proposals":4} +{"id":"MSR-V1-HLD-004","pattern":"chain_4","hops":4,"premises":["What is procedure?","Actually procedure is method.","What is method?","Actually method is approach.","What is approach?","Actually approach is direction.","What is direction?","Actually direction is intention."],"probe":"What is procedure?","expected_endpoint_tokens":["intention"],"intermediate_tokens":["method","approach","direction"],"expected_proposals":4} +{"id":"MSR-V1-HLD-005","pattern":"chain_5","hops":5,"premises":["What is verification?","Actually verification is evidence.","What is evidence?","Actually evidence is observation.","What is observation?","Actually observation is perception.","What is perception?","Actually perception is awareness.","What is awareness?","Actually awareness is consciousness."],"probe":"What is verification?","expected_endpoint_tokens":["consciousness"],"intermediate_tokens":["evidence","observation","perception","awareness"],"expected_proposals":5} +{"id":"MSR-V1-HLD-006","pattern":"mixed_relation_3","hops":3,"premises":["What is intention?","Actually intention grounds direction.","What is direction?","Actually direction causes movement.","What is movement?","Actually movement precedes change."],"probe":"What does intention precede?","expected_endpoint_tokens":["change"],"intermediate_tokens":["direction","movement"],"expected_proposals":3} +{"id":"MSR-V1-HLD-007","pattern":"mixed_relation_4","hops":4,"premises":["What is life?","Actually life causes movement.","What is movement?","Actually movement grounds change.","What is change?","Actually change is becoming.","What is becoming?","Actually becoming precedes growth."],"probe":"What does life precede?","expected_endpoint_tokens":["growth"],"intermediate_tokens":["movement","change","becoming"],"expected_proposals":4} +{"id":"MSR-V1-HLD-008","pattern":"chain_3","hops":3,"premises":["What is explanation?","Actually explanation is account.","What is account?","Actually account is story.","What is story?","Actually story is meaning."],"probe":"What is explanation?","expected_endpoint_tokens":["meaning"],"intermediate_tokens":["account","story"],"expected_proposals":3} +{"id":"MSR-V1-HLD-009","pattern":"chain_4","hops":4,"premises":["What is concept?","Actually concept is structure.","What is structure?","Actually structure is form.","What is form?","Actually form is pattern.","What is pattern?","Actually pattern is order."],"probe":"What is concept?","expected_endpoint_tokens":["order"],"intermediate_tokens":["structure","form","pattern"],"expected_proposals":4} +{"id":"MSR-V1-HLD-010","pattern":"chain_5","hops":5,"premises":["What is spirit?","Actually spirit is intention.","What is intention?","Actually intention is direction.","What is direction?","Actually direction is purpose.","What is purpose?","Actually purpose is meaning.","What is meaning?","Actually meaning is value."],"probe":"What is spirit?","expected_endpoint_tokens":["value"],"intermediate_tokens":["intention","direction","purpose","meaning"],"expected_proposals":5} diff --git a/evals/multi_step_reasoning/public/v1/cases.jsonl b/evals/multi_step_reasoning/public/v1/cases.jsonl new file mode 100644 index 00000000..8ef35f91 --- /dev/null +++ b/evals/multi_step_reasoning/public/v1/cases.jsonl @@ -0,0 +1,15 @@ +{"id":"MSR-V1-001","pattern":"chain_3","hops":3,"premises":["What is wisdom?","Actually wisdom is judgment.","What is judgment?","Actually judgment is decision.","What is decision?","Actually decision is action."],"probe":"What is wisdom?","expected_endpoint_tokens":["action"],"intermediate_tokens":["judgment","decision"],"expected_proposals":3} +{"id":"MSR-V1-002","pattern":"chain_3","hops":3,"premises":["What is light?","Actually light is clarity.","What is clarity?","Actually clarity is recognition.","What is recognition?","Actually recognition is naming."],"probe":"What is light?","expected_endpoint_tokens":["naming"],"intermediate_tokens":["clarity","recognition"],"expected_proposals":3} +{"id":"MSR-V1-003","pattern":"chain_3","hops":3,"premises":["What is creation?","Actually creation is movement.","What is movement?","Actually movement is change.","What is change?","Actually change is becoming."],"probe":"What is creation?","expected_endpoint_tokens":["becoming"],"intermediate_tokens":["movement","change"],"expected_proposals":3} +{"id":"MSR-V1-004","pattern":"chain_3","hops":3,"premises":["What is truth?","Actually truth is knowledge.","What is knowledge?","Actually knowledge is judgment.","What is judgment?","Actually judgment is wisdom."],"probe":"What is truth?","expected_endpoint_tokens":["wisdom"],"intermediate_tokens":["knowledge","judgment"],"expected_proposals":3} +{"id":"MSR-V1-005","pattern":"chain_4","hops":4,"premises":["What is principle?","Actually principle is order.","What is order?","Actually order is structure.","What is structure?","Actually structure is form.","What is form?","Actually form is meaning."],"probe":"What is principle?","expected_endpoint_tokens":["meaning"],"intermediate_tokens":["order","structure","form"],"expected_proposals":4} +{"id":"MSR-V1-006","pattern":"chain_4","hops":4,"premises":["What is question?","Actually question is inquiry.","What is inquiry?","Actually inquiry is thought.","What is thought?","Actually thought is reason.","What is reason?","Actually reason is inference."],"probe":"What is question?","expected_endpoint_tokens":["inference"],"intermediate_tokens":["inquiry","thought","reason"],"expected_proposals":4} +{"id":"MSR-V1-007","pattern":"chain_4","hops":4,"premises":["What is light?","Actually light is clarity.","What is clarity?","Actually clarity is recognition.","What is recognition?","Actually recognition is naming.","What is naming?","Actually naming is definition."],"probe":"What is light?","expected_endpoint_tokens":["definition"],"intermediate_tokens":["clarity","recognition","naming"],"expected_proposals":4} +{"id":"MSR-V1-008","pattern":"chain_5","hops":5,"premises":["What is wisdom?","Actually wisdom is judgment.","What is judgment?","Actually judgment is decision.","What is decision?","Actually decision is action.","What is action?","Actually action is effect.","What is effect?","Actually effect is consequence."],"probe":"What is wisdom?","expected_endpoint_tokens":["consequence"],"intermediate_tokens":["judgment","decision","action","effect"],"expected_proposals":5} +{"id":"MSR-V1-009","pattern":"chain_5","hops":5,"premises":["What is creation?","Actually creation is order.","What is order?","Actually order is structure.","What is structure?","Actually structure is form.","What is form?","Actually form is meaning.","What is meaning?","Actually meaning is purpose."],"probe":"What is creation?","expected_endpoint_tokens":["purpose"],"intermediate_tokens":["order","structure","form","meaning"],"expected_proposals":5} +{"id":"MSR-V1-010","pattern":"mixed_relation_3","hops":3,"premises":["What is light?","Actually light grounds clarity.","What is clarity?","Actually clarity causes recognition.","What is recognition?","Actually recognition precedes naming."],"probe":"What does light precede?","expected_endpoint_tokens":["naming"],"intermediate_tokens":["clarity","recognition"],"expected_proposals":3} +{"id":"MSR-V1-011","pattern":"mixed_relation_3","hops":3,"premises":["What is truth?","Actually truth grounds knowledge.","What is knowledge?","Actually knowledge causes judgment.","What is judgment?","Actually judgment precedes decision."],"probe":"What does truth precede?","expected_endpoint_tokens":["decision"],"intermediate_tokens":["knowledge","judgment"],"expected_proposals":3} +{"id":"MSR-V1-012","pattern":"mixed_relation_4","hops":4,"premises":["What is creation?","Actually creation causes order.","What is order?","Actually order grounds structure.","What is structure?","Actually structure is form.","What is form?","Actually form precedes meaning."],"probe":"What does creation precede?","expected_endpoint_tokens":["meaning"],"intermediate_tokens":["order","structure","form"],"expected_proposals":4} +{"id":"MSR-V1-013","pattern":"mixed_relation_4","hops":4,"premises":["What is principle?","Actually principle causes order.","What is order?","Actually order grounds coherence.","What is coherence?","Actually coherence is meaning.","What is meaning?","Actually meaning precedes purpose."],"probe":"What does principle precede?","expected_endpoint_tokens":["purpose"],"intermediate_tokens":["order","coherence","meaning"],"expected_proposals":4} +{"id":"MSR-V1-014","pattern":"chain_3","hops":3,"premises":["What is reason?","Actually reason is inference.","What is inference?","Actually inference is conclusion.","What is conclusion?","Actually conclusion is decision."],"probe":"What is reason?","expected_endpoint_tokens":["decision"],"intermediate_tokens":["inference","conclusion"],"expected_proposals":3} +{"id":"MSR-V1-015","pattern":"chain_4","hops":4,"premises":["What is memory?","Actually memory is recall.","What is recall?","Actually recall is recognition.","What is recognition?","Actually recognition is naming.","What is naming?","Actually naming is language."],"probe":"What is memory?","expected_endpoint_tokens":["language"],"intermediate_tokens":["recall","recognition","naming"],"expected_proposals":4} diff --git a/evals/multi_step_reasoning/runner.py b/evals/multi_step_reasoning/runner.py new file mode 100644 index 00000000..ec7c1f86 --- /dev/null +++ b/evals/multi_step_reasoning/runner.py @@ -0,0 +1,143 @@ +"""multi-step-reasoning eval lane runner. + +For each case: teach a 3- to 5-hop chain, probe the head, score +whether the final-hop entity appears in the response surface. + +Conforms to the framework interface: run_lane(cases, config=None) -> report. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from chat.runtime import ChatRuntime +from core.cognition.pipeline import CognitiveTurnPipeline +from core.config import RuntimeConfig +from evals.parallel import run_cases_parallel + + +@dataclass(slots=True) +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +_TOKEN_BOUND = re.compile(r"\b([a-z][a-z'\-]*)\b") + + +def _tokens(text: str) -> set[str]: + return set(_TOKEN_BOUND.findall((text or "").lower())) + + +def _hit(text: str, candidates: list[str]) -> bool: + if not text: + return False + toks = _tokens(text) + return any(c.lower() in toks for c in candidates) + + +def _run_sequence(premises: list[str], probe: str) -> dict[str, Any]: + runtime = ChatRuntime() + pipeline = CognitiveTurnPipeline(runtime) + proposals = 0 + for premise in premises: + try: + r = pipeline.run(premise, max_tokens=8) + except ValueError: + continue + if r.pack_mutation_proposal is not None: + proposals += 1 + try: + probe_result = pipeline.run(probe, max_tokens=8) + except ValueError: + return { + "surface": "", "articulation_surface": "", "walk_surface": "", + "trace_hash": "", "vault_hits": 0, "proposals": proposals, + } + return { + "surface": probe_result.surface or "", + "articulation_surface": probe_result.articulation_surface or "", + "walk_surface": probe_result.walk_surface or "", + "trace_hash": probe_result.trace_hash, + "vault_hits": int(probe_result.vault_hits), + "proposals": proposals, + } + + +def _run_case(case: dict[str, Any]) -> dict[str, Any]: + premises: list[str] = list(case.get("premises", [])) + probe: str = case["probe"] + endpoint_tokens: list[str] = list(case.get("expected_endpoint_tokens", [])) + intermediates: list[str] = list(case.get("intermediate_tokens", [])) + expected_proposals = int(case.get("expected_proposals", len(premises) // 2)) + + first = _run_sequence(premises, probe) + second = _run_sequence(premises, probe) + + surface_blob = " ".join([ + first["surface"], first["articulation_surface"], first["walk_surface"] + ]) + endpoint_hit = _hit(surface_blob, endpoint_tokens) + intermediate_hit = _hit(surface_blob, intermediates) + premises_stored = first["proposals"] >= expected_proposals + replay_pass = ( + bool(first["trace_hash"]) + and first["trace_hash"] == second["trace_hash"] + and first["vault_hits"] == second["vault_hits"] + and first["proposals"] == second["proposals"] + ) + + passed = endpoint_hit and premises_stored and replay_pass + + return { + "id": case.get("id", ""), + "pattern": case.get("pattern", ""), + "hops": int(case.get("hops", 0)), + "endpoint_tokens": endpoint_tokens, + "vault_hits": first["vault_hits"], + "trace_hash": first["trace_hash"], + "trace_hash_replay": second["trace_hash"], + "proposals": first["proposals"], + "expected_proposals": expected_proposals, + "endpoint_hit": endpoint_hit, + "intermediate_hit": intermediate_hit, + "premises_stored_pass": premises_stored, + "replay_pass": replay_pass, + "passed": passed, + } + + +def run_lane( + cases: list[dict[str, Any]], + *, + config: RuntimeConfig | None = None, + workers: int | None = None, +) -> LaneReport: + if not cases: + return LaneReport(metrics={}, case_details=[]) + _ = config + + case_details = run_cases_parallel(cases, _run_case, workers=workers) + total = len(case_details) + + endpoint = sum(1 for d in case_details if d["endpoint_hit"]) / total + intermediate = sum(1 for d in case_details if d["intermediate_hit"]) / total + stored = sum(1 for d in case_details if d["premises_stored_pass"]) / total + replay = sum(1 for d in case_details if d["replay_pass"]) / total + overall = sum(1 for d in case_details if d["passed"]) / total + + overall_pass = endpoint >= 0.50 and stored >= 0.95 and replay >= 0.95 + + metrics: dict[str, Any] = { + "chain_endpoint_recall_rate": round(endpoint, 4), + "intermediate_hop_visible_rate": round(intermediate, 4), + "premises_stored_rate": round(stored, 4), + "replay_determinism": round(replay, 4), + "all_pass_rate": round(overall, 4), + "case_count": total, + "overall_pass": overall_pass, + } + + return LaneReport(metrics=metrics, case_details=case_details)