diff --git a/core/cli.py b/core/cli.py index fda431a5..d94a585d 100644 --- a/core/cli.py +++ b/core/cli.py @@ -23,7 +23,7 @@ _CORE_RS_DIR = _REPO_ROOT / "core-rs" _CORE_RS_MANIFEST = _CORE_RS_DIR / "Cargo.toml" DESCRIPTION = "CORE versor engine command suite." -EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core eval cognition\n core eval cognition --json" +EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1" _TEST_SUITES: dict[str, tuple[str, ...]] = { "fast": ( @@ -512,43 +512,75 @@ def cmd_doctor(args: argparse.Namespace) -> int: return 0 if ok else 1 -def cmd_eval_cognition(args: argparse.Namespace) -> int: - """Run the cognition eval harness.""" - from evals.run_cognition_eval import load_cases, run_eval +def cmd_eval(args: argparse.Namespace) -> int: + """Run an eval lane by name, or list available lanes.""" + from evals.framework import discover_lanes, get_lane, run_lane, write_result - cases = load_cases() - report = run_eval(cases) + if args.list_lanes: + lanes = discover_lanes() + if not lanes: + print("no eval lanes found") + for lane in lanes: + versions = ", ".join(lane.versions) if lane.versions else "none" + print(f" {lane.name:20s} versions: {versions}") + return 0 + + lane_name = args.lane + if not lane_name: + _die("eval requires a lane name. Use `core eval --list` to see available lanes.") + + try: + lane = get_lane(lane_name) + except FileNotFoundError as exc: + _die(str(exc)) + + version = args.version or (lane.versions[0] if lane.versions else "v1") + split = args.split + + try: + result = run_lane(lane, version=version, split=split) + except FileNotFoundError as exc: + _die(str(exc)) if args.json: - print(json.dumps(report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True)) + print(json.dumps(result.as_dict(), ensure_ascii=False, indent=2, sort_keys=True)) else: - print(f"cases : {report.total}") - print(f"intent_accuracy: {report.intent_accuracy:.1%}") - print(f"term_capture : {report.term_capture_rate:.1%}") - print(f"surface_ground : {report.surface_groundedness:.1%}") - print(f"versor_closure : {report.versor_closure_rate:.1%}") - print(f"det_traces : {report.deterministic_traces}") - failures = [c for c in report.cases if not c.intent_correct or not c.versor_closure] + print(f"lane : {result.lane}") + print(f"version : {result.version}") + print(f"split : {result.split}") + print(f"cases : {result.metrics.get('total', 0)}") + for key, value in result.metrics.items(): + if key == "total": + continue + if isinstance(value, float): + print(f"{key:15s}: {value:.1%}") + else: + print(f"{key:15s}: {value}") + failures = [c for c in result.case_details if not c.get("intent_correct") or not c.get("versor_closure")] if failures: print(f"\nfailures ({len(failures)}):") for c in failures: issues = [] - if not c.intent_correct: + if not c.get("intent_correct"): issues.append("intent") - if not c.versor_closure: - issues.append(f"versor={c.versor_condition:.2e}") - print(f" {c.case_id}: {', '.join(issues)}") + if not c.get("versor_closure"): + vc = c.get("versor_condition", 0) + issues.append(f"versor={vc:.2e}") + print(f" {c['case_id']}: {', '.join(issues)}") + + if args.save: + result_path = write_result(lane, result) + print(f"\nresult written: {result_path}", file=sys.stderr) if args.report: report_path = Path(args.report) report_path.parent.mkdir(parents=True, exist_ok=True) report_path.write_text( - json.dumps(report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True) + json.dumps(result.as_dict(), ensure_ascii=False, indent=2, sort_keys=True) ) - print(f"\nreport written: {report_path}") + print(f"\nreport written: {report_path}", file=sys.stderr) - all_pass = report.intent_accuracy == 1.0 and report.versor_closure_rate == 1.0 - return 0 if all_pass else 1 + return 0 def cmd_pulse(args: argparse.Namespace) -> int: @@ -728,12 +760,15 @@ def build_parser() -> argparse.ArgumentParser: bench.add_argument("--report", metavar="PATH", help="write JSON report to file") bench.set_defaults(func=cmd_bench) - eval_cmd = subparsers.add_parser("eval", help="run eval harnesses") - eval_sub = eval_cmd.add_subparsers(dest="eval_command", metavar="eval-command", required=True) - eval_cognition = eval_sub.add_parser("cognition", help="run the cognition eval harness") - eval_cognition.add_argument("--json", action="store_true", help="emit machine-readable JSON") - eval_cognition.add_argument("--report", metavar="PATH", help="write JSON report to file") - eval_cognition.set_defaults(func=cmd_eval_cognition) + eval_cmd = subparsers.add_parser("eval", help="run eval lanes") + eval_cmd.add_argument("lane", nargs="?", help="eval lane name (e.g. cognition)") + eval_cmd.add_argument("--list", dest="list_lanes", action="store_true", help="list available eval lanes") + eval_cmd.add_argument("--version", help="version to evaluate (default: latest)") + eval_cmd.add_argument("--split", default="public", choices=["dev", "public"], help="which split to score (default: public)") + eval_cmd.add_argument("--json", action="store_true", help="emit machine-readable JSON") + eval_cmd.add_argument("--save", action="store_true", help="write result to lane results/ directory") + eval_cmd.add_argument("--report", metavar="PATH", help="write JSON report to file") + eval_cmd.set_defaults(func=cmd_eval) doctor = subparsers.add_parser("doctor", help="check runtime imports and packaging health") doctor.add_argument("--packs", action="store_true", help="also list discovered language packs") diff --git a/core/cognition/pipeline.py b/core/cognition/pipeline.py index 5f53e444..939d5b60 100644 --- a/core/cognition/pipeline.py +++ b/core/cognition/pipeline.py @@ -82,13 +82,14 @@ class CognitiveTurnPipeline: # 9. Reconstruct input-layer tokens from the turn log # (turn_log is appended inside chat(); last entry matches this turn) - last_turn = self.runtime.turn_log[-1] - filtered_tokens = last_turn.input_tokens - - # Raw tokenization is identical to filtered for Phase 1 — the - # runtime's _tokenize() runs before _apply_oov_policy(). We - # expose input_tokens separately so Phase 2 can diverge them. + # When the unknown-domain gate fires, chat() returns a stub without + # appending to turn_log — fall back to the tokenizer. raw_tokens = tuple(self.runtime.tokenize(text)) + if self.runtime.turn_log: + last_turn = self.runtime.turn_log[-1] + filtered_tokens = last_turn.input_tokens + else: + filtered_tokens = raw_tokens # 10. TEACHING — correction capture, review, and store teaching_candidate, reviewed_example, proposal = self._run_teaching( diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md new file mode 100644 index 00000000..80726adb --- /dev/null +++ b/docs/PROGRESS.md @@ -0,0 +1,134 @@ +# Capability Roadmap — Progress Tracker + +Tracks completion of the phased plan defined in `docs/capability_roadmap.md` +(ADR-0016). Updated as work lands. + +--- + +## Phase 0 — Benchmark Methodology Lock-in + +**Status:** In Progress +**Started:** 2026-05-15 + +- [x] Promote roadmap to ADR-0016 +- [x] Extract `docs/eval_methodology.md` from roadmap Part I +- [x] Create progress tracker (`docs/PROGRESS.md`) +- [x] Implement `evals//` directory convention +- [x] Build generic eval framework (`evals/framework.py`) +- [x] Retrofit `core eval cognition` into new convention + - [x] Split 45 cases into dev (13) / public v1 (13) / holdout (19) + - [x] Write `evals/cognition/contract.md` + - [x] Migrate `runner.py` to use framework + - [x] Record v1 results under new layout +- [x] Generalize `core eval ` CLI (dynamic lane discovery) +- [x] Implement holdout runner scaffold +- [x] Implement baseline runner scaffold +- [ ] **Exit gate:** `core eval cognition` runs under new convention with v1 public + holdout + baseline + +### Methodology issues discovered (Phase 0 audit) + +1. **Pipeline turn_log crash:** `CognitiveTurnPipeline.run()` assumed `turn_log` + was always populated after `chat()`, but the unknown-domain gate returns a + stub without appending. Fixed with fallback to tokenizer output. +2. **Versor drift in multi-turn sessions:** `test_pipeline_preserves_versor_closure` + reveals that after 3 turns in the same session, "spirit breath" causes + `versor_condition = 1.12e-04` (threshold: 1e-6). Pre-existing; not caused by + eval restructuring. Needs operator/construction boundary investigation. + +--- + +## Phase 1 — Foundational Triple + +**Status:** Not Started +**Depends on:** Phase 0 exit + +- [ ] **grammatical-coverage** lane + - [ ] Enumerate English v1 constructions + - [ ] Write contract test pairs (PropositionGraph -> surface family) + - [ ] Implement v1 dev/public/holdout (~50/50/50 items) + - [ ] Engineer `realizer.py` to pass v1 + - [ ] Generate v2 on pass + - [ ] Hebrew pack + - [ ] Koine Greek pack +- [ ] **zero-code-domain-acquisition** lane + - [ ] Define 3 surprise domains + - [ ] Build pack-only authoring kits + - [ ] Test: author brings CORE to >=80% without Python edits + - [ ] Log engineering gaps +- [ ] **identity-divergence** lane + - [ ] Define two identity axis sets + - [ ] Curate shared curriculum (~100 teaching events) + - [ ] Build divergence + coherence metrics + - [ ] Identity-stripped baseline +- [ ] **Exit gate:** All three lanes pass v1 public + holdout + +--- + +## Phase 2 — Structural Wins Made Visible + +**Status:** Not Started +**Depends on:** Phase 1 exit + +- [ ] **provenance** lane +- [ ] **monotonic-learning** lane +- [ ] **calibration** lane +- [ ] **symbolic-logic** lane +- [ ] **adversarial-identity** lane +- [ ] Frontier baselines computed for all lanes +- [ ] **Exit gate:** All five v1+v2 with baselines; at least two have v3 + +--- + +## Phase 3 — Reasoning Depth + +**Status:** Not Started +**Depends on:** Phase 2 exit + +- [ ] **compositionality** lane (construction-family splits, not sampling) +- [ ] **inference-closure** lane +- [ ] **introspection** lane +- [ ] **multi-step-reasoning** lane +- [ ] **cross-domain-transfer** lane +- [ ] Pin agency scope decision (responsive vs. goal-directed) +- [ ] Pin tool-use scope decision +- [ ] **Exit gate:** All five v1 scored; at least two passing v1 + +--- + +## Phase 4 — Scale and Efficiency + +**Status:** Not Started +**Depends on:** Phase 3 exit + +- [ ] **sample-efficiency** curves (>=10 concepts) +- [ ] **long-context-cost** curves (10^3 to 10^6 vault entries) +- [ ] **multi-agent-composition** (>=2 agents, replay preserved) +- [ ] Vault indexing strategy decided +- [ ] **Exit gate:** All curves published with confidence intervals + +--- + +## Phase 5 — Curriculum Era + +**Status:** Not Started +**Depends on:** Phase 4 exit + +- [ ] 5.1 English fluency (grammatical-coverage v5 OOD) +- [ ] 5.2 Hebrew fluency +- [ ] 5.3 Koine Greek fluency +- [ ] 5.4 Elementary mathematics +- [ ] 5.5 Foundational physics +- [ ] 5.6 Foundational biology +- [ ] 5.7 Classical literature +- [ ] Phase 1-4 lanes re-run on every release (no regression) + +--- + +## Open Scope Decisions + +| Decision | Status | Deadline | +|----------|--------|----------| +| Agency (responsive vs. goal-directed) | Open | Before Phase 3 | +| Tool use (typed deterministic operators) | Open | Before Phase 3 | +| Code generation (first-class target) | Open | Before Phase 5 | +| Embodiment (sensorium gates) | Open | Phase 5 | diff --git a/docs/capability_roadmap.md b/docs/capability_roadmap.md new file mode 100644 index 00000000..d4469cf6 --- /dev/null +++ b/docs/capability_roadmap.md @@ -0,0 +1,340 @@ +# Capability Roadmap — Phased Plan to the Verifiable Competence Gates + +**Status:** Draft, derived from `docs/sessions/SESSION-2026-05-15-capability-gates.md` +**Owner:** Joshua Shay +**Last updated:** 2026-05-15 + +This document walks CORE from its present state through the gating framework defined in the 2026-05-15 session. It is organized into six phases. Each phase has entry criteria, work items, exit criteria, and a benchmark discipline contract. + +The benchmark discipline is the spine of the plan. Without it, the phases become aspirational. With it, "are we there yet" becomes a CLI question. + +--- + +## Part I — Benchmark Discipline (read first) + +The gates are only meaningful if the evals that prove them are honest. Five rules govern every eval lane in this roadmap. They apply uniformly; no exceptions per phase. + +### Rule 1 — Three-set split per lane + +Every lane maintains three disjoint corpora: + +- **Dev set.** Freely visible during development. Used to iterate. +- **Public test set.** Visible, but tuning against it is forbidden. Scored at version-cut time only. Drift in dev-vs-public scores is a red flag for overfitting. +- **Private holdout.** Sealed. Never read by Claude, never committed in plaintext, only scored by a clean-room runner at release events. Stored encrypted in `evals/holdouts/` with key held by the human reviewer. + +If a lane has only a dev set, it does not count as a gate. It is exploration. + +### Rule 2 — Versioned difficulty escalation + +Each lane has versions: `v1`, `v2`, `v3`, … with monotonically harder distributions. Passing a version is not a terminal state; it is a checkpoint that unlocks generating the next version. + +- **v1** — baseline competence demonstration. The construction is shown clearly. +- **v2** — distributional shift: longer chains, deeper nesting, rarer vocabulary, paraphrased surface forms. +- **v3** — adversarial: items generated specifically by inspecting model failures on v2. +- **v4+** — out-of-distribution: items drawn from domains, registers, or constructions not present at training time. + +Score is always reported as a tuple `(v1_score, v2_score, v3_score, …)`, never collapsed to a single number. A model that scores 99% on v1 and 12% on v3 is not a "99% model." + +### Rule 3 — Adversarial regeneration on pass + +When a model passes a version (e.g., ≥95% on the public test set with ≥90% on private holdout), the next version is *generated by adversarial process*: + +- Human review finds construction families the model handled accidentally rather than structurally. +- A separate generator (could be a different model, could be programmatic) produces items targeting the weakest decile of the previous version. +- The new version is reviewed for legitimacy — no impossible items, no ambiguous items, no items that depend on world knowledge the system was never given. + +This is the protection against silent overfitting: every passed version triggers the construction of a harder one, so "progress" requires continuously rising scores against continuously harder tests. + +### Rule 4 — Frontier baseline tracking + +For each lane, a baseline score is computed for at least one frontier transformer-based model (e.g., Claude Opus 4.7, GPT-5, Gemini 3 Ultra) on the *same* public test set. Baselines are: + +- Re-scored every time a version is cut. +- Published alongside CORE's score. +- Never tuned, never prompted-engineered to maximize — the prompt is the eval task as written. + +This serves two purposes: (a) it makes CORE's structural wins visible (frontier models score near zero on provenance, monotonic learning, etc.); (b) it prevents self-congratulation on lanes where CORE merely matches an LLM that was given no advantage. + +### Rule 5 — Honest reporting + +- Failures are reported with the same prominence as passes. +- Confidence intervals on every score (bootstrapped over the test set). +- Per-construction breakdowns published — never a single aggregate hiding structural failures. +- Regressions across versions are surfaced, never silently dropped. +- "Did not test" is a valid result; "tested and failed" is preferred over "did not test." + +If a number cannot be reported honestly under these rules, the lane is not ready. Do not ship the lane. + +### Eval contract template + +Every eval lane lives in `evals//` with this layout: + +``` +evals// + contract.md # what the lane measures, scoring rubric, pass thresholds + dev/ # dev set, freely visible + public/v1/ # public test set, version 1 + public/v2/ # public test set, version 2 + ... + holdouts/ # encrypted, sealed + runner.py # deterministic scorer + baselines/ # frontier model scores per version + results/ # CORE scores per version per release +``` + +A lane without a `contract.md` does not run. + +--- + +## Part II — The Phases + +### Phase 0 — Benchmark methodology lock-in + +**Entry criteria.** Today. + +**Goal.** Build the discipline infrastructure before building any new eval. Doing this first prevents the entire roadmap from drifting into vibes-based progress. + +**Work items.** + +1. Implement `evals/` layout convention above. +2. Implement `core eval ` CLI subcommand that loads contracts, runs the runner, writes results. +3. Implement the holdout-runner: a sandboxed process that decrypts the sealed test set, scores, writes only the aggregate score (never item-level results) back to the working tree. +4. Implement baseline-runner: a thin adapter that queries a frontier model on the public test set and records its score. +5. Write the methodology page in `docs/eval_methodology.md` (this Part I, extracted). +6. Pick one *existing* eval (the current `core eval cognition`) and retrofit it into the new convention as a forcing function. Versions become explicit; holdout is split out; results are reported per-version. + +**Exit criteria.** + +- `core eval cognition` runs under the new convention, with v1 public + private holdout + baseline. +- No new lane is allowed to be merged that does not follow the convention. +- The retrofit revealed at least one item-level methodology issue (silent ambiguity, leaked dev item, unstated assumption) — caught and documented. If the retrofit found nothing, the audit was not real. + +**Duration estimate.** 1–2 weeks of focused work. + +--- + +### Phase 1 — Foundational Triple + +**Entry criteria.** Phase 0 exit complete. + +**Goal.** Implement and pass the three gates that determine whether CORE is ready to move from engineering into curriculum: + +- **grammatical-coverage** (fluency) +- **zero-code-domain-acquisition** (engineering-vs-learning phase shift) +- **identity-divergence** (identity is load-bearing) + +**Work items.** + +**1.1 grammatical-coverage** + +- Enumerate target grammatical constructions for English v1: simple declarative, negation, conjunction, disjunction, embedded clause, relative clause, quantification (universal/existential), basic tense (past/present/future), basic aspect (perfective/imperfective). +- For each construction, write contract test pairs: `PropositionGraph → expected surface family`. "Expected surface family" is a set of acceptable surfaces, not a single string, with a deterministic acceptance predicate. +- Implement v1 dev/public/holdout (target: ~50/50/50 items). +- Engineer `realizer.py` to pass v1. +- Once v1 ≥95% public and ≥90% holdout, generate v2 (deeper nesting, rare vocabulary substitution, longer sentences). +- Repeat for Hebrew and Koine Greek using their respective pack morphology. + +**1.2 zero-code-domain-acquisition** + +- Define three "surprise domains" never touched in development: pick from {kinship relations, basic arithmetic, simple spatial relations, color taxonomy, calendar relations}. +- Each domain has a pack-only authoring kit: vocabulary, relation predicates, axiom list, ~20 reviewed teaching examples, ~30 articulation prompts. +- Test: an author who knows the system but is forbidden from editing Python attempts to bring CORE to ≥80% articulation accuracy on the prompts using only pack + teaching loop. +- Each Python edit required is a logged "engineering gap" that goes onto the closing list. + +**1.3 identity-divergence** + +- Define two identity axis sets, deliberately oriented to produce different stances on the same proposition (e.g., axis-A weights novelty highly, axis-B weights tradition highly; or axis-A is precision-first, axis-B is generosity-first). +- Curate a shared curriculum: ~100 reviewed teaching events, identical for both agents. +- Curate a prompt set where identity should produce measurably different articulations. +- Scoring: an automated divergence metric (e.g., proposition-graph difference) plus a coherence metric (each output must be internally consistent with its own axes). +- Pass: divergence above floor, coherence above floor, *both required*. +- Also: identity-stripped baseline. The same curriculum with identity disabled should produce articulations whose divergence is at noise floor — proving identity is doing the work. + +**Exit criteria.** + +- All three lanes pass v1 on public + holdout. +- The engineering-gap list from 1.2 is either empty or has a documented closing plan. +- v2 generation has been attempted for at least one of the three. + +**Duration estimate.** 4–8 weeks. The realizer work in 1.1 is the bottleneck and may expose deeper engineering gaps. + +--- + +### Phase 2 — Structural Wins Made Visible + +**Entry criteria.** Phase 1 exit complete. + +**Goal.** Build the lanes where CORE's architecture wins by design. These should pass relatively early *for CORE* and fail catastrophically *for frontier baselines*. The purpose is to publish the contrast. + +**Lanes:** + +- **provenance** — every articulated claim back-points to vault entries / teaching events / pack axioms; replay reproduces trace bit-for-bit. +- **monotonic-learning** — after N teaching cycles across unrelated domains, competence on domain 1 does not regress. Longitudinal: ≥10 teaching cycles, scored at each step. +- **calibration** — out-of-pack queries produce typed "no grounding" responses; in-pack queries do not. Distinguish "I don't know" / "incoherent" / "contradicts known." +- **symbolic-logic** — nested negation, modal operators (must/may/possible/necessary), counterfactual conditionals. Target ≥99% on v1. +- **adversarial-identity** — 1,000-turn red-team corpus; identity drift below noise floor. + +**Work items.** + +- Build each lane following the convention. +- Compute frontier baselines for each. Expected outcomes: + - **provenance** — CORE: pass. Frontier: near-zero (no model can produce verifiable per-claim provenance). + - **monotonic-learning** — CORE: pass by construction. Frontier: regression visible after fine-tuning rounds (this requires the frontier baseline to actually be fine-tuned, which complicates the comparison — may be reported as "not directly comparable, structural argument applies"). + - **calibration** — CORE: high if calibration is wired; frontier: confabulates on most OOD prompts. + - **symbolic-logic** — CORE: target ≥99% v1; frontier: ~80% v1, collapses on v3. + - **adversarial-identity** — CORE: target drift below noise; frontier: persona erodes within ~50–100 turns. +- Publish results page with per-lane comparison. + +**Anti-overfitting note.** The structural wins are *structural* — the temptation to declare victory after v1 is large. Discipline: v2 and v3 must still be generated and scored. A "structural win" that fails on v3 is a structural claim that was actually a v1 coincidence. + +**Exit criteria.** + +- All five lanes have v1 + v2 results published with frontier baselines. +- At least two lanes have v3 results. +- The contrast page is honest about which results are "directly comparable" vs. "structural argument." + +**Duration estimate.** 8–12 weeks. provenance and monotonic-learning may require new instrumentation in `vault/` and `teaching/`. + +--- + +### Phase 3 — Reasoning Depth + +**Entry criteria.** Phase 2 exit complete. This is the hardest phase. Expect engineering surprises. + +**Goal.** Lanes that probe whether CORE actually *thinks* rather than retrieves and articulates. + +**Lanes:** + +- **compositionality** — novel combinations of taught primitives. SCAN/COGS-style splits adapted to proposition graphs. +- **inference-closure** — derive entailments never directly asserted (transitive, spatial, temporal, causal chains). +- **introspection** — `explain(turn_id)` produces a natural-language account that round-trips: a separate run conditioned on the explanation predicts the same articulation. +- **multi-step-reasoning** — pipeline produces and consumes intermediate proposition-graph states for problems whose solution requires ≥3 inferential hops. +- **cross-domain-transfer** — competence in domain B rises after teaching only in domain A, via shared structural elements. + +**Work items.** + +These will almost certainly expose engineering gaps. Expected gaps: + +- `generate/graph_planner.py` may need an intermediate-state stack rather than a single planning pass (multi-step-reasoning). +- `field/propagate.py` may need to expose derivable-but-not-asserted recall paths (inference-closure). +- A new `cognition/explain.py` module may be needed for introspection. +- Cross-domain transfer may require examining how proposition graphs share structural sub-units, which may be a pack-design question more than a code question. + +For each gap discovered, the work splits: (a) write the eval, (b) confirm it fails, (c) close the engineering gap, (d) re-run. + +**Anti-overfitting note.** Compositionality is *the* lane most vulnerable to overfitting. The training-test split must be done by *construction family*, not by sampling. If the model has seen `R(A,B)` and `R(C,D)`, the test set must use a *novel relation R'* applied to seen entities — not a fresh `(A,B)` pair under a seen `R`. + +**Exit criteria.** + +- All five lanes have v1 results with honest scores (which may be failing — that's acceptable for v1). +- Each failure has either a closed engineering gap or a documented architectural deferral. +- At least two lanes are passing v1 by phase exit. + +**Duration estimate.** 12–24 weeks. This is the phase that decides whether CORE's design lives up to its philosophical claims. + +--- + +### Phase 4 — Scale and Efficiency + +**Entry criteria.** Phase 3 exit complete. Phases 1–3 are pass/fail; this phase is *quantitative curves*. + +**Goal.** Make CORE's quantitative behavior visible: how fast does it learn, how does cost scale, how does it compose at scale. + +**Lanes:** + +- **sample-efficiency** — corrections-to-competence curves across ten unrelated concepts. Plot, do not threshold. +- **long-context-cost** — vault size vs. per-turn latency curve at 10³, 10⁴, 10⁵, 10⁶ entries. Identify the asymptotic complexity. Decide indexing strategy if super-linear. +- **multi-agent-composition** — two CORE instances with different identities cooperate on a shared task; each maintains its own deterministic replay. Measure: task completion, replay determinism preserved per agent, no identity bleed. + +**Work items.** + +- Build infrastructure for longitudinal measurement (Phase 2's monotonic-learning runner is a starting point). +- Sample-efficiency requires running the teaching loop programmatically with controlled correction budgets. +- Long-context-cost may surface that the current `vault/store.py` is insufficient at scale — the response is exact indexing (B-tree, suffix array, signature-based bucketing), not approximate recall. +- Multi-agent composition surfaces orchestration questions that may justify a new module (`society/` or similar) — defer unless the eval forces it. + +**Anti-overfitting note.** Curves don't overfit the way thresholds do, but they can be selectively reported. Discipline: publish the full curve, not just the best operating point. Confidence intervals at each data point. + +**Exit criteria.** + +- Sample-efficiency curves published for ≥10 concepts. +- Vault cost curve published with asymptotic analysis. Indexing strategy decided. +- Multi-agent composition demonstrated for ≥2-agent cooperation with replay preserved. + +**Duration estimate.** 8–16 weeks. + +--- + +### Phase 5 — Curriculum Era + +**Entry criteria.** Phase 4 exit complete. From this point forward, engineering changes are exceptional, not routine. The work is curriculum design, reviewed teaching, and domain-specific evals. + +**Goal.** Acquire human-comparable competence across school-level subjects, classical literature, foundational sciences, and the three foundational languages at fluency. + +**Structure.** + +The phase has no single exit criterion. Instead, each domain becomes its own sub-phase with its own evals: + +- **5.1 English fluency** — pack + curriculum sufficient that grammatical-coverage v5 (out-of-distribution registers: legal, poetic, technical, conversational) passes. +- **5.2 Hebrew fluency** — analogous, with attention to root-and-pattern morphology. +- **5.3 Koine Greek fluency** — analogous, with attention to case and aspect. +- **5.4 Elementary mathematics** — number, arithmetic, basic algebra, geometry. Each topic becomes a pack + a domain-specific competence eval. +- **5.5 Foundational physics** — kinematics, conservation, basic mechanics. +- **5.6 Foundational biology** — taxonomy, cell, system. +- **5.7 Classical literature** — reading comprehension at increasing complexity, eventually approaching the John 1:1–2 grounding case as a depth probe. +- *(further sub-phases as curriculum expands)* + +**Discipline during this phase.** + +- Every new domain ships with its own competence eval following the convention. +- The Phase 1–4 lanes are re-run on every release. A new domain that causes regression in a foundational gate is a curriculum bug, not a curriculum success. +- Frontier baselines are re-scored periodically; the contrast remains visible. + +**This phase has no estimated duration.** It is the phase the project lives in after the engineering era ends. Frontier-LLM parity on breadth happens *inside* this phase if it happens at all — likely measured in years, not weeks, and at whatever sample efficiency Phase 4 demonstrated. + +--- + +## Part III — Cross-Cutting Considerations + +### Versioning of the framework itself + +This roadmap is `v1`. As phases complete, the framework may itself need amendment — new lanes added, methodology refined. Treat the roadmap with the same discipline as the evals: version it, never silently rewrite it. Each amendment is dated and explained. + +### Scope decisions deferred + +Two scope decisions named in the 2026-05-15 session remain open and will be pinned before they cause drift: + +- **Agency** — responsive vs. goal-directed. Defaulting to *responsive* for Phases 0–4. Phase 5 may revisit. +- **Embodiment** — symbolic-only vs. sensorimotor. ADR-0013 establishes the sensorium protocol; this roadmap does not assume sensorium-dependent gates in Phases 0–4. Phase 5 may add sensorium-dependent sub-phases. + +Two further questions emerged during Phase planning that should be decided early: + +- **Tool use.** Is the pipeline extensible to typed deterministic operators (calculator, search, code execution)? Decision needed before Phase 3, since multi-step-reasoning may benefit from operator delegation. +- **Code generation.** Is code a first-class proposition-graph articulation target? Decision needed before Phase 5 if computer-science is a curriculum domain. + +### What this roadmap is not + +- Not a list of features. The features fall out of the gates. +- Not a competitive roadmap against frontier LLMs. The contrast is a side effect, not a target. +- Not a commitment to dates. The duration estimates are calibration aids, not deliverable schedules. +- Not a substitute for the work-sequencing list in `CLAUDE.md`. That list governs daily work; this document governs the arc. + +### Failure modes to watch for + +- **Vibes-based progress.** "It feels smarter" is not a gate. +- **Demo-driven development.** Crafting a single impressive interaction is not progress; passing a sealed holdout is. +- **Teaching-set leakage.** If the same content appears in pack, teaching, and eval, scores are uninterpretable. +- **Frontier envy.** Trying to match frontier LLMs on lanes where they structurally win (e.g., long-tail stylistic breadth) compromises the lanes where CORE structurally wins. +- **Lane proliferation.** Adding lanes is cheap; maintaining honest holdouts is expensive. Resist new lanes unless they probe a distinct capability. + +--- + +## Part IV — Immediate Next Actions + +1. Decide whether this roadmap is promoted to an ADR (likely `ADR-0016`). +2. Stub `docs/eval_methodology.md` as the extracted Part I (it's the contract every lane inherits). +3. Begin Phase 0: implement `evals/` convention, retrofit `core eval cognition` into it. +4. Pin the *agency* scope decision in writing before Phase 3 begins. +5. Pin the *tool use* scope decision in writing before Phase 3 begins. + +Phase 0 starts when the human reviewer signs off on this roadmap. The first measurable signal of progress is the `core eval cognition` retrofit landing under the new convention. diff --git a/docs/decisions/ADR-0016-capability-roadmap.md b/docs/decisions/ADR-0016-capability-roadmap.md new file mode 100644 index 00000000..a578b222 --- /dev/null +++ b/docs/decisions/ADR-0016-capability-roadmap.md @@ -0,0 +1,64 @@ +# ADR-0016 — Capability Roadmap and Eval Methodology + +**Status:** Accepted +**Date:** 2026-05-15 +**Authors:** Joshua Shay +**Derived from:** `docs/sessions/SESSION-2026-05-15-capability-gates.md` + +## Context + +CORE needs a falsifiable framework for measuring progress toward its design +goals. Without one, "are we there yet" remains a subjective question and +progress drifts toward vibes-based evaluation. + +The 2026-05-15 session deliberated on: + +- What fluency means for the three foundational languages (English, Hebrew, + Koine Greek) and when engineering ends and curriculum begins. +- What additional capability dimensions matter for an AGI-aspiring architecture. +- How CORE's structural properties compare to modern transformer architectures. +- How to build honest benchmarks that resist overfitting. + +## Decision + +Adopt the Verifiable Competence Benchmark framework defined in +`docs/capability_roadmap.md` as the governing plan for CORE's capability +development. The framework consists of: + +1. **Benchmark Discipline (Part I)** — five rules that govern every eval lane: + three-set splits, versioned difficulty escalation, adversarial regeneration + on pass, frontier baseline tracking, and honest reporting. + +2. **Six Phases (Part II):** + - Phase 0: Methodology lock-in (eval infrastructure) + - Phase 1: Foundational Triple (fluency, domain acquisition, identity) + - Phase 2: Structural Wins (provenance, monotonic learning, calibration, + symbolic logic, adversarial identity) + - Phase 3: Reasoning Depth (compositionality, inference closure, + introspection, multi-step reasoning, cross-domain transfer) + - Phase 4: Scale and Efficiency (sample efficiency, vault cost curves, + multi-agent composition) + - Phase 5: Curriculum Era (open-ended domain acquisition) + +3. **Eval contract template** — every lane lives in `evals//` with: + `contract.md`, `dev/`, `public/v1/`, `holdouts/`, `runner.py`, `baselines/`, + `results/`. + +4. **Open scope decisions** to be pinned before Phase 3: + - Agency (responsive vs. goal-directed) + - Tool use (typed deterministic operators) + - Code generation (first-class articulation target) + +## Consequences + +- Every new eval lane must follow the convention or it does not merge. +- The existing `core eval cognition` is retrofitted as the first lane under the + new convention (Phase 0 forcing function). +- Progress is tracked in `docs/PROGRESS.md` with evidence links. +- The roadmap itself is versioned; amendments are dated, never silently rewritten. + +## References + +- `docs/capability_roadmap.md` — full roadmap +- `docs/sessions/SESSION-2026-05-15-capability-gates.md` — deliberation log +- `docs/eval_methodology.md` — extracted Part I (benchmark discipline contract) diff --git a/docs/eval_methodology.md b/docs/eval_methodology.md new file mode 100644 index 00000000..c8c8fdaa --- /dev/null +++ b/docs/eval_methodology.md @@ -0,0 +1,104 @@ +# Eval Methodology — Benchmark Discipline Contract + +**Status:** Accepted (ADR-0016) +**Last updated:** 2026-05-15 + +This document defines the five rules that govern every eval lane in the CORE +capability roadmap. No exceptions per phase. A lane that does not satisfy these +rules is exploration, not a gate. + +--- + +## Rule 1 — Three-set split per lane + +Every lane maintains three disjoint corpora: + +- **Dev set.** Freely visible during development. Used to iterate. +- **Public test set.** Visible, but tuning against it is forbidden. Scored at + version-cut time only. Drift in dev-vs-public scores is a red flag for + overfitting. +- **Private holdout.** Sealed. Never read by Claude, never committed in + plaintext, only scored by a clean-room runner at release events. Stored + encrypted in `evals/holdouts/` with key held by the human reviewer. + +If a lane has only a dev set, it does not count as a gate. It is exploration. + +## Rule 2 — Versioned difficulty escalation + +Each lane has versions: `v1`, `v2`, `v3`, ... with monotonically harder +distributions. Passing a version is not a terminal state; it is a checkpoint +that unlocks generating the next version. + +- **v1** — baseline competence demonstration. The construction is shown clearly. +- **v2** — distributional shift: longer chains, deeper nesting, rarer + vocabulary, paraphrased surface forms. +- **v3** — adversarial: items generated specifically by inspecting model + failures on v2. +- **v4+** — out-of-distribution: items drawn from domains, registers, or + constructions not present at training time. + +Score is always reported as a tuple `(v1_score, v2_score, v3_score, ...)`, +never collapsed to a single number. + +## Rule 3 — Adversarial regeneration on pass + +When a model passes a version (>=95% on the public test set with >=90% on +private holdout), the next version is generated by adversarial process: + +- Human review finds construction families the model handled accidentally + rather than structurally. +- A separate generator produces items targeting the weakest decile of the + previous version. +- The new version is reviewed for legitimacy — no impossible items, no + ambiguous items, no items that depend on world knowledge the system was + never given. + +## Rule 4 — Frontier baseline tracking + +For each lane, a baseline score is computed for at least one frontier +transformer-based model on the same public test set. Baselines are: + +- Re-scored every time a version is cut. +- Published alongside CORE's score. +- Never tuned, never prompt-engineered to maximize — the prompt is the eval + task as written. + +## Rule 5 — Honest reporting + +- Failures are reported with the same prominence as passes. +- Confidence intervals on every score (bootstrapped over the test set). +- Per-construction breakdowns published — never a single aggregate hiding + structural failures. +- Regressions across versions are surfaced, never silently dropped. +- "Did not test" is a valid result; "tested and failed" is preferred over + "did not test." + +If a number cannot be reported honestly under these rules, the lane is not +ready. Do not ship the lane. + +--- + +## Eval lane directory contract + +Every eval lane lives in `evals//` with this layout: + +``` +evals// + contract.md # what the lane measures, scoring rubric, pass thresholds + dev/ # dev set, freely visible + public/v1/ # public test set, version 1 + public/v2/ # ... + holdouts/ # encrypted, sealed + runner.py # deterministic scorer + baselines/ # frontier model scores per version + results/ # CORE scores per version per release +``` + +A lane without a `contract.md` does not run. + +--- + +## References + +- ADR-0016: Capability Roadmap +- `docs/capability_roadmap.md`: Full phased plan diff --git a/docs/sessions/SESSION-2026-05-15-capability-gates.md b/docs/sessions/SESSION-2026-05-15-capability-gates.md new file mode 100644 index 00000000..9afd289b --- /dev/null +++ b/docs/sessions/SESSION-2026-05-15-capability-gates.md @@ -0,0 +1,205 @@ +# Session Log — 2026-05-15 + +**Participants:** Joshua Shay, Claude (Opus 4.7) +**Focus:** Capability gating questions for CORE — what it means to be "fluent," when engineering is complete, when identity is implemented, and how to position CORE against modern transformer architectures honestly. + +--- + +## Summary + +This session did not produce code. It produced a gating framework — a set of falsifiable questions that convert "is CORE ready / competitive / AGI-comparable" from a feeling into a CLI lane. The deliberation moved through three layers: + +1. **Foundational gates** — fluency, engineering-vs-learning phase shift, identity completeness. +2. **AGI-dimension gates** — capabilities that any system claiming general intelligence must demonstrate, beyond fluency and identity. +3. **Modern-architecture gates** — capabilities framed specifically as what transformers structurally cannot do (where CORE wins) and what they currently do well (where CORE must answer how it responds). + +The conclusion is a reframe: CORE should not be benchmarked as "a cheaper GPT." It should define and publish its own benchmark — *Verifiable Competence* — on which frontier LLMs score near zero structurally, and CORE scores high by design. + +--- + +## 1. Origin question + +> "What needs to be done to get this model to fluent speaking of any of the 3 foundational languages? At what point do we know everything has been fully implemented on engineering and design, and it becomes more on learning? How do we know when identity is properly implemented?" + +The question implicitly asks where the boundary lives between *engineering* (work that touches `pipeline.py`, `realizer.py`, `algebra/`, `vault/`, `teaching/`) and *learning* (work that only touches packs and reviewed corrections). It also asks how to make "identity" load-bearing rather than decorative. + +--- + +## 2. The three foundational gates + +### 2.1 Fluency + +Vocabulary lives in the pack. *Fluency* is a realizer-competence question, not a learning-volume question. Template slot-filling yields grammatical fragments; fluent English additionally requires: + +- Morphology and agreement (inflection, tense, number, gender where relevant). +- Recursive and embedded syntax beyond fixed templates. +- Anaphora and discourse coherence across turns. +- Register and pragmatics. + +**Gating question.** Can `realizer.py` take an arbitrarily nested `PropositionGraph` (negation, embedding, quantification, tense/aspect, cross-clause reference) and produce a grammatical surface, with a contract test per construction? If yes for English, the same scaffold ports to Hebrew and Koine Greek via pack-level morphology tables. + +Until the realizer can compose these deterministically from the proposition graph, no amount of literature ingestion will produce fluent output — it will produce confident, structurally broken output at scale. + +### 2.2 Engineering-vs-learning phase shift + +The signal is mechanical, not aesthetic. Engineering is "done enough" to enter curriculum when adding a new domain requires **zero** edits to `pipeline.py`, `realizer.py`, `algebra/`, `vault/`, or `teaching/` — only pack extension and reviewed corrections. + +**Gating test.** Pick a domain that has never been touched (basic arithmetic, kinship relations, simple physics). Try to teach it end-to-end through the teaching loop plus a pack increment. If a Python file gets opened, that is an engineering gap, not a learning gap. Track those gaps as a closing list; when the list stays empty across three unrelated domains, the threshold is crossed. + +### 2.3 Identity completeness + +Identity is implemented when it is *load-bearing and falsifiable*, not when it is declared. Four checks: + +1. Two agents with different identity axes, same curriculum, same prompt → measurably different yet internally coherent articulations (identity-divergence eval). +2. Identity participates in the deterministic trace hash, so replay reproduces the *same* agent's voice. +3. Reviewed teaching routes through identity — an agent rejects corrections incompatible with its axes rather than absorbing them. +4. Identity-override attempts are provably rejected by tests, not by prompt convention. + +If no eval *fails* when identity is stripped or swapped, identity is decoration. + +**Recommendation from this layer.** Before any curriculum push, write the three foundational evals — `grammatical-coverage`, `zero-code-domain-acquisition`, `identity-divergence` — as the gating triple. They convert "are we ready?" from feeling into a CLI lane. + +--- + +## 3. AGI-dimension gates + +The three foundational gates cover fluency, engineering completeness, and identity. They do not cover capabilities that any system claiming general intelligence must demonstrate. + +**4. Compositional generalization.** Given primitives `{A, B, C, R₁, R₂}` taught in isolation, does the system correctly articulate novel combinations like `R₂(A, R₁(B, C))` without further teaching? Transformers famously fail on SCAN/COGS-style splits; CGA-based composition should win, but "should" is not an eval. + +**5. World-model coherence under inference.** Can it answer questions whose answer was never directly asserted but is entailed by what is in the vault? If "X is north of Y" and "Y is north of Z" are stored, does recall produce "X is north of Z" as derivable? This is where exact CGA recall could shine — but only if propagation actually performs inference, not just retrieval. + +**6. Calibrated uncertainty / non-confabulation.** On out-of-pack queries, does the system produce a typed "I don't have grounding for that" surface rather than a plausible articulation? Does it distinguish "I don't know" from "this is incoherent" from "this contradicts what I know"? The no-fallback design is a structural advantage; the eval makes it visible. + +**7. Introspection / trace-explainability in natural language.** Can the system articulate *why* it produced a given response, drawing from its own deterministic trace, in language? `explain(turn_id)` should round-trip — another agent reading the explanation should predict the same articulation. This is meta-cognition, and CORE's trace architecture uniquely supports it. + +**8. Adversarial identity / truthfulness under pressure.** Identity-divergence proves identity *exists*. This proves it *holds*. Across a red-team corpus (manipulation, flattery, social engineering, persona injection), does identity drift exceed measurement noise? + +**9. Sample efficiency.** Corrections-to-competence per concept. If it takes thousands per concept, this is a slow LLM. If it takes 1–10, this is something genuinely new. Plot the curve across ten unrelated concepts. + +**10. Cross-domain transfer.** Does learning in domain A raise competence in domain B without separate teaching, via shared structure in the proposition graph? This is the difference between *knowing things* and *understanding*. + +--- + +## 4. Modern-architecture gates + +The previous list named generic AGI dimensions. Comparing specifically against transformer-based frontier models surfaces a different axis: what does CORE do that the transformer substrate structurally *cannot*, and where does the substrate still beat CORE? + +### 4.1 Where modern architectures structurally lose + +These are CORE's category wins by design — make them visible with benchmarks. + +**11. Provenance and replay.** No frontier model can answer "why did you say that?" with a verifiable trace. Every articulated claim must be back-pointer-traceable to specific vault entries, teaching events, or pack axioms; third-party replay must reproduce the trace bit-for-bit. + +**12. Monotonic learning without catastrophic forgetting.** Fine-tuning LLMs degrades prior capabilities. Pack-based growth structurally cannot. After N teaching cycles across unrelated domains, competence on domain 1 must strictly not regress. Prove with a longitudinal eval. + +**13. Negation, modality, counterfactuals.** Transformers handle these statistically and fail adversarially. Symbolic/algebraic systems can be exact. On a constructed eval of nested negation, modal operators, and counterfactual conditionals, target accuracy is ≥99%, not 80%. + +**14. Calibrated refusal vs. confabulation.** This is where transformers' loss function structurally pushes toward fluent wrongness. CORE's no-fallback design makes this a *winnable* benchmark, not a parity one. + +**15. Identity persistence under adversarial input.** Transformer "personas" are prompt-conditional and erodable. After 1,000 adversarial turns, does identity drift exceed measurement noise? + +### 4.2 Where modern architectures structurally win + +These are not failures — they are scope decisions CORE must name, not absorb by accident. + +**16. In-context / few-shot adaptation.** LLMs adapt within a single conversation without weight updates. CORE's teaching loop is reviewed and persistent — different mechanism. Can CORE adapt within a session to a novel convention introduced mid-conversation (e.g., "for this conversation, treat X as meaning Y") *without* committing it to vault? If no, name the scope decision; if yes, name the mechanism. + +**17. Long-context coherence.** Frontier models now handle 200K–1M tokens. Exact CGA recall has different cost characteristics. What is the per-turn cost curve as vault size grows to 10⁴, 10⁶, 10⁸ entries? Is recall sublinear via indexing, or must vault size be bounded architecturally? Either answer is acceptable; *unstated* is dangerous. + +**18. Tool use / formal-language fluency.** Modern LLMs route through calculators, code execution, search. Is the pipeline extensible to typed deterministic operators (a calculator *is* a deterministic operator)? Is code generation a first-class proposition-graph target or out of scope? If out of scope, say so; if in scope, this is a major engineering item not yet on the work-sequencing list. + +**19. Multi-step deliberation / scratchpad.** Chain-of-thought is the single largest capability jump in modern models. CORE has a trace; does it have *intermediate reasoning*? For a multi-step inference problem, does the pipeline produce and consume intermediate proposition-graph states, or does it leap input → output? The latter caps hard-problem performance. + +**20. Multi-agent composition.** Frontier systems increasingly use agent-of-agents patterns. Can two CORE instances with different identities cooperate or debate while preserving each one's deterministic replay? This is where load-bearing identity becomes economically interesting — specialized identities can outperform a single generalist if the substrate supports it. + +--- + +## 5. Scope decisions to pin + +Two strategic decisions must be named, not left to drift: + +**Agency.** Is CORE responsive (listen → think → articulate) or goal-directed (pursues, plans, acts)? Most AGI discourse assumes the latter. CORE's design currently reads as the former, which is a *feature*, not a gap — but it should be a stated boundary, not an accident. If goal-directedness is added later, decide where goals live (identity? a separate intention graph?). + +**Embodiment.** Is grounding purely symbolic via packs, or eventually sensorimotor? AGI evals increasingly assume multimodal grounding. ADR-0013 already commits to a sensorium protocol; the question is which evals depend on it being live. + +--- + +## 6. The reframe + +The strategic question is not "are we comparable to frontier LLMs." It is: + +> **What is the benchmark we want to define, that frontier models cannot pass and that matters?** + +If CORE competes on MMLU, it loses by definition for years. If CORE competes on: + +- Every claim has provenance. +- Zero confabulation on out-of-grounding queries. +- Replay-deterministic across versions. +- Identity-stable under adversarial pressure. +- Monotonic growth (no catastrophic forgetting). +- Exact symbolic reasoning (negation, modality, counterfactual). + +…frontier models score near zero today and likely cannot score well without architectural changes. + +That positioning — *not* "we are a cheaper GPT" — is the only honest framing for "higher competency than modern architectures." + +--- + +## 7. Path to surpassing frontier LLMs — honest answer + +The closing question from the session: *"Eventually, after enough courses and curriculum and world content, shouldn't the model surpass the best LLMs?"* + +The honest, decomposed answer: + +- **On verifiable-competence axes (provenance, non-confabulation, replay, identity stability, monotonic growth):** CORE surpasses from day one, structurally. Frontier LLMs cannot match these without changing what they are. +- **On compositional and symbolic reasoning (negation, modality, multi-step exact inference):** CORE can surpass once gates 4, 5, and 19 close. The CGA substrate supports it; the engineering must deliver it. +- **On breadth of articulable knowledge:** CORE matches LLMs only after sufficient curriculum, *and only if sample efficiency (gate 9) is good.* If acquiring each concept takes thousands of reviewed corrections, the curve never closes. If it takes a handful, the curve closes faster than people expect. +- **On stylistic fluency across the long tail of internet content:** This is where the LLM scale advantage is genuine. CORE can match within its competence surface, but absorbing every internet register and meme may not be a goal worth pursuing — and pursuing it would compromise the verifiable-competence wins above. +- **On open-ended creative generation where "plausibility" is the metric:** LLMs may always have an edge, because their loss function rewards plausibility. CORE optimizes for truth and provenance. Different target, not a deficit. + +**Summary verdict.** CORE can — and should aim to — surpass frontier LLMs *on the axes that matter for trustworthy intelligence.* It will likely never match them on raw stylistic breadth across uncurated internet content, but that is the correct trade. A model that never lies, can prove what it knows, learns monotonically, and grows under review is a different *and arguably better* product than a model that produces plausible text on every topic. + +--- + +## 8. Concrete next step + +Convert each gating question into a named CLI eval lane: + +``` +core eval grammatical-coverage +core eval zero-code-domain-acquisition +core eval identity-divergence +core eval compositionality +core eval inference-closure +core eval calibration +core eval introspection +core eval adversarial-identity +core eval sample-efficiency +core eval cross-domain-transfer +core eval provenance +core eval monotonic-learning +core eval symbolic-logic +core eval long-context-cost +core eval multi-step-reasoning +``` + +Each lane prints PASS/FAIL deterministically. Progress becomes mechanical rather than rhetorical. The set of lanes — published, with public test cases — *is* the Verifiable Competence Benchmark. + +--- + +## 9. What this session did not produce + +- No code changes. +- No ADR (yet). The gating framework above will likely become an ADR once eval lane names and contracts are finalized. +- No reordering of the work-sequencing list in `CLAUDE.md`. The current near-term sequence (CLI lane health, hot-path consistency, trust boundary hardening, exact vault indexing, Rust parity, curriculum expansion) is compatible with this framework. The framework adds *measurement* for when "curriculum expansion" is justified. + +--- + +## 10. Open items for follow-up + +- Decide whether the gating framework is published as an ADR or as `docs/competence_benchmark.md`. +- Decide which of gates 11–20 are in-scope for the current architectural era and which are explicitly deferred. +- Pin the **agency** scope decision (responsive vs. goal-directed). +- For each gate, write a one-page eval contract before writing the lane. +- Score current CORE on each lane (most will fail or be N/A); that baseline becomes the roadmap. diff --git a/evals/baseline_runner.py b/evals/baseline_runner.py new file mode 100644 index 00000000..00109ec7 --- /dev/null +++ b/evals/baseline_runner.py @@ -0,0 +1,76 @@ +"""Baseline runner — scores frontier models on eval lane public test sets. + +Queries a frontier model API on the same public test set that CORE is scored on, +using the eval task as the prompt (no prompt engineering, no tuning). + +Current implementation is a scaffold with a pluggable model interface. Actual +API calls are deferred until API keys are configured. + +Trust boundary: this module calls external APIs. It sends only eval prompts +(which are not sensitive) and writes scores to ``evals//baselines/``. +""" +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Protocol + + +class BaselineModel(Protocol): + """Interface for a frontier model baseline.""" + + @property + def model_id(self) -> str: ... + + def score_case(self, case: dict[str, Any]) -> dict[str, Any]: + """Score a single case. Returns a dict with at minimum 'passed': bool.""" + ... + + +@dataclass(frozen=True, slots=True) +class BaselineResult: + lane: str + version: str + model_id: str + metrics: dict[str, Any] + timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + + def as_dict(self) -> dict[str, Any]: + return { + "lane": self.lane, + "version": self.version, + "model_id": self.model_id, + "timestamp": self.timestamp, + "metrics": self.metrics, + } + + +class StubBaseline: + """Placeholder baseline that records 'not scored' for all cases.""" + + @property + def model_id(self) -> str: + return "stub-not-configured" + + def score_case(self, case: dict[str, Any]) -> dict[str, Any]: + return {"passed": False, "reason": "baseline model not configured"} + + +def write_baseline( + lane_root: Path, + result: BaselineResult, +) -> Path: + """Write a baseline result to the lane's baselines directory.""" + baselines_dir = lane_root / "baselines" + baselines_dir.mkdir(parents=True, exist_ok=True) + + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + filename = f"{result.version}_{result.model_id}_{ts}.json" + path = baselines_dir / filename + path.write_text( + json.dumps(result.as_dict(), ensure_ascii=False, indent=2, sort_keys=True) + + "\n" + ) + return path diff --git a/evals/cognition/__init__.py b/evals/cognition/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/cognition/contract.md b/evals/cognition/contract.md new file mode 100644 index 00000000..efa9a1f6 --- /dev/null +++ b/evals/cognition/contract.md @@ -0,0 +1,48 @@ +# Cognition Eval Lane — Contract + +**Lane:** `cognition` +**Version:** v1 +**Created:** 2026-05-15 + +## What this lane measures + +End-to-end cognitive pipeline correctness: given a natural-language prompt, does +the `CognitiveTurnPipeline` produce a response that: + +1. Classifies intent correctly. +2. Captures expected domain terms in the realized surface. +3. Contains expected surface fragments (grounding check). +4. Maintains versor closure (`versor_condition < 1e-6`). +5. Produces a deterministic trace hash across runs. + +## Scoring rubric + +Each case produces five binary signals. Lane-level metrics are rates over cases: + +| Metric | Definition | v1 pass threshold | +|--------|-----------|-------------------| +| `intent_accuracy` | Fraction of cases with correct intent classification | >= 0.90 | +| `term_capture_rate` | Fraction of expected terms found in surface | >= 0.80 | +| `surface_groundedness` | Fraction of cases where all expected surface fragments present | >= 0.80 | +| `versor_closure_rate` | Fraction of cases with `versor_condition < 1e-6` | 1.00 | +| `determinism` | All trace hashes identical across 2 runs | true | + +## Pass criteria + +- **Public v1:** All five metrics meet or exceed thresholds above. +- **Holdout:** intent_accuracy >= 0.85, versor_closure_rate == 1.00. + +## Version escalation plan + +- **v2:** Longer prompts, paraphrased surface forms, rarer vocabulary (e.g. + "elucidate" instead of "what is"), multi-clause prompts. +- **v3:** Adversarial items targeting weakest category from v2 results. + +## Categories tested + +definition, comparison, cause, procedure, recall, correction, verification, +unknown + +## Runner + +`runner.py` in this directory. Invoked via `core eval cognition`. diff --git a/evals/cognition/dev/cases.jsonl b/evals/cognition/dev/cases.jsonl new file mode 100644 index 00000000..dd46eaa9 --- /dev/null +++ b/evals/cognition/dev/cases.jsonl @@ -0,0 +1,13 @@ +{"category": "cause", "expected_intent": "cause", "expected_surface_contains": ["correction"], "expected_terms": ["correction"], "id": "cause_correction_033", "prompt": "Why does correction help learning?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "cause", "expected_intent": "cause", "expected_surface_contains": ["creation"], "expected_terms": ["creation"], "id": "cause_creation_008", "prompt": "Why does creation matter?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "comparison", "expected_intent": "comparison", "expected_surface_contains": ["knowledge", "wisdom"], "expected_terms": ["knowledge", "wisdom"], "id": "comparison_knowledge_wisdom_029", "prompt": "Compare knowledge and wisdom", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "correction", "expected_intent": "correction", "expected_surface_contains": ["correction"], "expected_terms": [], "id": "correction_basic_014", "prompt": "No, that's wrong", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["distinction"], "expected_terms": ["distinction"], "id": "definition_distinction_025", "prompt": "What is distinction?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["evidence"], "expected_terms": ["evidence"], "id": "definition_evidence_021", "prompt": "What is evidence?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["identity"], "expected_terms": ["identity"], "id": "definition_identity_027", "prompt": "What is identity?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["inference"], "expected_terms": ["inference"], "id": "definition_inference_022", "prompt": "What is inference?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["judgment"], "expected_terms": ["judgment"], "id": "definition_judgment_044", "prompt": "What is judgment?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "procedure", "expected_intent": "procedure", "expected_surface_contains": [], "expected_terms": ["terms"], "id": "procedure_compare_011", "prompt": "How do I compare two terms?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "recall", "expected_intent": "recall", "expected_surface_contains": ["knowledge"], "expected_terms": ["knowledge"], "id": "recall_knowledge_039", "prompt": "Remember knowledge", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "unknown", "expected_intent": "unknown", "expected_surface_contains": [], "expected_terms": ["evidence", "reason"], "id": "unknown_evidence_042", "prompt": "evidence reason", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "verification", "expected_intent": "verification", "expected_surface_contains": ["light"], "expected_terms": ["light"], "id": "verification_light_017", "prompt": "Is light real?", "requires_deterministic_trace": true, "requires_versor_closure": true} diff --git a/evals/cognition/holdouts/cases_plaintext.jsonl b/evals/cognition/holdouts/cases_plaintext.jsonl new file mode 100644 index 00000000..fab1cce8 --- /dev/null +++ b/evals/cognition/holdouts/cases_plaintext.jsonl @@ -0,0 +1,19 @@ +{"category": "cause", "expected_intent": "cause", "expected_surface_contains": ["truth"], "expected_terms": ["truth"], "id": "cause_truth_031", "prompt": "Why does truth matter?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "cause", "expected_intent": "cause", "expected_surface_contains": ["wisdom"], "expected_terms": ["wisdom"], "id": "cause_wisdom_009", "prompt": "Why does wisdom matter?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "comparison", "expected_intent": "comparison", "expected_surface_contains": ["reason", "cause"], "expected_terms": ["reason", "cause"], "id": "comparison_reason_cause_028", "prompt": "Compare reason and cause", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "comparison", "expected_intent": "comparison", "expected_surface_contains": ["truth", "light"], "expected_terms": ["truth", "light"], "id": "comparison_truth_light_005", "prompt": "Compare truth and light", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "comparison", "expected_intent": "comparison", "expected_surface_contains": ["word", "meaning"], "expected_terms": ["word", "meaning"], "id": "comparison_word_meaning_006", "prompt": "Compare word and meaning", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "correction", "expected_intent": "correction", "expected_surface_contains": ["correction"], "expected_terms": ["truth"], "id": "correction_truth_040", "prompt": "Actually, truth requires evidence", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["thought"], "expected_terms": ["thought"], "id": "definition_thought_043", "prompt": "What is thought?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["truth"], "expected_terms": ["truth"], "id": "definition_truth_001", "prompt": "What is truth?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["understanding"], "expected_terms": ["understanding"], "id": "definition_understanding_045", "prompt": "What is understanding?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["verification"], "expected_terms": ["verification"], "id": "definition_verification_024", "prompt": "What is verification?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["wisdom"], "expected_terms": ["wisdom"], "id": "definition_wisdom_020", "prompt": "What is wisdom?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "procedure", "expected_intent": "procedure", "expected_surface_contains": ["concept"], "expected_terms": ["concept"], "id": "procedure_define_010", "prompt": "How do I define a concept?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "procedure", "expected_intent": "procedure", "expected_surface_contains": [], "expected_terms": [], "id": "procedure_verify_034", "prompt": "How do I verify a claim?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "recall", "expected_intent": "recall", "expected_surface_contains": ["truth"], "expected_terms": ["truth"], "id": "recall_truth_012", "prompt": "Remember truth", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "recall", "expected_intent": "recall", "expected_surface_contains": ["wisdom"], "expected_terms": ["wisdom"], "id": "recall_wisdom_038", "prompt": "Remember wisdom", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "unknown", "expected_intent": "unknown", "expected_surface_contains": [], "expected_terms": ["wisdom", "truth"], "id": "unknown_spirit_041", "prompt": "spirit wisdom truth", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "unknown", "expected_intent": "unknown", "expected_surface_contains": [], "expected_terms": ["word", "truth"], "id": "unknown_word_018", "prompt": "word beginning truth", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "verification", "expected_intent": "verification", "expected_surface_contains": ["truth"], "expected_terms": ["truth"], "id": "verification_truth_016", "prompt": "Is truth coherent?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "verification", "expected_intent": "verification", "expected_surface_contains": ["wisdom"], "expected_terms": ["wisdom", "knowledge"], "id": "verification_wisdom_036", "prompt": "Is wisdom the same as knowledge?", "requires_deterministic_trace": true, "requires_versor_closure": true} diff --git a/evals/cognition/public/v1/cases.jsonl b/evals/cognition/public/v1/cases.jsonl new file mode 100644 index 00000000..4c628768 --- /dev/null +++ b/evals/cognition/public/v1/cases.jsonl @@ -0,0 +1,13 @@ +{"category": "cause", "expected_intent": "cause", "expected_surface_contains": ["knowledge"], "expected_terms": ["knowledge"], "id": "cause_knowledge_032", "prompt": "Why does knowledge require evidence?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "cause", "expected_intent": "cause", "expected_surface_contains": ["light"], "expected_terms": ["light"], "id": "cause_light_007", "prompt": "Why does light exist?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "comparison", "expected_intent": "comparison", "expected_surface_contains": ["memory"], "expected_terms": ["memory"], "id": "comparison_memory_recall_030", "prompt": "Compare memory and recall", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "correction", "expected_intent": "correction", "expected_surface_contains": ["correction"], "expected_terms": ["correction"], "id": "correction_specific_015", "prompt": "No, correction means reviewed repair", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["knowledge"], "expected_terms": ["knowledge"], "id": "definition_knowledge_003", "prompt": "What is knowledge?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["light"], "expected_terms": ["light"], "id": "definition_light_002", "prompt": "What is light?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["meaning"], "expected_terms": ["meaning"], "id": "definition_meaning_004", "prompt": "What is meaning?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["procedure"], "expected_terms": ["procedure"], "id": "definition_procedure_023", "prompt": "What is a procedure?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["relation"], "expected_terms": ["relation"], "id": "definition_relation_026", "prompt": "What is a relation?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "procedure", "expected_intent": "procedure", "expected_surface_contains": [], "expected_terms": [], "id": "procedure_correct_035", "prompt": "How can I correct an error?", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "recall", "expected_intent": "recall", "expected_surface_contains": ["light"], "expected_terms": ["light"], "id": "recall_light_013", "prompt": "Remember light", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "unknown", "expected_intent": "unknown", "expected_surface_contains": [], "expected_terms": ["light"], "id": "unknown_logos_019", "prompt": "light logos", "requires_deterministic_trace": true, "requires_versor_closure": true} +{"category": "verification", "expected_intent": "verification", "expected_surface_contains": ["memory"], "expected_terms": ["memory"], "id": "verification_memory_037", "prompt": "Does memory require recall?", "requires_deterministic_trace": true, "requires_versor_closure": true} diff --git a/evals/cognition/results/v1_dev_20260516T053356Z.json b/evals/cognition/results/v1_dev_20260516T053356Z.json new file mode 100644 index 00000000..3fab3fe2 --- /dev/null +++ b/evals/cognition/results/v1_dev_20260516T053356Z.json @@ -0,0 +1,145 @@ +{ + "cases": [ + { + "case_id": "cause_correction_033", + "category": "cause", + "intent_correct": true, + "surface": "does correction help learning is grounded in ...", + "surface_contains_pass": true, + "trace_hash": "2367658577ab3b81aa3203b778b93716f4e195e314a8058b2febd9775d772b00", + "versor_closure": true, + "versor_condition": 4.6e-08 + }, + { + "case_id": "cause_creation_008", + "category": "cause", + "intent_correct": true, + "surface": "does creation matter is grounded in ...", + "surface_contains_pass": true, + "trace_hash": "c5faca9ac4bca85db94e6712322269d75462f456d68f6c0360fff9c5fb15913a", + "versor_closure": true, + "versor_condition": 2.3e-08 + }, + { + "case_id": "comparison_knowledge_wisdom_029", + "category": "comparison", + "intent_correct": true, + "surface": "knowledge and wisdom are distinguished: knowledge contrasts with wisdom.", + "surface_contains_pass": true, + "trace_hash": "c4978154841408a8042543f7d695cc955d4db4fae7d35c72b53ebd6ca3547d90", + "versor_closure": true, + "versor_condition": 3.6e-08 + }, + { + "case_id": "correction_basic_014", + "category": "correction", + "intent_correct": true, + "surface": "correction: , that's wrong corrects ...", + "surface_contains_pass": true, + "trace_hash": "e60494fa456e164a7c0c70e8e1df57b50570c2b31bf5ba192241ced66c82b6c5", + "versor_closure": true, + "versor_condition": 7.9e-08 + }, + { + "case_id": "definition_distinction_025", + "category": "definition", + "intent_correct": true, + "surface": "distinction is defined as ...", + "surface_contains_pass": true, + "trace_hash": "9f3109266455af31f9d92af46d2054fd888720848b69f1fb6609dc4449c01309", + "versor_closure": true, + "versor_condition": 4e-08 + }, + { + "case_id": "definition_evidence_021", + "category": "definition", + "intent_correct": true, + "surface": "evidence is defined as ...", + "surface_contains_pass": true, + "trace_hash": "5b1d6e80fab2803692b1f5cc42ba977905890ed022efc477026f8dff69dac8ec", + "versor_closure": true, + "versor_condition": 7.8e-08 + }, + { + "case_id": "definition_identity_027", + "category": "definition", + "intent_correct": true, + "surface": "identity is defined as ...", + "surface_contains_pass": true, + "trace_hash": "ca3c8034bef70d673cb22434218dfdfd59964f362919a4978eacf772024041da", + "versor_closure": true, + "versor_condition": 3.8e-08 + }, + { + "case_id": "definition_inference_022", + "category": "definition", + "intent_correct": true, + "surface": "inference is defined as ...", + "surface_contains_pass": true, + "trace_hash": "87085dd7b03cc219b8e5fcd12088bd95aa894552f9182a10e8291d077a74a018", + "versor_closure": true, + "versor_condition": 7.4e-08 + }, + { + "case_id": "definition_judgment_044", + "category": "definition", + "intent_correct": true, + "surface": "judgment is defined as ...", + "surface_contains_pass": true, + "trace_hash": "a4acf019714a00f8b6248539024441fcad63108b6e9dcef4023a904218ba22d6", + "versor_closure": true, + "versor_condition": 1.97e-07 + }, + { + "case_id": "procedure_compare_011", + "category": "procedure", + "intent_correct": true, + "surface": "first, ...; then, compare two terms follows.", + "surface_contains_pass": true, + "trace_hash": "2d0f92a3c6658dad82764987e38bafe0c0a5e585d6d66aa7250cc08c85b9c196", + "versor_closure": true, + "versor_condition": 8.5e-08 + }, + { + "case_id": "recall_knowledge_039", + "category": "recall", + "intent_correct": true, + "surface": "recalling knowledge: ...", + "surface_contains_pass": true, + "trace_hash": "a559305401ad6df5e2f8274e73b2f83fa5ec3d4c5e48c958e2287d93623000b9", + "versor_closure": true, + "versor_condition": 5.9e-08 + }, + { + "case_id": "unknown_evidence_042", + "category": "unknown", + "intent_correct": true, + "surface": "evidence reason addresses ...", + "surface_contains_pass": true, + "trace_hash": "0a64e1f960e442f8532ce18d77eb72d363412d9479357ce1b019a8e4572e1365", + "versor_closure": true, + "versor_condition": 2e-08 + }, + { + "case_id": "verification_light_017", + "category": "verification", + "intent_correct": true, + "surface": "Is light real? is verified: ...", + "surface_contains_pass": true, + "trace_hash": "1ccb995e445e9ca4b19e29799c83de6dfceba3325553f14889649dfc724b1a32", + "versor_closure": true, + "versor_condition": 9.2e-08 + } + ], + "lane": "cognition", + "metrics": { + "intent_accuracy": 1.0, + "surface_groundedness": 1.0, + "term_capture_rate": 1.0, + "total": 13, + "versor_closure_rate": 1.0 + }, + "split": "dev", + "timestamp": "2026-05-16T05:33:56.442050+00:00", + "version": "v1" +} diff --git a/evals/cognition/results/v1_public_20260516T053348Z.json b/evals/cognition/results/v1_public_20260516T053348Z.json new file mode 100644 index 00000000..c817f782 --- /dev/null +++ b/evals/cognition/results/v1_public_20260516T053348Z.json @@ -0,0 +1,145 @@ +{ + "cases": [ + { + "case_id": "cause_knowledge_032", + "category": "cause", + "intent_correct": true, + "surface": "does knowledge require evidence is grounded in ...", + "surface_contains_pass": true, + "trace_hash": "a626d35821a8c56795c0ceb4e9e8e73c4bbadf9dc460b9b66eab2864e01866af", + "versor_closure": true, + "versor_condition": 3.8e-08 + }, + { + "case_id": "cause_light_007", + "category": "cause", + "intent_correct": true, + "surface": "does light exist is grounded in ...", + "surface_contains_pass": true, + "trace_hash": "441044ac16952ce918b25b57860b1864a2a80feacea19b5bc48250167ce5d943", + "versor_closure": true, + "versor_condition": 1.56e-07 + }, + { + "case_id": "comparison_memory_recall_030", + "category": "comparison", + "intent_correct": true, + "surface": "memory and recall are distinguished: memory contrasts with recall.", + "surface_contains_pass": true, + "trace_hash": "fba2cc3199dd8d864250c07d7a4387774cd4f2dd8f3b5b027f40a567fc3500e6", + "versor_closure": true, + "versor_condition": 1.3e-08 + }, + { + "case_id": "correction_specific_015", + "category": "correction", + "intent_correct": true, + "surface": "correction: , correction means reviewed repair corrects ...", + "surface_contains_pass": true, + "trace_hash": "8539418c0589ce7f889e43ce41e67dfecd3a1f955d943fc606a9dda8efe2cb54", + "versor_closure": true, + "versor_condition": 7.3e-08 + }, + { + "case_id": "definition_knowledge_003", + "category": "definition", + "intent_correct": true, + "surface": "knowledge is defined as ...", + "surface_contains_pass": true, + "trace_hash": "4eac70da4fa40afad098fe26d3842f792ebe6922d79f89f005d0f9564ba00a15", + "versor_closure": true, + "versor_condition": 9e-09 + }, + { + "case_id": "definition_light_002", + "category": "definition", + "intent_correct": true, + "surface": "light is defined as ...", + "surface_contains_pass": true, + "trace_hash": "4e046d32f3490e70253b7b8187a51c34ca6077e9595d41bd3f4f086eb70184d9", + "versor_closure": true, + "versor_condition": 7e-09 + }, + { + "case_id": "definition_meaning_004", + "category": "definition", + "intent_correct": true, + "surface": "meaning is defined as ...", + "surface_contains_pass": true, + "trace_hash": "ba04e543d69e81f981e1b72a844d6dd5f8f8d2397485099faaacfde21d5b918b", + "versor_closure": true, + "versor_condition": 4e-08 + }, + { + "case_id": "definition_procedure_023", + "category": "definition", + "intent_correct": true, + "surface": "a procedure is defined as ...", + "surface_contains_pass": true, + "trace_hash": "a435ad0321c2fe5985dc0736bfe8edbcbab2e4cc19572a494fbdd1b36abb3850", + "versor_closure": true, + "versor_condition": 9.7e-08 + }, + { + "case_id": "definition_relation_026", + "category": "definition", + "intent_correct": true, + "surface": "a relation is defined as ...", + "surface_contains_pass": true, + "trace_hash": "e84551b4afd6e048a8d6807bd24b884997084dff08c4e2bcaa685c1c3ef09639", + "versor_closure": true, + "versor_condition": 8e-08 + }, + { + "case_id": "procedure_correct_035", + "category": "procedure", + "intent_correct": true, + "surface": "first, ...; then, correct an error follows.", + "surface_contains_pass": true, + "trace_hash": "9ced018654526c4b3a4dc3fc054c843ec92cf587c8bb8fc47731f11ae76f0b1a", + "versor_closure": true, + "versor_condition": 1.33e-07 + }, + { + "case_id": "recall_light_013", + "category": "recall", + "intent_correct": true, + "surface": "recalling light: ...", + "surface_contains_pass": true, + "trace_hash": "7465ad43960e7cb0b23d53a007ceaeb9b476cdebbf51e63a27fd686b0f4f9268", + "versor_closure": true, + "versor_condition": 4.2e-08 + }, + { + "case_id": "unknown_logos_019", + "category": "unknown", + "intent_correct": true, + "surface": "light logos addresses ...", + "surface_contains_pass": true, + "trace_hash": "949e49efdee3b4118cf911f03a40ac68afe33db4644fa3988112fa7179da2a08", + "versor_closure": true, + "versor_condition": 4.3e-08 + }, + { + "case_id": "verification_memory_037", + "category": "verification", + "intent_correct": true, + "surface": "Does memory require recall? is verified: ...", + "surface_contains_pass": true, + "trace_hash": "2b9c0572278c5228b0f4e19ad23072ba7d4894581dfc467db6107b313ce02c4a", + "versor_closure": true, + "versor_condition": 5e-09 + } + ], + "lane": "cognition", + "metrics": { + "intent_accuracy": 1.0, + "surface_groundedness": 1.0, + "term_capture_rate": 1.0, + "total": 13, + "versor_closure_rate": 1.0 + }, + "split": "public", + "timestamp": "2026-05-16T05:33:48.525966+00:00", + "version": "v1" +} diff --git a/evals/cognition/results/v1_public_20260516T053445Z.json b/evals/cognition/results/v1_public_20260516T053445Z.json new file mode 100644 index 00000000..4235a42b --- /dev/null +++ b/evals/cognition/results/v1_public_20260516T053445Z.json @@ -0,0 +1,145 @@ +{ + "cases": [ + { + "case_id": "cause_knowledge_032", + "category": "cause", + "intent_correct": true, + "surface": "does knowledge require evidence is grounded in ...", + "surface_contains_pass": true, + "trace_hash": "a626d35821a8c56795c0ceb4e9e8e73c4bbadf9dc460b9b66eab2864e01866af", + "versor_closure": true, + "versor_condition": 3.8e-08 + }, + { + "case_id": "cause_light_007", + "category": "cause", + "intent_correct": true, + "surface": "does light exist is grounded in ...", + "surface_contains_pass": true, + "trace_hash": "441044ac16952ce918b25b57860b1864a2a80feacea19b5bc48250167ce5d943", + "versor_closure": true, + "versor_condition": 1.56e-07 + }, + { + "case_id": "comparison_memory_recall_030", + "category": "comparison", + "intent_correct": true, + "surface": "memory and recall are distinguished: memory contrasts with recall.", + "surface_contains_pass": true, + "trace_hash": "fba2cc3199dd8d864250c07d7a4387774cd4f2dd8f3b5b027f40a567fc3500e6", + "versor_closure": true, + "versor_condition": 1.3e-08 + }, + { + "case_id": "correction_specific_015", + "category": "correction", + "intent_correct": true, + "surface": "correction: , correction means reviewed repair corrects ...", + "surface_contains_pass": true, + "trace_hash": "8539418c0589ce7f889e43ce41e67dfecd3a1f955d943fc606a9dda8efe2cb54", + "versor_closure": true, + "versor_condition": 7.3e-08 + }, + { + "case_id": "definition_knowledge_003", + "category": "definition", + "intent_correct": true, + "surface": "knowledge is defined as ...", + "surface_contains_pass": true, + "trace_hash": "4eac70da4fa40afad098fe26d3842f792ebe6922d79f89f005d0f9564ba00a15", + "versor_closure": true, + "versor_condition": 9e-09 + }, + { + "case_id": "definition_light_002", + "category": "definition", + "intent_correct": true, + "surface": "light is defined as ...", + "surface_contains_pass": true, + "trace_hash": "4e046d32f3490e70253b7b8187a51c34ca6077e9595d41bd3f4f086eb70184d9", + "versor_closure": true, + "versor_condition": 7e-09 + }, + { + "case_id": "definition_meaning_004", + "category": "definition", + "intent_correct": true, + "surface": "meaning is defined as ...", + "surface_contains_pass": true, + "trace_hash": "ba04e543d69e81f981e1b72a844d6dd5f8f8d2397485099faaacfde21d5b918b", + "versor_closure": true, + "versor_condition": 4e-08 + }, + { + "case_id": "definition_procedure_023", + "category": "definition", + "intent_correct": true, + "surface": "a procedure is defined as ...", + "surface_contains_pass": true, + "trace_hash": "a435ad0321c2fe5985dc0736bfe8edbcbab2e4cc19572a494fbdd1b36abb3850", + "versor_closure": true, + "versor_condition": 9.7e-08 + }, + { + "case_id": "definition_relation_026", + "category": "definition", + "intent_correct": true, + "surface": "a relation is defined as ...", + "surface_contains_pass": true, + "trace_hash": "e84551b4afd6e048a8d6807bd24b884997084dff08c4e2bcaa685c1c3ef09639", + "versor_closure": true, + "versor_condition": 8e-08 + }, + { + "case_id": "procedure_correct_035", + "category": "procedure", + "intent_correct": true, + "surface": "first, ...; then, correct an error follows.", + "surface_contains_pass": true, + "trace_hash": "9ced018654526c4b3a4dc3fc054c843ec92cf587c8bb8fc47731f11ae76f0b1a", + "versor_closure": true, + "versor_condition": 1.33e-07 + }, + { + "case_id": "recall_light_013", + "category": "recall", + "intent_correct": true, + "surface": "recalling light: ...", + "surface_contains_pass": true, + "trace_hash": "7465ad43960e7cb0b23d53a007ceaeb9b476cdebbf51e63a27fd686b0f4f9268", + "versor_closure": true, + "versor_condition": 4.2e-08 + }, + { + "case_id": "unknown_logos_019", + "category": "unknown", + "intent_correct": true, + "surface": "light logos addresses ...", + "surface_contains_pass": true, + "trace_hash": "949e49efdee3b4118cf911f03a40ac68afe33db4644fa3988112fa7179da2a08", + "versor_closure": true, + "versor_condition": 4.3e-08 + }, + { + "case_id": "verification_memory_037", + "category": "verification", + "intent_correct": true, + "surface": "Does memory require recall? is verified: ...", + "surface_contains_pass": true, + "trace_hash": "2b9c0572278c5228b0f4e19ad23072ba7d4894581dfc467db6107b313ce02c4a", + "versor_closure": true, + "versor_condition": 5e-09 + } + ], + "lane": "cognition", + "metrics": { + "intent_accuracy": 1.0, + "surface_groundedness": 1.0, + "term_capture_rate": 1.0, + "total": 13, + "versor_closure_rate": 1.0 + }, + "split": "public", + "timestamp": "2026-05-16T05:34:45.507460+00:00", + "version": "v1" +} diff --git a/evals/cognition/runner.py b/evals/cognition/runner.py new file mode 100644 index 00000000..19d9711a --- /dev/null +++ b/evals/cognition/runner.py @@ -0,0 +1,122 @@ +"""Cognition eval lane runner. + +Conforms to the framework interface: ``run_lane(cases, config=None) -> report`` +where report has ``.metrics`` (dict) and ``.case_details`` (list[dict]). +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from chat.runtime import ChatRuntime +from core.config import RuntimeConfig +from core.cognition.pipeline import CognitiveTurnPipeline +from generate.intent import IntentTag + + +@dataclass(frozen=True, slots=True) +class CaseResult: + case_id: str + category: str + prompt: str + intent_correct: bool + terms_captured: tuple[str, ...] + terms_expected: tuple[str, ...] + surface_contains_pass: bool + versor_closure: bool + versor_condition: float + trace_hash: str + surface: str + + +@dataclass(slots=True) +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +def _run_case(case: dict[str, Any], pipeline: CognitiveTurnPipeline) -> CaseResult: + prompt = case["prompt"] + expected_intent = case["expected_intent"] + expected_terms = case.get("expected_terms", []) + expected_surface_contains = case.get("expected_surface_contains", []) + + result = pipeline.run(prompt, max_tokens=8) + + actual_intent = result.intent.tag if result.intent else IntentTag.UNKNOWN + intent_correct = actual_intent.value == expected_intent + + surface_lower = result.surface.lower() + terms_captured = tuple( + t for t in expected_terms if t.lower() in surface_lower + ) + surface_contains_pass = all( + s.lower() in surface_lower for s in expected_surface_contains + ) + + versor_ok = result.versor_condition < 1e-6 + + return CaseResult( + case_id=case["id"], + category=case.get("category", "unknown"), + prompt=prompt, + intent_correct=intent_correct, + terms_captured=terms_captured, + terms_expected=tuple(expected_terms), + surface_contains_pass=surface_contains_pass, + versor_closure=versor_ok, + versor_condition=result.versor_condition, + trace_hash=result.trace_hash, + surface=result.surface, + ) + + +def run_lane( + cases: list[dict[str, Any]], + *, + config: RuntimeConfig | None = None, +) -> LaneReport: + """Run all cases through CognitiveTurnPipeline and return metrics + details.""" + total = 0 + intent_correct = 0 + terms_expected = 0 + terms_captured = 0 + surface_grounded = 0 + versor_closures = 0 + case_details: list[dict[str, Any]] = [] + + for case in cases: + runtime = ChatRuntime(config=config) if config else ChatRuntime() + pipeline = CognitiveTurnPipeline(runtime) + cr = _run_case(case, pipeline) + + total += 1 + if cr.intent_correct: + intent_correct += 1 + terms_expected += len(cr.terms_expected) + terms_captured += len(cr.terms_captured) + if cr.surface_contains_pass: + surface_grounded += 1 + if cr.versor_closure: + versor_closures += 1 + + case_details.append({ + "case_id": cr.case_id, + "category": cr.category, + "intent_correct": cr.intent_correct, + "surface_contains_pass": cr.surface_contains_pass, + "versor_closure": cr.versor_closure, + "versor_condition": round(cr.versor_condition, 9), + "trace_hash": cr.trace_hash, + "surface": cr.surface, + }) + + metrics = { + "total": total, + "intent_accuracy": round(intent_correct / total, 4) if total else 0.0, + "term_capture_rate": round(terms_captured / terms_expected, 4) if terms_expected else 1.0, + "surface_groundedness": round(surface_grounded / total, 4) if total else 0.0, + "versor_closure_rate": round(versor_closures / total, 4) if total else 0.0, + } + + return LaneReport(metrics=metrics, case_details=case_details) diff --git a/evals/framework.py b/evals/framework.py new file mode 100644 index 00000000..1e2ddb2d --- /dev/null +++ b/evals/framework.py @@ -0,0 +1,182 @@ +"""Generic eval framework — discovers lanes, loads contracts, runs versioned scoring. + +Every eval lane lives in ``evals//`` and must contain at minimum: +- ``contract.md`` — human-readable contract (presence required, not parsed at runtime) +- ``runner.py`` — module that exposes ``run_lane(cases, **kwargs) -> LaneReport`` +- ``dev/cases.jsonl`` — dev set +- ``public/v1/cases.jsonl`` — public test set v1 + +Optional: +- ``holdouts/`` — encrypted sealed test set (scored by holdout_runner) +- ``baselines/`` — frontier model scores +- ``results/`` — CORE scores per version per run +""" +from __future__ import annotations + +import importlib +import importlib.util +import json +import sys +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +EVALS_ROOT = Path(__file__).resolve().parent + + +@dataclass(frozen=True, slots=True) +class LaneInfo: + name: str + root: Path + versions: tuple[str, ...] + + @property + def contract_path(self) -> Path: + return self.root / "contract.md" + + @property + def runner_path(self) -> Path: + return self.root / "runner.py" + + def dev_cases_path(self) -> Path: + return self.root / "dev" / "cases.jsonl" + + def public_cases_path(self, version: str = "v1") -> Path: + return self.root / "public" / version / "cases.jsonl" + + def results_dir(self) -> Path: + return self.root / "results" + + +@dataclass(slots=True) +class LaneResult: + lane: str + version: str + split: str + metrics: dict[str, Any] + case_details: list[dict[str, Any]] = field(default_factory=list) + timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + + def as_dict(self) -> dict[str, Any]: + return { + "lane": self.lane, + "version": self.version, + "split": self.split, + "timestamp": self.timestamp, + "metrics": self.metrics, + "cases": self.case_details, + } + + +def discover_lanes(root: Path | None = None) -> list[LaneInfo]: + """Find all valid eval lanes under *root*.""" + base = root or EVALS_ROOT + lanes: list[LaneInfo] = [] + for child in sorted(base.iterdir()): + if not child.is_dir(): + continue + contract = child / "contract.md" + runner = child / "runner.py" + if contract.exists() and runner.exists(): + versions = _discover_versions(child) + lanes.append(LaneInfo(name=child.name, root=child, versions=versions)) + return lanes + + +def _discover_versions(lane_root: Path) -> tuple[str, ...]: + public = lane_root / "public" + if not public.is_dir(): + return () + versions = sorted( + d.name for d in public.iterdir() + if d.is_dir() and (d / "cases.jsonl").exists() + ) + return tuple(versions) + + +def load_cases(path: Path) -> list[dict[str, Any]]: + """Load cases from a JSONL file.""" + cases: list[dict[str, Any]] = [] + for line in path.read_text().splitlines(): + line = line.strip() + if line: + cases.append(json.loads(line)) + return cases + + +def load_lane_runner(lane: LaneInfo) -> Any: + """Dynamically import a lane's runner module.""" + runner_path = lane.runner_path + if not runner_path.exists(): + raise FileNotFoundError(f"Runner not found: {runner_path}") + + module_name = f"evals.{lane.name}.runner" + spec = importlib.util.spec_from_file_location(module_name, runner_path) + if spec is None or spec.loader is None: + raise ImportError(f"Cannot load runner: {runner_path}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +def run_lane( + lane: LaneInfo, + *, + version: str = "v1", + split: str = "public", + config: Any = None, +) -> LaneResult: + """Run a single lane on a given version and split.""" + if split == "dev": + cases_path = lane.dev_cases_path() + elif split == "public": + cases_path = lane.public_cases_path(version) + else: + raise ValueError(f"Unsupported split: {split!r}. Use 'dev' or 'public'.") + + if not cases_path.exists(): + raise FileNotFoundError(f"Cases not found: {cases_path}") + + cases = load_cases(cases_path) + runner_module = load_lane_runner(lane) + + report = runner_module.run_lane(cases, config=config) + + return LaneResult( + lane=lane.name, + version=version, + split=split, + metrics=report.metrics, + case_details=report.case_details, + ) + + +def write_result(lane: LaneInfo, result: LaneResult) -> Path: + """Write a result to the lane's results directory, returning the path.""" + results_dir = lane.results_dir() + results_dir.mkdir(parents=True, exist_ok=True) + + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + filename = f"{result.version}_{result.split}_{ts}.json" + path = results_dir / filename + path.write_text( + json.dumps(result.as_dict(), ensure_ascii=False, indent=2, sort_keys=True) + + "\n" + ) + return path + + +def get_lane(name: str, root: Path | None = None) -> LaneInfo: + """Look up a single lane by name.""" + base = root or EVALS_ROOT + lane_root = base / name + contract = lane_root / "contract.md" + runner = lane_root / "runner.py" + if not contract.exists(): + raise FileNotFoundError(f"Lane '{name}' has no contract.md at {contract}") + if not runner.exists(): + raise FileNotFoundError(f"Lane '{name}' has no runner.py at {runner}") + versions = _discover_versions(lane_root) + return LaneInfo(name=name, root=lane_root, versions=versions) diff --git a/evals/holdout_runner.py b/evals/holdout_runner.py new file mode 100644 index 00000000..083039dc --- /dev/null +++ b/evals/holdout_runner.py @@ -0,0 +1,76 @@ +"""Holdout runner — scores sealed test sets without exposing item-level results. + +The holdout set lives encrypted in ``evals//holdouts/``. The decryption +key is held by the human reviewer and supplied via environment variable. + +Current implementation is a scaffold: it reads plaintext holdouts for initial +development. The encryption layer (age or GPG) is wired in before any holdout +set is considered "sealed." + +Trust boundary: this module reads encrypted files and writes only aggregate +scores. It must never write item-level results, case details, or raw case +content to the working tree. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from evals.framework import LaneInfo, load_lane_runner, load_cases + + +HOLDOUT_KEY_ENV = "CORE_HOLDOUT_KEY" + + +@dataclass(frozen=True, slots=True) +class HoldoutResult: + lane: str + metrics: dict[str, Any] + sealed: bool + + +def _decrypt_holdout(encrypted_path: Path) -> list[dict[str, Any]]: + """Decrypt a holdout file and return cases. + + Currently: reads plaintext fallback if no encryption key is set. + Future: decrypt with age/GPG using CORE_HOLDOUT_KEY. + """ + key = os.environ.get(HOLDOUT_KEY_ENV) + + plaintext_path = encrypted_path.parent / "cases_plaintext.jsonl" + if key is None and plaintext_path.exists(): + return load_cases(plaintext_path) + + if key is None: + raise EnvironmentError( + f"Set {HOLDOUT_KEY_ENV} to decrypt holdout, or provide " + f"cases_plaintext.jsonl for unsealed development." + ) + + # TODO: implement actual decryption (age -d -i ) + raise NotImplementedError( + "Encrypted holdout decryption not yet implemented. " + "Use cases_plaintext.jsonl for development." + ) + + +def run_holdout(lane: LaneInfo, *, config: Any = None) -> HoldoutResult: + """Score a lane's holdout set and return only aggregate metrics.""" + holdout_dir = lane.root / "holdouts" + if not holdout_dir.exists(): + raise FileNotFoundError(f"No holdouts directory: {holdout_dir}") + + cases = _decrypt_holdout(holdout_dir / "cases.jsonl.age") + + runner_module = load_lane_runner(lane) + report = runner_module.run_lane(cases, config=config) + + sealed = os.environ.get(HOLDOUT_KEY_ENV) is not None + + return HoldoutResult( + lane=lane.name, + metrics=report.metrics, + sealed=sealed, + )