diff --git a/evals/deterministic_fluency/contract.md b/evals/deterministic_fluency/contract.md new file mode 100644 index 00000000..19418cf1 --- /dev/null +++ b/evals/deterministic_fluency/contract.md @@ -0,0 +1,83 @@ +# Deterministic Fluency Eval Lane — Contract + +**Lane:** `deterministic_fluency` +**Version:** v1 +**Created:** 2026-05-19 + +## What this lane measures + +A small, deterministic, structural definition of "fluent" — no +subjective scoring, no embedding similarity, no LLM judge. Each +case is a prompt + a list of structural predicates the runtime's +final surface must satisfy. + +The 2026-05-19 design review observed that several existing eval +lanes (grammatical_coverage, english_fluency_ood) pass surfaces +like `"river flows valley"` and `"knowledge does not necessitates +force"`. Those surfaces are token-ordered but not English. This +lane closes that gap with checks that are testable as `bool` +predicates, not felt qualities. + +## The six structural predicates + +| Predicate | Definition | Implementation | +|---|---|---| +| `no_placeholder` | surface contains no `...`, ``, ``, `` | substring scan | +| `no_provenance_only` | surface is not bare structured disclosure like `"X — pack-grounded (pack_id): a; b; c. No session evidence yet."` | regex match: rejects surfaces matching `^[a-z_]+ — pack-grounded \(.*\): [^.]+\.\s*(No session evidence yet\|No prior turn in this session to correct yet)\.\s*$` | +| `complete_punctuation` | surface ends with `.`, `?`, `!`, or `;` after stripping whitespace | `rstrip().endswith(('.', '?', '!', ';'))` | +| `finite_predicate_shape` | surface contains at least one finite verb (is/are/was/were/has/have/does/do/did) OR an inflected verb form | regex scan for verb tokens | +| `no_dotted_domain_inventory` | surface does not contain three or more dotted-path tokens joined by `;` (e.g. `meta.x.y; meta.x; cognition.z`) | regex match | +| `surface_provenance_match` | actual `grounding_source` is consistent with the runtime's emitted surface tag | metadata cross-check | + +Each predicate emits a binary signal per case. Lane-level metrics +are rates across the predicate × case matrix. + +## Scoring rubric + +| Metric | Definition | v1 pass threshold | +|---|---|---| +| `no_placeholder_rate` | fraction of cases passing `no_placeholder` | 1.00 | +| `complete_punctuation_rate` | fraction of cases ending with terminal punctuation | 1.00 | +| `finite_predicate_rate` | fraction of cases with a finite-verb token | >= 0.90 | +| `no_provenance_only_rate` | fraction of cases NOT emitting a bare-disclosure surface | varies — see below | +| `no_dotted_inventory_rate` | fraction of cases NOT emitting dotted-path inventory | varies — see below | + +## The "varies" threshold note + +Pre-gloss, `no_provenance_only_rate` and `no_dotted_inventory_rate` +will be at the floor (most pack-grounded surfaces today ARE bare +provenance disclosure with dotted paths). This is expected and +documented — those two metrics are the lift target for the gloss +feature. After the gloss feature wires through: + + pre-gloss: no_provenance_only_rate ≈ 0.10, no_dotted_inventory_rate ≈ 0.10 + post-gloss: no_provenance_only_rate >= 0.85, no_dotted_inventory_rate >= 0.85 + +## Why this lane is not "subjective fluency" + +Every predicate above is decidable in code with no judgment. A +surface either contains `...` or does not. Either ends with a +terminal or not. Either contains `meta.x.y; meta.x; ...` or not. +This is *structural completeness*, not aesthetic quality. + +Subjective fluency (rhythm, idiom, register) is OUT OF SCOPE here. +It would require either an LLM judge (non-deterministic, doctrine +violation) or human review (not CI-pinnable). Either belongs in a +different lane. + +## Case schema + +```jsonl +{ + "id": "fluency_truth_001", + "prompt": "What is truth?", + "category": "pack_definition", + "expected_predicates": ["no_placeholder", "complete_punctuation", + "finite_predicate_shape"], + "post_gloss_predicates": ["no_provenance_only", "no_dotted_inventory"] +} +``` + +`expected_predicates` is the set of predicates that must hold today. +`post_gloss_predicates` is the set that will be enforced after the +gloss feature lands — currently informational, not asserted in v1. diff --git a/evals/deterministic_fluency/dev/cases.jsonl b/evals/deterministic_fluency/dev/cases.jsonl new file mode 100644 index 00000000..e417b1cf --- /dev/null +++ b/evals/deterministic_fluency/dev/cases.jsonl @@ -0,0 +1,2 @@ +{"id":"dev_fluency_truth","prompt":"What is truth?","category":"pack_definition","expected_predicates":["no_placeholder","complete_punctuation","finite_predicate_shape"]} +{"id":"dev_fluency_oov","prompt":"What is quasar?","category":"oov_invitation","expected_predicates":["no_placeholder","complete_punctuation"]} diff --git a/evals/deterministic_fluency/public/v1/cases.jsonl b/evals/deterministic_fluency/public/v1/cases.jsonl new file mode 100644 index 00000000..21194997 --- /dev/null +++ b/evals/deterministic_fluency/public/v1/cases.jsonl @@ -0,0 +1,15 @@ +{"id":"fluency_truth_001","prompt":"What is truth?","category":"pack_definition","expected_predicates":["no_placeholder","complete_punctuation","finite_predicate_shape"]} +{"id":"fluency_knowledge_002","prompt":"What is knowledge?","category":"pack_definition","expected_predicates":["no_placeholder","complete_punctuation","finite_predicate_shape"]} +{"id":"fluency_doubt_003","prompt":"What is doubt?","category":"pack_definition","expected_predicates":["no_placeholder","complete_punctuation","finite_predicate_shape"]} +{"id":"fluency_define_moment_004","prompt":"Define moment.","category":"pack_definition","expected_predicates":["no_placeholder","complete_punctuation","finite_predicate_shape"]} +{"id":"fluency_what_does_soon_mean_005","prompt":"What does soon mean?","category":"pack_definition","expected_predicates":["no_placeholder","complete_punctuation","finite_predicate_shape"]} +{"id":"fluency_what_is_true_006","prompt":"What is true?","category":"pack_definition","expected_predicates":["no_placeholder","complete_punctuation","finite_predicate_shape"]} +{"id":"fluency_define_evident_007","prompt":"Define evident.","category":"pack_definition","expected_predicates":["no_placeholder","complete_punctuation","finite_predicate_shape"]} +{"id":"fluency_what_does_important_mean_008","prompt":"What does important mean?","category":"pack_definition","expected_predicates":["no_placeholder","complete_punctuation","finite_predicate_shape"]} +{"id":"fluency_what_is_a_fact_009","prompt":"What is a fact?","category":"pack_definition","expected_predicates":["no_placeholder","complete_punctuation","finite_predicate_shape"]} +{"id":"fluency_what_is_self_010","prompt":"What is the self?","category":"pack_definition","expected_predicates":["no_placeholder","complete_punctuation","finite_predicate_shape"]} +{"id":"fluency_oov_hypothesis_011","prompt":"What is a hypothesis?","category":"oov_invitation","expected_predicates":["no_placeholder","complete_punctuation"]} +{"id":"fluency_oov_quasar_012","prompt":"What is quasar?","category":"oov_invitation","expected_predicates":["no_placeholder","complete_punctuation"]} +{"id":"fluency_cause_no_chain_013","prompt":"How does memory work?","category":"cause_no_chain_unknown_domain","expected_predicates":["no_placeholder","complete_punctuation"]} +{"id":"fluency_cause_no_chain_014","prompt":"What causes doubt?","category":"cause_no_chain_unknown_domain","expected_predicates":["no_placeholder","complete_punctuation"]} +{"id":"fluency_teaching_truth_015","prompt":"Why is truth important?","category":"teaching_grounded","expected_predicates":["no_placeholder","complete_punctuation","finite_predicate_shape"]} diff --git a/evals/deterministic_fluency/runner.py b/evals/deterministic_fluency/runner.py new file mode 100644 index 00000000..5040f162 --- /dev/null +++ b/evals/deterministic_fluency/runner.py @@ -0,0 +1,174 @@ +"""Deterministic fluency eval lane runner. + +Six structural predicates over the runtime's final surface — no +embedding, no LLM judge, no aesthetics. Each predicate is a +testable bool. This lane provides the substrate for the gloss +feature's lift target (the no_provenance_only and +no_dotted_inventory rates climb when glosses replace bare +disclosure surfaces). + +Framework contract: ``run_lane(cases, config=None) -> LaneReport``. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from chat.runtime import ChatRuntime + + +_PLACEHOLDER_MARKERS = ("...", "", "", "") + +# Bare structured-disclosure shape — e.g. +# "doubt — pack-grounded (en_core_meta_v1): meta.mental_state.uncertainty; meta.mental_state; cognition.epistemic. No session evidence yet." +# The shape is exactly: — pack-grounded (): . . +_PROVENANCE_ONLY_RE = re.compile( + r"^[a-z_][a-z_]* — pack-grounded \([a-z0-9_]+\): [^.]+\. " + r"(No session evidence yet|No prior turn in this session to correct yet)\.\s*$" +) + +# Three or more dotted-path tokens joined by `;` — the "domain inventory" +# shape that pre-gloss pack_grounded_surface emits. +_DOTTED_INVENTORY_RE = re.compile( + r"[a-z_]+\.[a-z_]+(?:\.[a-z_]+)?\s*;\s*[a-z_]+\.[a-z_]+(?:\.[a-z_]+)?\s*;\s*" + r"[a-z_]+\.[a-z_]+(?:\.[a-z_]+)?" +) + +_FINITE_VERB_PATTERNS = ( + # third-person singular forms + auxiliaries + irregulars + re.compile(r"\b(is|are|was|were|has|have|had|does|do|did|will|would|" + r"can|could|should|might|may|must|shall|been|being)\b"), + # regular -s present-third-singular + re.compile(r"\b[a-z]+(es|s)\b"), + # regular -ed simple past + re.compile(r"\b[a-z]+ed\b"), + # regular -ing present-participle + re.compile(r"\b[a-z]+ing\b"), +) + + +def _check_no_placeholder(surface: str) -> bool: + return not any(m in surface for m in _PLACEHOLDER_MARKERS) + + +def _check_no_provenance_only(surface: str) -> bool: + return _PROVENANCE_ONLY_RE.match(surface.strip()) is None + + +def _check_complete_punctuation(surface: str) -> bool: + stripped = surface.rstrip() + if not stripped: + return False + return stripped[-1] in (".", "?", "!", ";") + + +def _check_finite_predicate(surface: str) -> bool: + low = surface.lower() + return any(p.search(low) for p in _FINITE_VERB_PATTERNS) + + +def _check_no_dotted_inventory(surface: str) -> bool: + return _DOTTED_INVENTORY_RE.search(surface) is None + + +def _check_surface_provenance_match(surface: str, grounding: str) -> bool: + """The surface's text and the declared grounding_source must + agree. Specifically: when grounding_source != 'pack' / 'teaching', + the surface must NOT contain the 'pack-grounded' marker (would be + a metadata/text disagreement).""" + has_marker = "pack-grounded" in surface or "teaching-grounded" in surface + if grounding in {"pack", "teaching"}: + return True # marker present is allowed; absent is also allowed + # (gloss-backed surfaces may move the marker to a separate tag) + return not has_marker + + +_PREDICATE_FNS = { + "no_placeholder": lambda s, g: _check_no_placeholder(s), + "no_provenance_only": lambda s, g: _check_no_provenance_only(s), + "complete_punctuation": lambda s, g: _check_complete_punctuation(s), + "finite_predicate_shape": lambda s, g: _check_finite_predicate(s), + "no_dotted_inventory": lambda s, g: _check_no_dotted_inventory(s), + "surface_provenance_match": _check_surface_provenance_match, +} + + +@dataclass(frozen=True, slots=True) +class CaseResult: + case_id: str + category: str + prompt: str + surface: str + grounding_source: str + predicates: dict[str, bool] + expected_predicates: tuple[str, ...] + expected_pass: bool + + +@dataclass +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +def _run_case(case: dict[str, Any]) -> CaseResult: + prompt = case["prompt"] + expected = tuple(case.get("expected_predicates", ())) + + runtime = ChatRuntime() + response = runtime.chat(prompt) + surface = response.surface + grounding = response.grounding_source or "none" + + predicates = { + name: bool(fn(surface, grounding)) + for name, fn in _PREDICATE_FNS.items() + } + expected_pass = all(predicates[name] for name in expected) + + return CaseResult( + case_id=case["id"], + category=case.get("category", "uncategorised"), + prompt=prompt, + surface=surface, + grounding_source=grounding, + predicates=predicates, + expected_predicates=expected, + expected_pass=expected_pass, + ) + + +def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: # noqa: ARG001 + if not cases: + return LaneReport(metrics={}, case_details=[]) + + results = [_run_case(c) for c in cases] + total = len(results) + + rates: dict[str, Any] = {"cases": total} + for name in _PREDICATE_FNS: + passed = sum(1 for r in results if r.predicates[name]) + rates[f"{name}_rate"] = round(passed / total, 4) if total else 1.0 + + expected_pass = sum(1 for r in results if r.expected_pass) + rates["expected_predicates_pass_rate"] = round(expected_pass / total, 4) + + case_details = [ + { + "case_id": r.case_id, + "category": r.category, + "prompt": r.prompt, + "surface": r.surface, + "grounding_source": r.grounding_source, + "predicates": r.predicates, + "expected_predicates": list(r.expected_predicates), + "expected_pass": r.expected_pass, + } + for r in results + ] + + return LaneReport(metrics=rates, case_details=case_details) + + +__all__ = ["run_lane", "LaneReport", "CaseResult"] diff --git a/tests/test_deterministic_fluency_lane.py b/tests/test_deterministic_fluency_lane.py new file mode 100644 index 00000000..670a8c1d --- /dev/null +++ b/tests/test_deterministic_fluency_lane.py @@ -0,0 +1,160 @@ +"""Contract tests for the ``deterministic_fluency`` eval lane. + +The lane defines fluency as six deterministic structural predicates, +not as subjective quality. Tests here pin: + + - Case-set integrity. + - All six predicates are implemented and run cleanly. + - The runner produces all required metrics. + - Three hard invariants hold on current main (no_placeholder == 1, + complete_punctuation == 1, finite_predicate >= 0.90). + - The lift-target metrics (no_provenance_only, no_dotted_inventory) + are recorded but NOT pinned to a threshold yet — they are the + gloss feature's measurement substrate. + +The six predicates are: no_placeholder, no_provenance_only, +complete_punctuation, finite_predicate_shape, no_dotted_inventory, +surface_provenance_match. +""" + +from __future__ import annotations + +from pathlib import Path + +from evals.framework import ( + discover_lanes, + get_lane, + load_cases, + load_lane_runner, + run_lane, +) + + +LANE_NAME = "deterministic_fluency" +_EVAL_ROOT = Path(__file__).resolve().parent.parent / "evals" / LANE_NAME +_PUBLIC_CASES = _EVAL_ROOT / "public" / "v1" / "cases.jsonl" + +ALL_PREDICATES = ( + "no_placeholder", + "no_provenance_only", + "complete_punctuation", + "finite_predicate_shape", + "no_dotted_inventory", + "surface_provenance_match", +) + + +class TestCaseSetIntegrity: + def test_public_cases_file_exists(self) -> None: + assert _PUBLIC_CASES.exists() + + def test_case_count(self) -> None: + cases = load_cases(_PUBLIC_CASES) + assert len(cases) >= 10 + + def test_every_case_has_required_fields(self) -> None: + for case in load_cases(_PUBLIC_CASES): + assert "id" in case + assert "prompt" in case + assert "expected_predicates" in case + assert isinstance(case["expected_predicates"], list) + + def test_expected_predicates_are_known(self) -> None: + for case in load_cases(_PUBLIC_CASES): + for pred in case["expected_predicates"]: + assert pred in ALL_PREDICATES, (case["id"], pred) + + +class TestLaneDiscovery: + def test_lane_is_discoverable(self) -> None: + names = {lane.name for lane in discover_lanes()} + assert LANE_NAME in names + + def test_lane_runner_loads(self) -> None: + lane = get_lane(LANE_NAME) + runner = load_lane_runner(lane) + assert hasattr(runner, "run_lane") + + +class TestRunnerMetrics: + def test_all_predicate_rates_present(self) -> None: + lane = get_lane(LANE_NAME) + result = run_lane(lane, version="v1", split="public") + for pred in ALL_PREDICATES: + key = f"{pred}_rate" + assert key in result.metrics, ( + f"missing metric {key!r}; got: {sorted(result.metrics)}" + ) + + def test_per_case_predicates_dict_present(self) -> None: + lane = get_lane(LANE_NAME) + result = run_lane(lane, version="v1", split="public") + for case in result.case_details: + assert "predicates" in case + assert isinstance(case["predicates"], dict) + for pred in ALL_PREDICATES: + assert pred in case["predicates"], (case["case_id"], pred) + assert isinstance(case["predicates"][pred], bool) + + +class TestHardInvariants: + """Three predicates have hard 1.00 / >=0.90 thresholds that should + hold on current main. These are the structural-completeness floor + — any regression here is a doctrine violation, not a tunable.""" + + def test_no_placeholder_rate_is_one(self) -> None: + lane = get_lane(LANE_NAME) + result = run_lane(lane, version="v1", split="public") + rate = result.metrics["no_placeholder_rate"] + assert rate == 1.0, ( + f"no_placeholder_rate dropped below 1.0: {rate}. " + f"A user-facing surface containing ..., , or " + f"is a doctrine violation." + ) + + def test_complete_punctuation_rate_is_one(self) -> None: + lane = get_lane(LANE_NAME) + result = run_lane(lane, version="v1", split="public") + rate = result.metrics["complete_punctuation_rate"] + assert rate == 1.0, ( + f"complete_punctuation_rate dropped below 1.0: {rate}" + ) + + def test_finite_predicate_rate_at_least_ninety(self) -> None: + lane = get_lane(LANE_NAME) + result = run_lane(lane, version="v1", split="public") + rate = result.metrics["finite_predicate_shape_rate"] + assert rate >= 0.90, ( + f"finite_predicate_shape_rate dropped below 0.90: {rate}" + ) + + def test_expected_predicates_all_pass(self) -> None: + """Every case must satisfy at least its OWN declared + expected_predicates list. Cases that include an + as-yet-unmet predicate must move that predicate to + post_gloss_predicates instead.""" + lane = get_lane(LANE_NAME) + result = run_lane(lane, version="v1", split="public") + rate = result.metrics["expected_predicates_pass_rate"] + assert rate == 1.0, ( + f"some cases failed their own expected_predicates: rate={rate}" + ) + + +class TestLiftTargetMetricsAreRecorded: + """The two metrics the gloss feature is designed to lift are + recorded today but NOT pinned to a threshold. When the gloss + feature lands, these tests will be extended with thresholds.""" + + def test_no_provenance_only_rate_present(self) -> None: + lane = get_lane(LANE_NAME) + result = run_lane(lane, version="v1", split="public") + assert "no_provenance_only_rate" in result.metrics + + def test_no_dotted_inventory_rate_present(self) -> None: + lane = get_lane(LANE_NAME) + result = run_lane(lane, version="v1", split="public") + assert "no_dotted_inventory_rate" in result.metrics + # Sanity check: should be a float in [0, 1]. + rate = result.metrics["no_dotted_inventory_rate"] + assert 0.0 <= rate <= 1.0