b5d6ad6510
Closes the residual `novel_pair_under_seen_relation` pattern that neither `transitive_walk` nor `multi_relation_walk` could synthesise. - new `compose_relations(triples, head, frame, relation)` operator — pure lookup, returns both `R(head, ?)` and `R(frame, ?)` tails - new `FRAME_TRANSFER` intent + `_FRAME_TRANSFER_RE` regex tried before generic TRANSITIVE_QUERY so "in Y" isn't truncated; handles "X belong to in Y" → belongs_to normalisation - pipeline wiring: `_maybe_compose_relations`, `_fold_compose_into_surface`, `_serialize_compose` (folded into operator_invocation so trace_hash stays bit-identical across replay) - regression: inference_closure, multi_step_reasoning, cross_domain_transfer all still 100% on public + holdouts discourse_paragraph v2: - per-sentence grammar rubric (length, capitalization, subject alignment) gated on `require_per_sentence_grammar` - scaling cases at 10 / 20 / 50 sentences — 3/3 pass, 100% per-sentence - 3 runtime round-trip cases (`mode: runtime_roundtrip`) that prime vault, ask question, verify bit-identical across two fresh runtimes - new `per_sentence_grammar_pass_rate` lane metric Long-form replay benchmark (benchmarks/replay_vs_llm.py): - `replay_determinism_report(prompts, runs, priming)` — CORE-only - `compare_to_llm(prompts, llm_callable)` — BYO API client, no provider lock-in; reports per-prompt determinism on both sides - ships with default cognition-pack prompts; 100% bit-identical at runs=3 Lanes green: cognition 121/121, runtime 19/19, teaching 17/17, packs 6/6, compositionality 16/16 + 10/10, inference_closure 20/20 + 12/12, multi_step_reasoning 15/15 + 10/10, cross_domain_transfer 10/10 + 8/8, discourse_paragraph v1 12/12 + v2 6/6.
313 lines
11 KiB
Python
313 lines
11 KiB
Python
"""discourse_paragraph eval lane runner.
|
|
|
|
Exercises paragraph-scale realization: given a multi-step
|
|
ArticulationTarget, the deterministic realizer should produce a
|
|
multi-sentence surface with discourse markers (next, furthermore,
|
|
in contrast) and full subject coverage.
|
|
|
|
Bypasses ChatRuntime grounding so the paragraph claim is isolated
|
|
to the realizer. Runtime round-tripping is named as a v2 gap.
|
|
|
|
Conforms to the framework interface: run_lane(cases, config=None) -> report.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from generate.graph_planner import (
|
|
ArticulationStep,
|
|
ArticulationTarget,
|
|
GraphEdge,
|
|
GraphNode,
|
|
PropositionGraph,
|
|
Relation,
|
|
RhetoricalMove,
|
|
)
|
|
from generate.intent import IntentTag
|
|
from generate.realizer import realize_target
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class LaneReport:
|
|
metrics: dict[str, Any] = field(default_factory=dict)
|
|
case_details: list[dict[str, Any]] = field(default_factory=list)
|
|
|
|
|
|
_SENTENCE_SPLIT_RE = re.compile(r"[.!?]\s+|[.!?]$")
|
|
|
|
|
|
def _sentence_count(surface: str) -> int:
|
|
if not surface.strip():
|
|
return 0
|
|
parts = [p for p in _SENTENCE_SPLIT_RE.split(surface) if p.strip()]
|
|
return len(parts)
|
|
|
|
|
|
def _build_target_from_case(case: dict[str, Any]) -> tuple[ArticulationTarget, PropositionGraph]:
|
|
nodes_data = case["graph"]["nodes"]
|
|
edges_data = case["graph"].get("edges", [])
|
|
nodes = tuple(
|
|
GraphNode(
|
|
node_id=nd["node_id"],
|
|
subject=nd["subject"],
|
|
predicate=nd["predicate"],
|
|
obj=nd["obj"],
|
|
source_intent=IntentTag.UNKNOWN,
|
|
)
|
|
for nd in nodes_data
|
|
)
|
|
edges = tuple(
|
|
GraphEdge(
|
|
source=e["source"],
|
|
target=e["target"],
|
|
relation=Relation[e.get("relation", "SEQUENCE").upper()],
|
|
)
|
|
for e in edges_data
|
|
)
|
|
graph = PropositionGraph(nodes=nodes, edges=edges)
|
|
by_id = {n.node_id: n for n in nodes}
|
|
steps = tuple(
|
|
ArticulationStep(
|
|
node_id=s["node_id"],
|
|
subject=by_id[s["node_id"]].subject,
|
|
predicate=by_id[s["node_id"]].predicate,
|
|
move=RhetoricalMove[s["move"].upper()],
|
|
)
|
|
for s in case["steps"]
|
|
)
|
|
target = ArticulationTarget(steps=steps, source_intent=IntentTag.UNKNOWN)
|
|
return target, graph
|
|
|
|
|
|
_MIN_WORDS_PER_SENTENCE = 3
|
|
|
|
|
|
def _check_per_sentence_grammar(
|
|
sentences: list[str],
|
|
expected_steps: list[dict[str, Any]] | None,
|
|
) -> list[str]:
|
|
"""Per-sentence grammaticality rubric (v2).
|
|
|
|
For each emitted sentence, verifies:
|
|
- non-empty after strip
|
|
- at least ``_MIN_WORDS_PER_SENTENCE`` whitespace tokens
|
|
- starts with an uppercase alphabetic character (sentence-initial cap)
|
|
- if expected_steps is supplied, the subject of the aligned step
|
|
appears somewhere in the sentence (case-insensitive)
|
|
|
|
Returns a list of failure strings; empty if every sentence passes.
|
|
"""
|
|
failures: list[str] = []
|
|
for idx, sent in enumerate(sentences):
|
|
stripped = sent.strip()
|
|
if not stripped:
|
|
failures.append(f"sentence[{idx}] empty")
|
|
continue
|
|
words = stripped.split()
|
|
if len(words) < _MIN_WORDS_PER_SENTENCE:
|
|
failures.append(
|
|
f"sentence[{idx}] too short ({len(words)} words): {stripped[:40]!r}"
|
|
)
|
|
first_alpha = next((c for c in stripped if c.isalpha()), None)
|
|
if first_alpha is not None and not first_alpha.isupper():
|
|
failures.append(
|
|
f"sentence[{idx}] not capitalized: {stripped[:40]!r}"
|
|
)
|
|
if expected_steps is not None and idx < len(expected_steps):
|
|
subj = expected_steps[idx].get("subject", "").lower()
|
|
if subj and subj not in stripped.lower():
|
|
failures.append(
|
|
f"sentence[{idx}] missing aligned subject {subj!r}: {stripped[:40]!r}"
|
|
)
|
|
return failures
|
|
|
|
|
|
def _score_runtime_roundtrip_case(case: dict[str, Any]) -> dict[str, Any]:
|
|
"""Score a runtime round-trip case: prime vault, ask a question,
|
|
check the runtime's articulation surface is well-formed and
|
|
replay-deterministic.
|
|
|
|
Builds two fresh ``ChatRuntime`` instances, primes each with the
|
|
same sequence, and runs the same question — both surfaces must
|
|
match byte-identically.
|
|
|
|
This is a weaker structural claim than the realizer-direct
|
|
cases: the runtime/planner typically produces a single sentence
|
|
per turn, so we do not assert paragraph length here. Multi-
|
|
sentence-from-runtime is a v3 gap (requires planner extension).
|
|
"""
|
|
from chat.runtime import ChatRuntime
|
|
|
|
priming: list[str] = list(case.get("priming", []))
|
|
question: str = case["question"]
|
|
|
|
failures: list[str] = []
|
|
|
|
def run_once() -> tuple[str, int]:
|
|
rt = ChatRuntime()
|
|
for p in priming:
|
|
rt.chat(p)
|
|
resp = rt.chat(question)
|
|
surface = resp.articulation_surface or resp.surface or ""
|
|
return surface, int(getattr(resp, "vault_hits", 0))
|
|
|
|
surface_1, hits_1 = run_once()
|
|
surface_2, _ = run_once()
|
|
surface = surface_1.strip()
|
|
|
|
if not surface:
|
|
failures.append("empty runtime surface")
|
|
min_hits = int(case.get("min_vault_hits", 1))
|
|
if hits_1 < min_hits:
|
|
failures.append(f"vault_hits {hits_1} < min {min_hits} (gate likely fired)")
|
|
if surface_1 != surface_2:
|
|
failures.append(
|
|
f"runtime replay non-deterministic: {surface_1!r} != {surface_2!r}"
|
|
)
|
|
|
|
# Sentence-initial capitalization on the runtime surface too.
|
|
if surface:
|
|
first_alpha = next((c for c in surface if c.isalpha()), None)
|
|
if first_alpha is not None and not first_alpha.isupper():
|
|
failures.append(f"runtime surface not capitalized: {surface[:40]!r}")
|
|
|
|
must_contain = case.get("must_contain", [])
|
|
for token in must_contain:
|
|
if token.lower() not in surface.lower():
|
|
failures.append(f"missing required token {token!r} in {surface[:60]!r}")
|
|
|
|
sent_count = _sentence_count(surface)
|
|
|
|
return {
|
|
"id": case["id"],
|
|
"topic": case.get("topic", "runtime_roundtrip"),
|
|
"passed": not failures,
|
|
"surface": surface,
|
|
"sentence_count": sent_count,
|
|
"subject_coverage": 1.0 if not failures else 0.0,
|
|
"discourse_markers_found": [],
|
|
"replay_match": surface_1 == surface_2,
|
|
"per_sentence_failures": [],
|
|
"vault_hits": hits_1,
|
|
"failure_reasons": failures,
|
|
}
|
|
|
|
|
|
def _score_case(case: dict[str, Any]) -> dict[str, Any]:
|
|
if case.get("mode") == "runtime_roundtrip":
|
|
return _score_runtime_roundtrip_case(case)
|
|
target, graph = _build_target_from_case(case)
|
|
plan_1 = realize_target(target, graph)
|
|
plan_2 = realize_target(target, graph)
|
|
surface = plan_1.surface
|
|
surface_lower = surface.lower()
|
|
|
|
failures: list[str] = []
|
|
sent_count = _sentence_count(surface)
|
|
min_sentences = int(case["min_sentences"])
|
|
max_sentences = int(case.get("max_sentences", min_sentences + 2))
|
|
if sent_count < min_sentences:
|
|
failures.append(f"sentence_count {sent_count} < min {min_sentences}")
|
|
if sent_count > max_sentences:
|
|
failures.append(f"sentence_count {sent_count} > max {max_sentences}")
|
|
|
|
must_contain = case.get("must_contain_subjects", [])
|
|
present = [s for s in must_contain if s.lower() in surface_lower]
|
|
coverage = len(present) / max(1, len(must_contain))
|
|
if coverage < 0.75:
|
|
missing = [s for s in must_contain if s.lower() not in surface_lower]
|
|
failures.append(f"subject_coverage {coverage:.2f} < 0.75; missing={missing}")
|
|
|
|
expected_markers = case.get("discourse_markers", [])
|
|
if expected_markers:
|
|
found = [m for m in expected_markers if m.lower() in surface_lower]
|
|
if not found:
|
|
failures.append(
|
|
f"no discourse marker present; expected one of {expected_markers}"
|
|
)
|
|
else:
|
|
found = []
|
|
|
|
# Sentence-initial capitalization (G4): every sentence-leading
|
|
# alphabetic character must be uppercase. This is the gate that
|
|
# turned "wisdom grounds knowledge." into "Wisdom grounds
|
|
# knowledge." — addresses the open scope item.
|
|
sentences = [p.strip() for p in _SENTENCE_SPLIT_RE.split(surface) if p.strip()]
|
|
badly_cased: list[str] = []
|
|
for sent in sentences:
|
|
for ch in sent:
|
|
if ch.isalpha():
|
|
if not ch.isupper():
|
|
badly_cased.append(sent[:30])
|
|
break
|
|
if badly_cased:
|
|
failures.append(
|
|
f"sentence-initial capitalization missing in {len(badly_cased)} "
|
|
f"sentence(s): {badly_cased}"
|
|
)
|
|
|
|
replay_match = plan_1.surface == plan_2.surface
|
|
if not replay_match:
|
|
failures.append("replay determinism broken: surfaces differ")
|
|
|
|
per_sentence_failures: list[str] = []
|
|
if case.get("require_per_sentence_grammar"):
|
|
# v2: align emitted sentences to the case steps (one sentence per
|
|
# step in non-folded cases) and run the per-sentence rubric.
|
|
expected_steps_aligned: list[dict[str, Any]] | None = (
|
|
case.get("steps") if case.get("align_steps_to_sentences") else None
|
|
)
|
|
per_sentence_failures = _check_per_sentence_grammar(
|
|
sentences, expected_steps_aligned
|
|
)
|
|
if per_sentence_failures:
|
|
failures.extend(per_sentence_failures)
|
|
|
|
passed = not failures
|
|
return {
|
|
"id": case["id"],
|
|
"topic": case.get("topic", ""),
|
|
"passed": passed,
|
|
"surface": surface,
|
|
"sentence_count": sent_count,
|
|
"subject_coverage": coverage,
|
|
"discourse_markers_found": found,
|
|
"replay_match": replay_match,
|
|
"per_sentence_failures": per_sentence_failures,
|
|
"failure_reasons": failures,
|
|
}
|
|
|
|
|
|
def run_lane(cases: list[dict[str, Any]], *, config: Any = None) -> LaneReport:
|
|
details = [_score_case(c) for c in cases]
|
|
total = len(details)
|
|
passed = sum(1 for d in details if d["passed"])
|
|
return LaneReport(
|
|
metrics={
|
|
"total": total,
|
|
"passed": passed,
|
|
"accuracy": round(passed / total, 4) if total else 0.0,
|
|
"mean_sentence_count": round(
|
|
sum(d["sentence_count"] for d in details) / max(1, total), 3
|
|
),
|
|
"mean_subject_coverage": round(
|
|
sum(d["subject_coverage"] for d in details) / max(1, total), 4
|
|
),
|
|
"replay_determinism_rate": round(
|
|
sum(1 for d in details if d["replay_match"]) / max(1, total), 4
|
|
),
|
|
"per_sentence_grammar_pass_rate": round(
|
|
sum(
|
|
1
|
|
for d in details
|
|
if not d.get("per_sentence_failures")
|
|
)
|
|
/ max(1, total),
|
|
4,
|
|
),
|
|
},
|
|
case_details=details,
|
|
)
|