From 9fcc029a13f8c7fe0481a7cd64444438bc2c8b27 Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 18 Jun 2026 21:05:37 -0700 Subject: [PATCH] feat(kernel): bind ProblemFrame relations and measure contract readiness (#831) --- HANDOFF-gpt55-2026-06-18.md | 110 +++++++ ...-bindings-contract-readiness-2026-06-18.md | 107 +++++++ generate/kernel_facts.py | 51 +++- generate/problem_frame.py | 43 +++ generate/problem_frame_builder.py | 268 ++++++++++++++++-- generate/problem_frame_contracts.py | 132 +++++++++ scripts/gsm8k_problem_frame_adequacy.py | 79 ++++++ scripts/gsm8k_substrate_morphology.py | 32 ++- ..._gsm8k_morphology_missing_kernel_labels.py | 10 +- tests/test_gsm8k_problem_frame_adequacy.py | 14 + tests/test_problem_frame_builder.py | 31 +- tests/test_problem_frame_contracts.py | 20 ++ 12 files changed, 863 insertions(+), 34 deletions(-) create mode 100644 HANDOFF-gpt55-2026-06-18.md create mode 100644 docs/analysis/problemframe-bindings-contract-readiness-2026-06-18.md create mode 100644 generate/problem_frame_contracts.py create mode 100644 scripts/gsm8k_problem_frame_adequacy.py create mode 100644 tests/test_gsm8k_problem_frame_adequacy.py create mode 100644 tests/test_problem_frame_contracts.py diff --git a/HANDOFF-gpt55-2026-06-18.md b/HANDOFF-gpt55-2026-06-18.md new file mode 100644 index 00000000..e9d0640a --- /dev/null +++ b/HANDOFF-gpt55-2026-06-18.md @@ -0,0 +1,110 @@ +# HANDOFF — gpt55 — 2026-06-18 + +## Agent and Session + +- **Agent:** gpt55 +- **Date:** 2026-06-18 +- **Reasoning effort used:** high +- **Grok Build mode used:** not applicable; Codex default mode +- **Session entry point:** Implement PR #831 brief: bind ProblemFrame relations and measure organ-contract readiness; write results to the requested handoff path. + +## Smoke Suite + Bootstrap Status + +```text +uv run python -m core.cli test --suite smoke -q +108 passed in 123.64s +``` + +The repository has no callable `core-bootstrap` skill in this session. The GPT55 checklist was completed manually: instructions, runtime contracts, recent handoff, dirty state, baseline smoke, scope, pre-edit sweep, and invariant statement. + +## Modules Touched + +| File | Change type | Summary | +|---|---|---| +| `generate/kernel_facts.py` | modified | Added frozen mention, binding, bound-role, and bound-relation primitives. | +| `generate/problem_frame.py` | modified | Added bound question target and canonical frame collections. | +| `generate/problem_frame_builder.py` | modified | Added conservative span-grounded extraction, binding, relation construction, target grounding, and centralized punctuation-safe surface matching. | +| `generate/problem_frame_contracts.py` | created | Added pure diagnostic contract assessments and gap-based migration recommendation. | +| `scripts/gsm8k_problem_frame_adequacy.py` | created | Added gold-independent adequacy reporting. | +| `scripts/gsm8k_substrate_morphology.py` | modified | Replaced trigger-priority recommendations with contract-gap evidence and centralized boundary recognition. | +| `tests/test_problem_frame_builder.py` | modified | Added mention/span, quantity/entity/unit binding, transfer relation, target, and determinism coverage. | +| `tests/test_problem_frame_contracts.py` | created | Added missing/runnable percent contract tests. | +| `tests/test_gsm8k_problem_frame_adequacy.py` | created | Added adequacy report smoke coverage. | +| `tests/test_gsm8k_morphology_missing_kernel_labels.py` | modified | Added punctuation boundary and contract-gap planner expectations. | +| `docs/analysis/problemframe-bindings-contract-readiness-2026-06-18.md` | created | Recorded model, metrics, deferral rationale, and migration conditions. | +| `HANDOFF-gpt55-2026-06-18.md` | created | Session continuity and verification record. | + +## Invariants Verified + +| Invariant | Check performed | Result | Notes | +|---|---|---|---| +| `versor_condition(F) < 1e-6` | Scope trace + smoke suite | preserved | No algebra, field state, operator, or normalization path changed. | +| versor_apply / cga_inner exactness | Import/call-site sweep | untouched | No algebra or recall imports added. | +| Normalization boundaries respected | Diff review | pass | No normalization/unitization/repair added. | +| No approximate recall | Diff review | pass | No vault or ranking changes. | +| No answer admission | Train/holdout serving probes | pass | Train 30 correct / 20 refused / 0 wrong; holdout 5 / 495 / 0. | +| No unreviewed mutation | Diff/status review | pass | Diagnostic-only; no teaching, pack, report, or sealed artifact mutation. | +| Exact source evidence | Focused tests | pass | Mention and contract evidence uses original source spans. | + +## Subagent / Arena Reconciliation + +- Number of subagents spawned: 0 +- Each subagent independently verified versor closure: not applicable +- Reconciliation: not applicable + +## Tests Run + +```text +tests/test_problem_frame_builder.py: 12 passed +tests/test_gsm8k_morphology_missing_kernel_labels.py: 8 passed +tests/test_kernel_no_new_legacy_derivation_surfaces.py: 2 passed +tests/test_problem_frame_contracts.py: 2 passed +tests/test_gsm8k_problem_frame_adequacy.py: 1 passed +core test --suite smoke -q: 108 passed +git diff --check: pass +``` + +Safety probes: + +```text +train_sample: correct=30 refused=20 wrong=0; wrong_ids=[] +holdout_dev: correct=5 refused=495 wrong=0; wrong_ids=[] +``` + +Adequacy after implementation: + +```text +train 50: entity=50 quantity_binding=46 relation=16 bound_target=42 candidates=42 runnable=1 +holdout 500: entity=494 quantity_binding=452 relation=124 bound_target=402 candidates=423 runnable=1 +``` + +## Open Tasks / Next Session Entry Point + +1. Review the conservative local grammar against adequacy gap distributions; widen only with confuser tests and exact spans. +2. Bind container and labor-rate roles before allowing their contracts to become runnable. +3. Do not migrate `percent_partition` until runnable coverage and no-fallback organ consumption are proven beyond case 0046. + +## Known Hazards / Do Not Touch + +- `contract_runnable_count=1` in each corpus is intentionally conservative; do not replace it with trigger presence. +- Case `gsm8k-train-sample-v1-0046` is diagnostic evidence only, not serving authorization. +- Do not add raw-text fallback inside derivation organs or mutate `report.json`/sealed artifacts. + +## Architectural Decisions Made This Session + +- The canonical `ProblemFrame` owns mentions, bindings, relations, and target state; no parallel diagnostic graph. +- `subgroup_partition` and `percent_of` are distinct, and readiness requires matching subgroup referents. +- Contract assessment is a pure projection over typed frame evidence. +- Punctuation-safe token boundary matching is centralized in the ProblemFrame builder and reused by morphology. + +## What Must Not Be Forgotten + +`contract_runnable_count`, not process-trigger presence, is the honest migration-readiness metric. Serving remains unchanged. + +## Skills Used This Session + +- core-bootstrap: unavailable; checklist performed manually +- versor-coherence-guardian: unavailable; scope + smoke verification performed manually +- pre-edit-sweep: unavailable; import/caller/eval/test sweep performed manually +- claim-proposal-guardian: not applicable; no claim or proposal mutation +- Other: none diff --git a/docs/analysis/problemframe-bindings-contract-readiness-2026-06-18.md b/docs/analysis/problemframe-bindings-contract-readiness-2026-06-18.md new file mode 100644 index 00000000..fd4128a2 --- /dev/null +++ b/docs/analysis/problemframe-bindings-contract-readiness-2026-06-18.md @@ -0,0 +1,107 @@ +# ProblemFrame Bindings and Contract Readiness — 2026-06-18 + +## Limitation + +PR #830 recognized scalar, unit, hazard, and process-frame presence but left +semantic roles declarative. On train 50 / holdout 500, actor/object bindings, +bound question targets, and runnable organ contracts were all zero. A process +trigger therefore could not prove that an organ had the facts it required. + +## Map + +Three directions were considered: + +1. Extend the canonical `ProblemFrame` with span-grounded mentions, bindings, + bound relations, and a bound question target. +2. Build a parallel diagnostic graph beside `ProblemFrame`. +3. Infer readiness in reporting scripts from raw text. + +Direction 1 was selected. It preserves one source of structural truth and lets +contract assessment operate only on typed frame evidence. Directions 2 and 3 +would permit disagreement or raw-text fallback at the readiness boundary. + +## Binding Model + +`GroundedMention` retains kind, exact `SourceSpan`, original source text, and an +optional substrate fact ID. `MentionBinding` currently represents only the two +load-bearing edges: quantity-to-entity and quantity-to-unit. IDs and collection +order derive from source position and stable tie-breaks. + +The builder recognizes a deliberately narrow set of local forms. It does not +derive answers, inspect case IDs, or introduce organ-local parsers. + +## Bound Relation Model + +`BoundRole` points a declared semantic role at a grounded mention or fact. +`BoundRelation` groups those role edges with their evidence spans. The first +diagnostic relations are: + +- `transfer`: agent, patient, quantity, object; +- `subgroup_partition`: whole, part, fractional scale; +- `percent_of`: whole, part, percent scale. + +Subgroup and percent relations are distinct so readiness cannot be fabricated +by pooling role names from unrelated candidates. + +## Question Target Model + +`BoundQuestionTarget` records the requested surface, target kind, evidence, and +the grounded target mention. An interrogative that cannot be grounded remains +an explicit target with `target_mention_id=None`; absence and unresolved state +are not conflated. + +## Contract Assessment Model + +`ContractAssessment` is a diagnostic projection with candidate organ, missing +bindings, unresolved hazards, evidence spans, explanation, and `runnable`. +`percent_partition` is runnable only when: + +- a whole and subgroup are grounded by a subgroup relation; +- a percent relation refers to that same subgroup surface; +- the question target is grounded; +- no relevant blocking hazard remains. + +Container and temporal contracts are emitted as explicit future-facing gaps. +Assessment does not admit an organ to serving. + +## Adequacy Metrics + +The new report reads committed case text and optional existing verdict metadata. +Gold answers are not used to synthesize bindings or contracts. + +| Metric | Before train 50 | After train 50 | Before holdout 500 | After holdout 500 | +|---|---:|---:|---:|---:| +| frame built | 50 | 50 | 500 | 500 | +| scalar present | 47 | 47 | 470 | 470 | +| unit present | 21 | 21 | 202 | 202 | +| entity mention present | 0 | 50 | 0 | 494 | +| quantity binding present | 0 | 46 | 0 | 452 | +| bound process relation present | 0 | 16 | 0 | 124 | +| bound question target present | 0 | 42 | 0 | 402 | +| contract candidates | 0 | 42 | 0 | 423 | +| `contract_runnable_count` | 0 | 1 | 0 | 1 | + +`contract_runnable_count` is the honest readiness metric. Trigger presence is +not readiness. + +## Why `percent_partition` Remains Deferred + +Case `gsm8k-train-sample-v1-0046` now has five quantity bindings, four bound +relations, a grounded question target, and a runnable diagnostic contract. That +is progress, not a serving migration. Only 1/42 train candidates and 1/423 +holdout candidates meet the conservative contract, so broad admission would +still require fallback or overclaim coverage. + +Migration becomes safe when the target corpus has adequate runnable coverage, +the remaining gap taxonomy is explicitly addressed, an organ consumes only +`ProblemFrame` evidence, and the serving safety lanes remain wrong-free. This +change intentionally leaves serving, answer admission, `report.json`, and +sealed artifacts unchanged. + +## Masterstroke + +Readiness is represented as a conjugate of construction: the builder propagates +source evidence into typed relations, while contract assessment exposes exactly +where that propagation fails to close. The design makes a nominal frame call +insufficient; an organ is runnable only when its intrinsic role geometry is +actually bound. diff --git a/generate/kernel_facts.py b/generate/kernel_facts.py index f9d95006..cf3de4b4 100644 --- a/generate/kernel_facts.py +++ b/generate/kernel_facts.py @@ -19,7 +19,7 @@ from __future__ import annotations from dataclasses import dataclass from fractions import Fraction -from typing import Union +from typing import Literal, Union # --------------------------------------------------------------------------- # Provenance kinds — closed set per Tranche 1 brief. @@ -207,6 +207,55 @@ class CandidateRelation: hazards: tuple[KernelHazard, ...] = () +# --------------------------------------------------------------------------- +# Span-grounded mention and binding primitives. +# --------------------------------------------------------------------------- +MentionKind = Literal["entity", "actor", "object", "quantity", "unit"] +BindingKind = Literal["quantity_entity", "quantity_unit"] + + +@dataclass(frozen=True, slots=True) +class GroundedMention: + """A deterministic, source-grounded mention; never a derived answer.""" + + mention_id: str + kind: MentionKind + surface: str + span: SourceSpan + fact_id: str | None = None + + +@dataclass(frozen=True, slots=True) +class MentionBinding: + """A typed edge between two grounded mentions.""" + + binding_id: str + binding_type: BindingKind + source_mention_id: str + target_mention_id: str + evidence_spans: tuple[SourceSpan, ...] + + +@dataclass(frozen=True, slots=True) +class BoundRole: + """A declared relation role bound to a mention or substrate fact.""" + + role: str + target_id: str + target_kind: str + evidence_spans: tuple[SourceSpan, ...] + + +@dataclass(frozen=True, slots=True) +class BoundRelation: + """A candidate relation whose roles have explicit grounded referents.""" + + relation_id: str + relation_type: str + roles: tuple[BoundRole, ...] + evidence_spans: tuple[SourceSpan, ...] + + # --------------------------------------------------------------------------- # Substrate fact — the canonical union wrapper. # --------------------------------------------------------------------------- diff --git a/generate/problem_frame.py b/generate/problem_frame.py index a6a55767..1d650001 100644 --- a/generate/problem_frame.py +++ b/generate/problem_frame.py @@ -13,11 +13,15 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: from generate.kernel_facts import ( + BoundRelation, GroundedScalar, + GroundedMention, GroundedUnit, + MentionBinding, CandidateRelation, KernelHazard, KernelProvenance, + SourceSpan, ) from language_packs.scalar_equivalence import ScalarCandidate from generate.process_frames import ProcessFrame @@ -39,6 +43,21 @@ class QuestionTarget: ) +@dataclass(frozen=True, slots=True) +class BoundQuestionTarget: + """Question target grounded to a mention, or explicitly unresolved.""" + + target_type: str + requested_surface: str + target_mention_id: str | None + unknown_slot: str + evidence_spans: tuple[SourceSpan, ...] + + @property + def grounded(self) -> bool: + return self.target_mention_id is not None + + @dataclass(frozen=True, slots=True) class ProblemFrame: """Immutable target representation of a mathematical word problem. @@ -56,6 +75,10 @@ class ProblemFrame: question_target: QuestionTarget | None hazards: tuple[KernelHazard, ...] provenance: tuple[KernelProvenance, ...] + mentions: tuple[GroundedMention, ...] = () + bindings: tuple[MentionBinding, ...] = () + bound_relations: tuple[BoundRelation, ...] = () + bound_question_target: BoundQuestionTarget | None = None class ProblemFrameBuilder: @@ -72,6 +95,10 @@ class ProblemFrameBuilder: self._question_target: QuestionTarget | None = None self._hazards: list[KernelHazard] = [] self._provenance: list[KernelProvenance] = [] + self._mentions: list[GroundedMention] = [] + self._bindings: list[MentionBinding] = [] + self._bound_relations: list[BoundRelation] = [] + self._bound_question_target: BoundQuestionTarget | None = None def add_quantity(self, scalar: GroundedScalar) -> None: """Add a GroundedScalar to the frame, collecting hazards and provenance.""" @@ -123,6 +150,18 @@ class ProblemFrameBuilder: """Add a provenance record directly to the frame.""" self._provenance.append(provenance) + def add_mention(self, mention: GroundedMention) -> None: + self._mentions.append(mention) + + def add_binding(self, binding: MentionBinding) -> None: + self._bindings.append(binding) + + def add_bound_relation(self, relation: BoundRelation) -> None: + self._bound_relations.append(relation) + + def set_bound_question_target(self, target: BoundQuestionTarget) -> None: + self._bound_question_target = target + def build(self) -> ProblemFrame: """Produce the immutable ProblemFrame.""" return ProblemFrame( @@ -136,4 +175,8 @@ class ProblemFrameBuilder: question_target=self._question_target, hazards=tuple(self._hazards), provenance=tuple(self._provenance), + mentions=tuple(self._mentions), + bindings=tuple(self._bindings), + bound_relations=tuple(self._bound_relations), + bound_question_target=self._bound_question_target, ) diff --git a/generate/problem_frame_builder.py b/generate/problem_frame_builder.py index 2dd35b99..5aa60f1f 100644 --- a/generate/problem_frame_builder.py +++ b/generate/problem_frame_builder.py @@ -16,15 +16,24 @@ import re from fractions import Fraction from generate.kernel_facts import ( + BoundRelation, + BoundRole, CandidateRelation, + GroundedMention, GroundedScalar, GroundedUnit, KernelHazard, KernelProvenance, + MentionBinding, RelationRole, SourceSpan, ) -from generate.problem_frame import ProblemFrame, ProblemFrameBuilder, QuestionTarget +from generate.problem_frame import ( + BoundQuestionTarget, + ProblemFrame, + ProblemFrameBuilder, + QuestionTarget, +) from generate.process_frames import ProcessFrame, all_frames from language_packs.ambiguity_hazards import ( AmbiguityHazard, @@ -51,15 +60,13 @@ _ORDINAL_SUFFIX_RE: re.Pattern[str] = re.compile( ) -def _surface_in_text(surface: str, text_lower: str) -> bool: - token = surface.lower() - padded = f" {text_lower} " - return ( - f" {token} " in padded - or text_lower.startswith(f"{token} ") - or text_lower.endswith(f" {token}") - or text_lower == token - ) +def surface_in_text(surface: str, text: str) -> bool: + """Match a registered surface at lexical, including punctuation, boundaries.""" + return re.search( + rf"(? KernelHazard: @@ -111,7 +118,7 @@ def _extract_hazards(text: str) -> tuple[KernelHazard, ...]: seen: set[str] = set() for surface in all_registered_surfaces(): - if not _surface_in_text(surface, text_lower): + if not surface_in_text(surface, text_lower): continue for hazard in lookup_hazards(surface): if hazard.hazard_id in seen: @@ -159,12 +166,14 @@ def _filter_scalar_candidates( def _trigger_span(text: str, trigger: str) -> SourceSpan | None: - text_lower = text.lower() - trigger_lower = trigger.lower() - idx = text_lower.find(trigger_lower) - if idx < 0: + match = re.search( + rf"(? tuple[ProcessFrame, ...]: @@ -173,7 +182,7 @@ def _extract_process_frame_candidates(text: str) -> tuple[ProcessFrame, ...]: for frame in all_frames(): for trigger in frame.trigger_surfaces: - if _surface_in_text(trigger, text_lower): + if surface_in_text(trigger, text_lower): matched[frame.name] = frame break @@ -268,6 +277,210 @@ def _detect_question_target(text: str) -> QuestionTarget | None: return None +_ENTITY_AFTER_QUANTITY_RE = re.compile( + r"(?P\d+(?:\.\d+)?\s*%?)\s+(?:of\s+(?:the\s+)?)?" + r"(?P[A-Za-z][A-Za-z'-]*)", + re.IGNORECASE, +) +_FRACTION_ENTITY_RE = re.compile( + r"\b(?Phalf|third|quarter)\b\s+(?:of\s+(?:the\s+)?|are\s+|the\s+)?" + r"(?P[A-Za-z][A-Za-z'-]*)", + re.IGNORECASE, +) +_QUESTION_ENTITY_RE = re.compile( + r"\bhow\s+(?:many|much)\s+(?:more\s+)?(?P[A-Za-z][A-Za-z'-]*)", + re.IGNORECASE, +) +_ACTOR_VERB_RE = re.compile( + r"\b(?P[A-Z][A-Za-z'-]*)\s+" + r"(?:gave|gives|give|received|receives|spent|spends|ate|eats|bought|buys|sold|sells)\b" +) +_TRANSFER_RE = re.compile( + r"\b(?P[A-Z][A-Za-z'-]*)\s+(?:gave|gives|give|handed|passed)\s+" + r"(?P[A-Z][A-Za-z'-]*)\s+" + r"(?P\d+(?:\.\d+)?)\s+(?P[A-Za-z][A-Za-z'-]*)", +) + + +def _extract_mentions( + text: str, + quantities: tuple[GroundedScalar, ...], + units: tuple[GroundedUnit, ...], +) -> tuple[GroundedMention, ...]: + records: dict[tuple[str, int, int], GroundedMention] = {} + + def add(kind: str, start: int, end: int, *, fact_id: str | None = None) -> None: + key = (kind, start, end) + if key in records: + return + records[key] = GroundedMention( + mention_id="", kind=kind, surface=text[start:end], + span=SourceSpan(text[start:end], start, end), fact_id=fact_id, + ) + + for quantity in quantities: + span = quantity.provenance.source_spans[0] + add("quantity", span.start, span.end, fact_id=quantity.fact_id) + for unit in units: + span = unit.provenance.source_spans[0] + add("unit", span.start, span.end, fact_id=unit.fact_id) + for pattern in (_ENTITY_AFTER_QUANTITY_RE, _FRACTION_ENTITY_RE, _QUESTION_ENTITY_RE): + for match in pattern.finditer(text): + add("object", match.start("entity"), match.end("entity")) + for match in _ACTOR_VERB_RE.finditer(text): + add("actor", match.start("actor"), match.end("actor")) + for match in _TRANSFER_RE.finditer(text): + add("actor", match.start("agent"), match.end("agent")) + add("actor", match.start("patient"), match.end("patient")) + add("object", match.start("object"), match.end("object")) + + ordered = sorted( + records.values(), + key=lambda m: (m.span.start, m.span.end, m.kind, m.surface.lower()), + ) + return tuple( + GroundedMention( + mention_id=f"mention-{index:04d}", kind=m.kind, surface=m.surface, + span=m.span, fact_id=m.fact_id, + ) + for index, m in enumerate(ordered) + ) + + +def _extract_bindings( + text: str, + mentions: tuple[GroundedMention, ...], +) -> tuple[MentionBinding, ...]: + by_span_kind = {(m.span.start, m.span.end, m.kind): m for m in mentions} + quantities = [m for m in mentions if m.kind == "quantity"] + bindings: list[MentionBinding] = [] + seen: set[tuple[str, str, str]] = set() + + def bind(binding_type: str, source: GroundedMention, target: GroundedMention) -> None: + key = (binding_type, source.mention_id, target.mention_id) + if key in seen: + return + seen.add(key) + bindings.append(MentionBinding( + binding_id="", binding_type=binding_type, + source_mention_id=source.mention_id, target_mention_id=target.mention_id, + evidence_spans=(source.span, target.span), + )) + + for pattern in (_ENTITY_AFTER_QUANTITY_RE, _FRACTION_ENTITY_RE): + for match in pattern.finditer(text): + entity = by_span_kind.get((match.start("entity"), match.end("entity"), "object")) + if entity is None: + continue + candidates = [q for q in quantities if q.span.start == match.start("quantity")] + if candidates: + bind("quantity_entity", candidates[0], entity) + units = [m for m in mentions if m.kind == "unit"] + for quantity in quantities: + following = [ + unit + for unit in units + if unit.span.start >= quantity.span.end + and not text[quantity.span.end:unit.span.start].strip() + ] + if following: + bind("quantity_unit", quantity, min(following, key=lambda u: u.span.start)) + + ordered = sorted(bindings, key=lambda b: (b.evidence_spans[0].start, b.binding_type, b.target_mention_id)) + return tuple(MentionBinding( + binding_id=f"binding-{index:04d}", binding_type=b.binding_type, + source_mention_id=b.source_mention_id, target_mention_id=b.target_mention_id, + evidence_spans=b.evidence_spans, + ) for index, b in enumerate(ordered)) + + +def _bound_relations( + text: str, + mentions: tuple[GroundedMention, ...], + bindings: tuple[MentionBinding, ...], +) -> tuple[BoundRelation, ...]: + by_id = {m.mention_id: m for m in mentions} + relations: list[BoundRelation] = [] + quantity_entity = [b for b in bindings if b.binding_type == "quantity_entity"] + whole = next( + ( + binding + for binding in quantity_entity + if "%" not in by_id[binding.source_mention_id].surface + and by_id[binding.source_mention_id].surface.lower() + not in {"half", "third", "quarter"} + ), + None, + ) + for binding in quantity_entity: + quantity = by_id[binding.source_mention_id] + part = by_id[binding.target_mention_id] + if "%" not in quantity.surface and quantity.surface.lower() not in {"half", "third", "quarter"}: + continue + roles = [ + BoundRole("part", part.mention_id, part.kind, (part.span,)), + BoundRole("scale", quantity.mention_id, quantity.kind, (quantity.span,)), + ] + if whole is not None: + whole_entity = by_id[whole.target_mention_id] + roles.insert(0, BoundRole("whole", whole_entity.mention_id, whole_entity.kind, (whole_entity.span,))) + relation_type = "percent_of" if "%" in quantity.surface else "subgroup_partition" + relations.append(BoundRelation( + relation_id="", relation_type=relation_type, roles=tuple(roles), + evidence_spans=tuple(span for role in roles for span in role.evidence_spans), + )) + + for match in _TRANSFER_RE.finditer(text): + def at(group: str, kind: str) -> GroundedMention | None: + return next((m for m in mentions if m.kind == kind and m.span.start == match.start(group)), None) + agent = at("agent", "actor") + patient = at("patient", "actor") + quantity = at("quantity", "quantity") + obj = at("object", "object") + if all((agent, patient, quantity, obj)): + assert agent and patient and quantity and obj + roles = tuple( + BoundRole(name, mention.mention_id, mention.kind, (mention.span,)) + for name, mention in ( + ("agent", agent), ("patient", patient), + ("quantity", quantity), ("object", obj), + ) + ) + relations.append(BoundRelation( + "", "transfer", roles, + tuple(m.span for m in (agent, patient, quantity, obj)), + )) + + relations.sort(key=lambda r: (r.evidence_spans[0].start, r.relation_type)) + return tuple( + BoundRelation( + f"bound-rel-{index:04d}", relation.relation_type, + relation.roles, relation.evidence_spans, + ) + for index, relation in enumerate(relations) + ) + + +def _bound_question_target(text: str, mentions: tuple[GroundedMention, ...]) -> BoundQuestionTarget | None: + question = _QUESTION_ENTITY_RE.search(text) + if question is None: + if "?" not in text: + return None + qmark = text.index("?") + return BoundQuestionTarget( + "unknown", "?", None, "unresolved", + (SourceSpan("?", qmark, qmark + 1),), + ) + entity = next((m for m in mentions if m.kind == "object" and m.span.start == question.start("entity")), None) + prefix = text[max(0, question.start() - 24):question.end()].lower() + target_type = "difference" if "more" in question.group(0).lower() else "remaining" if any(x in prefix for x in ("remaining", "left")) else "total" if any(x in prefix for x in ("total", "altogether")) else "count" + span = SourceSpan(text[question.start():question.end()], question.start(), question.end()) + return BoundQuestionTarget( + target_type, question.group("entity"), + entity.mention_id if entity else None, target_type, (span,), + ) + + def build_problem_frame(problem_text: str) -> ProblemFrame: """Build a substrate-backed ProblemFrame from raw problem text. @@ -280,12 +493,15 @@ def build_problem_frame(problem_text: str) -> ProblemFrame: for scalar in scalars: builder.add_scalar(scalar) + grounded_quantities: list[GroundedScalar] = [] for index, scalar in enumerate(scalars): grounded = _scalar_to_grounded(scalar, problem_text, index) if grounded is not None: builder.add_quantity(grounded) + grounded_quantities.append(grounded) - for unit in _extract_unit_candidates(problem_text): + units = _extract_unit_candidates(problem_text) + for unit in units: builder.add_unit(unit) for hazard in _extract_hazards(problem_text): @@ -302,6 +518,22 @@ def build_problem_frame(problem_text: str) -> ProblemFrame: if question_target is not None: builder.set_question_target(question_target) + mentions = _extract_mentions(problem_text, tuple(grounded_quantities), units) + bindings = _extract_bindings(problem_text, mentions) + for mention in mentions: + builder.add_mention(mention) + if mention.kind == "actor": + builder.add_actor(mention.surface) + elif mention.kind == "object": + builder.add_object(mention.surface) + for binding in bindings: + builder.add_binding(binding) + for relation in _bound_relations(problem_text, mentions, bindings): + builder.add_bound_relation(relation) + bound_target = _bound_question_target(problem_text, mentions) + if bound_target is not None: + builder.set_bound_question_target(bound_target) + return builder.build() diff --git a/generate/problem_frame_contracts.py b/generate/problem_frame_contracts.py new file mode 100644 index 00000000..2b29dcc1 --- /dev/null +++ b/generate/problem_frame_contracts.py @@ -0,0 +1,132 @@ +"""Diagnostic organ-contract readiness derived only from ProblemFrame evidence.""" +from __future__ import annotations + +from dataclasses import dataclass + +from generate.kernel_facts import BoundRelation, SourceSpan +from generate.problem_frame import ProblemFrame + + +@dataclass(frozen=True, slots=True) +class ContractAssessment: + candidate_organ: str + missing_bindings: tuple[str, ...] + unresolved_hazards: tuple[str, ...] + runnable: bool + explanation: str + evidence_spans: tuple[SourceSpan, ...] + + +def _roles(frame: ProblemFrame, relation_type: str) -> set[str]: + return { + role.role + for relation in frame.bound_relations + if relation.relation_type == relation_type + for role in relation.roles + } + + +def _evidence(frame: ProblemFrame, relation_type: str) -> tuple[SourceSpan, ...]: + spans = { + (span.start, span.end, span.text): span + for relation in frame.bound_relations + if relation.relation_type == relation_type + for span in relation.evidence_spans + } + if frame.bound_question_target is not None: + for span in frame.bound_question_target.evidence_spans: + spans[(span.start, span.end, span.text)] = span + return tuple(spans[key] for key in sorted(spans)) + + +def assess_percent_partition(frame: ProblemFrame) -> ContractAssessment: + mentions = {mention.mention_id: mention for mention in frame.mentions} + subgroups = [relation for relation in frame.bound_relations if relation.relation_type == "subgroup_partition"] + percentages = [relation for relation in frame.bound_relations if relation.relation_type == "percent_of"] + + def role_target(relation: BoundRelation, role_name: str) -> str | None: + return next((role.target_id for role in relation.roles if role.role == role_name), None) + + linked_pairs = [] + for subgroup in subgroups: + subgroup_part = role_target(subgroup, "part") + if subgroup_part is None or subgroup_part not in mentions: + continue + subgroup_surface = mentions[subgroup_part].surface.lower() + for percent in percentages: + percent_part = role_target(percent, "part") + if percent_part is not None and percent_part in mentions and mentions[percent_part].surface.lower() == subgroup_surface: + linked_pairs.append((subgroup, percent)) + + missing: list[str] = [] + if not any(role_target(relation, "whole") for relation in subgroups): + missing.append("grounded_whole_entity") + if not subgroups: + missing.append("grounded_partition_subgroup") + if not linked_pairs: + missing.append("percent_or_fraction_linked_to_subgroup") + question_target = frame.bound_question_target + if question_target is None or not question_target.grounded: + missing.append("grounded_question_target") + + unresolved: set[str] = set() + categories = {hazard.category for hazard in frame.hazards} + if "grounded_whole_entity" in missing and "unbound_base_quantity" in categories: + unresolved.add("unbound_base_quantity") + if "grounded_partition_subgroup" in missing and "percent_change_vs_percent_of" in categories: + unresolved.add("percent_change_vs_percent_of") + runnable = not missing and not unresolved + return ContractAssessment( + candidate_organ="percent_partition", + missing_bindings=tuple(missing), + unresolved_hazards=tuple(sorted(unresolved)), + runnable=runnable, + explanation=( + "all percent-partition roles and the question target are grounded" + if runnable else "diagnostic candidate is not runnable: " + ", ".join((*missing, *sorted(unresolved))) + ), + evidence_spans=tuple(sorted( + { + (span.start, span.end, span.text): span + for pair in linked_pairs + for relation in pair + for span in relation.evidence_spans + }.values(), + key=lambda span: (span.start, span.end, span.text), + )) + (() if question_target is None else question_target.evidence_spans), + ) + + +def assess_contracts(frame: ProblemFrame) -> tuple[ContractAssessment, ...]: + """Return deterministic diagnostic assessments; never admits serving.""" + frame_names = {candidate.name for candidate in frame.process_frames} + results: list[ContractAssessment] = [] + if frame_names & {"partition", "consumption"}: + results.append(assess_percent_partition(frame)) + if "container_packing" in frame_names and frame.bound_question_target is not None: + roles = _roles(frame, "container_packing") + missing = tuple(name for name in ("container", "content", "count_per") if name not in roles) + results.append(ContractAssessment( + "nested_fraction_remainder_total", missing, (), not missing, + "container contract grounded" if not missing else "missing container bindings: " + ", ".join(missing), + _evidence(frame, "container_packing"), + )) + if "labor_rate" in frame_names: + roles = _roles(frame, "labor_rate") + missing = tuple(name for name in ("worker", "rate", "duration") if name not in roles) + results.append(ContractAssessment( + "temporal_tariff", missing, (), not missing, + "temporal tariff contract grounded" if not missing else "missing tariff bindings: " + ", ".join(missing), + _evidence(frame, "labor_rate"), + )) + return tuple(sorted(results, key=lambda item: item.candidate_organ)) + + +def recommended_migration_target(assessments: tuple[ContractAssessment, ...]) -> str: + runnable = [item.candidate_organ for item in assessments if item.runnable] + if runnable: + return sorted(runnable)[0] + if assessments: + best = min(assessments, key=lambda item: (len(item.missing_bindings) + len(item.unresolved_hazards), item.candidate_organ)) + return f"substrate:contract_gap:{best.candidate_organ}" + return "substrate:problem_frame_builder" diff --git a/scripts/gsm8k_problem_frame_adequacy.py b/scripts/gsm8k_problem_frame_adequacy.py new file mode 100644 index 00000000..4ad458c7 --- /dev/null +++ b/scripts/gsm8k_problem_frame_adequacy.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Report ProblemFrame binding and organ-contract adequacy without solving.""" +from __future__ import annotations + +import argparse +import json +from collections.abc import Iterable, Mapping +from pathlib import Path +from typing import Any + +from generate.problem_frame_builder import build_problem_frame +from generate.problem_frame_contracts import assess_contracts, recommended_migration_target + + +def assess_case(case: Mapping[str, Any], *, current_verdict: str | None = None) -> dict[str, Any]: + text = str(case.get("question") or case.get("problem") or case.get("problem_text") or "") + case_id = str(case.get("case_id") or case.get("id") or "unknown") + frame = build_problem_frame(text) + contracts = assess_contracts(frame) + return { + "case_id": case_id, + "current_verdict": current_verdict, + "frame_built": True, + "scalar_count": len(frame.quantities), + "unit_count": len(frame.units), + "entity_mention_count": sum(m.kind in {"entity", "actor", "object"} for m in frame.mentions), + "quantity_binding_count": sum(b.binding_type == "quantity_entity" for b in frame.bindings), + "process_relation_count": len(frame.bound_relations), + "bound_question_target_present": bool(frame.bound_question_target and frame.bound_question_target.grounded), + "candidate_organ_contracts": [item.candidate_organ for item in contracts], + "runnable_contracts": [item.candidate_organ for item in contracts if item.runnable], + "missing_binding_taxonomy": sorted({gap for item in contracts for gap in item.missing_bindings}), + "unresolved_hazards": sorted({gap for item in contracts for gap in item.unresolved_hazards}), + "recommended_next_migration_target": recommended_migration_target(contracts), + } + + +def build_report(cases: Iterable[Mapping[str, Any]], *, verdicts: Mapping[str, str] | None = None) -> dict[str, Any]: + verdicts = verdicts or {} + per_case = [ + assess_case(case, current_verdict=verdicts.get(str(case.get("case_id") or case.get("id") or "unknown"))) + for case in cases + ] + return { + "schema_version": 1, + "case_count": len(per_case), + "counts": { + "frame_built": sum(row["frame_built"] for row in per_case), + "scalar_present": sum(row["scalar_count"] > 0 for row in per_case), + "unit_present": sum(row["unit_count"] > 0 for row in per_case), + "entity_mention_present": sum(row["entity_mention_count"] > 0 for row in per_case), + "quantity_binding_present": sum(row["quantity_binding_count"] > 0 for row in per_case), + "process_relation_present": sum(row["process_relation_count"] > 0 for row in per_case), + "bound_question_target_present": sum(row["bound_question_target_present"] for row in per_case), + "contract_candidate_count": sum(len(row["candidate_organ_contracts"]) for row in per_case), + "contract_runnable_count": sum(len(row["runnable_contracts"]) for row in per_case), + }, + "per_case": per_case, + } + + +def _load_jsonl(path: Path) -> list[dict[str, Any]]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cases", type=Path, required=True) + parser.add_argument("--limit", type=int) + args = parser.parse_args(argv) + cases = _load_jsonl(args.cases) + if args.limit is not None: + cases = cases[:args.limit] + print(json.dumps(build_report(cases), indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gsm8k_substrate_morphology.py b/scripts/gsm8k_substrate_morphology.py index fa68cdbe..0419a729 100644 --- a/scripts/gsm8k_substrate_morphology.py +++ b/scripts/gsm8k_substrate_morphology.py @@ -14,8 +14,10 @@ from generate.problem_frame_builder import ( recognized_process_frame_names, recognized_scalar_surfaces, recognized_unit_surfaces, + surface_in_text, ) -from generate.process_frames import lookup_frame +from generate.problem_frame_contracts import assess_contracts, recommended_migration_target as contract_target +from generate.process_frames import frame_by_name, lookup_frame from language_packs.loader import lookup_container from language_packs.scalar_equivalence import list_unsupported_surfaces from language_packs.unit_dimensions import classify_dimension @@ -50,16 +52,14 @@ _STOPWORDS = { def _surface_in_text(surface: str, text_lower: str) -> bool: - padded = f" {text_lower} " - token = surface.lower() - return f" {token} " in padded or text_lower.startswith(f"{token} ") or text_lower.endswith(f" {token}") or text_lower == token + return surface_in_text(surface, text_lower) def _registered_frame_present(text_lower: str, expected: set[str]) -> bool: for frame_name in expected: - for frame in lookup_frame(frame_name): - if any(_surface_in_text(trigger, text_lower) for trigger in frame.trigger_surfaces): - return True + frame = frame_by_name(frame_name) + if frame is not None and any(_surface_in_text(trigger, text_lower) for trigger in frame.trigger_surfaces): + return True for trigger in text_lower.split(): if any(frame.name in expected for frame in lookup_frame(trigger)): return True @@ -131,10 +131,9 @@ def _target_for_process_frames(process_frames: tuple[str, ...]) -> str | None: def recommend_migration_target(problem_text: str, process_frames: tuple[str, ...], missing_labels: tuple[str, ...]) -> str: lowered = problem_text.lower() - if "%" in problem_text and ("half" in lowered or "partition" in process_frames or "consumption" in process_frames): - return "percent_partition" - if "other half" in lowered and "%" in problem_text: - return "percent_partition" + assessments = assess_contracts(build_problem_frame(problem_text)) + if assessments: + return contract_target(assessments) if "missing_scalar_equivalence" in missing_labels: return "substrate:scalar_equivalence" if "missing_unit_dimension" in missing_labels: @@ -159,6 +158,7 @@ def plan_substrate_case(*, case_id: str, problem_text: str, current_verdict: str frame = build_problem_frame(problem_text) missing_labels = classify_missing_substrate(problem_text) process_frames = recognized_process_frame_names(frame) + assessments = assess_contracts(frame) return { "case_id": case_id, "current_verdict": current_verdict, @@ -166,9 +166,17 @@ def plan_substrate_case(*, case_id: str, problem_text: str, current_verdict: str "recognized_units": recognized_unit_surfaces(frame), "recognized_process_frames": process_frames, "recognized_hazards": recognized_hazard_ids(frame), + "entity_mention_count": sum(m.kind in {"entity", "actor", "object"} for m in frame.mentions), + "quantity_binding_count": sum(b.binding_type == "quantity_entity" for b in frame.bindings), + "bound_process_relation_count": len(frame.bound_relations), + "bound_question_target_present": bool(frame.bound_question_target and frame.bound_question_target.grounded), + "candidate_organ_contracts": tuple(a.candidate_organ for a in assessments), + "runnable_contracts": tuple(a.candidate_organ for a in assessments if a.runnable), + "missing_bindings": tuple(sorted({gap for a in assessments for gap in a.missing_bindings})), + "unresolved_contract_hazards": tuple(sorted({gap for a in assessments for gap in a.unresolved_hazards})), "missing_substrate_labels": missing_labels, "legacy_parser_dependency": _legacy_parser_dependency(problem_text, process_frames, missing_labels), - "recommended_migration_target": recommend_migration_target(problem_text, process_frames, missing_labels), + "recommended_migration_target": contract_target(assessments) if assessments else recommend_migration_target(problem_text, process_frames, missing_labels), } diff --git a/tests/test_gsm8k_morphology_missing_kernel_labels.py b/tests/test_gsm8k_morphology_missing_kernel_labels.py index 0ea962ef..a67aa64c 100644 --- a/tests/test_gsm8k_morphology_missing_kernel_labels.py +++ b/tests/test_gsm8k_morphology_missing_kernel_labels.py @@ -100,7 +100,7 @@ def test_planner_v2_recognizes_substrate_without_solving() -> None: assert "consumption" in record["recognized_process_frames"] assert record["recognized_hazards"] assert isinstance(record["legacy_parser_dependency"], tuple) - assert record["recommended_migration_target"] == "percent_partition" + assert record["recommended_migration_target"] == "substrate:contract_gap:percent_partition" assert record["recommended_migration_target"] not in _RAW_PROCESS_FRAME_NAMES @@ -115,7 +115,13 @@ def test_planner_v2_recommends_percent_partition_for_half_percent_split() -> Non ("partition", "consumption"), classify_missing_substrate(text), ) - assert target == "percent_partition" + assert target in {"percent_partition", "substrate:contract_gap:percent_partition"} + + +def test_punctuation_boundary_registered_container() -> None: + assert "missing_container_frame" not in classify_missing_substrate( + "There are 10 bloops in the box." + ) def test_planner_fallback_never_returns_raw_process_frame_name() -> None: diff --git a/tests/test_gsm8k_problem_frame_adequacy.py b/tests/test_gsm8k_problem_frame_adequacy.py new file mode 100644 index 00000000..0e1a00c3 --- /dev/null +++ b/tests/test_gsm8k_problem_frame_adequacy.py @@ -0,0 +1,14 @@ +from scripts.gsm8k_problem_frame_adequacy import build_report + + +def test_adequacy_report_smoke_does_not_require_gold_answers() -> None: + report = build_report([ + { + "case_id": "diagnostic-1", + "question": "There are 100 students. Half are girls. How many students are girls?", + } + ]) + assert report["case_count"] == 1 + assert report["counts"]["frame_built"] == 1 + assert "contract_runnable_count" in report["counts"] + assert report["per_case"][0]["current_verdict"] is None diff --git a/tests/test_problem_frame_builder.py b/tests/test_problem_frame_builder.py index a23f29a0..52c0cb0f 100644 --- a/tests/test_problem_frame_builder.py +++ b/tests/test_problem_frame_builder.py @@ -102,4 +102,33 @@ def test_deterministic_ordering_across_repeated_runs() -> None: assert frame_a.units == frame_b.units assert frame_a.process_frames == frame_b.process_frames assert frame_a.hazards == frame_b.hazards - assert frame_a.candidate_relations == frame_b.candidate_relations \ No newline at end of file + assert frame_a.candidate_relations == frame_b.candidate_relations + assert frame_a.mentions == frame_b.mentions + assert frame_a.bindings == frame_b.bindings + assert frame_a.bound_relations == frame_b.bound_relations + + +def test_mentions_bind_quantities_and_units_with_exact_spans() -> None: + text = "A runner traveled 5 miles in 2 hours. How many miles?" + frame = build_problem_frame(text) + + assert any(binding.binding_type == "quantity_entity" for binding in frame.bindings) + assert any(binding.binding_type == "quantity_unit" for binding in frame.bindings) + for mention in frame.mentions: + assert text[mention.span.start:mention.span.end] == mention.span.text + + +def test_transfer_roles_and_question_target_are_bound() -> None: + frame = build_problem_frame( + "Tom gave Ana 3 marbles. How many marbles does Ana have?" + ) + relation = next(item for item in frame.bound_relations if item.relation_type == "transfer") + assert {role.role for role in relation.roles} == {"agent", "patient", "quantity", "object"} + assert frame.bound_question_target is not None + assert frame.bound_question_target.grounded + + +def test_question_target_is_explicitly_unbound_when_not_groundable() -> None: + frame = build_problem_frame("What is the answer?") + assert frame.bound_question_target is not None + assert not frame.bound_question_target.grounded diff --git a/tests/test_problem_frame_contracts.py b/tests/test_problem_frame_contracts.py new file mode 100644 index 00000000..8ab5b882 --- /dev/null +++ b/tests/test_problem_frame_contracts.py @@ -0,0 +1,20 @@ +from generate.problem_frame_builder import build_problem_frame +from generate.problem_frame_contracts import assess_percent_partition + + +def test_percent_partition_missing_bindings_is_not_runnable() -> None: + assessment = assess_percent_partition(build_problem_frame("Mia spent 50% of her money.")) + assert not assessment.runnable + assert "grounded_whole_entity" in assessment.missing_bindings + assert "grounded_question_target" in assessment.missing_bindings + + +def test_tightly_grounded_percent_partition_is_diagnostically_runnable() -> None: + frame = build_problem_frame( + "There are 100 students. Half are girls. 30% of the girls own pets. " + "How many students own pets?" + ) + assessment = assess_percent_partition(frame) + assert assessment.runnable + assert assessment.missing_bindings == () + assert assessment.evidence_spans