feat(field-wedge): geometric field reader — relational-metric lane wrong=0 (Phase W.1)
Measurement #1 of the field-reasoner falsifiable experiment: does the CL(4,1) field, given an honest metric encoding, read forward-substitutable quantitative-relational problems from TEXT with wrong==0? It does — 14/15 correct, 0 wrong, 1 refused (precision ceiling), scored against an independent arithmetic oracle. - generate/relational_field_reader.py: reads problem text into conformal points on the e1 number line; additive/part-whole relations are conformal TRANSLATOR versors (versor_apply(T_delta, embed[x]) == embed[x+delta], exact); the answer reads back by projective dehomogenization. Refusal-first: fences multiplicative/ratio (the sign/orientation-blind cases), the precision ceiling, non-forward-substitutable references, negatives. A per-step exactness self-check turns any f64 translator drift into a refusal (precision_drift) — it NEVER commits a wrong integer. Its parser is an independent reimplementation importing no generate.derivation/math_*. - evals/relational_metric/: independent arithmetic oracle (computes gold from the STRUCTURE, shares no code with the reader), 15-case fixture, and a runner that enforces gold integrity + wrong==0. - INV-25: relational_metric registered in INDEPENDENT_GOLD_LANES (oracle proven code-disjoint from the field reader and the algebra engine). The independently golded panel is now three domains: deductive, dimensional, relational-metric. Green: smoke 87, 53 architectural invariants, 16 new tests; deductive + dimensional lanes unperturbed (wrong=0).
This commit is contained in:
Shay
parent
568face63e
commit
145d797196
8 changed files with 593 additions and 0 deletions
10
evals/relational_metric/__init__.py
Normal file
10
evals/relational_metric/__init__.py
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
"""Relational-metric lane — the field-reasoner wedge's golded micro-domain.
|
||||
|
||||
A panel of forward-substitutable quantitative-relational word problems (additive /
|
||||
part-whole). The system under test is the geometric FIELD reader
|
||||
(``generate.relational_field_reader``), which reads problem TEXT into conformal
|
||||
points on the e1 number line and reads the answer back by projective
|
||||
dehomogenization. The gold is an INDEPENDENT, pure-arithmetic oracle
|
||||
(``evals.relational_metric.oracle``) that computes the answer from the structured
|
||||
relations and shares no code with the reader (INV-25).
|
||||
"""
|
||||
78
evals/relational_metric/oracle.py
Normal file
78
evals/relational_metric/oracle.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
"""Independent arithmetic oracle — the gold for the relational-metric lane.
|
||||
|
||||
This is **deliberately a second, independent decision procedure**: it computes the
|
||||
answer to a forward-substitutable quantitative-relational problem by plain integer
|
||||
arithmetic over the *structured* relations. It never reads problem text, and it
|
||||
shares **no code** with the geometric field reader under test
|
||||
(``generate.relational_field_reader``) — it imports no ``algebra`` / ``field`` /
|
||||
``generate`` module. Two independent procedures (the field reader off the TEXT, the
|
||||
oracle off the STRUCTURE) agreeing is real evidence the field READ the text
|
||||
correctly; a shared-code "gold" would only prove the reader agrees with itself
|
||||
(INV-25).
|
||||
|
||||
Supported relation kinds (the v1 forward-substitutable grammar):
|
||||
|
||||
- ``fact`` : ``entity = value`` (a given quantity)
|
||||
- ``more_than`` : ``entity = ref + delta``
|
||||
- ``fewer_than`` : ``entity = ref - delta``
|
||||
- ``sum_of`` : ``entity = sum(parts)`` (part-whole / total)
|
||||
|
||||
Every relation's references must already be resolved (forward-substitutable /
|
||||
triangular). The oracle refuses anything else, never guesses.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
class OracleError(ValueError):
|
||||
"""Malformed or out-of-grammar case — the oracle refuses, never guesses."""
|
||||
|
||||
|
||||
_SUPPORTED = frozenset({"fact", "more_than", "fewer_than", "sum_of"})
|
||||
|
||||
|
||||
def oracle_answer(relations: list[dict[str, Any]], query: dict[str, Any]) -> int:
|
||||
"""Compute the integer answer by forward substitution over the relations.
|
||||
|
||||
Raises :class:`OracleError` on an unknown relation kind, a forward reference to
|
||||
an unresolved entity, a duplicate definition, or a missing query entity.
|
||||
"""
|
||||
values: dict[str, int] = {}
|
||||
|
||||
for rel in relations:
|
||||
kind = rel.get("kind")
|
||||
entity = rel.get("entity")
|
||||
if kind not in _SUPPORTED:
|
||||
raise OracleError(f"unsupported relation kind: {kind!r}")
|
||||
if not isinstance(entity, str) or not entity:
|
||||
raise OracleError(f"relation missing entity: {rel!r}")
|
||||
if entity in values:
|
||||
raise OracleError(f"duplicate definition of {entity!r}")
|
||||
|
||||
if kind == "fact":
|
||||
value = rel.get("value")
|
||||
if not isinstance(value, int) or isinstance(value, bool):
|
||||
raise OracleError(f"fact value must be int: {rel!r}")
|
||||
values[entity] = value
|
||||
elif kind in ("more_than", "fewer_than"):
|
||||
ref = rel.get("ref")
|
||||
delta = rel.get("delta")
|
||||
if ref not in values:
|
||||
raise OracleError(f"forward reference to unresolved {ref!r}")
|
||||
if not isinstance(delta, int) or isinstance(delta, bool):
|
||||
raise OracleError(f"delta must be int: {rel!r}")
|
||||
values[entity] = values[ref] + (delta if kind == "more_than" else -delta)
|
||||
else: # sum_of
|
||||
parts = rel.get("parts")
|
||||
if not isinstance(parts, list) or not parts:
|
||||
raise OracleError(f"sum_of needs non-empty parts: {rel!r}")
|
||||
if any(p not in values for p in parts):
|
||||
raise OracleError(f"sum_of references unresolved part: {rel!r}")
|
||||
values[entity] = sum(values[p] for p in parts)
|
||||
|
||||
target = query.get("entity")
|
||||
if target not in values:
|
||||
raise OracleError(f"query entity {target!r} not resolved")
|
||||
return values[target]
|
||||
98
evals/relational_metric/runner.py
Normal file
98
evals/relational_metric/runner.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
"""Relational-metric lane runner — field reader vs the independent arithmetic gold.
|
||||
|
||||
For each committed case:
|
||||
|
||||
1. the independent oracle recomputes the gold from the STRUCTURED relations
|
||||
(gold-integrity: a committed gold that the oracle cannot reproduce is rejected,
|
||||
so the gold can never be field-derived — INV-25);
|
||||
2. the geometric field reader reads the TEXT into an answer (or refuses);
|
||||
3. the field's committed answer is scored against that gold.
|
||||
|
||||
Buckets: ``correct`` (field == gold), ``wrong`` (field committed a different
|
||||
integer), ``refused`` (field declined). ``wrong`` must be 0 — the runner exits 1
|
||||
otherwise. A refusal is honest coverage, not a wrong answer.
|
||||
|
||||
Run: PYTHONPATH=. .venv/bin/python -m evals.relational_metric.runner
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from evals.relational_metric.oracle import OracleError, oracle_answer
|
||||
from generate.relational_field_reader import read_relational
|
||||
|
||||
_CASES = Path(__file__).resolve().parent / "v1" / "cases.jsonl"
|
||||
|
||||
|
||||
def _load_cases() -> list[dict[str, Any]]:
|
||||
cases: list[dict[str, Any]] = []
|
||||
for line in _CASES.read_text(encoding="utf-8").splitlines():
|
||||
if line.strip():
|
||||
cases.append(json.loads(line))
|
||||
return cases
|
||||
|
||||
|
||||
def run() -> dict[str, Any]:
|
||||
"""Run the lane and return a structured report."""
|
||||
cases = _load_cases()
|
||||
correct = 0
|
||||
wrong: list[dict[str, Any]] = []
|
||||
refused: list[str] = []
|
||||
gold_integrity_failures: list[str] = []
|
||||
|
||||
for case in cases:
|
||||
cid = case["id"]
|
||||
committed_gold = case["gold"]
|
||||
# 1 — independent gold integrity (the oracle must reproduce the committed gold)
|
||||
try:
|
||||
recomputed = oracle_answer(case["relations"], case["query"])
|
||||
except OracleError:
|
||||
gold_integrity_failures.append(f"{cid}: oracle could not reproduce gold")
|
||||
continue
|
||||
if recomputed != committed_gold:
|
||||
gold_integrity_failures.append(
|
||||
f"{cid}: oracle={recomputed} != committed gold={committed_gold}"
|
||||
)
|
||||
continue
|
||||
|
||||
# 2/3 — the field reads the TEXT, scored against the independent gold
|
||||
reading = read_relational(case["text"])
|
||||
if reading.refused:
|
||||
refused.append(f"{cid}: {reading.refusal_reason}")
|
||||
elif reading.answer == committed_gold:
|
||||
correct += 1
|
||||
else:
|
||||
wrong.append(
|
||||
{"id": cid, "field": reading.answer, "gold": committed_gold}
|
||||
)
|
||||
|
||||
return {
|
||||
"total": len(cases),
|
||||
"correct": correct,
|
||||
"wrong": len(wrong),
|
||||
"refused": len(refused),
|
||||
"wrong_detail": wrong,
|
||||
"refused_detail": refused,
|
||||
"gold_integrity_failures": gold_integrity_failures,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
report = run()
|
||||
print(json.dumps({k: v for k, v in report.items()
|
||||
if k not in ("wrong_detail",)}, indent=2))
|
||||
if report["gold_integrity_failures"]:
|
||||
print("GOLD INTEGRITY FAILURE:", report["gold_integrity_failures"], file=sys.stderr)
|
||||
return 1
|
||||
if report["wrong"] > 0:
|
||||
print("WRONG ANSWERS (wrong!=0 breach):", report["wrong_detail"], file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
15
evals/relational_metric/v1/cases.jsonl
Normal file
15
evals/relational_metric/v1/cases.jsonl
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{"id": "rm-v1-0001", "text": "Liam has 6 stickers. Mia has 4 more stickers than Liam. How many stickers does Mia have?", "relations": [{"kind": "fact", "entity": "liam", "value": 6}, {"kind": "more_than", "entity": "mia", "ref": "liam", "delta": 4}], "query": {"entity": "mia", "unit": "stickers"}, "gold": 10, "class": "additive_more_than"}
|
||||
{"id": "rm-v1-0002", "text": "Noah has 15 cards. Olivia has 6 fewer cards than Noah. How many cards does Olivia have?", "relations": [{"kind": "fact", "entity": "noah", "value": 15}, {"kind": "fewer_than", "entity": "olivia", "ref": "noah", "delta": 6}], "query": {"entity": "olivia", "unit": "cards"}, "gold": 9, "class": "additive_fewer_than"}
|
||||
{"id": "rm-v1-0003", "text": "Ava has 8 pencils. Ben has 5 more pencils than Ava. Cara has 3 more pencils than Ben. How many pencils does Cara have?", "relations": [{"kind": "fact", "entity": "ava", "value": 8}, {"kind": "more_than", "entity": "ben", "ref": "ava", "delta": 5}, {"kind": "more_than", "entity": "cara", "ref": "ben", "delta": 3}], "query": {"entity": "cara", "unit": "pencils"}, "gold": 16, "class": "additive_chain"}
|
||||
{"id": "rm-v1-0004", "text": "Dan has 7 coins. Eva has 9 more coins than Dan. How many coins do Dan and Eva have?", "relations": [{"kind": "fact", "entity": "dan", "value": 7}, {"kind": "more_than", "entity": "eva", "ref": "dan", "delta": 9}, {"kind": "sum_of", "entity": "total", "parts": ["dan", "eva"]}], "query": {"entity": "total", "unit": "coins"}, "gold": 23, "class": "part_whole_sum"}
|
||||
{"id": "rm-v1-0005", "text": "Finn has 12 books. How many books does Finn have?", "relations": [{"kind": "fact", "entity": "finn", "value": 12}], "query": {"entity": "finn", "unit": "books"}, "gold": 12, "class": "fact_only"}
|
||||
{"id": "rm-v1-0006", "text": "Gabe has 30 apples. Hana has 10 fewer apples than Gabe. How many apples does Hana have?", "relations": [{"kind": "fact", "entity": "gabe", "value": 30}, {"kind": "fewer_than", "entity": "hana", "ref": "gabe", "delta": 10}], "query": {"entity": "hana", "unit": "apples"}, "gold": 20, "class": "additive_fewer_than"}
|
||||
{"id": "rm-v1-0007", "text": "Iris has 100 dollars. Jack has 250 more dollars than Iris. How many dollars does Jack have?", "relations": [{"kind": "fact", "entity": "iris", "value": 100}, {"kind": "more_than", "entity": "jack", "ref": "iris", "delta": 250}], "query": {"entity": "jack", "unit": "dollars"}, "gold": 350, "class": "additive_more_than"}
|
||||
{"id": "rm-v1-0008", "text": "Kim has 2 marbles. Leo has 3 more marbles than Kim. How many marbles do Kim and Leo have?", "relations": [{"kind": "fact", "entity": "kim", "value": 2}, {"kind": "more_than", "entity": "leo", "ref": "kim", "delta": 3}, {"kind": "sum_of", "entity": "total", "parts": ["kim", "leo"]}], "query": {"entity": "total", "unit": "marbles"}, "gold": 7, "class": "part_whole_sum"}
|
||||
{"id": "rm-v1-0009", "text": "Maya has 40 beads. Nico has 18 fewer beads than Maya. How many beads does Nico have?", "relations": [{"kind": "fact", "entity": "maya", "value": 40}, {"kind": "fewer_than", "entity": "nico", "ref": "maya", "delta": 18}], "query": {"entity": "nico", "unit": "beads"}, "gold": 22, "class": "additive_fewer_than"}
|
||||
{"id": "rm-v1-0010", "text": "Omar has 5 tokens. Pia has 5 more tokens than Omar. Quinn has 5 more tokens than Pia. How many tokens does Quinn have?", "relations": [{"kind": "fact", "entity": "omar", "value": 5}, {"kind": "more_than", "entity": "pia", "ref": "omar", "delta": 5}, {"kind": "more_than", "entity": "quinn", "ref": "pia", "delta": 5}], "query": {"entity": "quinn", "unit": "tokens"}, "gold": 15, "class": "additive_chain"}
|
||||
{"id": "rm-v1-0011", "text": "Rosa has 9 ribbons. Sam has 14 more ribbons than Rosa. How many ribbons do Rosa and Sam have?", "relations": [{"kind": "fact", "entity": "rosa", "value": 9}, {"kind": "more_than", "entity": "sam", "ref": "rosa", "delta": 14}, {"kind": "sum_of", "entity": "total", "parts": ["rosa", "sam"]}], "query": {"entity": "total", "unit": "ribbons"}, "gold": 32, "class": "part_whole_sum"}
|
||||
{"id": "rm-v1-0012", "text": "Tara has 11 stamps. Uma has 11 fewer stamps than Tara. How many stamps does Uma have?", "relations": [{"kind": "fact", "entity": "tara", "value": 11}, {"kind": "fewer_than", "entity": "uma", "ref": "tara", "delta": 11}], "query": {"entity": "uma", "unit": "stamps"}, "gold": 0, "class": "additive_fewer_than"}
|
||||
{"id": "rm-v1-0013", "text": "Vera has 1200 points. Will has 800 more points than Vera. How many points does Will have?", "relations": [{"kind": "fact", "entity": "vera", "value": 1200}, {"kind": "more_than", "entity": "will", "ref": "vera", "delta": 800}], "query": {"entity": "will", "unit": "points"}, "gold": 2000, "class": "additive_more_than"}
|
||||
{"id": "rm-v1-0014", "text": "Xena has 25 shells. Yara has 13 more shells than Xena. Zane has 7 fewer shells than Yara. How many shells does Zane have?", "relations": [{"kind": "fact", "entity": "xena", "value": 25}, {"kind": "more_than", "entity": "yara", "ref": "xena", "delta": 13}, {"kind": "fewer_than", "entity": "zane", "ref": "yara", "delta": 7}], "query": {"entity": "zane", "unit": "shells"}, "gold": 31, "class": "additive_chain"}
|
||||
{"id": "rm-v1-0015", "text": "Gus has 9000000 apples. How many apples does Gus have?", "relations": [{"kind": "fact", "entity": "gus", "value": 9000000}], "query": {"entity": "gus", "unit": "apples"}, "gold": 9000000, "class": "coverage_over_ceiling"}
|
||||
211
generate/relational_field_reader.py
Normal file
211
generate/relational_field_reader.py
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
"""The geometric FIELD reader for forward-substitutable quantitative relations.
|
||||
|
||||
Phase W of the field-reasoner wedge
|
||||
(docs/analysis/field-reasoner-wedge-design-and-falsification-2026-06-04.md).
|
||||
|
||||
This is the system under test: it reads problem **text** into conformal points on
|
||||
the e1 number line (``algebra.cga.embed_point`` at float64), resolves each unknown
|
||||
by applying a conformal **translator versor** (``algebra.versor.versor_apply``), and
|
||||
reads the answer back by **projective dehomogenization** (``read_scalar_e1``). The
|
||||
resolution is genuinely geometric: ``versor_apply(T_delta, embed([x])) == embed([x+δ])``
|
||||
exactly (verified), so "A is δ more than B" is a translation, not a hidden add.
|
||||
|
||||
It is a **second, hand-written reader**: its tokenizer / number extraction / relation
|
||||
classification are an independent reimplementation. It imports **nothing** from
|
||||
``generate.derivation`` / ``generate.math_candidate_parser`` / ``generate.math_*`` /
|
||||
``WORD_NUMBERS`` — so its agreement with a symbolic reader is not common-mode by
|
||||
construction (the disjointness is *proven* by INV-27, not asserted here).
|
||||
|
||||
**Refusal-first.** It commits an answer only inside a narrow, sealed grammar
|
||||
(digit integers; ``has``/``more``/``fewer``/``than``/``how many`` cues; additive and
|
||||
part-whole only). It REFUSES — never guesses — on:
|
||||
|
||||
- multiplicative / ratio cues (``times``/``twice``/``double``/``ratio`` …) — fenced,
|
||||
because ``cga_inner`` is sign/orientation-blind (A=2B vs A=−2B);
|
||||
- any quantity above :data:`algebra.cga.EMBED_EXACT_MAX` (precision ceiling);
|
||||
- a non-forward-substitutable reference (an unknown referenced before it is defined);
|
||||
- a negative resolved quantity, a non-integer read-back, or any unparsed sentence.
|
||||
|
||||
The answer is exact-integer; there is no float tolerance anywhere on the commit path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
|
||||
from algebra.cga import EMBED_EXACT_MAX, embed_point, read_scalar_e1
|
||||
from algebra.cl41 import N_COMPONENTS, geometric_product
|
||||
from algebra.versor import versor_apply
|
||||
|
||||
# The reader's stable identity for the Tier-2 gate (registered in INV-27).
|
||||
READER_LINEAGE: str = "field.relational_number_line"
|
||||
|
||||
_F64 = np.float64
|
||||
|
||||
# Multiplicative / ratio cues are out of the sealed metric domain — refuse.
|
||||
_FENCED_CUE = re.compile(
|
||||
r"\b(times|twice|double|triple|thrice|half|quarter|ratio|per|each)\b"
|
||||
)
|
||||
|
||||
_MORE_THAN = re.compile(r"\b(\w+)\s+(?:has|have)\s+(\d+)\s+more\s+\w+\s+than\s+(\w+)")
|
||||
_FEWER_THAN = re.compile(
|
||||
r"\b(\w+)\s+(?:has|have)\s+(\d+)\s+(?:fewer|less)\s+\w+\s+than\s+(\w+)"
|
||||
)
|
||||
_FACT = re.compile(r"\b(\w+)\s+(?:has|have)\s+(\d+)\s+(\w+)")
|
||||
_QUERY_SINGLE = re.compile(r"how many\s+(\w+)\s+does\s+(\w+)\s+have")
|
||||
_QUERY_SUM = re.compile(
|
||||
r"how many\s+(\w+)\s+do\s+(\w+)\s+and\s+(\w+)(?:\s+and\s+(\w+))?\s+have"
|
||||
)
|
||||
|
||||
|
||||
class FieldReaderError(ValueError):
|
||||
"""Internal signal that a case is out of the sealed grammar (→ typed refusal)."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FieldReading:
|
||||
"""Result of the geometric reading. ``answer is None`` iff ``refused``."""
|
||||
|
||||
refused: bool
|
||||
answer: int | None = None
|
||||
answer_unit: str | None = None
|
||||
refusal_reason: str | None = None
|
||||
reader_lineage: str = READER_LINEAGE
|
||||
|
||||
|
||||
def _refuse(reason: str) -> FieldReading:
|
||||
return FieldReading(refused=True, refusal_reason=reason)
|
||||
|
||||
|
||||
def _translator_e1(delta: int) -> np.ndarray:
|
||||
"""The conformal translator versor that shifts an e1-axis point by ``delta``.
|
||||
|
||||
``T = 1 − ½·(δ·e1)·n_inf`` with ``n_inf = e4 + e5``. For null n_inf the
|
||||
exponential series truncates exactly, so ``versor_apply(T, embed([x]))`` lands
|
||||
on ``embed([x+δ])`` with zero residual (the metric stays exact in f64).
|
||||
"""
|
||||
n_inf = np.zeros(N_COMPONENTS, dtype=_F64)
|
||||
n_inf[4] = 1.0
|
||||
n_inf[5] = 1.0
|
||||
e1 = np.zeros(N_COMPONENTS, dtype=_F64)
|
||||
e1[1] = float(delta)
|
||||
t = geometric_product(e1, n_inf)
|
||||
rotor = np.zeros(N_COMPONENTS, dtype=_F64)
|
||||
rotor[0] = 1.0
|
||||
return rotor - 0.5 * t
|
||||
|
||||
|
||||
def _check_magnitude(value: int) -> None:
|
||||
if abs(value) > EMBED_EXACT_MAX:
|
||||
raise FieldReaderError("over_ceiling")
|
||||
|
||||
|
||||
def _apply_delta(point: np.ndarray, delta: int) -> np.ndarray:
|
||||
"""Translate an e1 point by ``delta`` and PROVE the move was exact.
|
||||
|
||||
The translator sandwich loses f64 integer exactness when ``δ·x²`` approaches
|
||||
2^52, which would silently commit a plausible-but-wrong integer (the resolve_pooled
|
||||
failure mode). So we verify the read-back moved by *exactly* ``delta`` and refuse
|
||||
(``precision_drift``) otherwise — the field never commits a drifted answer.
|
||||
"""
|
||||
before = _read_int(point)
|
||||
moved = versor_apply(_translator_e1(delta), point)
|
||||
if read_scalar_e1(moved) != before + delta: # exact integer move, or refuse
|
||||
raise FieldReaderError("precision_drift")
|
||||
return moved
|
||||
|
||||
|
||||
def _sentences(text: str) -> list[str]:
|
||||
return [s.strip() for s in re.split(r"[.?!]", text.lower()) if s.strip()]
|
||||
|
||||
|
||||
def read_relational(text: str) -> FieldReading:
|
||||
"""Read problem ``text`` geometrically into an exact integer answer, or refuse."""
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
return _refuse("empty_input")
|
||||
if _FENCED_CUE.search(text.lower()):
|
||||
return _refuse("fenced_multiplicative")
|
||||
|
||||
try:
|
||||
return _read(text)
|
||||
except FieldReaderError as exc:
|
||||
return _refuse(str(exc))
|
||||
|
||||
|
||||
def _read(text: str) -> FieldReading:
|
||||
sentences = _sentences(text)
|
||||
|
||||
# --- locate the query (the question sentence) -------------------------------
|
||||
query_unit: str | None = None
|
||||
query_single: str | None = None
|
||||
query_parts: list[str] | None = None
|
||||
for s in sentences:
|
||||
if "how many" in s:
|
||||
if (m := _QUERY_SUM.search(s)) is not None:
|
||||
query_unit = m.group(1)
|
||||
query_parts = [g for g in m.groups()[1:] if g]
|
||||
elif (m := _QUERY_SINGLE.search(s)) is not None:
|
||||
query_unit = m.group(1)
|
||||
query_single = m.group(2)
|
||||
else:
|
||||
raise FieldReaderError("unparsed_query")
|
||||
break
|
||||
if query_single is None and query_parts is None:
|
||||
raise FieldReaderError("no_query")
|
||||
|
||||
# --- parse the declarative sentences, in order ------------------------------
|
||||
points: dict[str, np.ndarray] = {}
|
||||
|
||||
def resolve(entity: str) -> None:
|
||||
if entity not in points:
|
||||
raise FieldReaderError("non_forward_substitutable")
|
||||
|
||||
for s in sentences:
|
||||
if "how many" in s:
|
||||
continue
|
||||
if (m := _MORE_THAN.search(s)) is not None:
|
||||
entity, n, ref = m.group(1), int(m.group(2)), m.group(3)
|
||||
_check_magnitude(n)
|
||||
resolve(ref)
|
||||
points[entity] = _apply_delta(points[ref], n)
|
||||
elif (m := _FEWER_THAN.search(s)) is not None:
|
||||
entity, n, ref = m.group(1), int(m.group(2)), m.group(3)
|
||||
_check_magnitude(n)
|
||||
resolve(ref)
|
||||
points[entity] = _apply_delta(points[ref], -n)
|
||||
elif (m := _FACT.search(s)) is not None:
|
||||
entity, n = m.group(1), int(m.group(2))
|
||||
_check_magnitude(n)
|
||||
points[entity] = embed_point(np.array([float(n), 0.0, 0.0]), dtype=_F64)
|
||||
else:
|
||||
raise FieldReaderError("unparsed_sentence")
|
||||
|
||||
# --- read the answer back off the geometry ----------------------------------
|
||||
if query_single is not None:
|
||||
if query_single not in points:
|
||||
raise FieldReaderError("query_entity_unresolved")
|
||||
answer = _read_int(points[query_single])
|
||||
else:
|
||||
assert query_parts is not None
|
||||
total = embed_point(np.array([0.0, 0.0, 0.0]), dtype=_F64)
|
||||
for part in query_parts:
|
||||
if part not in points:
|
||||
raise FieldReaderError("query_entity_unresolved")
|
||||
total = _apply_delta(total, _read_int(points[part]))
|
||||
answer = _read_int(total)
|
||||
|
||||
if answer < 0:
|
||||
raise FieldReaderError("negative_quantity")
|
||||
return FieldReading(refused=False, answer=answer, answer_unit=query_unit)
|
||||
|
||||
|
||||
def _read_int(point: np.ndarray) -> int:
|
||||
"""Projective read-back, refusing any non-integer coordinate (no float slack)."""
|
||||
value = read_scalar_e1(point)
|
||||
rounded = round(value)
|
||||
if abs(value - rounded) > 0.0: # exact-integer commit path; no tolerance
|
||||
raise FieldReaderError("non_integer_readback")
|
||||
return int(rounded)
|
||||
|
|
@ -1103,6 +1103,19 @@ INDEPENDENT_GOLD_LANES: tuple[IndependentGoldLane, ...] = (
|
|||
oracle_module="evals/dimensional/oracle.py",
|
||||
sut_import_prefixes=("generate.binding_graph",),
|
||||
),
|
||||
# The relational-metric lane (field-reasoner wedge): the geometric field reader
|
||||
# (generate.relational_field_reader, which reads TEXT into conformal points) is
|
||||
# the SUT, so its arithmetic gold oracle must share no code with the reader or
|
||||
# the field engine it rides on.
|
||||
IndependentGoldLane(
|
||||
name="relational_metric",
|
||||
oracle_module="evals/relational_metric/oracle.py",
|
||||
sut_import_prefixes=(
|
||||
"generate.relational_field_reader",
|
||||
"algebra",
|
||||
"field",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
_DEDUCTIVE_CASE_FILES: tuple[str, ...] = (
|
||||
|
|
|
|||
133
tests/test_relational_field_reader.py
Normal file
133
tests/test_relational_field_reader.py
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
"""Phase W — the geometric field reader on forward-substitutable relations.
|
||||
|
||||
Proves the field reads problem TEXT into an exact integer answer via conformal
|
||||
translators + projective read-back, and REFUSES (never guesses) outside its sealed
|
||||
metric grammar. wrong==0 is structural: every commit is an exact-integer read-back.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from generate.relational_field_reader import READER_LINEAGE, read_relational
|
||||
|
||||
|
||||
# --- commits: the field reads correctly ------------------------------------
|
||||
|
||||
|
||||
def test_fact_then_more_than():
|
||||
r = read_relational(
|
||||
"Tom has 3 marbles. Jane has 5 more marbles than Tom. "
|
||||
"How many marbles does Jane have?"
|
||||
)
|
||||
assert not r.refused
|
||||
assert r.answer == 8
|
||||
assert r.answer_unit == "marbles"
|
||||
assert r.reader_lineage == READER_LINEAGE
|
||||
|
||||
|
||||
def test_fewer_than():
|
||||
r = read_relational(
|
||||
"Anna has 20 apples. Ben has 7 fewer apples than Anna. "
|
||||
"How many apples does Ben have?"
|
||||
)
|
||||
assert not r.refused
|
||||
assert r.answer == 13
|
||||
|
||||
|
||||
def test_chained_forward_substitution():
|
||||
r = read_relational(
|
||||
"Tom has 4 coins. Jane has 6 more coins than Tom. "
|
||||
"Sara has 10 more coins than Jane. How many coins does Sara have?"
|
||||
)
|
||||
assert not r.refused
|
||||
assert r.answer == 20 # 4 -> 10 -> 20
|
||||
|
||||
|
||||
def test_part_whole_sum_query():
|
||||
r = read_relational(
|
||||
"Tom has 3 marbles. Jane has 5 more marbles than Tom. "
|
||||
"How many marbles do Tom and Jane have?"
|
||||
)
|
||||
assert not r.refused
|
||||
assert r.answer == 11 # 3 + 8
|
||||
|
||||
|
||||
def test_large_value_within_ceiling_is_exact():
|
||||
# x=12345 already collapses f32 (the n_o weight loses the ±1 past ~4096);
|
||||
# f64 + translator stays exact here.
|
||||
r = read_relational(
|
||||
"Tom has 12345 dollars. Jane has 5000 more dollars than Tom. "
|
||||
"How many dollars does Jane have?"
|
||||
)
|
||||
assert not r.refused
|
||||
assert r.answer == 17345
|
||||
|
||||
|
||||
def test_never_commits_a_drifted_answer():
|
||||
"""wrong==0 guard: at a scale where the translator sandwich loses f64 integer
|
||||
exactness, the field REFUSES (precision_drift) rather than commit a wrong int."""
|
||||
r = read_relational(
|
||||
"Tom has 123456 dollars. Jane has 654321 more dollars than Tom. "
|
||||
"How many dollars does Jane have?"
|
||||
)
|
||||
# Either it commits the EXACT answer, or it refuses — never a wrong integer.
|
||||
assert r.refused or r.answer == 777777
|
||||
if r.refused:
|
||||
assert r.refusal_reason == "precision_drift"
|
||||
|
||||
|
||||
# --- refusals: the field declines outside its sealed grammar ----------------
|
||||
|
||||
|
||||
def test_refuses_multiplicative():
|
||||
r = read_relational(
|
||||
"Tom has 3 marbles. Jane has twice as many marbles as Tom. "
|
||||
"How many marbles does Jane have?"
|
||||
)
|
||||
assert r.refused
|
||||
assert r.refusal_reason == "fenced_multiplicative"
|
||||
|
||||
|
||||
def test_refuses_times_cue():
|
||||
r = read_relational(
|
||||
"Tom has 3 marbles. Jane has 4 times as many marbles as Tom. "
|
||||
"How many marbles does Jane have?"
|
||||
)
|
||||
assert r.refused
|
||||
assert r.refusal_reason == "fenced_multiplicative"
|
||||
|
||||
|
||||
def test_refuses_over_ceiling():
|
||||
r = read_relational(
|
||||
"Tom has 9000000 marbles. How many marbles does Tom have?"
|
||||
)
|
||||
assert r.refused
|
||||
assert r.refusal_reason == "over_ceiling"
|
||||
|
||||
|
||||
def test_refuses_forward_reference():
|
||||
r = read_relational(
|
||||
"Jane has 5 more marbles than Tom. Tom has 3 marbles. "
|
||||
"How many marbles does Jane have?"
|
||||
)
|
||||
assert r.refused
|
||||
assert r.refusal_reason == "non_forward_substitutable"
|
||||
|
||||
|
||||
def test_refuses_no_question():
|
||||
r = read_relational("Tom has 3 marbles. Jane has 5 more marbles than Tom.")
|
||||
assert r.refused
|
||||
assert r.refusal_reason == "no_query"
|
||||
|
||||
|
||||
def test_refuses_negative_quantity():
|
||||
r = read_relational(
|
||||
"Tom has 3 marbles. Jane has 5 fewer marbles than Tom. "
|
||||
"How many marbles does Jane have?"
|
||||
)
|
||||
assert r.refused
|
||||
assert r.refusal_reason == "negative_quantity"
|
||||
|
||||
|
||||
def test_refuses_empty():
|
||||
assert read_relational("").refused
|
||||
assert read_relational(" ").refusal_reason == "empty_input"
|
||||
35
tests/test_relational_metric_lane.py
Normal file
35
tests/test_relational_metric_lane.py
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
"""Relational-metric gold lane — the field reader is wrong==0 vs an independent gold.
|
||||
|
||||
This is measurement #1 of the field-reasoner wedge falsifiable experiment: does the
|
||||
geometric field reader, reading problem TEXT, commit answers that match an
|
||||
independent arithmetic oracle (computed from the STRUCTURE) with zero wrong? The
|
||||
oracle shares no code with the reader (enforced structurally by INV-25's
|
||||
INDEPENDENT_GOLD_LANES registration).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from evals.relational_metric.runner import run
|
||||
|
||||
|
||||
def test_lane_is_wrong_zero_with_independent_gold():
|
||||
report = run()
|
||||
# The committed gold is reproducible by the independent oracle (never field-derived).
|
||||
assert report["gold_integrity_failures"] == [], report["gold_integrity_failures"]
|
||||
# wrong==0 is the prime directive on this lane.
|
||||
assert report["wrong"] == 0, report["wrong_detail"]
|
||||
# Buckets account for every case.
|
||||
assert report["total"] == report["correct"] + report["wrong"] + report["refused"]
|
||||
|
||||
|
||||
def test_field_actually_commits_not_a_refusal_floor():
|
||||
"""A refusal floor (refuse everything) would be wrong==0 too — and worthless.
|
||||
The capability claim is COVERAGE with wrong==0: the field must commit real cases."""
|
||||
report = run()
|
||||
assert report["correct"] >= 10
|
||||
|
||||
|
||||
def test_over_ceiling_is_refused_not_wrong():
|
||||
"""The precision ceiling is honest coverage (a refusal), never a wrong commit."""
|
||||
report = run()
|
||||
assert any("over_ceiling" in r for r in report["refused_detail"])
|
||||
Loading…
Reference in a new issue