core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge pull request #619 from AssetOverflow/feat/inverse-solve-contract

Browse source 
feat: narrow reverse-solve oracle contract (PR-7a, contract only)
This commit is contained in:
Shay 2026-06-07 06:44:07 -07:00 committed by GitHub
commit 0e6a7f9ac0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 164 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
evals/relational_metric/oracle.py
View file

@ -19,7 +19,13 @@ Supported relation kinds (the v1 + PR-6b forward-substitutable grammar):
- ``divide_by`` : ``entity = ref // divisor`` (exact integer division only)
- ``sum_of`` : ``entity = sum(parts)`` (part-whole / total)
Every relation's references must already be resolved (forward-substitutable /
Plus one narrow off-grammar extension — a reverse-solve (PR-7a): the base of a single
more_than/fewer_than whose other side is grounded (``Nia has 9 more than Omar. Nia has
15. -> Omar = 6``). It is deliberately tiny — single base constraint, base == query
target (no chains), base not otherwise grounded, non-negative count result, more/fewer
only (never times/divide). Everything wider refuses.
Every forward relation's references must already be resolved (forward-substitutable /
triangular). The oracle refuses anything else, never guesses. PR-6b/6c keep this
off-serving: they let setup-correct R1 cases compute answers in the eval lane only.
@ -45,11 +51,23 @@ _SUPPORTED = frozenset(
def oracle_answer(relations: list[dict[str, Any]], query: dict[str, Any]) -> int:
"""Compute the integer answer by forward substitution over the relations.
Plus one narrow reverse-solve (PR-7a): a single more_than/fewer_than whose entity
is grounded and whose ref is the (otherwise ungrounded) query target is inverted to
that base — ``base = entity -/+ delta``. The reverse-solve refuses on anything wider:
more than one inverse constraint, an inverse base that is not the query target (a
chain), an already-grounded base, a negative result (count domain), or an inverse
over times/divide. The +/- inversion is always integer-exact.
Raises :class:`OracleError` on an unknown relation kind, a forward reference to
an unresolved entity, a duplicate definition, malformed scalar, or a missing
query entity.
an unresolved entity, a duplicate definition, malformed scalar, an out-of-contract
reverse-solve, or a missing query entity.
"""
values: dict[str, int] = {}
# Deferred narrow inverse (PR-7a): a more_than/fewer_than whose ENTITY is already
# grounded (by a prior fact) and whose REF is unresolved is NOT a duplicate
# definition — it pins its ref (the unknown base). Collected here, resolved after
# the forward pass so the single-base / no-chains guardrails apply to the whole set.
inverse: list[tuple[str, str, str, int]] = [] # (kind, entity, ref, delta)
for rel in relations:
kind = rel.get("kind")
@ -58,10 +76,10 @@ def oracle_answer(relations: list[dict[str, Any]], query: dict[str, Any]) -> int
raise OracleError(f"unsupported relation kind: {kind!r}")
if not isinstance(entity, str) or not entity:
raise OracleError(f"relation missing entity: {rel!r}")
if entity in values:
raise OracleError(f"duplicate definition of {entity!r}")
if kind == "fact":
if entity in values:
raise OracleError(f"duplicate definition of {entity!r}")
value = rel.get("value")
if not isinstance(value, int) or isinstance(value, bool):
raise OracleError(f"fact value must be int: {rel!r}")
@ -69,12 +87,25 @@ def oracle_answer(relations: list[dict[str, Any]], query: dict[str, Any]) -> int
elif kind in ("more_than", "fewer_than"):
ref = rel.get("ref")
delta = rel.get("delta")
if ref not in values:
raise OracleError(f"forward reference to unresolved {ref!r}")
if not isinstance(ref, str) or not ref:
raise OracleError(f"relation missing ref: {rel!r}")
if not isinstance(delta, int) or isinstance(delta, bool):
raise OracleError(f"delta must be int: {rel!r}")
values[entity] = values[ref] + (delta if kind == "more_than" else -delta)
entity_known = entity in values
ref_known = ref in values
if entity_known and not ref_known:
# INVERSE: the unknown is the ref (base). Solved in phase 2.
inverse.append((kind, entity, ref, delta))
elif not entity_known and ref_known:
values[entity] = values[ref] + (delta if kind == "more_than" else -delta)
elif not entity_known and not ref_known:
raise OracleError(f"forward reference to unresolved {ref!r}")
else: # both sides grounded -> over-determined, refuse rather than ignore
raise OracleError(f"over-determined relation (both sides known): {rel!r}")
elif kind == "times_as_many":
if entity in values:
# Reverse-solve over times/divide is NOT in the PR-7a contract.
raise OracleError(f"duplicate definition of {entity!r}")
ref = rel.get("ref")
factor = rel.get("factor")
if ref not in values:
@ -83,6 +114,8 @@ def oracle_answer(relations: list[dict[str, Any]], query: dict[str, Any]) -> int
raise OracleError(f"factor must be int: {rel!r}")
values[entity] = values[ref] * factor
elif kind == "divide_by":
if entity in values:
raise OracleError(f"duplicate definition of {entity!r}")
ref = rel.get("ref")
divisor = rel.get("divisor")
if ref not in values:
@ -94,6 +127,8 @@ def oracle_answer(relations: list[dict[str, Any]], query: dict[str, Any]) -> int
raise OracleError(f"non-exact division {values[ref]}/{divisor}: {rel!r}")
values[entity] = values[ref] // divisor
else: # sum_of
if entity in values:
raise OracleError(f"duplicate definition of {entity!r}")
parts = rel.get("parts")
if not isinstance(parts, list) or not parts:
raise OracleError(f"sum_of needs non-empty parts: {rel!r}")
@ -102,6 +137,30 @@ def oracle_answer(relations: list[dict[str, Any]], query: dict[str, Any]) -> int
values[entity] = sum(values[p] for p in parts)
target = query.get("entity")
# Phase 2: narrow reverse-solve (PR-7a). The single unknown is the base of exactly
# one more_than/fewer_than whose other side is grounded. Anything wider refuses.
if inverse:
if len(inverse) != 1:
raise OracleError(
f"reverse-solve supports a single base constraint only: {inverse!r}"
)
kind, ent, ref, delta = inverse[0]
if target in values:
raise OracleError("reverse-solve constraint present but target already determined")
if ref != target:
raise OracleError(
f"reverse-solve base {ref!r} is not the query target {target!r} (no chains)"
)
if ref in values:
raise OracleError(f"reverse-solve base {ref!r} is already grounded")
# more_than: entity = base + delta -> base = entity - delta;
# fewer_than: entity = base - delta -> base = entity + delta. (Always exact for +/-.)
result = values[ent] - delta if kind == "more_than" else values[ent] + delta
if result < 0:
raise OracleError(f"reverse-solve base is negative (count domain): {result}")
values[ref] = result
if target not in values:
raise OracleError(f"query entity {target!r} not resolved")
return values[target]

2
evals/setup_oracle/r1_gold.jsonl
View file

@ -4,7 +4,7 @@
{"id": "r1-04-fewer-total", "text": "Gail has 20 cards. Hank has 6 fewer cards than Gail. How many cards do Gail and Hank have in total?", "relations": [{"kind": "fact", "entity": "gail", "value": 20}, {"kind": "fewer_than", "entity": "hank", "ref": "gail", "delta": 6}, {"kind": "sum_of", "entity": "total", "parts": ["gail", "hank"]}], "expected_units": {"gail": "item", "hank": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "gold": 34, "notes": "Additive(fewer) + aggregate (aggregate-query slice): the query closes with a two-token qualifier 'in total' after 'have'; the recognizer strips it and reads the multi-part sum, honored ONLY for the multi-part form. hank = gail - 6 = 14; total = gail + hank = 34."}
{"id": "r1-05-chain", "text": "Ivy has 4 pens. Jon has 3 times as many pens as Ivy. Kim has 2 more pens than Jon. How many pens does Kim have?", "relations": [{"kind": "fact", "entity": "ivy", "value": 4}, {"kind": "times_as_many", "entity": "jon", "ref": "ivy", "factor": 3}, {"kind": "more_than", "entity": "kim", "ref": "jon", "delta": 2}], "expected_units": {"ivy": "item", "jon": "item", "kim": "item"}, "query": {"entity": "kim", "unit": "item"}, "gold": 14, "notes": "Multi-step derived chain: jon=3*ivy (intermediate), kim=jon+2. The multiplicative middle step has no template -> must REFUSE the whole reading, never read a partial/wrong chain."}
{"id": "r1-06-subtotal-reused", "text": "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them equally into 3 boxes. How many hats are in each box?", "relations": [{"kind": "fact", "entity": "lee", "value": 5}, {"kind": "fact", "entity": "mae", "value": 7}, {"kind": "sum_of", "entity": "total", "parts": ["lee", "mae"]}, {"kind": "divide_by", "entity": "per_box", "ref": "total", "divisor": 3}], "expected_units": {"lee": "item", "mae": "item", "total": "item", "per_box": "item"}, "query": {"entity": "per_box", "unit": "item"}, "gold": 4, "notes": "Aggregate-then-divide partition (PR-6d): semantic source is equal PARTITION ('split equally into 3 boxes'); the mathematical setup is total = lee + mae (sum_of) then per_box = total / 3 (divide_by), reusing SumOf + Div with NO new relation kind. First derived-subtotal reuse: the divide's ref is itself a derived symbol. Exact-divisibility still gates the answer (total % 3 == 0; 12 / 3 = 4)."}
{"id": "r1-07-inverse", "text": "Nia has 9 more beads than Omar. Nia has 15 beads. How many beads does Omar have?", "relations": [{"kind": "fact", "entity": "nia", "value": 15}, {"kind": "more_than", "entity": "nia", "ref": "omar", "delta": 9}, {"kind": "ask_base", "entity": "omar"}], "expected_units": {"nia": "item", "omar": "item"}, "query": {"entity": "omar", "unit": "item"}, "notes": "Inverse target: omar = nia - 9 = 6, asked via the base of a more_than. The reader may read the STRUCTURE (nia=15, nia=omar+9, ask omar) — that setup is correct; only the solver can't invert. So setup_correct OR refuse, but NEVER setup_wrong."}
{"id": "r1-07-inverse", "text": "Nia has 9 more beads than Omar. Nia has 15 beads. How many beads does Omar have?", "relations": [{"kind": "fact", "entity": "nia", "value": 15}, {"kind": "more_than", "entity": "nia", "ref": "omar", "delta": 9}], "expected_units": {"nia": "item", "omar": "item"}, "query": {"entity": "omar", "unit": "item"}, "gold": 6, "notes": "Inverse target: omar = nia - 9 = 6, asked via the base of a more_than (nia=15, nia=omar+9, ask omar). PR-7a pins the ORACLE reverse-solve: the gold relations feed oracle_answer (a more_than whose entity is grounded inverts to its ref). The reader is unchanged and STILL refuses (admissibility_refused), so this stays setup_refused / answer-refused until PR-7b reads the inverse frame; gold=6 is dormant until then. The non-consumed ask_base annotation was dropped to match the reader's future projection. setup_correct OR refuse, but NEVER setup_wrong."}
{"id": "r1-08-ambiguous-referent", "text": "Pat has 5 marbles. He has 3 more than her. How many marbles does she have?", "relations": [], "expected_units": {}, "query": {"entity": "she", "unit": "item"}, "notes": "Ambiguous pronoun referents (he/her/she) with no grounded base for 'her'/'she' -> must REFUSE (no honest reading), never bind a guessed referent."}
{"id": "r1-09-missing-base", "text": "Quinn has twice as many toys as Rosa. How many toys does Quinn have?", "relations": [{"kind": "times_as_many", "entity": "quinn", "ref": "rosa", "factor": 2}], "expected_units": {"quinn": "item", "rosa": "item"}, "query": {"entity": "quinn", "unit": "item"}, "notes": "Missing base quantity: Rosa's count is never given, so Quinn is underdetermined -> must REFUSE."}
{"id": "r1-10-distractor", "text": "Sam has 7 pencils and 3 erasers. Tom has 4 more pencils than Sam. How many pencils does Tom have?", "relations": [{"kind": "fact", "entity": "sam", "value": 7}, {"kind": "more_than", "entity": "tom", "ref": "sam", "delta": 4}], "expected_units": {"sam": "item", "tom": "item"}, "query": {"entity": "tom", "unit": "item"}, "notes": "Distractor quantity (3 erasers) in a compound clause the 'X has N unit' template can't parse -> must REFUSE rather than mis-bind the distractor."}

96
tests/test_setup_oracle.py
View file

@ -306,3 +306,99 @@ def test_oracle_divide_by_one_is_identity() -> None:
],
{"entity": "dora"},
) == 8
# --------------------------------------------------------------------------- #
# PR-7a — narrow reverse-solve oracle contract (the base of one more/fewer_than).
# Pins the EXACT semantics before the reader learns the inverse frame (PR-7b). The
# reader is unchanged here: r1-07 still refuses; these exercise the oracle directly.
# Each refusal is meaningful-fail — drop its guardrail and the case computes a value.
# --------------------------------------------------------------------------- #
def _F(e, v):
return {"kind": "fact", "entity": e, "value": v}
def _M(e, r, d):
return {"kind": "more_than", "entity": e, "ref": r, "delta": d}
def _W(e, r, d):
return {"kind": "fewer_than", "entity": e, "ref": r, "delta": d}
def test_oracle_reverse_solves_more_than_base() -> None:
# Nia has 9 more beads than Omar. Nia has 15. -> omar = 15 - 9 = 6. (r1-07 gold)
assert oracle_answer([_F("nia", 15), _M("nia", "omar", 9)], {"entity": "omar"}) == 6
def test_oracle_reverse_solves_fewer_than_base() -> None:
# Pat has 3 fewer than Quinn. Pat has 4. -> quinn = 4 + 3 = 7.
assert oracle_answer([_F("pat", 4), _W("pat", "quinn", 3)], {"entity": "quinn"}) == 7
def test_r1_07_gold_relations_reverse_solve_to_six() -> None:
# The exact gold relations the PR-7b answer lane will feed the oracle compute gold=6.
from evals.setup_oracle.runner import _load_r1_gold
fx = next(f for f in run_r1()["details"] if f["id"] == "r1-07-inverse")
assert fx["outcome"] == "refused" # reader still refuses in PR-7a (contract only)
gold = next(g for g in _load_r1_gold() if g["id"] == "r1-07-inverse")
assert oracle_answer(gold["relations"], gold["query"]) == gold["gold"] == 6
def test_oracle_reverse_solve_refuses_negative_count() -> None:
# Nia has 9 more than Omar. Nia has 5. -> omar = -4 < 0: refuse, never a negative count.
with pytest.raises(OracleError):
oracle_answer([_F("nia", 5), _M("nia", "omar", 9)], {"entity": "omar"})
def test_oracle_reverse_solve_refuses_multiple_bases() -> None:
# Two inverse constraints -> not a single base: refuse (no multi-inverse / no system).
with pytest.raises(OracleError):
oracle_answer(
[_F("a", 10), _F("b", 8), _M("a", "x", 2), _M("b", "x", 1)], {"entity": "x"}
)
def test_oracle_reverse_solve_refuses_grounded_base() -> None:
# The base is otherwise grounded -> over-determined: refuse rather than ignore a side.
with pytest.raises(OracleError):
oracle_answer(
[_F("nia", 15), _F("omar", 6), _M("nia", "omar", 9)], {"entity": "omar"}
)
def test_oracle_reverse_solve_refuses_base_not_target() -> None:
# The inverse base is not the asked entity (a chain): refuse, never solve through.
with pytest.raises(OracleError):
oracle_answer([_F("nia", 15), _M("nia", "omar", 9)], {"entity": "zed"})
def test_oracle_reverse_solve_refuses_over_times_as_many() -> None:
# No reverse-solve over times_as_many: Nia has twice as many as Omar; Nia has 14 -> refuse.
with pytest.raises(OracleError):
oracle_answer(
[_F("nia", 14), {"kind": "times_as_many", "entity": "nia", "ref": "omar", "factor": 2}],
{"entity": "omar"},
)
def test_oracle_forward_paths_unchanged_by_reverse_solve() -> None:
# Regression guard: every forward path still computes (the duplicate-check refactor
# must not perturb forward more/fewer/times/divide/sum).
assert oracle_answer([_F("a", 6), _M("b", "a", 4)], {"entity": "b"}) == 10
assert oracle_answer([_F("a", 6), _W("b", "a", 4)], {"entity": "b"}) == 2
assert oracle_answer(
[_F("a", 6), {"kind": "times_as_many", "entity": "b", "ref": "a", "factor": 3}],
{"entity": "b"},
) == 18
assert oracle_answer(
[_F("a", 8), {"kind": "divide_by", "entity": "b", "ref": "a", "divisor": 2}],
{"entity": "b"},
) == 4
assert oracle_answer(
[_F("a", 6), _F("b", 4), {"kind": "sum_of", "entity": "t", "parts": ["a", "b"]}],
{"entity": "t"},
) == 10