diff --git a/evals/relational_metric/oracle.py b/evals/relational_metric/oracle.py index da7cede1..d17e6de4 100644 --- a/evals/relational_metric/oracle.py +++ b/evals/relational_metric/oracle.py @@ -19,7 +19,13 @@ Supported relation kinds (the v1 + PR-6b forward-substitutable grammar): - ``divide_by`` : ``entity = ref // divisor`` (exact integer division only) - ``sum_of`` : ``entity = sum(parts)`` (part-whole / total) -Every relation's references must already be resolved (forward-substitutable / +Plus one narrow off-grammar extension — a reverse-solve (PR-7a): the base of a single +more_than/fewer_than whose other side is grounded (``Nia has 9 more than Omar. Nia has +15. -> Omar = 6``). It is deliberately tiny — single base constraint, base == query +target (no chains), base not otherwise grounded, non-negative count result, more/fewer +only (never times/divide). Everything wider refuses. + +Every forward relation's references must already be resolved (forward-substitutable / triangular). The oracle refuses anything else, never guesses. PR-6b/6c keep this off-serving: they let setup-correct R1 cases compute answers in the eval lane only. @@ -45,11 +51,23 @@ _SUPPORTED = frozenset( def oracle_answer(relations: list[dict[str, Any]], query: dict[str, Any]) -> int: """Compute the integer answer by forward substitution over the relations. + Plus one narrow reverse-solve (PR-7a): a single more_than/fewer_than whose entity + is grounded and whose ref is the (otherwise ungrounded) query target is inverted to + that base — ``base = entity -/+ delta``. The reverse-solve refuses on anything wider: + more than one inverse constraint, an inverse base that is not the query target (a + chain), an already-grounded base, a negative result (count domain), or an inverse + over times/divide. The +/- inversion is always integer-exact. + Raises :class:`OracleError` on an unknown relation kind, a forward reference to - an unresolved entity, a duplicate definition, malformed scalar, or a missing - query entity. + an unresolved entity, a duplicate definition, malformed scalar, an out-of-contract + reverse-solve, or a missing query entity. """ values: dict[str, int] = {} + # Deferred narrow inverse (PR-7a): a more_than/fewer_than whose ENTITY is already + # grounded (by a prior fact) and whose REF is unresolved is NOT a duplicate + # definition — it pins its ref (the unknown base). Collected here, resolved after + # the forward pass so the single-base / no-chains guardrails apply to the whole set. + inverse: list[tuple[str, str, str, int]] = [] # (kind, entity, ref, delta) for rel in relations: kind = rel.get("kind") @@ -58,10 +76,10 @@ def oracle_answer(relations: list[dict[str, Any]], query: dict[str, Any]) -> int raise OracleError(f"unsupported relation kind: {kind!r}") if not isinstance(entity, str) or not entity: raise OracleError(f"relation missing entity: {rel!r}") - if entity in values: - raise OracleError(f"duplicate definition of {entity!r}") if kind == "fact": + if entity in values: + raise OracleError(f"duplicate definition of {entity!r}") value = rel.get("value") if not isinstance(value, int) or isinstance(value, bool): raise OracleError(f"fact value must be int: {rel!r}") @@ -69,12 +87,25 @@ def oracle_answer(relations: list[dict[str, Any]], query: dict[str, Any]) -> int elif kind in ("more_than", "fewer_than"): ref = rel.get("ref") delta = rel.get("delta") - if ref not in values: - raise OracleError(f"forward reference to unresolved {ref!r}") + if not isinstance(ref, str) or not ref: + raise OracleError(f"relation missing ref: {rel!r}") if not isinstance(delta, int) or isinstance(delta, bool): raise OracleError(f"delta must be int: {rel!r}") - values[entity] = values[ref] + (delta if kind == "more_than" else -delta) + entity_known = entity in values + ref_known = ref in values + if entity_known and not ref_known: + # INVERSE: the unknown is the ref (base). Solved in phase 2. + inverse.append((kind, entity, ref, delta)) + elif not entity_known and ref_known: + values[entity] = values[ref] + (delta if kind == "more_than" else -delta) + elif not entity_known and not ref_known: + raise OracleError(f"forward reference to unresolved {ref!r}") + else: # both sides grounded -> over-determined, refuse rather than ignore + raise OracleError(f"over-determined relation (both sides known): {rel!r}") elif kind == "times_as_many": + if entity in values: + # Reverse-solve over times/divide is NOT in the PR-7a contract. + raise OracleError(f"duplicate definition of {entity!r}") ref = rel.get("ref") factor = rel.get("factor") if ref not in values: @@ -83,6 +114,8 @@ def oracle_answer(relations: list[dict[str, Any]], query: dict[str, Any]) -> int raise OracleError(f"factor must be int: {rel!r}") values[entity] = values[ref] * factor elif kind == "divide_by": + if entity in values: + raise OracleError(f"duplicate definition of {entity!r}") ref = rel.get("ref") divisor = rel.get("divisor") if ref not in values: @@ -94,6 +127,8 @@ def oracle_answer(relations: list[dict[str, Any]], query: dict[str, Any]) -> int raise OracleError(f"non-exact division {values[ref]}/{divisor}: {rel!r}") values[entity] = values[ref] // divisor else: # sum_of + if entity in values: + raise OracleError(f"duplicate definition of {entity!r}") parts = rel.get("parts") if not isinstance(parts, list) or not parts: raise OracleError(f"sum_of needs non-empty parts: {rel!r}") @@ -102,6 +137,30 @@ def oracle_answer(relations: list[dict[str, Any]], query: dict[str, Any]) -> int values[entity] = sum(values[p] for p in parts) target = query.get("entity") + + # Phase 2: narrow reverse-solve (PR-7a). The single unknown is the base of exactly + # one more_than/fewer_than whose other side is grounded. Anything wider refuses. + if inverse: + if len(inverse) != 1: + raise OracleError( + f"reverse-solve supports a single base constraint only: {inverse!r}" + ) + kind, ent, ref, delta = inverse[0] + if target in values: + raise OracleError("reverse-solve constraint present but target already determined") + if ref != target: + raise OracleError( + f"reverse-solve base {ref!r} is not the query target {target!r} (no chains)" + ) + if ref in values: + raise OracleError(f"reverse-solve base {ref!r} is already grounded") + # more_than: entity = base + delta -> base = entity - delta; + # fewer_than: entity = base - delta -> base = entity + delta. (Always exact for +/-.) + result = values[ent] - delta if kind == "more_than" else values[ent] + delta + if result < 0: + raise OracleError(f"reverse-solve base is negative (count domain): {result}") + values[ref] = result + if target not in values: raise OracleError(f"query entity {target!r} not resolved") return values[target] diff --git a/evals/setup_oracle/r1_gold.jsonl b/evals/setup_oracle/r1_gold.jsonl index 76007365..62c4bc83 100644 --- a/evals/setup_oracle/r1_gold.jsonl +++ b/evals/setup_oracle/r1_gold.jsonl @@ -4,7 +4,7 @@ {"id": "r1-04-fewer-total", "text": "Gail has 20 cards. Hank has 6 fewer cards than Gail. How many cards do Gail and Hank have in total?", "relations": [{"kind": "fact", "entity": "gail", "value": 20}, {"kind": "fewer_than", "entity": "hank", "ref": "gail", "delta": 6}, {"kind": "sum_of", "entity": "total", "parts": ["gail", "hank"]}], "expected_units": {"gail": "item", "hank": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "gold": 34, "notes": "Additive(fewer) + aggregate (aggregate-query slice): the query closes with a two-token qualifier 'in total' after 'have'; the recognizer strips it and reads the multi-part sum, honored ONLY for the multi-part form. hank = gail - 6 = 14; total = gail + hank = 34."} {"id": "r1-05-chain", "text": "Ivy has 4 pens. Jon has 3 times as many pens as Ivy. Kim has 2 more pens than Jon. How many pens does Kim have?", "relations": [{"kind": "fact", "entity": "ivy", "value": 4}, {"kind": "times_as_many", "entity": "jon", "ref": "ivy", "factor": 3}, {"kind": "more_than", "entity": "kim", "ref": "jon", "delta": 2}], "expected_units": {"ivy": "item", "jon": "item", "kim": "item"}, "query": {"entity": "kim", "unit": "item"}, "gold": 14, "notes": "Multi-step derived chain: jon=3*ivy (intermediate), kim=jon+2. The multiplicative middle step has no template -> must REFUSE the whole reading, never read a partial/wrong chain."} {"id": "r1-06-subtotal-reused", "text": "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them equally into 3 boxes. How many hats are in each box?", "relations": [{"kind": "fact", "entity": "lee", "value": 5}, {"kind": "fact", "entity": "mae", "value": 7}, {"kind": "sum_of", "entity": "total", "parts": ["lee", "mae"]}, {"kind": "divide_by", "entity": "per_box", "ref": "total", "divisor": 3}], "expected_units": {"lee": "item", "mae": "item", "total": "item", "per_box": "item"}, "query": {"entity": "per_box", "unit": "item"}, "gold": 4, "notes": "Aggregate-then-divide partition (PR-6d): semantic source is equal PARTITION ('split equally into 3 boxes'); the mathematical setup is total = lee + mae (sum_of) then per_box = total / 3 (divide_by), reusing SumOf + Div with NO new relation kind. First derived-subtotal reuse: the divide's ref is itself a derived symbol. Exact-divisibility still gates the answer (total % 3 == 0; 12 / 3 = 4)."} -{"id": "r1-07-inverse", "text": "Nia has 9 more beads than Omar. Nia has 15 beads. How many beads does Omar have?", "relations": [{"kind": "fact", "entity": "nia", "value": 15}, {"kind": "more_than", "entity": "nia", "ref": "omar", "delta": 9}, {"kind": "ask_base", "entity": "omar"}], "expected_units": {"nia": "item", "omar": "item"}, "query": {"entity": "omar", "unit": "item"}, "notes": "Inverse target: omar = nia - 9 = 6, asked via the base of a more_than. The reader may read the STRUCTURE (nia=15, nia=omar+9, ask omar) — that setup is correct; only the solver can't invert. So setup_correct OR refuse, but NEVER setup_wrong."} +{"id": "r1-07-inverse", "text": "Nia has 9 more beads than Omar. Nia has 15 beads. How many beads does Omar have?", "relations": [{"kind": "fact", "entity": "nia", "value": 15}, {"kind": "more_than", "entity": "nia", "ref": "omar", "delta": 9}], "expected_units": {"nia": "item", "omar": "item"}, "query": {"entity": "omar", "unit": "item"}, "gold": 6, "notes": "Inverse target: omar = nia - 9 = 6, asked via the base of a more_than (nia=15, nia=omar+9, ask omar). PR-7a pins the ORACLE reverse-solve: the gold relations feed oracle_answer (a more_than whose entity is grounded inverts to its ref). The reader is unchanged and STILL refuses (admissibility_refused), so this stays setup_refused / answer-refused until PR-7b reads the inverse frame; gold=6 is dormant until then. The non-consumed ask_base annotation was dropped to match the reader's future projection. setup_correct OR refuse, but NEVER setup_wrong."} {"id": "r1-08-ambiguous-referent", "text": "Pat has 5 marbles. He has 3 more than her. How many marbles does she have?", "relations": [], "expected_units": {}, "query": {"entity": "she", "unit": "item"}, "notes": "Ambiguous pronoun referents (he/her/she) with no grounded base for 'her'/'she' -> must REFUSE (no honest reading), never bind a guessed referent."} {"id": "r1-09-missing-base", "text": "Quinn has twice as many toys as Rosa. How many toys does Quinn have?", "relations": [{"kind": "times_as_many", "entity": "quinn", "ref": "rosa", "factor": 2}], "expected_units": {"quinn": "item", "rosa": "item"}, "query": {"entity": "quinn", "unit": "item"}, "notes": "Missing base quantity: Rosa's count is never given, so Quinn is underdetermined -> must REFUSE."} {"id": "r1-10-distractor", "text": "Sam has 7 pencils and 3 erasers. Tom has 4 more pencils than Sam. How many pencils does Tom have?", "relations": [{"kind": "fact", "entity": "sam", "value": 7}, {"kind": "more_than", "entity": "tom", "ref": "sam", "delta": 4}], "expected_units": {"sam": "item", "tom": "item"}, "query": {"entity": "tom", "unit": "item"}, "notes": "Distractor quantity (3 erasers) in a compound clause the 'X has N unit' template can't parse -> must REFUSE rather than mis-bind the distractor."} diff --git a/tests/test_setup_oracle.py b/tests/test_setup_oracle.py index 2f04ed08..7b442e53 100644 --- a/tests/test_setup_oracle.py +++ b/tests/test_setup_oracle.py @@ -306,3 +306,99 @@ def test_oracle_divide_by_one_is_identity() -> None: ], {"entity": "dora"}, ) == 8 + + +# --------------------------------------------------------------------------- # +# PR-7a — narrow reverse-solve oracle contract (the base of one more/fewer_than). +# Pins the EXACT semantics before the reader learns the inverse frame (PR-7b). The +# reader is unchanged here: r1-07 still refuses; these exercise the oracle directly. +# Each refusal is meaningful-fail — drop its guardrail and the case computes a value. +# --------------------------------------------------------------------------- # + + +def _F(e, v): + return {"kind": "fact", "entity": e, "value": v} + + +def _M(e, r, d): + return {"kind": "more_than", "entity": e, "ref": r, "delta": d} + + +def _W(e, r, d): + return {"kind": "fewer_than", "entity": e, "ref": r, "delta": d} + + +def test_oracle_reverse_solves_more_than_base() -> None: + # Nia has 9 more beads than Omar. Nia has 15. -> omar = 15 - 9 = 6. (r1-07 gold) + assert oracle_answer([_F("nia", 15), _M("nia", "omar", 9)], {"entity": "omar"}) == 6 + + +def test_oracle_reverse_solves_fewer_than_base() -> None: + # Pat has 3 fewer than Quinn. Pat has 4. -> quinn = 4 + 3 = 7. + assert oracle_answer([_F("pat", 4), _W("pat", "quinn", 3)], {"entity": "quinn"}) == 7 + + +def test_r1_07_gold_relations_reverse_solve_to_six() -> None: + # The exact gold relations the PR-7b answer lane will feed the oracle compute gold=6. + from evals.setup_oracle.runner import _load_r1_gold + + fx = next(f for f in run_r1()["details"] if f["id"] == "r1-07-inverse") + assert fx["outcome"] == "refused" # reader still refuses in PR-7a (contract only) + gold = next(g for g in _load_r1_gold() if g["id"] == "r1-07-inverse") + assert oracle_answer(gold["relations"], gold["query"]) == gold["gold"] == 6 + + +def test_oracle_reverse_solve_refuses_negative_count() -> None: + # Nia has 9 more than Omar. Nia has 5. -> omar = -4 < 0: refuse, never a negative count. + with pytest.raises(OracleError): + oracle_answer([_F("nia", 5), _M("nia", "omar", 9)], {"entity": "omar"}) + + +def test_oracle_reverse_solve_refuses_multiple_bases() -> None: + # Two inverse constraints -> not a single base: refuse (no multi-inverse / no system). + with pytest.raises(OracleError): + oracle_answer( + [_F("a", 10), _F("b", 8), _M("a", "x", 2), _M("b", "x", 1)], {"entity": "x"} + ) + + +def test_oracle_reverse_solve_refuses_grounded_base() -> None: + # The base is otherwise grounded -> over-determined: refuse rather than ignore a side. + with pytest.raises(OracleError): + oracle_answer( + [_F("nia", 15), _F("omar", 6), _M("nia", "omar", 9)], {"entity": "omar"} + ) + + +def test_oracle_reverse_solve_refuses_base_not_target() -> None: + # The inverse base is not the asked entity (a chain): refuse, never solve through. + with pytest.raises(OracleError): + oracle_answer([_F("nia", 15), _M("nia", "omar", 9)], {"entity": "zed"}) + + +def test_oracle_reverse_solve_refuses_over_times_as_many() -> None: + # No reverse-solve over times_as_many: Nia has twice as many as Omar; Nia has 14 -> refuse. + with pytest.raises(OracleError): + oracle_answer( + [_F("nia", 14), {"kind": "times_as_many", "entity": "nia", "ref": "omar", "factor": 2}], + {"entity": "omar"}, + ) + + +def test_oracle_forward_paths_unchanged_by_reverse_solve() -> None: + # Regression guard: every forward path still computes (the duplicate-check refactor + # must not perturb forward more/fewer/times/divide/sum). + assert oracle_answer([_F("a", 6), _M("b", "a", 4)], {"entity": "b"}) == 10 + assert oracle_answer([_F("a", 6), _W("b", "a", 4)], {"entity": "b"}) == 2 + assert oracle_answer( + [_F("a", 6), {"kind": "times_as_many", "entity": "b", "ref": "a", "factor": 3}], + {"entity": "b"}, + ) == 18 + assert oracle_answer( + [_F("a", 8), {"kind": "divide_by", "entity": "b", "ref": "a", "divisor": 2}], + {"entity": "b"}, + ) == 4 + assert oracle_answer( + [_F("a", 6), _F("b", 4), {"kind": "sum_of", "entity": "t", "parts": ["a", "b"]}], + {"entity": "t"}, + ) == 10