From 121362c52abdcf179a14cb82124b92a99bf03449 Mon Sep 17 00:00:00 2001 From: Shay Date: Sun, 7 Jun 2026 07:26:00 -0700 Subject: [PATCH] feat(answer-choices): multiple-choice verifier with contradiction flag (R2 C4) generate/answer_choices/{parse,verify}.py: parse_options normalizes {label:value} to {label:int} (int or single-integer string; ambiguous/empty refuse). verify_answer_choice ties a PROVEN value to exactly one option -> ChoiceVerdict(consistent); a disagreeing key -> ChoiceVerdict(contradiction) naming both the consistent answer and the wrong key (truth discipline, not a refusal). Refuses no_matching_option / ambiguous_options / unknown_provided_label. End-to-end with C2 gold + C3 solver: every solved fixture solves, ties to its labeled answer, confirms consistent. Off-serving. 9 tests incl. contradiction-flag meaningful-fail. --- generate/answer_choices/__init__.py | 24 +++++++++ generate/answer_choices/parse.py | 49 +++++++++++++++++ generate/answer_choices/verify.py | 81 ++++++++++++++++++++++++++++ tests/test_answer_choices.py | 84 +++++++++++++++++++++++++++++ 4 files changed, 238 insertions(+) create mode 100644 generate/answer_choices/__init__.py create mode 100644 generate/answer_choices/parse.py create mode 100644 generate/answer_choices/verify.py create mode 100644 tests/test_answer_choices.py diff --git a/generate/answer_choices/__init__.py b/generate/answer_choices/__init__.py new file mode 100644 index 00000000..2fd25dd8 --- /dev/null +++ b/generate/answer_choices/__init__.py @@ -0,0 +1,24 @@ +"""Multiple-choice answer verification (off-serving). + +Ties a PROVEN value to exactly one labeled option and flags answer-key contradictions — the +engine asserts the consistent answer and names a wrong key, never silently accepting it. Used +by the R2 constraint organ (and reusable by any lane that proves an integer answer). Imports no +``generate.derivation`` / ``core.reliability_gate``. +""" + +from __future__ import annotations + +from generate.answer_choices.parse import parse_option_value, parse_options +from generate.answer_choices.verify import ( + ChoiceVerdict, + VERDICT_STATUSES, + verify_answer_choice, +) + +__all__ = [ + "ChoiceVerdict", + "VERDICT_STATUSES", + "parse_option_value", + "parse_options", + "verify_answer_choice", +] diff --git a/generate/answer_choices/parse.py b/generate/answer_choices/parse.py new file mode 100644 index 00000000..9c65e309 --- /dev/null +++ b/generate/answer_choices/parse.py @@ -0,0 +1,49 @@ +"""Parse a multiple-choice option map into normalized integer values (R2 C4). + +Options arrive as ``{label: value}``. A value may be a bare integer (the R2 gold form) or a +string carrying exactly one integer (``"11"``, ``"11 chickens"``, ``"$11"``). A string with +zero or several integers denotes no single value and REFUSES — the verifier must never guess +which number an ambiguous option meant. Off-serving; deterministic. +""" + +from __future__ import annotations + +import re +from typing import Any + +from generate.meaning_graph.reader import Refusal + +_INT_RE = re.compile(r"-?\d+") + + +def parse_option_value(value: Any) -> int | None: + """The integer an option denotes, or ``None`` if it denotes no single integer. + + An ``int`` is taken verbatim; a ``str`` is accepted iff it carries exactly one integer + (so ``"between 5 and 10"`` -> ``None``). ``bool`` is rejected (``True`` is not a count). + """ + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, str): + found = _INT_RE.findall(value) + if len(found) == 1: + return int(found[0]) + return None + + +def parse_options(raw: Any) -> dict[str, int] | Refusal: + """Normalize ``{label: value}`` into ``{label: int}``; refuse an empty or unparseable map.""" + if not isinstance(raw, dict) or not raw: + return Refusal("no_options") + out: dict[str, int] = {} + for label, value in raw.items(): + parsed = parse_option_value(value) + if parsed is None: + return Refusal("unparseable_option", f"{label}: {value!r}") + out[str(label)] = parsed + return out + + +__all__ = ["parse_option_value", "parse_options"] diff --git a/generate/answer_choices/verify.py b/generate/answer_choices/verify.py new file mode 100644 index 00000000..992d39b8 --- /dev/null +++ b/generate/answer_choices/verify.py @@ -0,0 +1,81 @@ +"""Verify a computed answer against multiple-choice options, flagging key contradictions (R2 C4). + +Truth discipline (the user's Phase 5): the engine ties its PROVEN value to exactly one labeled +option. If a provided answer key disagrees with the proof, that is not a refusal — it is a +confident **contradiction** verdict ("the math says A; the key says C — the key is wrong"). The +verifier refuses only when the proof cannot be tied to exactly one option (no match, or a +duplicate-valued match). Off-serving; deterministic. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from generate.answer_choices.parse import parse_options +from generate.meaning_graph.reader import Refusal + +#: A confident verdict status — NOT a refusal. ``contradiction`` asserts the key is wrong while +#: the engine's value stands; ``consistent`` confirms (or, with no key, simply labels) it. +VERDICT_STATUSES = frozenset({"consistent", "contradiction"}) + + +@dataclass(frozen=True, slots=True) +class ChoiceVerdict: + """The outcome of tying a proven value to the options. ``computed_label`` is the option the + proof matches; ``provided_label`` is the supplied key (or ``None``); ``message`` is the + user-facing sentence.""" + + computed_value: int + computed_label: str + provided_label: str | None + status: str + message: str + + +def _suffix(noun: str) -> str: + return f" {noun}" if noun else "" + + +def verify_answer_choice( + computed_value: int, options: Any, provided_label: str | None = None, *, noun: str = "" +) -> ChoiceVerdict | Refusal: + """Match the solver's proven value to the options; confirm or contradict a provided key. + + Returns a :class:`ChoiceVerdict` (``consistent`` / ``contradiction``) when the value ties to + exactly one option, else a typed :class:`Refusal` (``no_options`` / ``unparseable_option`` / + ``no_matching_option`` / ``ambiguous_options`` / ``unknown_provided_label``). + """ + parsed = parse_options(options) + if isinstance(parsed, Refusal): + return parsed + matches = sorted(label for label, value in parsed.items() if value == computed_value) + if not matches: + return Refusal("no_matching_option", f"no option equals {computed_value}") + if len(matches) > 1: + return Refusal("ambiguous_options", f"{matches} all equal {computed_value}") + + computed_label = matches[0] + suffix = _suffix(noun) + if provided_label is None or provided_label == computed_label: + return ChoiceVerdict( + computed_value, + computed_label, + provided_label, + "consistent", + f"The mathematically consistent answer is {computed_label}. {computed_value}{suffix}.", + ) + if provided_label not in parsed: + return Refusal("unknown_provided_label", str(provided_label)) + return ChoiceVerdict( + computed_value, + computed_label, + provided_label, + "contradiction", + f"The mathematically consistent answer is {computed_label} ({computed_value}{suffix}). " + f"The supplied answer key says {provided_label} ({parsed[provided_label]}{suffix}), " + f"which contradicts the equations.", + ) + + +__all__ = ["ChoiceVerdict", "VERDICT_STATUSES", "verify_answer_choice"] diff --git a/tests/test_answer_choices.py b/tests/test_answer_choices.py new file mode 100644 index 00000000..2be8c4ae --- /dev/null +++ b/tests/test_answer_choices.py @@ -0,0 +1,84 @@ +"""Tests for the R2 multiple-choice verifier (C4). + +Pins the truth-discipline behavior: a proven value ties to exactly one option (else refuse), +and a disagreeing key is flagged as a CONTRADICTION (a confident verdict, not a refusal). Ties +to the C2 gold + C3 solver end-to-end: every solved fixture solves, ties to its labeled answer, +and confirms consistent. +""" + +from __future__ import annotations + +from evals.constraint_oracle.runner import _load_r2_gold, gold_to_problem +from generate.answer_choices.parse import parse_option_value, parse_options +from generate.answer_choices.verify import ChoiceVerdict, verify_answer_choice +from generate.constraint_comprehension.solver import answer_constraint_problem +from generate.meaning_graph.reader import Refusal + + +def _solved() -> list[dict]: + return [f for f in _load_r2_gold() if f["expect"] == "solved"] + + +def test_parse_option_value_int_and_string() -> None: + assert parse_option_value(11) == 11 + assert parse_option_value("11") == 11 + assert parse_option_value("11 chickens") == 11 + assert parse_option_value("$11") == 11 + assert parse_option_value("between 5 and 10") is None # two integers -> ambiguous + assert parse_option_value(True) is None # a bool is not a count + + +def test_parse_options_refuses_empty_and_unparseable() -> None: + assert isinstance(parse_options({}), Refusal) + assert isinstance(parse_options({"A": "lots"}), Refusal) + assert parse_options({"A": 2, "B": "3 buses"}) == {"A": 2, "B": 3} + + +def test_every_solved_gold_key_is_consistent() -> None: + for fx in _solved(): + v = verify_answer_choice(fx["gold"], fx["options"], fx["answer"]) + assert isinstance(v, ChoiceVerdict), fx["id"] + assert v.status == "consistent" + assert v.computed_label == fx["answer"] + + +def test_solve_then_verify_end_to_end() -> None: + # The full off-serving chain that the reader (C5+) will feed: solve -> tie to the option. + for fx in _solved(): + computed = answer_constraint_problem(gold_to_problem(fx)) + v = verify_answer_choice(computed, fx["options"], fx["answer"], noun=fx["query"]["unit"]) + assert isinstance(v, ChoiceVerdict) and v.status == "consistent" + assert v.computed_value == fx["gold"] and v.computed_label == fx["answer"] + + +def test_disagreeing_key_is_flagged_as_contradiction() -> None: + # chickens: proven 11 == option A; a key of "D" (13) contradicts the equations. + fx = next(f for f in _solved() if f["id"] == "r2-002-chickens") + v = verify_answer_choice(11, fx["options"], "D", noun="animals") + assert isinstance(v, ChoiceVerdict) + assert v.status == "contradiction" + assert v.computed_label == "A" and v.provided_label == "D" + # The message names BOTH the consistent answer and the contradicted key. + assert "A" in v.message and "11" in v.message and "D" in v.message and "13" in v.message + assert "contradicts" in v.message + + +def test_no_matching_option_refuses() -> None: + out = verify_answer_choice(99, {"A": 2, "B": 3, "C": 4}, "A") + assert isinstance(out, Refusal) and out.reason == "no_matching_option" + + +def test_ambiguous_duplicate_options_refuse() -> None: + out = verify_answer_choice(4, {"A": 4, "B": 4}, None) + assert isinstance(out, Refusal) and out.reason == "ambiguous_options" + + +def test_unknown_provided_label_refuses() -> None: + out = verify_answer_choice(4, {"A": 2, "B": 4}, "Z") + assert isinstance(out, Refusal) and out.reason == "unknown_provided_label" + + +def test_consistent_without_a_provided_key_still_labels() -> None: + v = verify_answer_choice(4, {"A": 2, "B": 4}, None, noun="buses") + assert isinstance(v, ChoiceVerdict) and v.status == "consistent" + assert v.computed_label == "B" and "4 buses" in v.message