diff --git a/teaching/math_composition_ratification.py b/teaching/math_composition_ratification.py index bd98b727..9811515e 100644 --- a/teaching/math_composition_ratification.py +++ b/teaching/math_composition_ratification.py @@ -236,6 +236,7 @@ def apply_composition_claim( surface_pattern: str | None = None, pack_root: Path | None = None, evidence_source: str = "math_audit", + ratifier_kind: str = "cli", ) -> CompositionRatificationReceipt: """Apply a reviewed composition claim to a math composition source file. @@ -343,6 +344,7 @@ def apply_composition_claim( ) existing_evidence.append(claim.evidence_hash) existing["evidence_hashes"] = sorted(set(existing_evidence)) + existing["ratifier_kind"] = ratifier_kind is_duplicate_evidence = True else: entries.append( @@ -352,6 +354,7 @@ def apply_composition_claim( "polarity": polarity, "provenance": provenance, "evidence_hashes": [claim.evidence_hash], + "ratifier_kind": ratifier_kind, } ) diff --git a/teaching/math_frame_ratification.py b/teaching/math_frame_ratification.py index a2133c52..f565d7c9 100644 --- a/teaching/math_frame_ratification.py +++ b/teaching/math_frame_ratification.py @@ -223,6 +223,7 @@ def apply_frame_claim( reviewer: str, pack_root: Path | None = None, evidence_source: str = "math_audit", + ratifier_kind: str = "cli", ) -> FrameRatificationReceipt: """Apply a reviewed frame claim to a math frame source file. @@ -320,6 +321,7 @@ def apply_frame_claim( ) existing_evidence.append(claim.evidence_hash) existing["evidence_hashes"] = sorted(set(existing_evidence)) + existing["ratifier_kind"] = ratifier_kind is_duplicate_evidence = True else: entries.append( @@ -329,6 +331,7 @@ def apply_frame_claim( "polarity": polarity, "provenance": provenance, "evidence_hashes": [claim.evidence_hash], + "ratifier_kind": ratifier_kind, } ) diff --git a/teaching/math_lexical_ratification.py b/teaching/math_lexical_ratification.py index fbe28e22..bb3a06db 100644 --- a/teaching/math_lexical_ratification.py +++ b/teaching/math_lexical_ratification.py @@ -197,6 +197,7 @@ def apply_lexical_claim( category: str, reviewer: str, pack_root: Path | None = None, + ratifier_kind: str = "cli", ) -> RatificationReceipt: """Apply a reviewed lexical claim to a math source lexicon file. @@ -267,6 +268,7 @@ def apply_lexical_claim( ) aliases.append(lemma) parent["aliases"] = sorted(set(aliases)) + parent["ratifier_kind"] = ratifier_kind is_alias = True aliased_to = parent_lemma else: @@ -276,6 +278,7 @@ def apply_lexical_claim( "category": category, "aliases": [], "provenance": provenance, + "ratifier_kind": ratifier_kind, } ) diff --git a/tests/test_workbench_operator_telemetry.py b/tests/test_workbench_operator_telemetry.py new file mode 100644 index 00000000..89ec392a --- /dev/null +++ b/tests/test_workbench_operator_telemetry.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import json +from pathlib import Path +import pytest + +from chat.telemetry import JsonlBufferSink +from tests.workbench_test_helper import setup_isolated_workbench, make_and_write_proposal + + +def test_workbench_operator_telemetry_emission_and_redaction(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + api = setup_isolated_workbench(tmp_path, monkeypatch) + sink = JsonlBufferSink() + api.attach_telemetry_sink(sink) + + # Generate proposal + proposal = make_and_write_proposal( + api, + proposed_change_kind="vocabulary_addition", + evidence_surface="testlemma", + evidence_sub_type="lexical", + missing_operator="drain_token", + refusal_reason="unknown_word", + ) + + # 1. Test operator_ratify success + body = json.dumps({"category": "drain_token"}).encode("utf-8") + response_ratify = api.handle("POST", f"/math-proposals/{proposal.proposal_id}/ratify", body) + assert response_ratify.status == 200 + + assert len(sink.lines) == 1 + event = json.loads(sink.lines[-1]) + assert event["event"] == "operator_ratify" + assert event["proposal_id"] == proposal.proposal_id + assert event["handler"] == "LexicalClaim" + assert event["outcome"] == "applied" + assert event["ratifier_kind"] == "workbench" + # Ensure no evidence surface is leaked + assert "testlemma" not in sink.lines[-1] + + # 2. Test operator_ratify failure (rejected_precondition) + # Re-ratifying will fail (AlreadyRatified) + api.handle("POST", f"/math-proposals/{proposal.proposal_id}/ratify", body) + assert len(sink.lines) == 2 + event_fail = json.loads(sink.lines[-1]) + assert event_fail["event"] == "operator_ratify" + assert event_fail["outcome"] == "rejected_precondition" + assert "testlemma" not in sink.lines[-1] + + # 3. Test operator_reject + body_reject = json.dumps({"note": "Spam proposal"}).encode("utf-8") + response_reject = api.handle("POST", f"/math-proposals/{proposal.proposal_id}/reject", body_reject) + assert response_reject.status == 200 + + assert len(sink.lines) == 3 + event_reject = json.loads(sink.lines[-1]) + assert event_reject["event"] == "operator_reject" + assert event_reject["note"] == "Spam proposal" + assert event_reject["handler"] == "LexicalClaim" + assert "testlemma" not in sink.lines[-1] + + # 4. Test operator_defer + response_defer = api.handle("POST", f"/math-proposals/{proposal.proposal_id}/defer", b"") + assert response_defer.status == 200 + + assert len(sink.lines) == 4 + event_defer = json.loads(sink.lines[-1]) + assert event_defer["event"] == "operator_defer" + assert event_defer["handler"] == "LexicalClaim" + assert "testlemma" not in sink.lines[-1] diff --git a/tests/test_workbench_ratify_case_0050_hazard_pin.py b/tests/test_workbench_ratify_case_0050_hazard_pin.py new file mode 100644 index 00000000..e82711c4 --- /dev/null +++ b/tests/test_workbench_ratify_case_0050_hazard_pin.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import json +import functools +from pathlib import Path +import pytest + +from generate.comprehension import lexicon as comprehension_lexicon +from generate.comprehension import lifecycle +from generate.comprehension.audit import audit_problem +from generate.comprehension.state import ReaderRefusal +from tests.workbench_test_helper import setup_isolated_workbench, make_and_write_proposal + + +REPO_ROOT = Path(__file__).resolve().parents[1] +CASES_PATH = REPO_ROOT / "evals" / "gsm8k_math" / "train_sample" / "v1" / "cases.jsonl" + + +def _use_pack_for_reader(monkeypatch: pytest.MonkeyPatch, pack_root: Path) -> None: + comprehension_lexicon._CACHE.clear() + + @functools.cache + def _tmp_lexicon(): + return comprehension_lexicon.load_lexicon(pack_root) + + monkeypatch.setattr(lifecycle, "_get_lexicon", _tmp_lexicon) + + +def test_case_0050_remains_refused_after_ui_ratification(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + api = setup_isolated_workbench(tmp_path, monkeypatch) + pack_root = tmp_path / "en_core_math_v1" + + # 1. UI Ratify Lexical Claim + proposal_lex = make_and_write_proposal( + api, + proposed_change_kind="vocabulary_addition", + evidence_surface="testlemma", + evidence_sub_type="lexical", + missing_operator="drain_token", + refusal_reason="unknown_word", + ) + body_lex = json.dumps({"category": "drain_token"}).encode("utf-8") + resp_lex = api.handle("POST", f"/math-proposals/{proposal_lex.proposal_id}/ratify", body_lex) + assert resp_lex.status == 200 + + # 2. UI Ratify Frame Claim + proposal_frame = make_and_write_proposal( + api, + proposed_change_kind="frame_reclassification", + evidence_surface="gives", + evidence_sub_type="frame", + missing_operator="transfer_frame", + refusal_reason="unexpected_category", + ) + body_frame = json.dumps({"category": "transfer_frame", "polarity": "affirms"}).encode("utf-8") + resp_frame = api.handle("POST", f"/math-proposals/{proposal_frame.proposal_id}/ratify", body_frame) + assert resp_frame.status == 200 + + # 3. UI Ratify Composition Claim + proposal_comp = make_and_write_proposal( + api, + proposed_change_kind="composition_reclassification", + evidence_surface="each", + evidence_sub_type="composition", + missing_operator="quantity_extraction", + refusal_reason="incomplete_operation", + ) + body_comp = json.dumps({"category": "multiplicative_composition", "polarity": "affirms"}).encode("utf-8") + resp_comp = api.handle("POST", f"/math-proposals/{proposal_comp.proposal_id}/ratify", body_comp) + assert resp_comp.status == 200 + + # Hook the reader to use the mutated pack copy + _use_pack_for_reader(monkeypatch, pack_root) + + # Load and check case 0050 + cases = [json.loads(line) for line in CASES_PATH.read_text().splitlines()] + case = next(c for c in cases if c["case_id"] == "gsm8k-train-sample-v1-0050") + + result, _rows = audit_problem(case["question"], case_id=case["case_id"]) + + assert isinstance(result, ReaderRefusal), "case 0050 must remain refused after ratification" + assert result.sentence_index == 0 diff --git a/tests/test_workbench_ratify_category_allowlist.py b/tests/test_workbench_ratify_category_allowlist.py new file mode 100644 index 00000000..f62acc72 --- /dev/null +++ b/tests/test_workbench_ratify_category_allowlist.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import json +from pathlib import Path +import pytest + +from tests.workbench_test_helper import setup_isolated_workbench, make_and_write_proposal + + +def test_workbench_ratify_category_allowlist_enforcement(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + api = setup_isolated_workbench(tmp_path, monkeypatch) + pack_root = tmp_path / "en_core_math_v1" + + # 1. Test missing category + proposal_lex = make_and_write_proposal( + api, + proposed_change_kind="vocabulary_addition", + evidence_surface="testlemma", + evidence_sub_type="lexical", + ) + + # Sending POST with empty body should be dry-run (returns 200, applied=False) + response_dry = api.handle("POST", f"/math-proposals/{proposal_lex.proposal_id}/ratify", b"") + assert response_dry.status == 200 + assert response_dry.payload["data"]["applied"] is False + + # Sending invalid/empty category in body should raise 400 + body_empty = json.dumps({"category": ""}).encode("utf-8") + response_empty = api.handle("POST", f"/math-proposals/{proposal_lex.proposal_id}/ratify", body_empty) + assert response_empty.status == 400 + assert response_empty.payload["ok"] is False + + bad_file = pack_root / "lexicon" / "accumulation_verb.jsonl" + before_content = bad_file.read_bytes() if bad_file.exists() else b"" + + # 2. Test off-allowlist category + body_bad = json.dumps({"category": "accumulation_verb"}).encode("utf-8") # Lexical safe list is only drain_token + response_bad = api.handle("POST", f"/math-proposals/{proposal_lex.proposal_id}/ratify", body_bad) + + assert response_bad.status == 400 + assert response_bad.payload["ok"] is False + assert "drain-class" in response_bad.payload["error"]["message"] or "restricted" in response_bad.payload["error"]["message"] + + # Verify nothing was written to accumulation_verb.jsonl + after_content = bad_file.read_bytes() if bad_file.exists() else b"" + assert before_content == after_content diff --git a/tests/test_workbench_ratify_composition.py b/tests/test_workbench_ratify_composition.py new file mode 100644 index 00000000..7ae9eba3 --- /dev/null +++ b/tests/test_workbench_ratify_composition.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import json +import getpass +from pathlib import Path +import pytest + +from teaching.math_composition_ratification import apply_composition_claim +from tests.workbench_test_helper import setup_isolated_workbench, make_and_write_proposal + + +def test_workbench_ratify_composition_byte_equivalence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + api = setup_isolated_workbench(tmp_path, monkeypatch) + pack_root = tmp_path / "en_core_math_v1" + target_file = pack_root / "compositions" / "multiplicative_composition.jsonl" + + # Write a clean target file + target_file.write_text("", encoding="utf-8") + + # Generate proposal + proposal = make_and_write_proposal( + api, + proposed_change_kind="composition_reclassification", + evidence_surface="each", + evidence_sub_type="composition", + missing_operator="quantity_extraction", + refusal_reason="incomplete_operation", + ) + + # 1. UI Ratify + body = json.dumps({"category": "multiplicative_composition", "polarity": "affirms"}).encode("utf-8") + response = api.handle("POST", f"/math-proposals/{proposal.proposal_id}/ratify", body) + + assert response.status == 200 + assert response.payload["ok"] is True + assert response.payload["data"]["applied"] is True + + ui_lines = target_file.read_bytes().splitlines() + assert len(ui_lines) == 1 + + # Clear target file + target_file.write_text("", encoding="utf-8") + + # 2. CLI Ratify + claim = proposal.evidence_pointers[0] + apply_composition_claim( + claim=claim, + composition_category="multiplicative_composition", + polarity="affirms", + reviewer=getpass.getuser(), + pack_root=pack_root, + ratifier_kind="cli", + ) + + cli_lines = target_file.read_bytes().splitlines() + assert len(cli_lines) == 1 + + # 3. Assert byte equivalence except for ratifier_kind + ui_line = ui_lines[0].decode("utf-8") + cli_line = cli_lines[0].decode("utf-8") + + assert ui_line.replace('"ratifier_kind": "workbench"', '"ratifier_kind": "cli"').replace('"ratifier_kind":"workbench"', '"ratifier_kind":"cli"') == cli_line + diff --git a/tests/test_workbench_ratify_cors.py b/tests/test_workbench_ratify_cors.py new file mode 100644 index 00000000..6287eb5a --- /dev/null +++ b/tests/test_workbench_ratify_cors.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import json +from unittest.mock import Mock +import pytest +from workbench.server import WorkbenchRequestHandler + + +def test_cors_host_and_origin_verification() -> None: + handler = Mock(spec=WorkbenchRequestHandler) + handler.headers = {"Host": "malicious.com", "Origin": "http://malicious-origin.com"} + handler.command = "GET" + handler.path = "/health" + handler.wfile = Mock() + + # Bind the real method to the mock instance + handler._handle = WorkbenchRequestHandler._handle.__get__(handler, WorkbenchRequestHandler) + handler._send_common_headers = Mock() + + # Run the handler + handler._handle() + + # Assert response was 400 Bad Request + handler.send_response.assert_called_with(400) + + # Verify wfile.write payload + handler.wfile.write.assert_called() + payload = json.loads(handler.wfile.write.call_args[0][0].decode("utf-8")) + assert payload["ok"] is False + assert "CORS check failed" in payload["error"] + + +def test_cors_loopback_host_and_origin_allowed() -> None: + handler = Mock(spec=WorkbenchRequestHandler) + handler.headers = {"Host": "127.0.0.1:8765", "Origin": "http://localhost:5173"} + handler.command = "GET" + handler.path = "/health" + handler.wfile = Mock() + handler.api = Mock() + handler.api.handle.return_value = Mock(status=200, payload={"ok": True}) + + # Bind the real method + handler._handle = WorkbenchRequestHandler._handle.__get__(handler, WorkbenchRequestHandler) + handler._send_common_headers = Mock() + + # Run the handler + handler._handle() + + # Assert response was 200 OK (not 400) + handler.send_response.assert_called_with(200) diff --git a/tests/test_workbench_ratify_exception_surface.py b/tests/test_workbench_ratify_exception_surface.py new file mode 100644 index 00000000..f198cb86 --- /dev/null +++ b/tests/test_workbench_ratify_exception_surface.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import json +from pathlib import Path +import pytest + +from tests.workbench_test_helper import setup_isolated_workbench, make_and_write_proposal + + +def test_workbench_ratify_exception_surface_verbatim(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + api = setup_isolated_workbench(tmp_path, monkeypatch) + + # 1. Trigger WrongClaimSubType: Lexical proposal but frame sub_type + proposal_lex = make_and_write_proposal( + api, + proposed_change_kind="vocabulary_addition", + evidence_surface="testlemma", + evidence_sub_type="frame", # wrong sub_type for LexicalClaim + missing_operator="drain_token", + refusal_reason="unknown_word", + ) + body = json.dumps({"category": "drain_token"}).encode("utf-8") + response_sub = api.handle("POST", f"/math-proposals/{proposal_lex.proposal_id}/ratify", body) + + assert response_sub.status == 400 + assert response_sub.payload["ok"] is False + # Verbatim error message from WrongClaimSubType + assert "Lexical ratification requires sub_type='lexical'" in response_sub.payload["error"]["message"] + + # 2. Trigger WrongCompositionCategory + proposal_comp = make_and_write_proposal( + api, + proposed_change_kind="composition_reclassification", + evidence_surface="each", + evidence_sub_type="composition", + missing_operator="quantity_extraction", + refusal_reason="incomplete_operation", + ) + body_bad_comp = json.dumps({"category": "distributive_composition", "polarity": "affirms"}).encode("utf-8") + response_comp = api.handle("POST", f"/math-proposals/{proposal_comp.proposal_id}/ratify", body_bad_comp) + + assert response_comp.status == 400 + assert response_comp.payload["ok"] is False + assert "CompositionClaim ratification is allowlist-only" in response_comp.payload["error"]["message"] diff --git a/tests/test_workbench_ratify_frame.py b/tests/test_workbench_ratify_frame.py new file mode 100644 index 00000000..4da66be9 --- /dev/null +++ b/tests/test_workbench_ratify_frame.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import json +import getpass +from pathlib import Path +import pytest + +from teaching.math_frame_ratification import apply_frame_claim +from tests.workbench_test_helper import setup_isolated_workbench, make_and_write_proposal + + +def test_workbench_ratify_frame_byte_equivalence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + api = setup_isolated_workbench(tmp_path, monkeypatch) + pack_root = tmp_path / "en_core_math_v1" + target_file = pack_root / "frames" / "transfer_frame.jsonl" + + # Write a clean target file + target_file.write_text("", encoding="utf-8") + + # Generate proposal + proposal = make_and_write_proposal( + api, + proposed_change_kind="frame_reclassification", + evidence_surface="gives", + evidence_sub_type="frame", + missing_operator="transfer_frame", + refusal_reason="unexpected_category", + ) + + # 1. UI Ratify + body = json.dumps({"category": "transfer_frame", "polarity": "affirms"}).encode("utf-8") + response = api.handle("POST", f"/math-proposals/{proposal.proposal_id}/ratify", body) + + assert response.status == 200 + assert response.payload["ok"] is True + assert response.payload["data"]["applied"] is True + + ui_lines = target_file.read_bytes().splitlines() + assert len(ui_lines) == 1 + + # Clear target file + target_file.write_text("", encoding="utf-8") + + # 2. CLI Ratify + claim = proposal.evidence_pointers[0] + apply_frame_claim( + claim=claim, + frame_category="transfer_frame", + polarity="affirms", + reviewer=getpass.getuser(), + pack_root=pack_root, + ratifier_kind="cli", + ) + + cli_lines = target_file.read_bytes().splitlines() + assert len(cli_lines) == 1 + + # 3. Assert byte equivalence except for ratifier_kind + ui_line = ui_lines[0].decode("utf-8") + cli_line = cli_lines[0].decode("utf-8") + + assert ui_line.replace('"ratifier_kind": "workbench"', '"ratifier_kind": "cli"').replace('"ratifier_kind":"workbench"', '"ratifier_kind":"cli"') == cli_line + diff --git a/tests/test_workbench_ratify_idempotent.py b/tests/test_workbench_ratify_idempotent.py new file mode 100644 index 00000000..28f41076 --- /dev/null +++ b/tests/test_workbench_ratify_idempotent.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import json +from pathlib import Path +import pytest + +from tests.workbench_test_helper import setup_isolated_workbench, make_and_write_proposal + + +def test_workbench_ratify_idempotent_raises_already_ratified(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + api = setup_isolated_workbench(tmp_path, monkeypatch) + + # Generate proposal + proposal = make_and_write_proposal( + api, + proposed_change_kind="vocabulary_addition", + evidence_surface="testlemma", + evidence_sub_type="lexical", + missing_operator="drain_token", + refusal_reason="unknown_word", + ) + body = json.dumps({"category": "drain_token"}).encode("utf-8") + + # First ratification succeeds + response1 = api.handle("POST", f"/math-proposals/{proposal.proposal_id}/ratify", body) + assert response1.status == 200 + assert response1.payload["data"]["applied"] is True + + # Second ratification fails with 409 Conflict (AlreadyRatified) + response2 = api.handle("POST", f"/math-proposals/{proposal.proposal_id}/ratify", body) + assert response2.status == 409 + assert response2.payload["ok"] is False + assert "already ratified" in response2.payload["error"]["message"].lower() diff --git a/tests/test_workbench_ratify_lexical.py b/tests/test_workbench_ratify_lexical.py new file mode 100644 index 00000000..d575399f --- /dev/null +++ b/tests/test_workbench_ratify_lexical.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import json +import getpass +from pathlib import Path +import pytest + +from teaching.math_lexical_ratification import apply_lexical_claim +from tests.workbench_test_helper import setup_isolated_workbench, make_and_write_proposal + + +def test_workbench_ratify_lexical_byte_equivalence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + api = setup_isolated_workbench(tmp_path, monkeypatch) + pack_root = tmp_path / "en_core_math_v1" + target_file = pack_root / "lexicon" / "drain_token.jsonl" + + # Write a clean target file + target_file.write_text("", encoding="utf-8") + + # Generate proposal + proposal = make_and_write_proposal( + api, + proposed_change_kind="vocabulary_addition", + evidence_surface="testlemma", + evidence_sub_type="lexical", + missing_operator="drain_token", + refusal_reason="unknown_word", + ) + + # 1. UI Ratify + body = json.dumps({"category": "drain_token"}).encode("utf-8") + response = api.handle("POST", f"/math-proposals/{proposal.proposal_id}/ratify", body) + + assert response.status == 200 + assert response.payload["ok"] is True + assert response.payload["data"]["applied"] is True + + ui_lines = target_file.read_bytes().splitlines() + assert len(ui_lines) == 1 + + # Clear target file + target_file.write_text("", encoding="utf-8") + + # 2. CLI Ratify + claim = proposal.evidence_pointers[0] + apply_lexical_claim( + claim=claim, + category="drain_token", + reviewer=getpass.getuser(), + pack_root=pack_root, + ratifier_kind="cli", + ) + + cli_lines = target_file.read_bytes().splitlines() + assert len(cli_lines) == 1 + + # 3. Assert byte equivalence except for ratifier_kind + ui_line = ui_lines[0].decode("utf-8") + cli_line = cli_lines[0].decode("utf-8") + + # Replacing ratifier_kind="workbench" with "cli" in the UI-serialized line should make it identical to the CLI-serialized line. + assert ui_line.replace('"ratifier_kind": "workbench"', '"ratifier_kind": "cli"').replace('"ratifier_kind":"workbench"', '"ratifier_kind":"cli"') == cli_line + diff --git a/tests/test_workbench_ratify_no_auto.py b/tests/test_workbench_ratify_no_auto.py new file mode 100644 index 00000000..9e326758 --- /dev/null +++ b/tests/test_workbench_ratify_no_auto.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from pathlib import Path +import pytest + +from tests.workbench_test_helper import setup_isolated_workbench, make_and_write_proposal + + +def test_workbench_ratify_no_auto_transition(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + api = setup_isolated_workbench(tmp_path, monkeypatch) + pack_root = tmp_path / "en_core_math_v1" + + # 1. Create a replay-equivalent proposal in the queue + proposal = make_and_write_proposal( + api, + proposed_change_kind="vocabulary_addition", + evidence_surface="testlemma", + evidence_sub_type="lexical", + missing_operator="drain_token", + refusal_reason="unknown_word", + ) + + # 2. Check that no entries have been written to the target file + lex_file = pack_root / "lexicon" / "drain_token.jsonl" + # It might contain existing lemmas from the copy, but should NOT contain "testlemma" + content_before = lex_file.read_text(encoding="utf-8") if lex_file.exists() else "" + assert "testlemma" not in content_before + + # 3. Emulate polling/idle state: assert that even after time/polling, nothing mutates the target file + content_after_idle = lex_file.read_text(encoding="utf-8") if lex_file.exists() else "" + assert "testlemma" not in content_after_idle diff --git a/tests/test_workbench_ratify_partition.py b/tests/test_workbench_ratify_partition.py new file mode 100644 index 00000000..c55a9d3f --- /dev/null +++ b/tests/test_workbench_ratify_partition.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import json +from pathlib import Path +import pytest + +from tests.workbench_test_helper import setup_isolated_workbench, make_and_write_proposal + + +def test_workbench_ratify_partition_enforcement(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + api = setup_isolated_workbench(tmp_path, monkeypatch) + + # Create a proposal with domain="math" initially (so build_proposal succeeds) + proposal = make_and_write_proposal( + api, + domain="math", + proposed_change_kind="vocabulary_addition", + evidence_surface="testlemma", + evidence_sub_type="lexical", + missing_operator="drain_token", + refusal_reason="unknown_word", + ) + + # Overwrite the domain key directly on disk to "cognition" to test partition isolation + import workbench.readers as readers + path = readers.MATH_PROPOSALS_JSONL + lines = path.read_text(encoding="utf-8").splitlines() + modified_lines = [] + for line in lines: + if line.strip(): + record = json.loads(line) + if record["proposal_id"] == proposal.proposal_id: + record["domain"] = "cognition" + modified_lines.append(json.dumps(record, ensure_ascii=False, separators=(",", ":"))) + path.write_text("\n".join(modified_lines) + "\n", encoding="utf-8") + + + # Try to ratify it via the math API + body = json.dumps({"category": "drain_token"}).encode("utf-8") + response = api.handle("POST", f"/math-proposals/{proposal.proposal_id}/ratify", body) + + # Should be rejected with 400 Bad Request + assert response.status == 400 + assert response.payload["ok"] is False + assert "partition" in response.payload["error"]["message"].lower() diff --git a/tests/workbench_test_helper.py b/tests/workbench_test_helper.py new file mode 100644 index 00000000..c05a16b7 --- /dev/null +++ b/tests/workbench_test_helper.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any +import pytest + +from evals.refusal_taxonomy.shape_categories import ShapeCategory +from generate.comprehension.audit import AuditRow +from teaching.math_contemplation_proposal import build_proposal, to_jsonl_record +from teaching.math_evidence import from_audit_row +from teaching.math_reasoning_trace import ReasoningStep, build_trace +from workbench.api import WorkbenchApi +from workbench import readers + +REPO_ROOT = Path(__file__).resolve().parents[1] +PACK_ROOT = REPO_ROOT / "language_packs" / "data" / "en_core_math_v1" + +def setup_isolated_workbench(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> WorkbenchApi: + # 1. Copy en_core_math_v1 to tmp_path + target_pack = tmp_path / "en_core_math_v1" + shutil.copytree(PACK_ROOT, target_pack) + (target_pack / "lexicon").mkdir(exist_ok=True) + (target_pack / "frames").mkdir(exist_ok=True) + (target_pack / "compositions").mkdir(exist_ok=True) + + # Clear caches + from generate.comprehension import lexicon as comprehension_lexicon + from generate.comprehension import lifecycle + comprehension_lexicon._CACHE.clear() + lifecycle._get_lexicon.cache_clear() + + # 2. Redirect pack root in ratification handlers + monkeypatch.setattr("teaching.math_lexical_ratification._default_pack_root", lambda: target_pack) + monkeypatch.setattr("teaching.math_frame_ratification._default_pack_root", lambda: target_pack) + monkeypatch.setattr("teaching.math_composition_ratification._default_pack_root", lambda: target_pack) + + # Redirect proposal file + proposals_jsonl = tmp_path / "proposals.jsonl" + proposals_jsonl.touch() + monkeypatch.setattr(readers, "MATH_PROPOSALS_JSONL", proposals_jsonl) + + # Create WorkbenchApi + api = WorkbenchApi() + return api + +def build_test_claim(surface: str, sub_type: str, case_id: str | None = None, missing_operator: str = "lexicon_entry", refusal_reason: str = "unknown_word") -> Any: + row = AuditRow( + case_id=case_id or f"case-{surface}", + sentence_index=0, + token_index=2, + token_text=surface, + recognized_terms=("Ava", "counts"), + skipped_frame=None, + missing_operator=missing_operator, + refusal_reason=refusal_reason, + refusal_detail=f"test refusal for '{surface}'", + ) + return from_audit_row(row, sub_type) + +def write_proposal_record(api: WorkbenchApi, proposal_id: str, record: dict[str, Any]) -> None: + # Append to the mocked proposals.jsonl + path = readers.MATH_PROPOSALS_JSONL + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n") + +def make_and_write_proposal( + api: WorkbenchApi, + *, + domain: str = "math", + shape_category: ShapeCategory = ShapeCategory.UNCATEGORIZED, + proposed_change_kind: str = "vocabulary_addition", + evidence_surface: str = "widgets", + evidence_sub_type: str = "lexical", + missing_operator: str = "lexicon_entry", + refusal_reason: str = "unknown_word", + wrong_zero_assertion: str = "Proposal wrong zero assertion must be at least forty chars long.", +) -> Any: + ev1 = build_test_claim(evidence_surface, evidence_sub_type, case_id="case-1", missing_operator=missing_operator, refusal_reason=refusal_reason) + ev2 = build_test_claim(evidence_surface, evidence_sub_type, case_id="case-2", missing_operator=missing_operator, refusal_reason=refusal_reason) + steps = ( + ReasoningStep( + step_index=0, + step_kind="observation", + input_pointers=("case-1", "case-2"), + claim="refusals share pattern", + justification="grouped by key", + output_payload={"evidence_count": 2}, + ), + ReasoningStep( + step_index=1, + step_kind="grouping", + input_pointers=("case-1", "case-2"), + claim="group key", + justification="pair equality", + output_payload={"k": "v"}, + ), + ReasoningStep( + step_index=2, + step_kind="hypothesis", + input_pointers=("case-1", "case-2"), + claim="change fits", + justification="fits shape", + output_payload={"proposed_change_kind": proposed_change_kind}, + ), + ReasoningStep( + step_index=3, + step_kind="conclusion", + input_pointers=("case-1", "case-2"), + claim=f"propose {proposed_change_kind}", + justification="evidence-only proposal", + output_payload={"proposed_change_kind": proposed_change_kind}, + ), + ) + trace = build_trace(steps) + proposal = build_proposal( + domain=domain, # type: ignore[arg-type] + shape_category=shape_category, + structural_commonality="test commonality", + evidence_pointers=(ev1, ev2), + proposed_change_kind=proposed_change_kind, + proposed_change_payload={"k": "v"}, + wrong_zero_assertion=wrong_zero_assertion, + replay_equivalence_hash="0" * 64, + reasoning_trace=trace, + ) + record = to_jsonl_record(proposal) + write_proposal_record(api, proposal.proposal_id, record) + return proposal diff --git a/workbench-ui/api-schema-snapshot.json b/workbench-ui/api-schema-snapshot.json index fd448833..912c0a7c 100644 --- a/workbench-ui/api-schema-snapshot.json +++ b/workbench-ui/api-schema-snapshot.json @@ -104,7 +104,7 @@ "path": "str", "original": "Any", "replay": "Any", - "severity": "Literal['info', 'warning', 'failure']" + "severity": "ReplayDivergenceSeverity" } }, "ReplayComparison": { @@ -115,6 +115,51 @@ "equivalent": "bool", "divergences": "list[ReplayDivergence]" } + }, + "MathReasoningStep": { + "fields": { + "step_index": "int", + "step_kind": "str", + "claim": "str", + "justification": "str", + "input_pointers": "list[str]", + "output_payload": "Any" + } + }, + "MathProposalSummary": { + "fields": { + "proposal_id": "str", + "domain": "Literal['math']", + "shape_category": "str", + "proposed_change_kind": "str", + "structural_commonality": "str", + "evidence_count": "int", + "replay_equivalence_hash": "str" + } + }, + "MathProposalDetail": { + "fields": { + "wrong_zero_assertion": "str", + "proposed_change_payload": "Any", + "reasoning_trace_id": "str", + "reasoning_trace_steps": "list[MathReasoningStep]", + "evidence_hashes": "list[str]", + "handler_name": "str | None", + "suggested_ratify_cli": "str | None" + } + }, + "MathRatifyResult": { + "fields": { + "proposal_id": "str", + "change_kind": "str", + "handler_name": "str", + "routing_status": "Literal['routed', 'not_implemented']", + "message": "str", + "suggested_cli": "str | None", + "applied": "bool", + "target_path": "str | None", + "evidence_hash": "str | None" + } } } } diff --git a/workbench-ui/src/api/client.ts b/workbench-ui/src/api/client.ts index 7f7c9682..126ec28a 100644 --- a/workbench-ui/src/api/client.ts +++ b/workbench-ui/src/api/client.ts @@ -10,6 +10,9 @@ import type { ProposalDetail, ProposalState, ProposalSummary, + MathProposalSummary, + MathProposalDetail, + MathRatifyResult, } from "../types/api"; export class WorkbenchApiError extends Error { @@ -103,3 +106,52 @@ export async function fetchProposals( export async function fetchProposalDetail(proposalId: string): Promise { return apiFetch(`/proposals/${encodeURIComponent(proposalId)}`); } + +export async function fetchMathProposals(): Promise { + const envelope = await apiFetch>("/math-proposals"); + return envelope.items; +} + +export async function fetchMathProposalDetail(proposalId: string): Promise { + return apiFetch(`/math-proposals/${encodeURIComponent(proposalId)}`); +} + +export async function ratifyMathProposal( + proposalId: string, + category?: string, + polarity?: string, + dryRun?: boolean, +): Promise { + return apiFetch(`/math-proposals/${encodeURIComponent(proposalId)}/ratify`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ category, polarity, dry_run: dryRun }), + }); +} + +export async function rejectMathProposal( + proposalId: string, + note?: string, +): Promise<{ proposal_id: string; rejected: boolean }> { + return apiFetch<{ proposal_id: string; rejected: boolean }>( + `/math-proposals/${encodeURIComponent(proposalId)}/reject`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ note }), + }, + ); +} + +export async function deferMathProposal( + proposalId: string, +): Promise<{ proposal_id: string; deferred: boolean }> { + return apiFetch<{ proposal_id: string; deferred: boolean }>( + `/math-proposals/${encodeURIComponent(proposalId)}/defer`, + { + method: "POST", + }, + ); +} + + diff --git a/workbench-ui/src/api/queries.ts b/workbench-ui/src/api/queries.ts index c530fe2a..97415ffb 100644 --- a/workbench-ui/src/api/queries.ts +++ b/workbench-ui/src/api/queries.ts @@ -13,6 +13,11 @@ import { fetchReplayComparison, fetchProposalDetail, fetchProposals, + fetchMathProposals, + fetchMathProposalDetail, + ratifyMathProposal, + rejectMathProposal, + deferMathProposal, type ProposalStateFilter, } from "./client"; import type { WorkbenchApiError } from "./client"; @@ -27,6 +32,9 @@ import type { ChatTurnResult, EvalRunRequest, ReplayComparison, + MathProposalSummary, + MathProposalDetail, + MathRatifyResult, } from "../types/api"; export { QueryClientProvider }; @@ -136,3 +144,76 @@ export function useChatTurn() { }); } +export function useMathProposals() { + return useQuery({ + queryKey: ["api", "math-proposals"], + queryFn: fetchMathProposals, + staleTime: 30_000, + refetchOnWindowFocus: false, + }); +} + +export function useMathProposalDetail(proposalId: string) { + return useQuery({ + queryKey: ["api", "math-proposal", proposalId], + queryFn: () => fetchMathProposalDetail(proposalId), + enabled: !!proposalId, + staleTime: 30_000, + refetchOnWindowFocus: false, + }); +} + +export function useMathRatify() { + return useMutation< + MathRatifyResult, + WorkbenchApiError, + { proposalId: string; category?: string; polarity?: string; dryRun?: boolean } + >({ + mutationKey: ["math-ratify"], + mutationFn: ({ proposalId, category, polarity, dryRun }) => + ratifyMathProposal(proposalId, category, polarity, dryRun), + onSuccess: (data, variables) => { + queryClient.invalidateQueries({ queryKey: ["api", "proposals"] }); + queryClient.invalidateQueries({ queryKey: ["api", "math-proposals"] }); + queryClient.invalidateQueries({ queryKey: ["api", "proposal", variables.proposalId] }); + queryClient.invalidateQueries({ queryKey: ["api", "math-proposal", variables.proposalId] }); + }, + }); +} + +export function useMathReject() { + return useMutation< + { proposal_id: string; rejected: boolean }, + WorkbenchApiError, + { proposalId: string; note?: string } + >({ + mutationKey: ["math-reject"], + mutationFn: ({ proposalId, note }) => rejectMathProposal(proposalId, note), + onSuccess: (data, variables) => { + queryClient.invalidateQueries({ queryKey: ["api", "proposals"] }); + queryClient.invalidateQueries({ queryKey: ["api", "math-proposals"] }); + queryClient.invalidateQueries({ queryKey: ["api", "proposal", variables.proposalId] }); + queryClient.invalidateQueries({ queryKey: ["api", "math-proposal", variables.proposalId] }); + }, + }); +} + +export function useMathDefer() { + return useMutation< + { proposal_id: string; deferred: boolean }, + WorkbenchApiError, + { proposalId: string } + >({ + mutationKey: ["math-defer"], + mutationFn: ({ proposalId }) => deferMathProposal(proposalId), + onSuccess: (data, variables) => { + queryClient.invalidateQueries({ queryKey: ["api", "proposals"] }); + queryClient.invalidateQueries({ queryKey: ["api", "math-proposals"] }); + queryClient.invalidateQueries({ queryKey: ["api", "proposal", variables.proposalId] }); + queryClient.invalidateQueries({ queryKey: ["api", "math-proposal", variables.proposalId] }); + }, + }); +} + + + diff --git a/workbench-ui/src/app/proposals/ProposalTable.tsx b/workbench-ui/src/app/proposals/ProposalTable.tsx index 14ac246f..cc7574d8 100644 --- a/workbench-ui/src/app/proposals/ProposalTable.tsx +++ b/workbench-ui/src/app/proposals/ProposalTable.tsx @@ -11,10 +11,12 @@ const ROW_INCREMENT = 40; export function ProposalTable({ proposals, selectedProposalId, + focusedProposalId, onSelect, }: { proposals: ProposalSummary[]; selectedProposalId: string | null; + focusedProposalId?: string | null; onSelect: (proposalId: string) => void; }) { const [visibleCount, setVisibleCount] = useState(INITIAL_ROWS); @@ -44,14 +46,22 @@ export function ProposalTable({
{visibleProposals.map((proposal) => { const selected = proposal.proposal_id === selectedProposalId; + const focused = proposal.proposal_id === focusedProposalId; return (
onSelect(proposal.proposal_id)} + onKeyDown={(event) => { if (event.key === "Enter" || event.key === " ") { event.preventDefault(); diff --git a/workbench-ui/src/app/proposals/ProposalsRoute.tsx b/workbench-ui/src/app/proposals/ProposalsRoute.tsx index c7946e5b..03e9c4fa 100644 --- a/workbench-ui/src/app/proposals/ProposalsRoute.tsx +++ b/workbench-ui/src/app/proposals/ProposalsRoute.tsx @@ -1,7 +1,9 @@ import { useEffect, useMemo, useState } from "react"; import { useSearchParams } from "react-router-dom"; +import { AlertTriangle } from "lucide-react"; import { WorkbenchApiError, type ProposalStateFilter } from "../../api/client"; -import { useProposalDetail, useProposals } from "../../api/queries"; +import { useProposalDetail, useProposals, useMathProposals, useMathProposalDetail } from "../../api/queries"; +import type { ProposalSummary, MathProposalDetail, ProposalState, DownstreamEffect, MathReasoningStep } from "../../types/api"; import { Button } from "../../design/components/primitives/Button"; import { EmptyState } from "../../design/components/states/EmptyState"; import { ErrorState } from "../../design/components/states/ErrorState"; @@ -11,11 +13,19 @@ import { ProposalProvenanceViewer } from "./ProposalProvenanceViewer"; import { ProposalSummaryCard } from "./ProposalSummaryCard"; import { ProposalTable } from "./ProposalTable"; import { ReplayEvidenceCard } from "./ReplayEvidenceCard"; +import { RatificationCommandPanel } from "./RatificationCommandPanel"; const filters: ProposalStateFilter[] = ["pending", "accepted", "rejected", "all"]; function isProposalFilter(value: string | null): value is ProposalStateFilter { - return value === "pending" || value === "accepted" || value === "rejected" || value === "withdrawn" || value === "unknown" || value === "all"; + return ( + value === "pending" || + value === "accepted" || + value === "rejected" || + value === "withdrawn" || + value === "unknown" || + value === "all" + ); } function errorMessage(error: unknown) { @@ -26,13 +36,32 @@ export function ProposalsRoute() { const [searchParams, setSearchParams] = useSearchParams(); const selectedFromUrl = searchParams.get("proposal_id"); const filterFromUrl = searchParams.get("state"); + const domainFromUrl = searchParams.get("domain"); + + const [domain, setDomain] = useState<"math" | "cognition">( + domainFromUrl === "math" ? "math" : "cognition" + ); + const [filter, setFilter] = useState( isProposalFilter(filterFromUrl) ? filterFromUrl : "pending", ); - const proposalsQuery = useProposals(filter); - const selectedProposalId = selectedFromUrl ?? null; - const detailQuery = useProposalDetail(selectedProposalId ?? ""); + const [focusedIndex, setFocusedIndex] = useState(0); + + // Queries + const mathProposalsQuery = useMathProposals(); + const cognitionProposalsQuery = useProposals(filter); + + const selectedProposalId = selectedFromUrl ?? null; + + const mathDetailQuery = useMathProposalDetail( + domain === "math" ? (selectedProposalId ?? "") : "" + ); + const cognitionDetailQuery = useProposalDetail( + domain === "cognition" ? (selectedProposalId ?? "") : "" + ); + + // Synchronize state from URL useEffect(() => { const urlFilter = searchParams.get("state"); if (isProposalFilter(urlFilter) && urlFilter !== filter) { @@ -40,12 +69,62 @@ export function ProposalsRoute() { } }, [filter, searchParams]); - const proposals = useMemo(() => proposalsQuery.data ?? [], [proposalsQuery.data]); + useEffect(() => { + const urlDomain = searchParams.get("domain"); + if ((urlDomain === "math" || urlDomain === "cognition") && urlDomain !== domain) { + setDomain(urlDomain as "math" | "cognition"); + } + }, [domain, searchParams]); - function updateRoute(next: { proposalId?: string | null; state?: ProposalStateFilter }) { + // Load appropriate proposals list + const rawProposals = useMemo(() => { + if (domain === "math") { + return mathProposalsQuery.data ?? []; + } else { + return cognitionProposalsQuery.data ?? []; + } + }, [domain, mathProposalsQuery.data, cognitionProposalsQuery.data]); + + // Map to unified ProposalSummary structure + const proposals: ProposalSummary[] = useMemo(() => { + if (domain === "math") { + return (rawProposals as any[]).map((mp) => ({ + proposal_id: mp.proposal_id, + state: "pending" as ProposalState, + source_kind: `math / ${mp.proposed_change_kind}`, + replay_equivalent: true, + created_at: null, + downstream_effect: "unknown" as DownstreamEffect, + })); + } else { + return (rawProposals as ProposalSummary[]) ?? []; + } + }, [domain, rawProposals]); + + const focusedProposalId = useMemo(() => { + if (proposals.length > 0 && focusedIndex >= 0 && focusedIndex < proposals.length) { + return proposals[focusedIndex].proposal_id; + } + return null; + }, [proposals, focusedIndex]); + + // Reset focus when domain or filter changes + useEffect(() => { + setFocusedIndex(0); + }, [domain, filter]); + + function updateRoute(next: { proposalId?: string | null; state?: ProposalStateFilter; domain?: "math" | "cognition" }) { const params = new URLSearchParams(searchParams); + const nextDomain = next.domain ?? domain; const nextState = next.state ?? filter; + + if (nextDomain === "math") { + params.set("domain", "math"); + } else { + params.delete("domain"); + } params.set("state", nextState); + if (next.proposalId === null) { params.delete("proposal_id"); } else if (next.proposalId) { @@ -59,60 +138,162 @@ export function ProposalsRoute() { updateRoute({ state: nextFilter, proposalId: null }); } + function changeDomain(nextDomain: "math" | "cognition") { + setDomain(nextDomain); + updateRoute({ domain: nextDomain, proposalId: null }); + } + function selectProposal(proposalId: string) { + const index = proposals.findIndex((p) => p.proposal_id === proposalId); + if (index !== -1) { + setFocusedIndex(index); + } updateRoute({ proposalId }); } - if (proposalsQuery.isLoading) { + // Keyboard navigation wires + useEffect(() => { + const handleKeyDown = (e: KeyboardEvent) => { + if (document.activeElement?.tagName === "INPUT" || document.activeElement?.tagName === "TEXTAREA") { + return; + } + + if (e.key === "j" || e.key === "ArrowDown") { + e.preventDefault(); + setFocusedIndex((prev) => Math.min(prev + 1, proposals.length - 1)); + } else if (e.key === "k" || e.key === "ArrowUp") { + e.preventDefault(); + setFocusedIndex((prev) => Math.max(prev - 1, 0)); + } else if (e.key === "Enter") { + e.preventDefault(); + if (proposals[focusedIndex]) { + selectProposal(proposals[focusedIndex].proposal_id); + } + } else if (e.key === "Escape") { + e.preventDefault(); + updateRoute({ proposalId: null }); + } + }; + + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }, [proposals, focusedIndex]); + + // Auto-advance focus on success + const autoAdvance = () => { + let nextIndex = -1; + for (let i = focusedIndex + 1; i < proposals.length; i++) { + if (proposals[i].state === "pending" || domain === "math") { + nextIndex = i; + break; + } + } + if (nextIndex === -1) { + for (let i = 0; i < focusedIndex; i++) { + if (proposals[i].state === "pending" || domain === "math") { + nextIndex = i; + break; + } + } + } + + if (nextIndex !== -1) { + setFocusedIndex(nextIndex); + selectProposal(proposals[nextIndex].proposal_id); + } else { + updateRoute({ proposalId: null }); + } + }; + + const isLoading = domain === "math" ? mathProposalsQuery.isLoading : cognitionProposalsQuery.isLoading; + const isError = domain === "math" ? mathProposalsQuery.isError : cognitionProposalsQuery.isError; + const error = domain === "math" ? mathProposalsQuery.error : cognitionProposalsQuery.error; + + if (isLoading) { return ; } - if (proposalsQuery.isError) { + if (isError) { return ( ); } + const detailQuery = domain === "math" ? mathDetailQuery : cognitionDetailQuery; + return (
-

Proposal Queue

-
- {filters.map((state) => ( - - ))} + Math Corridor + + +
+ + {domain === "cognition" && ( +
+ {filters.map((state) => ( + + ))} +
+ )}
{proposals.length === 0 ? ( ) : ( )} -
+
{!selectedProposalId ? ( ) : detailQuery.data ? ( -
- - - - -
+ domain === "math" ? ( + + ) : ( +
+ + + + +
+ ) ) : null}
); } + +interface MathProposalDetailViewProps { + proposal: MathProposalDetail; + state: string; + replayEquivalent: boolean | null; + onSuccess: () => void; + onDefer: () => void; +} + +function MathProposalDetailView({ + proposal, + state, + replayEquivalent, + onSuccess, + onDefer, +}: MathProposalDetailViewProps) { + const steps = proposal.reasoning_trace_steps || []; + const hashes = proposal.evidence_hashes || []; + + return ( +
+ {/* Summary Card */} +
+
+

+ {proposal.proposal_id} +

+ + {proposal.shape_category} + + + math_contemplation + +
+

+ Proposed change: {proposal.proposed_change_kind} +

+
+
+
Structural Commonality
+
{proposal.structural_commonality}
+
+
+
Replay Equivalence Hash
+
+ {proposal.replay_equivalence_hash} +
+
+
+
+ + {/* Wrong Zero Assertion Card */} +
+

+ + Wrong Zero Assertion +

+

+ {proposal.wrong_zero_assertion} +

+
+ + {/* Proposed Change Payload Card */} +
+

Proposed Change Payload

+ 
+          {JSON.stringify(proposal.proposed_change_payload, null, 2)}
+
+
+ + {/* Reasoning Trace Steps */} +
+

+ Reasoning Steps ({steps.length}) +

+
+ {steps.map((step: MathReasoningStep) => ( +
+
+ Step {step.step_index}: {step.step_kind} + [{step.input_pointers?.join(", ")}] +
+

+ {step.claim} +

+

+ {step.justification} +

+
+ ))} +
+
+ + {/* Evidence Hashes */} +
+

Evidence Hashes

+
+ {hashes.map((hash: string, i: number) => ( +
+ #{i + 1} + + {hash} + +
+ ))} +
+
+ + {/* Ratification Command Panel */} + +
+ ); +} diff --git a/workbench-ui/src/app/proposals/RatificationCommandPanel.tsx b/workbench-ui/src/app/proposals/RatificationCommandPanel.tsx new file mode 100644 index 00000000..04c233fd --- /dev/null +++ b/workbench-ui/src/app/proposals/RatificationCommandPanel.tsx @@ -0,0 +1,353 @@ +import React, { useState, useEffect } from "react"; +import { Check, X, Clock, Terminal, AlertTriangle } from "lucide-react"; +import { useMathRatify, useMathReject, useMathDefer } from "../../api/queries"; +import type { MathProposalDetail } from "../../types/api"; +import { Button } from "../../design/components/primitives/Button"; +import { copyText } from "../../design/lib"; + +interface RatificationCommandPanelProps { + proposal: MathProposalDetail; + state: string; + replayEquivalent: boolean | null; + onSuccess?: () => void; + onDefer?: () => void; +} + +const CATEGORY_ALLOWLISTS: Record = { + LexicalClaim: ["drain_token"], + FrameClaim: ["increment_frame", "decrement_frame", "transfer_frame", "remainder_frame"], + CompositionClaim: ["multiplicative_composition", "additive_composition", "subtractive_composition"], +}; + +export function RatificationCommandPanel({ + proposal, + state, + replayEquivalent, + onSuccess, + onDefer, +}: RatificationCommandPanelProps) { + const handlerName = proposal.handler_name ?? ""; + const isPending = state === "pending"; + const isReplayEquivalent = replayEquivalent === true; + const isSupportedHandler = ["LexicalClaim", "FrameClaim", "CompositionClaim"].includes(handlerName); + + const isEnabled = isPending && isReplayEquivalent && isSupportedHandler; + + const allowedCategories = CATEGORY_ALLOWLISTS[handlerName] || []; + const [selectedCategory, setSelectedCategory] = useState(""); + const [selectedPolarity, setSelectedPolarity] = useState("affirms"); + const [showNoteInput, setShowNoteInput] = useState(false); + const [note, setNote] = useState(""); + const [statusMessage, setStatusMessage] = useState(null); + const [statusType, setStatusType] = useState<"success" | "error" | "info" | null>(null); + + const ratifyMutation = useMathRatify(); + const rejectMutation = useMathReject(); + const deferMutation = useMathDefer(); + + // Reset states when proposal changes + useEffect(() => { + if (allowedCategories.length > 0) { + setSelectedCategory(allowedCategories[0]); + } else { + setSelectedCategory(""); + } + setSelectedPolarity("affirms"); + setShowNoteInput(false); + setNote(""); + setStatusMessage(null); + setStatusType(null); + }, [proposal.proposal_id, handlerName]); + + const handleRatify = async () => { + if (!isEnabled) { + if (!isPending) { + setStatusMessage(`Cannot ratify: proposal state is ${state}`); + setStatusType("error"); + } else if (!isReplayEquivalent) { + setStatusMessage("Cannot ratify: proposal is not replay equivalent"); + setStatusType("error"); + } else { + setStatusMessage("Cannot ratify: unsupported handler name"); + setStatusType("error"); + } + return; + } + + setStatusMessage("Executing ratification..."); + setStatusType("info"); + + ratifyMutation.mutate( + { + proposalId: proposal.proposal_id, + category: selectedCategory || undefined, + polarity: handlerName !== "LexicalClaim" ? selectedPolarity : undefined, + dryRun: false, + }, + { + onSuccess: (result) => { + if (result.applied) { + setStatusMessage(`Ratification succeeded: ${result.message}`); + setStatusType("success"); + setTimeout(() => { + if (onSuccess) onSuccess(); + }, 800); + } else { + setStatusMessage(`Dry run validation routed but not applied: ${result.message}`); + setStatusType("info"); + } + }, + onError: (err) => { + setStatusMessage(err.message); + setStatusType("error"); + }, + } + ); + }; + + const handleReject = async () => { + setStatusMessage("Recording rejection..."); + setStatusType("info"); + + rejectMutation.mutate( + { + proposalId: proposal.proposal_id, + note: note || undefined, + }, + { + onSuccess: () => { + setStatusMessage("Proposal rejected successfully"); + setStatusType("success"); + setShowNoteInput(false); + setNote(""); + setTimeout(() => { + if (onSuccess) onSuccess(); + }, 800); + }, + onError: (err) => { + setStatusMessage(err.message); + setStatusType("error"); + }, + } + ); + }; + + const handleDefer = async () => { + setStatusMessage("Recording deferral..."); + setStatusType("info"); + + deferMutation.mutate( + { + proposalId: proposal.proposal_id, + }, + { + onSuccess: () => { + setStatusMessage("Proposal deferred successfully"); + setStatusType("success"); + setTimeout(() => { + if (onDefer) onDefer(); + }, 800); + }, + onError: (err) => { + setStatusMessage(err.message); + setStatusType("error"); + }, + } + ); + }; + + const handleCopyCli = async () => { + if (proposal.suggested_ratify_cli) { + await copyText(proposal.suggested_ratify_cli); + setStatusMessage("Suggested CLI command copied to clipboard"); + setStatusType("success"); + setTimeout(() => { + setStatusMessage(null); + setStatusType(null); + }, 3000); + } else { + setStatusMessage("No suggested CLI command available"); + setStatusType("error"); + } + }; + + // Keyboard shortcut listener + useEffect(() => { + const handleKeyDown = (e: KeyboardEvent) => { + // If typing in input, don't trigger global shortcuts + if (document.activeElement?.tagName === "INPUT" || document.activeElement?.tagName === "TEXTAREA") { + if (e.key === "Escape") { + e.preventDefault(); + setShowNoteInput(false); + } + return; + } + + if (e.key === "r") { + e.preventDefault(); + handleRatify(); + } else if (e.key === "x") { + e.preventDefault(); + setShowNoteInput(true); + } else if (e.key === "d") { + e.preventDefault(); + handleDefer(); + } else if (e.key === "y") { + e.preventDefault(); + handleCopyCli(); + } + }; + + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }, [proposal.proposal_id, isEnabled, selectedCategory, selectedPolarity, note, state, replayEquivalent]); + + const handleNoteKeyDown = (e: React.KeyboardEvent) => { + if (e.key === "Enter") { + e.preventDefault(); + handleReject(); + } + }; + + return ( +
+
+

+ + Ratification Corridor +

+ + {handlerName || "No handler"} + +
+ + {!isEnabled ? ( +
+ +
+ Corridor Gated:{" "} + {!isPending + ? `This proposal is already ${state}.` + : !isReplayEquivalent + ? "Replay verification failed or is not equivalent. Ratification disabled." + : !isSupportedHandler + ? `Ratification handler '${handlerName}' not admitted.` + : "Preconditions failed."} +
+
+ ) : ( +
+
+ + +
+ + {handlerName !== "LexicalClaim" && ( +
+ + +
+ )} +
+ )} + + {showNoteInput && isEnabled && ( +
+ +
+ setNote(e.target.value)} + onKeyDown={handleNoteKeyDown} + placeholder="e.g. unexpected slot constraints, wrong zero hazard" + className="flex-1 bg-[var(--color-surface-base)] border border-[var(--color-border-subtle)] rounded-md px-3 py-1 text-sm text-[var(--color-text-primary)] focus:border-[var(--color-focus-ring)] focus:outline-none" + /> + + +
+
+ )} + +
+
+ + + + + +
+ + +
+ + {statusMessage && ( +
+ {statusMessage} +
+ )} +
+ ); +} diff --git a/workbench-ui/src/types/api.ts b/workbench-ui/src/types/api.ts index b4862972..0dfc2e4a 100644 --- a/workbench-ui/src/types/api.ts +++ b/workbench-ui/src/types/api.ts @@ -183,3 +183,45 @@ export interface ApiError { } export type ApiResponse = ApiOk | ApiError; + +export interface MathReasoningStep { + step_index: number; + step_kind: string; + claim: string; + justification: string; + input_pointers: string[]; + output_payload: unknown; +} + +export interface MathProposalSummary { + proposal_id: string; + domain: "math"; + shape_category: string; + proposed_change_kind: string; + structural_commonality: string; + evidence_count: number; + replay_equivalence_hash: string; +} + +export interface MathProposalDetail extends MathProposalSummary { + wrong_zero_assertion: string; + proposed_change_payload: unknown; + reasoning_trace_id: string; + reasoning_trace_steps: MathReasoningStep[]; + evidence_hashes: string[]; + handler_name: string | null; + suggested_ratify_cli: string | null; +} + +export interface MathRatifyResult { + proposal_id: string; + change_kind: string; + handler_name: string; + routing_status: "routed" | "not_implemented"; + message: string; + suggested_cli: string | null; + applied: boolean; + target_path: string | null; + evidence_hash: string | null; +} + diff --git a/workbench/api.py b/workbench/api.py index dd9e5caf..9740761c 100644 --- a/workbench/api.py +++ b/workbench/api.py @@ -33,6 +33,36 @@ class ApiResponse: class WorkbenchApi: + def __init__(self, telemetry_sink: Any | None = None) -> None: + self._telemetry_sink = telemetry_sink + + def attach_telemetry_sink(self, sink: Any | None) -> None: + self._telemetry_sink = sink + + def _emit_operator_telemetry( + self, + event_name: str, + proposal_id: str, + outcome: str | None = None, + handler: str | None = None, + note: str | None = None, + ) -> None: + if self._telemetry_sink is None: + return + payload: dict[str, Any] = { + "event": event_name, + "proposal_id": proposal_id, + "ratifier_kind": "workbench", + } + if handler is not None: + payload["handler"] = handler + if outcome is not None: + payload["outcome"] = outcome + if note is not None: + payload["note"] = note + line = json.dumps(payload, sort_keys=True, separators=(",", ":")) + self._telemetry_sink.emit(line) + def handle(self, method: str, raw_path: str, body: bytes = b"") -> ApiResponse: parsed = urlparse(raw_path) path = parsed.path.rstrip("/") or "/" @@ -42,7 +72,11 @@ class WorkbenchApi: except json.JSONDecodeError as exc: return ApiResponse(400, error("bad_request", "invalid JSON body", detail=str(exc))) except ValueError as exc: - return ApiResponse(400, error("bad_request", str(exc))) + status = 400 + msg = str(exc) + if "already ratified" in msg.lower(): + status = 409 + return ApiResponse(status, error("bad_request", msg)) except FileNotFoundError as exc: missing = str(exc) or "resource" return ApiResponse(404, error("not_found", f"not found: {missing}")) @@ -79,7 +113,13 @@ class WorkbenchApi: return ApiResponse(200, ok({"items": readers.list_math_proposals()})) if method == "POST" and path.endswith("/ratify") and path.startswith("/math-proposals/"): proposal_id = unquote(path.removeprefix("/math-proposals/").removesuffix("/ratify")) - return self._math_ratify(proposal_id) + return self._math_ratify(proposal_id, body) + if method == "POST" and path.endswith("/reject") and path.startswith("/math-proposals/"): + proposal_id = unquote(path.removeprefix("/math-proposals/").removesuffix("/reject")) + return self._math_reject(proposal_id, body) + if method == "POST" and path.endswith("/defer") and path.startswith("/math-proposals/"): + proposal_id = unquote(path.removeprefix("/math-proposals/").removesuffix("/defer")) + return self._math_defer(proposal_id) if method == "GET" and path.startswith("/math-proposals/"): proposal_id = unquote(path.removeprefix("/math-proposals/")) return ApiResponse(200, ok(readers.read_math_proposal(proposal_id))) @@ -111,14 +151,104 @@ class WorkbenchApi: return ApiResponse(501, error("unsupported", "route is deferred beyond W-026")) return ApiResponse(404, error("not_found", f"route not found: {method} {path}")) - def _math_ratify(self, proposal_id: str) -> ApiResponse: - """Route ratification by change_kind; 501 for unimplemented handlers.""" + def _math_ratify(self, proposal_id: str, body: bytes) -> ApiResponse: + """Route ratification by change_kind; in-process execution with allowlist checks.""" + category: str | None = None + polarity: str | None = None + dry_run: bool = False + + if body: + try: + req = json.loads(body.decode("utf-8") or "{}") + if isinstance(req, dict): + category = req.get("category") + polarity = req.get("polarity") + dry_run = bool(req.get("dry_run", False)) + except Exception as exc: + return ApiResponse(400, error("bad_request", "invalid JSON body", detail=str(exc))) + + import getpass + reviewer = getpass.getuser() + try: - result: MathRatifyResult = readers.ratify_math_proposal(proposal_id) + result: MathRatifyResult = readers.ratify_math_proposal( + proposal_id, + category=category, + polarity=polarity, + reviewer=reviewer, + dry_run=dry_run, + ) except NotImplementedError as exc: return ApiResponse(501, error("unsupported", str(exc))) + except (ValueError, FileNotFoundError) as exc: + msg = str(exc) + handler = "unknown" + try: + prop = readers.read_math_proposal(proposal_id) + handler = prop.handler_name or "unknown" + except Exception: + pass + status_code = 400 + exc_class_name = exc.__class__.__name__ + if exc_class_name == "AlreadyRatified" or "already ratified" in msg.lower(): + status_code = 409 + + self._emit_operator_telemetry( + event_name="operator_ratify", + proposal_id=proposal_id, + outcome="rejected_precondition", + handler=handler, + ) + return ApiResponse(status_code, error("bad_request", msg)) + except Exception as exc: + return ApiResponse(500, error("runtime_unavailable", f"internal error: {exc}")) + + if result.applied: + self._emit_operator_telemetry( + event_name="operator_ratify", + proposal_id=proposal_id, + outcome="applied", + handler=result.handler_name, + ) return ApiResponse(200, ok(result)) + def _math_reject(self, proposal_id: str, body: bytes) -> ApiResponse: + note: str = "" + if body: + try: + req = json.loads(body.decode("utf-8") or "{}") + if isinstance(req, dict): + note = str(req.get("note", "")) + except Exception as exc: + return ApiResponse(400, error("bad_request", "invalid JSON body", detail=str(exc))) + try: + prop = readers.read_math_proposal(proposal_id) + handler = prop.handler_name or "unknown" + except FileNotFoundError as exc: + return ApiResponse(404, error("not_found", str(exc))) + + self._emit_operator_telemetry( + event_name="operator_reject", + proposal_id=proposal_id, + handler=handler, + note=note, + ) + return ApiResponse(200, ok({"proposal_id": proposal_id, "rejected": True})) + + def _math_defer(self, proposal_id: str) -> ApiResponse: + try: + prop = readers.read_math_proposal(proposal_id) + handler = prop.handler_name or "unknown" + except FileNotFoundError as exc: + return ApiResponse(404, error("not_found", str(exc))) + + self._emit_operator_telemetry( + event_name="operator_defer", + proposal_id=proposal_id, + handler=handler, + ) + return ApiResponse(200, ok({"proposal_id": proposal_id, "deferred": True})) + def _chat_turn(self, body: bytes) -> ApiResponse: """Execute one live runtime turn. diff --git a/workbench/readers.py b/workbench/readers.py index 8a46900e..d735793c 100644 --- a/workbench/readers.py +++ b/workbench/readers.py @@ -328,7 +328,8 @@ def _math_proposal_summary(record: dict[str, Any]) -> MathProposalSummary: def list_math_proposals(*, jsonl_path: Path | None = None) -> list[MathProposalSummary]: path = jsonl_path or MATH_PROPOSALS_JSONL records = _load_math_proposals_raw(path) - return [_math_proposal_summary(r) for r in records] + return [_math_proposal_summary(r) for r in records if r.get("domain") == "math"] + def _math_trace_steps_from_record(record: dict[str, Any]) -> list[MathReasoningStep]: @@ -376,6 +377,9 @@ def read_math_proposal( record = next((r for r in records if r.get("proposal_id") == proposal_id), None) if record is None: raise FileNotFoundError(proposal_id) + if record.get("domain") != "math": + raise ValueError(f"Partition isolation violation: proposal domain must be 'math', got {record.get('domain')!r}") + change_kind = str(record.get("proposed_change_kind", "")) handler_name = _HANDLER_DISPATCH.get(change_kind) @@ -435,23 +439,25 @@ def read_math_proposal( def ratify_math_proposal( proposal_id: str, *, + category: str | None = None, + polarity: str | None = None, + reviewer: str | None = None, + dry_run: bool = False, jsonl_path: Path | None = None, ) -> MathRatifyResult: - """Dispatch ratification by change_kind; fail loudly for unimplemented handlers. + """Dispatch ratification by change_kind. - ADR-0160 "Proposal before mutation" doctrine: this function validates - routing and returns the handler name + suggested CLI without applying - the change. Mutation requires an explicit operator action outside the - workbench (e.g. calling apply_lexical_claim() directly). - - Raises FileNotFoundError if proposal_id not found. - Raises NotImplementedError with a clear message for unhandled change_kinds. + If dry_run is False and category is provided, this applies the ratification mutation in-process. + Otherwise, it validates routing and returns the handler name + suggested CLI. """ path = jsonl_path or MATH_PROPOSALS_JSONL records = _load_math_proposals_raw(path) record = next((r for r in records if r.get("proposal_id") == proposal_id), None) if record is None: raise FileNotFoundError(proposal_id) + if record.get("domain") != "math": + raise ValueError(f"Partition isolation violation: proposal domain must be 'math', got {record.get('domain')!r}") + change_kind = str(record.get("proposed_change_kind", "")) handler_name = _HANDLER_DISPATCH.get(change_kind) @@ -483,14 +489,95 @@ def ratify_math_proposal( f"polarity='affirms', reviewer='')" ) - return MathRatifyResult( - proposal_id=proposal_id, - change_kind=change_kind, - handler_name=handler_name, - routing_status="routed", - message=f"routed to {handler_name} handler", - suggested_cli=suggested_cli, - ) + if dry_run or category is None: + return MathRatifyResult( + proposal_id=proposal_id, + change_kind=change_kind, + handler_name=handler_name, + routing_status="routed", + message=f"routed to {handler_name} handler", + suggested_cli=suggested_cli, + applied=False, + ) + + # In-process application + from teaching.math_contemplation_proposal import from_jsonl_record + proposal = from_jsonl_record(record) + if not proposal.evidence_pointers: + raise ValueError(f"Proposal {proposal_id} has no evidence pointers") + claim = proposal.evidence_pointers[0] + + import getpass + effective_reviewer = reviewer or getpass.getuser() + + if handler_name == "LexicalClaim": + from teaching.math_lexical_ratification import apply_lexical_claim + receipt = apply_lexical_claim( + claim=claim, + category=category, + reviewer=effective_reviewer, + ratifier_kind="workbench", + ) + return MathRatifyResult( + proposal_id=proposal_id, + change_kind=change_kind, + handler_name=handler_name, + routing_status="routed", + message=f"Applied LexicalClaim to {receipt.target_file}", + suggested_cli=suggested_cli, + applied=True, + target_path=receipt.target_file, + evidence_hash=receipt.evidence_hash, + ) + + elif handler_name == "FrameClaim": + from teaching.math_frame_ratification import apply_frame_claim + if not polarity: + raise ValueError("Polarity is required for FrameClaim ratification") + receipt = apply_frame_claim( + claim=claim, + frame_category=category, + polarity=polarity, + reviewer=effective_reviewer, + ratifier_kind="workbench", + ) + return MathRatifyResult( + proposal_id=proposal_id, + change_kind=change_kind, + handler_name=handler_name, + routing_status="routed", + message=f"Applied FrameClaim to {receipt.target_file}", + suggested_cli=suggested_cli, + applied=True, + target_path=receipt.target_file, + evidence_hash=receipt.evidence_hash, + ) + + elif handler_name == "CompositionClaim": + from teaching.math_composition_ratification import apply_composition_claim + if not polarity: + raise ValueError("Polarity is required for CompositionClaim ratification") + receipt = apply_composition_claim( + claim=claim, + composition_category=category, + polarity=polarity, + reviewer=effective_reviewer, + ratifier_kind="workbench", + ) + return MathRatifyResult( + proposal_id=proposal_id, + change_kind=change_kind, + handler_name=handler_name, + routing_status="routed", + message=f"Applied CompositionClaim to {receipt.target_file}", + suggested_cli=suggested_cli, + applied=True, + target_path=receipt.target_file, + evidence_hash=receipt.evidence_hash, + ) + + else: + raise NotImplementedError(f"handler {handler_name} application not implemented") def run_safe_eval_lane( diff --git a/workbench/schemas.py b/workbench/schemas.py index cc9e5d57..c4231e59 100644 --- a/workbench/schemas.py +++ b/workbench/schemas.py @@ -243,3 +243,6 @@ class MathRatifyResult: routing_status: Literal["routed", "not_implemented"] message: str suggested_cli: str | None = None + applied: bool = False + target_path: str | None = None + evidence_hash: str | None = None diff --git a/workbench/server.py b/workbench/server.py index fe682a5c..7044859d 100644 --- a/workbench/server.py +++ b/workbench/server.py @@ -34,6 +34,30 @@ class WorkbenchRequestHandler(BaseHTTPRequestHandler): ) def _handle(self) -> None: + host = self.headers.get("Host", "") + origin = self.headers.get("Origin", "") + + host_name = host.split(":")[0] if ":" in host else host + if host_name not in {"127.0.0.1", "localhost"}: + payload = json.dumps({"ok": False, "error": "CORS check failed: Host is non-local"}).encode("utf-8") + self.send_response(400) + self._send_common_headers(len(payload)) + self.end_headers() + self.wfile.write(payload) + return + + if origin: + import urllib.parse + parsed_origin = urllib.parse.urlparse(origin) + origin_host = parsed_origin.hostname + if origin_host not in {"127.0.0.1", "localhost"}: + payload = json.dumps({"ok": False, "error": "CORS check failed: Origin is non-local"}).encode("utf-8") + self.send_response(400) + self._send_common_headers(len(payload)) + self.end_headers() + self.wfile.write(payload) + return + try: length = max(0, int(self.headers.get("Content-Length") or "0")) except ValueError: