From 4d26e1503b98b14169a6372e3880568067c939fe Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 20 May 2026 15:13:38 -0700 Subject: [PATCH] fix(tests): make frontier_compare viewer test resilient to copy refreshes (#67) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_frontier_compare_report_viewer_exists was failing on main against the current report_viewer.html because two verbatim substring checks no longer matched the viewer's UI copy: - "Drop report JSON" → viewer now says "Drop JSON report" (order swapped) - "No network calls" → viewer now says "no network calls" (lowercase) Both copy refreshes were behavior-preserving — drop-zone affordance and network-free trust boundary are both intact in the viewer. The test was coupling to verbatim phrasing rather than to the load-bearing affordances. Switch to case-insensitive substring checks that pin what actually matters: - "frontier compare" — viewer identity - "drop" AND "json" together — drop-zone affordance, order-independent - "no network calls" — trust boundary (case-insensitive) - fetch(/XMLHttpRequest still hard-banned (case-sensitive — these are JS API surface, not human-readable copy) Pre-existing failure flagged in PR #66's body as out-of-scope cleanup; this is that cleanup. --- tests/test_frontier_compare_wave1.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/test_frontier_compare_wave1.py b/tests/test_frontier_compare_wave1.py index a57ac86d..b0343a76 100644 --- a/tests/test_frontier_compare_wave1.py +++ b/tests/test_frontier_compare_wave1.py @@ -81,8 +81,18 @@ def test_frontier_compare_report_viewer_exists() -> None: viewer = Path("evals/frontier_compare/ui/report_viewer.html") text = viewer.read_text(encoding="utf-8") - assert "Frontier Compare" in text - assert "Drop report JSON" in text - assert "No network calls" in text + # Case-insensitive substring checks — pin the viewer's load-bearing + # affordances and trust-boundary copy without coupling to verbatim + # phrasing. Earlier verbatim checks ("Drop report JSON", + # "No network calls") broke when the viewer's copy was refreshed + # to "Drop JSON report" / "no network calls" without any change in + # behavior. + lowered = text.lower() + assert "frontier compare" in lowered + # Drop-zone affordance: order-independent — both tokens must be + # present so a user can paste a report into the viewer. + assert "drop" in lowered and "json" in lowered + # Trust boundary: viewer must remain network-free. + assert "no network calls" in lowered assert "fetch(" not in text assert "XMLHttpRequest" not in text