Merge remote-tracking branch 'forgejo/main' into feat/adr-0242-carry-items
All checks were successful
lane-shas / verify pinned lane SHAs (pull_request) Successful in 17m8s
smoke / smoke (-m "not quarantine") (pull_request) Successful in 5m18s

This commit is contained in:
Shay 2026-07-16 06:57:12 -07:00
commit 8d58481566
11 changed files with 3327 additions and 36 deletions

1
.gitignore vendored
View file

@ -15,7 +15,6 @@ workbench_data/*
core-rs/target/
core-rs/Cargo.lock
uv.lock
# Environment secrets — never commit real keys
.env

View file

@ -10,7 +10,7 @@ Base: `forgejo/main` @ `54228d45`
|------|---------|--------|
| Fast | `pytest -m "not quarantine and not slow" -n auto -q` | **5 failed**, 11660 passed, 106 skipped (pre-fix) |
| Fast recheck | same after fix | (see verification below) |
| Slow | `pytest -m "slow and not quarantine" -n auto -q` | running / recorded at PR time |
| Slow | `pytest -m "slow and not quarantine" -n auto -q` | **1 failed, 909 passed, 1 skipped, 1 xfailed** (37m06s, 2026-07-15 sweep @ `8f67590d`+#47). The 1 red = `test_public_showcase.py::TestRuntimeBudget` — stale constant pin (30) vs the deliberate 640dbe8f re-budget (60); fixed alongside this row. Slow lane is otherwise clean → the historical "~31 reds" figure is fully retired. |
| Full | `pytest -m "not quarantine"` | union of fast + slow |
The briefs “~31 reds” appears to reflect an older main tip. On current `forgejo/main`, the **non-quarantine fast lane** surfaces **exactly 5** dedicated failures. Smoke/pack/lane gates stay green because they never execute these nodes.

View file

@ -25,6 +25,7 @@ single scoring path.
from __future__ import annotations
import json
import os
import sys
from pathlib import Path
from typing import Any
@ -33,7 +34,13 @@ from evals.gsm8k_math.runner import _score_one_candidate_graph
_HERE = Path(__file__).resolve().parent
_CASES_PATH = _HERE / "cases.jsonl"
_REPORT_PATH = _HERE / "report.json"
# Overridable so concurrent test invocations (xdist workers) can each point
# the CLI entrypoint at an isolated tmp path instead of racing on the shared
# committed report.json. Unset (the default) preserves the real operator
# contract: `python -m ...runner` writes the tracked repo file.
_REPORT_PATH = Path(
os.environ.get("CORE_GSM8K_TRAIN_SAMPLE_REPORT_PATH", str(_HERE / "report.json"))
)
_SAMPLE_REL = "evals/gsm8k_math/train_sample/v1/cases.jsonl"
_EXPECTED_COUNT = 50
_CORRECT_MIN = 10

View file

@ -4,7 +4,8 @@ Does not re-execute the full physics suite. Asserts the load-bearing
governance surfaces for ADR-0241/0242 cohesion remain present and honest:
* runtime_contracts documents off-serve quarantine + epistemic standing
* acceptance checklist maps C0C8 to tests
* ADRs stay Proposed (ready for Joshua) not self-Accepted
* ADR statuses carry recorded ruling provenance (ratified 2026-07-15;
a silent status flip in either direction fails)
* cohesion suite still names I-01I-05 pins
"""
@ -82,21 +83,37 @@ def test_cohesion_suite_names_entity_invariants():
assert name in text, f"missing entity pin {name}"
def test_adrs_ready_for_acceptance_not_self_accepted():
"""P12: implementation complete → Proposed + ready; Joshua alone Accepts."""
def test_adrs_accepted_with_recorded_ruling_provenance():
"""P12 (post-ratification): status flips are valid ONLY with provenance.
Joshua Shay ruled "Ratify" on 2026-07-15 (D10 acceptance packet §8), so
the anti-self-Accept guard evolves rather than dies: each ADR must say
Accepted WITH the ratification provenance inline, and the packet must
carry the ruling record. A silent status flip in either direction an
Accept without provenance, or a quiet demotion fails here.
"""
packet = (
_ROOT / "docs/audit/adr-0241-0242-acceptance-packet-2026-07-15.md"
).read_text(encoding="utf-8")
assert "## 8. RULING RECORD" in packet, "ruling record section missing"
assert "RATIFIED — Joshua Shay, 2026-07-15" in packet, "ruling attribution missing"
for rel in (
"docs/adr/ADR-0241-wave-field-driven-hyperbolic-atlas-and-resonant-cognition.md",
"docs/adr/ADR-0242-atlas-packing-and-fibonacci.md",
):
text = (_ROOT / rel).read_text(encoding="utf-8")
# First status line must remain Proposed until human Accept.
status_line = next(
(ln for ln in text.splitlines() if ln.startswith("**Status**")),
"",
)
assert "Proposed" in status_line, f"{rel} lost Proposed status"
assert "Accepted" not in status_line, f"{rel} must not self-Accept"
assert "Joshua" in status_line or "ready" in status_line.lower()
assert "Accepted" in status_line, f"{rel} lost Accepted status"
assert "ratified by Joshua Shay" in status_line, (
f"{rel} status lacks ratification provenance"
)
assert "acceptance-packet" in status_line, (
f"{rel} status must cite the ruling packet"
)
def test_serve_quarantine_list_matches_cohesion_ast_pin():

View file

@ -10,6 +10,24 @@ import pytest
from core import cli
@pytest.fixture(autouse=True)
def _isolate_demo_results_dir(tmp_path, monkeypatch):
"""Redirect the demo results dir to an isolated per-test tmp dir.
``core.cli._DEMO_RESULTS_DIR`` is a module-level constant pointing at the
shared repo dir ``evals/forward_semantic_control/results/`` that every
``core demo`` subcommand reads, writes, and glob-scans (including the
``index.json`` rebuild in ``_write_results_index``). Multiple tests below
invoke ``cli.main(["demo", ...])``; left unpatched they race on that one
shared directory under ``-n auto`` (concurrent report writes plus a
scan-then-rewrite of ``index.json``). Follows the same monkeypatch-the-
module-attribute isolation idiom as the root-conftest engine-state fixture
(#782). ``TestResultsReadme`` reads the real checked-in README by a
hardcoded path, so it is unaffected by this redirection.
"""
monkeypatch.setattr(cli, "_DEMO_RESULTS_DIR", tmp_path / "results")
class TestADR0024SuiteAliases:
"""Layer 1: pin the new suite aliases so they don't drift."""
@ -93,7 +111,7 @@ class TestDemoSubcommand:
assert "ALL THREE CONDITIONS" in captured.out
assert "PASS" in captured.out
# Report file present and well-formed.
report = Path("evals/forward_semantic_control/results/phase6_demo_report.json")
report = cli._DEMO_RESULTS_DIR / "phase6_demo_report.json"
assert report.exists()
data = json.loads(report.read_text())
assert data["metrics"]["all_three_conditions_pass"] is True
@ -109,6 +127,12 @@ class TestDemoSubcommand:
assert "all_three_conditions_pass" in payload["metrics"]
def test_demo_list_results_indexes_reports(self, capsys) -> None:
# Self-contained: write a report before indexing rather than relying
# on a sibling test's shared-directory side effect (that implicit
# ordering coupling only "worked" by accident of file-declaration
# order and would break under `-n auto` reordering).
cli.main(["demo", "phase6"])
capsys.readouterr()
rc = cli.main(["demo", "list-results"])
assert rc == 0
captured = capsys.readouterr()
@ -116,6 +140,10 @@ class TestDemoSubcommand:
assert "phase6_demo_report.json" in captured.out
def test_demo_list_results_json_well_formed(self, capsys) -> None:
# Self-contained for the same reason as
# test_demo_list_results_indexes_reports above.
cli.main(["demo", "phase6"])
capsys.readouterr()
rc = cli.main(["demo", "list-results", "--json"])
assert rc == 0
captured = capsys.readouterr()
@ -127,7 +155,7 @@ class TestDemoSubcommand:
def test_demo_index_file_refreshed_after_run(self) -> None:
cli.main(["demo", "phase6"])
index_path = Path("evals/forward_semantic_control/results/index.json")
index_path = cli._DEMO_RESULTS_DIR / "index.json"
assert index_path.exists()
data = json.loads(index_path.read_text())
names = [e["file"] for e in data["reports"]]

View file

@ -162,5 +162,11 @@ class TestPureCompositionGate:
class TestRuntimeBudget:
def test_budget_is_thirty_seconds(self) -> None:
assert MAX_RUNTIME_SECONDS == 30
def test_budget_is_sixty_seconds(self) -> None:
# 30 → 60 was a deliberate re-budget (commit 640dbe8f: post-CGA
# substrate work makes a cold RegisterTour alone ~30s+; see
# evals/public_demo/contract.md "Known Environment Caveat"). This slow
# pin was missed then because the slow lane hadn't been run since
# (nightly lives on the billing-locked GitHub side) — caught by the
# 2026-07 slow-lane sweep. Pin the deliberate value.
assert MAX_RUNTIME_SECONDS == 60

View file

@ -114,19 +114,19 @@ def test_canonical_pack_admits_cross_sentence_composition_when_seeded():
)
def test_wrong_zero_preserved_on_train_sample():
def test_wrong_zero_preserved_on_train_sample(tmp_path):
"""The full train_sample eval must preserve wrong == 0 after RAT-1."""
import os
import subprocess
import sys
report_path = tmp_path / "report.json"
result = subprocess.run(
[sys.executable, "-m", "evals.gsm8k_math.train_sample.v1.runner"],
cwd=_repo_root(),
capture_output=True,
text=True,
)
report_path = (
_repo_root() / "evals" / "gsm8k_math" / "train_sample" / "v1" / "report.json"
env={**os.environ, "CORE_GSM8K_TRAIN_SAMPLE_REPORT_PATH": str(report_path)},
)
assert report_path.exists(), "train_sample runner must emit report.json"
report = json.loads(report_path.read_text())
@ -134,18 +134,18 @@ def test_wrong_zero_preserved_on_train_sample():
assert counts["wrong"] == 0, f"wrong-zero invariant violated: {counts}"
def test_case_0050_remains_refused_after_rat1():
def test_case_0050_remains_refused_after_rat1(tmp_path):
"""Hazard pin: case 0050 must not admit after any RAT-1 wiring."""
import os
import subprocess
import sys
report_path = tmp_path / "report.json"
subprocess.run(
[sys.executable, "-m", "evals.gsm8k_math.train_sample.v1.runner"],
cwd=_repo_root(),
capture_output=True,
)
report_path = (
_repo_root() / "evals" / "gsm8k_math" / "train_sample" / "v1" / "report.json"
env={**os.environ, "CORE_GSM8K_TRAIN_SAMPLE_REPORT_PATH": str(report_path)},
)
report = json.loads(report_path.read_text())
case = next(

View file

@ -9,6 +9,7 @@ from unittest.mock import patch
import pytest
import engine_state
from core.cli import main
from teaching.discovery import DiscoveryCandidate, EvidencePointer
from teaching.proposals import ProposalLog, ReplayEvidence, build_proposal
@ -515,11 +516,21 @@ def snapshot_dir(directory: Path) -> dict[Path, bytes]:
def test_read_only_invariant():
"""Directories the hitl-queue commands must not mutate.
``engine_state`` is read via ``engine_state._DEFAULT_DIR`` (not a
hardcoded ``project_root / "engine_state"``) because the root-conftest
``_isolate_engine_state_default`` autouse fixture repoints it at a
per-test tmp dir; a hardcoded real path would instead snapshot the
shared repo dir and false-fail under ``-n auto`` whenever an unrelated
concurrent worker wrote to it (observed: AssertionError on this exact
directory under a full `-n auto` fast-lane run).
"""
project_root = Path(__file__).resolve().parent.parent
dirs = [
project_root / "teaching" / "proposals",
project_root / "packs",
project_root / "engine_state",
Path(engine_state._DEFAULT_DIR),
project_root / "contemplation" / "runs",
]

View file

@ -171,41 +171,43 @@ def _has_wave_a_seed() -> bool:
)
def test_wrong_zero_preserved():
def test_wrong_zero_preserved(tmp_path):
"""The full train_sample eval keeps wrong == 0 after WAVE-A."""
import os
import subprocess
import sys
here = Path(__file__).resolve()
while here.parent != here and not (here / "pyproject.toml").exists():
here = here.parent
report_path = tmp_path / "report.json"
subprocess.run(
[sys.executable, "-m", "evals.gsm8k_math.train_sample.v1.runner"],
cwd=here,
capture_output=True,
env={**os.environ, "CORE_GSM8K_TRAIN_SAMPLE_REPORT_PATH": str(report_path)},
)
report = json.loads(
(here / "evals" / "gsm8k_math" / "train_sample" / "v1" / "report.json").read_text()
)
report = json.loads(report_path.read_text())
assert report["counts"]["wrong"] == 0
def test_case_0050_remains_refused():
def test_case_0050_remains_refused(tmp_path):
"""Hazard pin."""
import os
import subprocess
import sys
here = Path(__file__).resolve()
while here.parent != here and not (here / "pyproject.toml").exists():
here = here.parent
report_path = tmp_path / "report.json"
subprocess.run(
[sys.executable, "-m", "evals.gsm8k_math.train_sample.v1.runner"],
cwd=here,
capture_output=True,
env={**os.environ, "CORE_GSM8K_TRAIN_SAMPLE_REPORT_PATH": str(report_path)},
)
report = json.loads(
(here / "evals" / "gsm8k_math" / "train_sample" / "v1" / "report.json").read_text()
)
report = json.loads(report_path.read_text())
case_0050 = next(
(c for c in report["per_case"] if c["case_id"].endswith("-0050")),
None,

View file

@ -12,14 +12,13 @@ from pathlib import Path
import pytest
import engine_state
from chat.runtime import ChatRuntime
from workbench import api as workbench_api
from workbench.api import WorkbenchApi, _run_sealed_chat_turn, _with_turn_cost_and_id
from workbench.journal import TurnJournal, TurnJournalEntry
from workbench.replay import CRITICAL_FIELDS, INFORMATIONAL_FIELDS, replay_turn
_ENGINE_STATE_DIR = Path("engine_state")
def _snapshot(root: Path) -> dict[str, bytes]:
snap: dict[str, bytes] = {}
@ -150,19 +149,26 @@ def test_hashless_legacy_turn_is_not_replayable(tmp_path, recorded, monkeypatch)
def test_replay_leaves_no_trace(tmp_path, recorded) -> None:
"""Obligation 4: GET /replay writes nothing — journal and engine state."""
"""Obligation 4: GET /replay writes nothing — journal and engine state.
``engine_state._DEFAULT_DIR`` is read fresh (not a module-level constant)
because the root-conftest ``_isolate_engine_state_default`` autouse
fixture repoints it at a per-test tmp dir; a hardcoded ``Path("engine_state")``
would instead snapshot the real shared repo dir and false-fail under
``-n auto`` whenever an unrelated concurrent worker wrote to it.
"""
entry, _ = recorded
journal = TurnJournal(journal_dir=tmp_path / "workbench_data")
journal.append(entry)
api = WorkbenchApi(journal=journal)
journal_before = _snapshot(tmp_path / "workbench_data")
engine_state_before = _snapshot(_ENGINE_STATE_DIR)
engine_state_before = _snapshot(Path(engine_state._DEFAULT_DIR))
response = api.handle("GET", "/replay/1", b"")
assert response.status == 200
assert _snapshot(tmp_path / "workbench_data") == journal_before
assert _snapshot(_ENGINE_STATE_DIR) == engine_state_before
assert _snapshot(Path(engine_state._DEFAULT_DIR)) == engine_state_before
def test_wall_clock_divergence_does_not_break_equivalence(tmp_path, recorded) -> None:

3215
uv.lock Normal file

File diff suppressed because it is too large Load diff