core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feat(ADR-0172/W3): core eval math-contemplation CLI lane (#384)

Browse source 
Wires `decompose_audit()` into a new `core eval math-contemplation`
subcommand:

- `cmd_eval_math_contemplation` in `core/cli.py` dispatched via `cmd_eval`
  when `lane == "math-contemplation"`
- `--audit` (default: audit_brief_11.json) + `--output` (default:
  teaching/math_proposals/proposals.jsonl) with path-traversal validation
  (absolute paths and directory-escaping relative paths → exit 2)
- exit 0 success / exit 1 audit-not-found / exit 2 parse-error or rejection
- `--json` flag for machine-readable output
- idempotent: re-run on same audit writes byte-identical JSONL
- output sorted by proposal_id (inherits decomposer sort contract)
- forbidden: no auto-apply, no writes outside teaching/math_proposals/,
  no audit-file mutation
- `teaching/math_proposals/.gitkeep` directory scaffold committed
- `.gitignore` entry for `teaching/math_proposals/proposals.jsonl`
- 11 tests in `tests/test_adr_0172_w3_cli_lane.py`; runtime suite green
This commit is contained in:
Shay 2026-05-27 12:58:31 -07:00 committed by GitHub
parent af3821f0ed
commit fbbc57edff
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 351 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.gitignore vendored
View file

@ -36,3 +36,7 @@ evals/expert_demos/
evals/gsm8k_math/holdouts/v1/cases.jsonl
evals/gsm8k_math/holdouts/v1/cases_plaintext.jsonl
evals/gsm8k_math/holdouts/v1/cases-train*
# ADR-0172 W3 — math-contemplation proposals are generated on demand; the
# directory skeleton (.gitkeep) is committed, the generated JSONL is not.
teaching/math_proposals/proposals.jsonl

138
core/cli.py
View file

@ -23,7 +23,7 @@ _CORE_RS_DIR = _REPO_ROOT / "core-rs"
_CORE_RS_MANIFEST = _CORE_RS_DIR / "Cargo.toml"
DESCRIPTION = "CORE versor engine command suite."
EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite all\n core bench --suite all --json --report bench_all.json\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core teaching audit\n core teaching audit --json\n core teaching gaps --top 10\n core teaching queue --threshold 3\n core teaching hitl-queue list\n core teaching hitl-queue list --state all --json\n core teaching hitl-queue show <proposal_id>\n core teaching propose <candidate-jsonl-path>\n core teaching propose-from-exemplars teaching/admissibility_exemplars/rate_with_currency_v1.jsonl\n core teaching propose-from-exemplars --all\n core teaching proposals --state pending\n core teaching review <proposal_id> --accept --review-date 2026-05-18\n core teaching supersede cause_light_reveals_truth --subject light --intent cause --connective grounds --object truth --review-date 2026-05-18\n core teaching supersessions\n core teaching supersessions --json\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo register-tour\n core demo anchor-lens-tour\n core demo orthogonality-tour\n core demo pack-measurements\n core demo long-context-comparison\n core demo anti-regression\n core demo learning-loop\n core demo learning-arc\n core demo articulation\n core demo conversation\n core demo conversation --no-stream\n core demo all\n core demo adr-0024-chain\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout\n core eval contemplation_quality\n core eval contemplation_quality --json --save\n core workbench api\n core workbench api --port 9000\n core workbench api --host 0.0.0.0 --allow-nonlocal-bind"
EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite all\n core bench --suite all --json --report bench_all.json\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core teaching audit\n core teaching audit --json\n core teaching gaps --top 10\n core teaching queue --threshold 3\n core teaching hitl-queue list\n core teaching hitl-queue list --state all --json\n core teaching hitl-queue show <proposal_id>\n core teaching propose <candidate-jsonl-path>\n core teaching propose-from-exemplars teaching/admissibility_exemplars/rate_with_currency_v1.jsonl\n core teaching propose-from-exemplars --all\n core teaching proposals --state pending\n core teaching review <proposal_id> --accept --review-date 2026-05-18\n core teaching supersede cause_light_reveals_truth --subject light --intent cause --connective grounds --object truth --review-date 2026-05-18\n core teaching supersessions\n core teaching supersessions --json\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo register-tour\n core demo anchor-lens-tour\n core demo orthogonality-tour\n core demo pack-measurements\n core demo long-context-comparison\n core demo anti-regression\n core demo learning-loop\n core demo learning-arc\n core demo articulation\n core demo conversation\n core demo conversation --no-stream\n core demo all\n core demo adr-0024-chain\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout\n core eval contemplation_quality\n core eval contemplation_quality --json --save\n core eval math-contemplation\n core eval math-contemplation --audit evals/gsm8k_math/train_sample/v1/audit_brief_11.json\n core eval math-contemplation --output teaching/math_proposals/proposals.jsonl\n core workbench api\n core workbench api --port 9000\n core workbench api --host 0.0.0.0 --allow-nonlocal-bind"
_TEST_SUITES: dict[str, tuple[str, ...]] = {
"fast": (
@ -2071,6 +2071,9 @@ def cmd_doctor(args: argparse.Namespace) -> int:
def cmd_eval(args: argparse.Namespace) -> int:
"""Run an eval lane by name, or list available lanes."""
if getattr(args, "lane", None) == "math-contemplation":
return cmd_eval_math_contemplation(args)
from evals._parallel import normalize_workers
from evals.framework import (
discover_lanes,
@ -2175,6 +2178,120 @@ def cmd_eval(args: argparse.Namespace) -> int:
return 0
# ---------------------------------------------------------------------------
# ADR-0172 W3 — math-contemplation CLI lane
# ---------------------------------------------------------------------------
_MATH_PROPOSALS_DIR = _REPO_ROOT / "teaching" / "math_proposals"
_DEFAULT_AUDIT_PATH = (
_REPO_ROOT
/ "evals"
/ "gsm8k_math"
/ "train_sample"
/ "v1"
/ "audit_brief_11.json"
)
_DEFAULT_OUTPUT_PATH = _MATH_PROPOSALS_DIR / "proposals.jsonl"
def _validate_output_path(raw: str | None) -> Path:
"""Reject output paths that escape teaching/math_proposals/.
Mirrors :func:`language_packs.compiler._validate_pack_id` trust-boundary
discipline: path-traversal and absolute paths are rejected before any
filesystem access.
Exit code 2 on rejection (parse/path-rejection class).
"""
if raw is None:
return _DEFAULT_OUTPUT_PATH
candidate = Path(raw)
if candidate.is_absolute():
_die(
f"--output must be a relative path inside teaching/math_proposals/; "
f"got absolute path: {raw!r}",
code=2,
)
resolved = (_REPO_ROOT / candidate).resolve()
allowed_root = _MATH_PROPOSALS_DIR.resolve()
try:
resolved.relative_to(allowed_root)
except ValueError:
_die(
f"--output must resolve inside teaching/math_proposals/; "
f"got: {raw!r}",
code=2,
)
return resolved
def cmd_eval_math_contemplation(args: argparse.Namespace) -> int:
"""ADR-0172 W3 — decompose an audit brief into refusal-shape proposals.
Reads ``--audit`` (default: ``evals/gsm8k_math/train_sample/v1/audit_brief_11.json``),
runs :func:`teaching.math_contemplation.decompose_audit`, and writes one
``canonical_bytes()`` JSON line per proposal to ``--output``
(default: ``teaching/math_proposals/proposals.jsonl``).
Idempotency: re-running on the same audit overwrites byte-identical bytes.
Output is sorted by ``proposal_id`` (matches the decomposer sort contract).
Exit codes:
0 success
1 audit file not found
2 parse error or path-traversal rejection
Forbidden by design: no proposal is auto-applied, no file outside
``teaching/math_proposals/`` is written, the audit file is not mutated.
"""
from teaching.math_contemplation import decompose_audit
from teaching.math_contemplation_proposal import canonical_bytes
audit_raw = getattr(args, "audit", None)
output_raw = getattr(args, "output", None)
audit_path = Path(audit_raw) if audit_raw else _DEFAULT_AUDIT_PATH
if not audit_path.is_absolute():
audit_path = (_REPO_ROOT / audit_path).resolve()
if not audit_path.exists():
_die(f"audit file not found: {audit_path}", code=1)
output_path = _validate_output_path(output_raw)
try:
proposals = decompose_audit(audit_path)
except json.JSONDecodeError as exc:
_die(f"parse error in audit file {audit_path}: {exc}", code=2)
output_path.parent.mkdir(parents=True, exist_ok=True)
lines = [canonical_bytes(p) + b"\n" for p in proposals]
output_path.write_bytes(b"".join(lines))
if not getattr(args, "json", False):
print(f"proposals : {len(proposals)}")
print(f"output : {output_path}")
else:
print(
json.dumps(
{
"proposals": len(proposals),
"output": str(output_path),
},
ensure_ascii=False,
sort_keys=True,
)
)
return 0
def cmd_workbench(args: argparse.Namespace) -> int:
"""Run CORE Workbench local operator surfaces."""
if args.workbench_command == "api":
@ -4386,6 +4503,25 @@ def build_parser() -> argparse.ArgumentParser:
eval_cmd.add_argument("--json", action="store_true", help="emit machine-readable JSON")
eval_cmd.add_argument("--save", action="store_true", help="write result to lane results/ directory")
eval_cmd.add_argument("--report", metavar="PATH", help="write JSON report to file")
eval_cmd.add_argument(
"--audit",
metavar="PATH",
default=None,
help=(
"math-contemplation lane: path to audit JSON "
"(default: evals/gsm8k_math/train_sample/v1/audit_brief_11.json)"
),
)
eval_cmd.add_argument(
"--output",
metavar="PATH",
default=None,
help=(
"math-contemplation lane: output JSONL path "
"(default: teaching/math_proposals/proposals.jsonl); "
"must resolve inside teaching/math_proposals/"
),
)
eval_cmd.set_defaults(func=cmd_eval)
from formation.cli import register as _register_formation

0
teaching/math_proposals/.gitkeep Normal file
View file

210
tests/test_adr_0172_w3_cli_lane.py Normal file
View file

@ -0,0 +1,210 @@
"""ADR-0172 W3 — math-contemplation CLI lane tests.
Covers:
- subcommand is wired in core/cli.py and exits 0 on the real audit brief;
- output file is written to teaching/math_proposals/ and contains valid JSONL;
- idempotency: second run overwrites byte-identical bytes;
- output is sorted by proposal_id (matches decomposer contract);
- --output accepts a valid relative path inside teaching/math_proposals/;
- --output rejects absolute paths (exit 2);
- --output rejects relative path-traversal that escapes the allowed root (exit 2);
- --audit exit-1 when the audit file does not exist;
- --json flag emits machine-readable JSON to stdout.
"""
from __future__ import annotations
import json
from contextlib import contextmanager
from pathlib import Path
from typing import Generator
import pytest
import core.cli as cli_module
from core.cli import cmd_eval, cmd_eval_math_contemplation
REAL_AUDIT_PATH = (
Path(__file__).resolve().parent.parent
/ "evals"
/ "gsm8k_math"
/ "train_sample"
/ "v1"
/ "audit_brief_11.json"
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
class _Args:
"""Minimal argparse.Namespace stand-in for direct cmd_eval_math_contemplation calls."""
def __init__(self, *, audit=None, output=None, json_mode=False):
self.lane = "math-contemplation"
self.audit = audit
self.output = output
self.json = json_mode
# Fields required by other eval branches but unused here:
self.list_lanes = False
self.version = None
self.split = "public"
self.workers = 4
self.save = False
self.report = None
@contextmanager
def _patch_proposals_dir(tmp_path: Path) -> Generator[Path, None, None]:
"""Redirect _MATH_PROPOSALS_DIR and _DEFAULT_OUTPUT_PATH to tmp_path."""
original_dir = cli_module._MATH_PROPOSALS_DIR
original_default = cli_module._DEFAULT_OUTPUT_PATH
cli_module._MATH_PROPOSALS_DIR = tmp_path
cli_module._DEFAULT_OUTPUT_PATH = tmp_path / "proposals.jsonl"
try:
yield tmp_path / "proposals.jsonl"
finally:
cli_module._MATH_PROPOSALS_DIR = original_dir
cli_module._DEFAULT_OUTPUT_PATH = original_default
# ---------------------------------------------------------------------------
# Subcommand routing
# ---------------------------------------------------------------------------
def test_eval_math_contemplation_routes_from_cmd_eval(tmp_path: Path) -> None:
with _patch_proposals_dir(tmp_path) as output:
args = _Args(audit=str(REAL_AUDIT_PATH))
rc = cmd_eval(args)
assert rc == 0
assert output.exists()
# ---------------------------------------------------------------------------
# Happy path — real audit brief
# ---------------------------------------------------------------------------
def test_cli_lane_writes_jsonl_with_real_audit(tmp_path: Path) -> None:
with _patch_proposals_dir(tmp_path) as output:
args = _Args(audit=str(REAL_AUDIT_PATH))
rc = cmd_eval_math_contemplation(args)
assert rc == 0
assert output.exists()
lines = output.read_bytes().splitlines()
assert len(lines) >= 1
for line in lines:
obj = json.loads(line)
assert isinstance(obj, dict)
def test_cli_lane_output_sorted_by_proposal_id(tmp_path: Path) -> None:
# canonical_bytes() omits proposal_id by design; verify by comparing the
# written bytes to the decomposer output (which is itself sorted by proposal_id).
from teaching.math_contemplation import decompose_audit
from teaching.math_contemplation_proposal import canonical_bytes
proposals = decompose_audit(REAL_AUDIT_PATH)
expected = b"".join(canonical_bytes(p) + b"\n" for p in proposals)
with _patch_proposals_dir(tmp_path) as output:
args = _Args(audit=str(REAL_AUDIT_PATH))
cmd_eval_math_contemplation(args)
assert output.read_bytes() == expected, (
"output bytes must match decomposer output in proposal_id order"
)
def test_cli_lane_idempotent(tmp_path: Path) -> None:
with _patch_proposals_dir(tmp_path) as output:
args = _Args(audit=str(REAL_AUDIT_PATH))
cmd_eval_math_contemplation(args)
first_bytes = output.read_bytes()
cmd_eval_math_contemplation(args)
second_bytes = output.read_bytes()
assert first_bytes == second_bytes, "re-run must produce byte-identical output"
def test_cli_lane_json_flag(tmp_path: Path, capsys: pytest.CaptureFixture) -> None:
with _patch_proposals_dir(tmp_path) as output:
args = _Args(audit=str(REAL_AUDIT_PATH), json_mode=True)
rc = cmd_eval_math_contemplation(args)
assert rc == 0
captured = capsys.readouterr()
obj = json.loads(captured.out)
assert "proposals" in obj
assert isinstance(obj["proposals"], int)
assert obj["proposals"] >= 1
# ---------------------------------------------------------------------------
# Path-traversal rejection (exit 2)
# ---------------------------------------------------------------------------
def test_absolute_output_rejected() -> None:
args = _Args(audit=str(REAL_AUDIT_PATH), output="/etc/passwd")
with pytest.raises(SystemExit) as exc_info:
cmd_eval_math_contemplation(args)
assert exc_info.value.code == 2
def test_relative_traversal_rejected() -> None:
args = _Args(audit=str(REAL_AUDIT_PATH), output="../../../etc/passwd")
with pytest.raises(SystemExit) as exc_info:
cmd_eval_math_contemplation(args)
assert exc_info.value.code == 2
def test_sibling_directory_rejected() -> None:
# teaching/vault/ is a sibling, not inside teaching/math_proposals/
args = _Args(audit=str(REAL_AUDIT_PATH), output="teaching/vault/proposals.jsonl")
with pytest.raises(SystemExit) as exc_info:
cmd_eval_math_contemplation(args)
assert exc_info.value.code == 2
# ---------------------------------------------------------------------------
# Missing audit file (exit 1)
# ---------------------------------------------------------------------------
def test_missing_audit_exits_1(tmp_path: Path) -> None:
args = _Args(audit=str(tmp_path / "nope.json"))
with pytest.raises(SystemExit) as exc_info:
cmd_eval_math_contemplation(args)
assert exc_info.value.code == 1
# ---------------------------------------------------------------------------
# No writes outside teaching/math_proposals/
# ---------------------------------------------------------------------------
def test_no_writes_outside_allowed_root(tmp_path: Path) -> None:
with _patch_proposals_dir(tmp_path) as output:
args = _Args(audit=str(REAL_AUDIT_PATH))
cmd_eval_math_contemplation(args)
written = list(tmp_path.iterdir())
assert written == [output], f"unexpected files written: {written}"
# ---------------------------------------------------------------------------
# .gitkeep scaffold exists
# ---------------------------------------------------------------------------
def test_math_proposals_gitkeep_exists() -> None:
repo_root = Path(__file__).resolve().parent.parent
gitkeep = repo_root / "teaching" / "math_proposals" / ".gitkeep"
assert gitkeep.exists(), "teaching/math_proposals/.gitkeep must be committed"