core/scripts/scaffold_phase5_domain_lanes.py

"""Write contract.md, runner.py, __init__.py, and gaps.md for each
Phase 5.4–5.7 domain lane.

The lane scaffolds delegate scoring to the grammatical_coverage runner
(same pattern as english_fluency_ood) so the rubric stays consistent
across lanes.  Vocabulary is the only thing that changes.

Run after `generate_phase5_domain_lanes.py`.
"""

from __future__ import annotations

from pathlib import Path

LANES: dict[str, dict[str, str]] = {
    "elementary_mathematics_ood": {
        "phase": "5.4",
        "title": "Elementary mathematics fluency OOD",
        "public_domains": "arithmetic, set, geometry",
        "holdout_domain": "probability",
        "claim": "the deterministic realizer produces grammatical English surfaces over elementary-mathematics vocabulary the seed pack does not contain",
    },
    "foundational_physics_ood": {
        "phase": "5.5",
        "title": "Foundational physics fluency OOD",
        "public_domains": "mechanics, electricity, thermodynamics",
        "holdout_domain": "optics",
        "claim": "the deterministic realizer produces grammatical English surfaces over foundational-physics vocabulary the seed pack does not contain",
    },
    "foundational_biology_ood": {
        "phase": "5.6",
        "title": "Foundational biology fluency OOD",
        "public_domains": "cell, organism, ecosystem",
        "holdout_domain": "genetics",
        "claim": "the deterministic realizer produces grammatical English surfaces over foundational-biology vocabulary the seed pack does not contain",
    },
    "classical_literature_ood": {
        "phase": "5.7",
        "title": "Classical literature fluency OOD",
        "public_domains": "epic, tragedy, lyric",
        "holdout_domain": "comedy",
        "claim": "the deterministic realizer produces grammatical English surfaces over classical-literature vocabulary the seed pack does not contain",
    },
}


def _contract(lane: str, spec: dict[str, str]) -> str:
    return f"""# {spec['title']} eval lane (Phase {spec['phase']})

## What it measures

Whether {spec['claim']}.

The structural claim under test: fluency is mechanistic in the
realizer (templates over typed graph nodes), not lexical
(pack-bound).  If the claim holds, this OOD vocabulary passes the
same syntactic gates the seed vocabulary did at v1/v2 of
`grammatical_coverage` — and the same 13 constructions covered by
the Phase 5.1 `english_fluency_ood` lane.

## Inputs

`public/v1/cases.jsonl`
:  117 cases — 13 constructions × 3 public domains ({spec['public_domains']})
   × 3 lexical items per domain.

`holdouts/v1/cases.jsonl`
:  39 cases — 13 constructions × 1 held-out domain
   ({spec['holdout_domain']}) × 3 items.

`dev/cases.jsonl`
:  13 cases — one of each construction from the first public domain.

Each case is one PropositionGraph JSON with realised constraints,
generated by `scripts/generate_phase5_domain_lanes.py`.

## Scoring rubric

Delegated to `evals/grammatical_coverage/runner.py` so the rubric
stays consistent across all fluency-OOD lanes:

  - surface satisfies `accept_surfaces` or `constraints` (must_contain,
    word_order, max_words)
  - surface is not in `reject_surfaces`

## Target

100% on public + holdouts.  Any miss indicates a real realizer or
inflection gap and is filed in `gaps.md`.

## Provenance

Same harness as `evals/english_fluency_ood/` (Phase 5.1).
Vocabulary is the only material change between lanes.
"""


_RUNNER = '''"""Phase {phase} domain fluency OOD eval lane runner.

Verifies the deterministic realizer produces grammatical English
surfaces across the 13 grammatical-coverage constructions when the
(subject, predicate, object) vocabulary is **out of the
en_core_cognition_v1 distribution** — drawn from the {public_domains}
public domains and the held-out {holdout_domain} domain.

Scoring is delegated to the grammatical_coverage runner so the
rubric stays consistent across all fluency-OOD lanes.

Conforms to the framework interface: run_lane(cases, config=None) -> report.
"""

from __future__ import annotations

from typing import Any

from evals.grammatical_coverage.runner import LaneReport, run_lane as _run


def run_lane(cases: list[dict[str, Any]], *, config: Any = None) -> LaneReport:
    return _run(cases, config=config)
'''


def _gaps(lane: str, spec: dict[str, str]) -> str:
    return f"""# {spec['title']} — gaps

## v1 (current)

- Constructions covered: all 13 (C01–C13) via grammatical_coverage rubric.
- Domains covered: public — {spec['public_domains']}; holdout —
  {spec['holdout_domain']}.
- Predicates default to regular verbs to keep morphology gaps from
  confounding the structural fluency claim.

## Known gaps for v2

The Phase 5.1 English fluency OOD lane already names three realizer
gaps that apply equally here:

  G1. Irregular past tense — past_tense() handles regular -ed but not
      irregular forms (e.g. "ran", "stood").
  G2. Plural agreement — the realizer does not pluralise nouns or
      conjugate to plural subjects.
  G3. Punctuation strictness — the rubric pins comma-bounded relative
      clauses exactly; small punctuation differences still fail.

These are not lane-specific.  When 5.1 closes them, every domain lane
gains the same coverage for free.

## Out of scope for this lane

- Domain-specific semantic correctness (whether "ribosome assembles
  protein" is *true* biology).  This lane measures fluency, not
  factual correctness.  Truth checks live in the
  `epistemic_status` surface (ADR-0021) and future curriculum
  lanes.
"""


if __name__ == "__main__":
    root = Path(__file__).resolve().parent.parent
    for lane, spec in LANES.items():
        lane_dir = root / "evals" / lane
        (lane_dir / "__init__.py").write_text("")
        (lane_dir / "contract.md").write_text(_contract(lane, spec))
        (lane_dir / "runner.py").write_text(_RUNNER.format(**spec))
        (lane_dir / "gaps.md").write_text(_gaps(lane, spec))
        print(f"scaffolded: evals/{lane}/")