From 21b10028b54736ec0d5fa72b327599eae4afd895 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 20 May 2026 05:59:51 -0700 Subject: [PATCH] fix(evals): introspect run_lane signature before passing workers kwarg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #46 added the `workers` kwarg to framework dispatch (evals/framework.py:176) but only the cognition runner was updated to accept it. The three serial lanes (cold_start_grounding, deterministic_fluency, warmed_session_consistency) — and ~30 other runners — raised TypeError on every framework invocation, producing 18 test failures across the full suite. Fix at the dispatch site rather than per-runner: inspect the target run_lane signature and pass `workers=` only when it accepts the kwarg (or has **kwargs). This keeps the framework contract backward-compatible with the legacy two-arg shape and forward-compatible with future parallelized runners — no runner needs updating. Full lane: 2859 passed, 3 skipped, 0 failed (was 2841/18 failed). Cognition eval byte-identical: 100/100/91.7/100. --- evals/framework.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/evals/framework.py b/evals/framework.py index 4a9b1a1b..74bd3a7c 100644 --- a/evals/framework.py +++ b/evals/framework.py @@ -15,6 +15,7 @@ from __future__ import annotations import importlib import importlib.util +import inspect import json import sys from dataclasses import dataclass, field @@ -173,7 +174,19 @@ def run_lane( cases = load_cases(cases_path) runner_module = load_lane_runner(lane) - report = runner_module.run_lane(cases, config=config, workers=workers) + # PR #46 added the `workers` kwarg to enable parallel execution on + # lanes that opt in (currently: cognition). Most lane runners are + # serial and predate this contract. Pass `workers` only when the + # target runner's signature accepts it — otherwise call the legacy + # two-arg form. This keeps the framework dispatch backward-compatible + # without forcing every runner to accept a no-op kwarg. + runner_params = inspect.signature(runner_module.run_lane).parameters + if "workers" in runner_params or any( + p.kind == inspect.Parameter.VAR_KEYWORD for p in runner_params.values() + ): + report = runner_module.run_lane(cases, config=config, workers=workers) + else: + report = runner_module.run_lane(cases, config=config) return LaneResult( lane=lane.name,