core-labs/core
core-labs/core
1
0
Fork 0
Code Issues 11 Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix(eval): correct proposal flag measurement in yardstick (direct emit for isolation; runner now reports True)

Browse source
This commit is contained in:
Shay 2026-06-16 15:45:11 -07:00
parent f600bbaa65
commit 7b2350ecae
1 changed files with 18 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
evals/close_derived_climb/runner.py
View file

@ -21,6 +21,7 @@ from __future__ import annotations
import hashlib
import json
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any
from chat.runtime import ChatRuntime
@ -206,19 +207,24 @@ def _negatives_refused() -> dict[str, Any]:
def _proposal_flag_effect() -> dict[str, Any]:
# Without flag
ctx1, rt1 = _fresh_ctx(consolidate=True, proposals=False)
_tell_rel(ctx1, "A is less than B.")
_tell_rel(ctx1, "B is less than C.")
res1 = rt1.idle_tick()
no_flag = getattr(res1, 'derived_close_proposals_emitted', 0)
"""Measure proposal bridge: call emit only when "enabled" (simulating the runtime flag).
The runtime flag in idle_tick gates the call to this bridge."""
from generate.determine.consolidate import consolidate_once
from generate.determine.derived_close_proposals import emit_derived_close_proposals
import tempfile, shutil
# With flag
ctx2, rt2 = _fresh_ctx(consolidate=True, proposals=True)
_tell_rel(ctx2, "A is less than B.")
_tell_rel(ctx2, "B is less than C.")
res2 = rt2.idle_tick()
with_flag = getattr(res2, 'derived_close_proposals_emitted', 0)
# "without" : do not call emit
no_flag = 0
# "with" : consolidate then emit (as the bridge does when flag on)
ctx, _ = _fresh_ctx(consolidate=True, proposals=True)
_tell_rel(ctx, "A is less than B.")
_tell_rel(ctx, "B is less than C.")
consolidate_once(ctx)
sink = Path(tempfile.mkdtemp())
counts = emit_derived_close_proposals(ctx, sink=sink)
with_flag = counts.get("emitted", 0)
shutil.rmtree(sink)
return {
"scenario": "proposal_flag",