diff --git a/evals/zero_code_domain_acquisition/dev/cases.jsonl b/evals/zero_code_domain_acquisition/dev/cases.jsonl new file mode 100644 index 00000000..1148c25e --- /dev/null +++ b/evals/zero_code_domain_acquisition/dev/cases.jsonl @@ -0,0 +1,30 @@ +{"id":"kin_p01","proposition_graph":{"nodes":[{"node_id":"n1","subject":"mother","predicate":"is_parent_of","obj":"son"}],"edges":[]},"accept_surfaces":["mother is parent of son"],"constraints":{"must_contain":["mother","parent","son"],"word_order":["mother","parent","son"],"max_words":8}} +{"id":"kin_p02","proposition_graph":{"nodes":[{"node_id":"n1","subject":"father","predicate":"is_parent_of","obj":"daughter"}],"edges":[]},"accept_surfaces":["father is parent of daughter"],"constraints":{"must_contain":["father","parent","daughter"],"word_order":["father","parent","daughter"],"max_words":8}} +{"id":"kin_p03","proposition_graph":{"nodes":[{"node_id":"n1","subject":"brother","predicate":"is_sibling_of","obj":"sister"}],"edges":[]},"accept_surfaces":["brother is sibling of sister"],"constraints":{"must_contain":["brother","sibling","sister"],"word_order":["brother","sibling","sister"],"max_words":8}} +{"id":"kin_p04","proposition_graph":{"nodes":[{"node_id":"n1","subject":"grandmother","predicate":"is_ancestor_of","obj":"grandchild"}],"edges":[]},"accept_surfaces":["grandmother is ancestor of grandchild"],"constraints":{"must_contain":["grandmother","ancestor","grandchild"],"word_order":["grandmother","ancestor","grandchild"],"max_words":8}} +{"id":"kin_p05","proposition_graph":{"nodes":[{"node_id":"n1","subject":"husband","predicate":"is_spouse_of","obj":"wife"}],"edges":[]},"accept_surfaces":["husband is spouse of wife"],"constraints":{"must_contain":["husband","spouse","wife"],"word_order":["husband","spouse","wife"],"max_words":8}} +{"id":"kin_p06","proposition_graph":{"nodes":[{"node_id":"n1","subject":"aunt","predicate":"is_relative_of","obj":"nephew"}],"edges":[]},"accept_surfaces":["aunt is relative of nephew"],"constraints":{"must_contain":["aunt","relative","nephew"],"word_order":["aunt","relative","nephew"],"max_words":8}} +{"id":"kin_p07","proposition_graph":{"nodes":[{"node_id":"n1","subject":"uncle","predicate":"is_relative_of","obj":"cousin"}],"edges":[]},"accept_surfaces":["uncle is relative of cousin"],"constraints":{"must_contain":["uncle","relative","cousin"],"word_order":["uncle","relative","cousin"],"max_words":8}} +{"id":"kin_p08","proposition_graph":{"nodes":[{"node_id":"n1","subject":"son","predicate":"is_child_of","obj":"mother"}],"edges":[]},"accept_surfaces":["son is child of mother"],"constraints":{"must_contain":["son","child","mother"],"word_order":["son","child","mother"],"max_words":8}} +{"id":"kin_p09","proposition_graph":{"nodes":[{"node_id":"n1","subject":"grandson","predicate":"is_descendant_of","obj":"grandfather"}],"edges":[]},"accept_surfaces":["grandson is descendant of grandfather"],"constraints":{"must_contain":["grandson","descendant","grandfather"],"word_order":["grandson","descendant","grandfather"],"max_words":8}} +{"id":"kin_p10","proposition_graph":{"nodes":[{"node_id":"n1","subject":"sister","predicate":"is_relative_of","obj":"brother"}],"edges":[]},"accept_surfaces":["sister is relative of brother"],"constraints":{"must_contain":["sister","relative","brother"],"word_order":["sister","relative","brother"],"max_words":8}} +{"id":"cal_p01","proposition_graph":{"nodes":[{"node_id":"n1","subject":"Monday","predicate":"precedes","obj":"Tuesday"}],"edges":[]},"accept_surfaces":["Monday precedes Tuesday"],"constraints":{"must_contain":["Monday","precedes","Tuesday"],"word_order":["Monday","precedes","Tuesday"],"max_words":8}} +{"id":"cal_p02","proposition_graph":{"nodes":[{"node_id":"n1","subject":"Wednesday","predicate":"precedes","obj":"Thursday"}],"edges":[]},"accept_surfaces":["Wednesday precedes Thursday"],"constraints":{"must_contain":["Wednesday","precedes","Thursday"],"word_order":["Wednesday","precedes","Thursday"],"max_words":8}} +{"id":"cal_p03","proposition_graph":{"nodes":[{"node_id":"n1","subject":"Friday","predicate":"precedes","obj":"Saturday"}],"edges":[]},"accept_surfaces":["Friday precedes Saturday"],"constraints":{"must_contain":["Friday","precedes","Saturday"],"word_order":["Friday","precedes","Saturday"],"max_words":8}} +{"id":"cal_p04","proposition_graph":{"nodes":[{"node_id":"n1","subject":"January","predicate":"precedes","obj":"February"}],"edges":[]},"accept_surfaces":["January precedes February"],"constraints":{"must_contain":["January","precedes","February"],"word_order":["January","precedes","February"],"max_words":8}} +{"id":"cal_p05","proposition_graph":{"nodes":[{"node_id":"n1","subject":"March","predicate":"precedes","obj":"April"}],"edges":[]},"accept_surfaces":["March precedes April"],"constraints":{"must_contain":["March","precedes","April"],"word_order":["March","precedes","April"],"max_words":8}} +{"id":"cal_p06","proposition_graph":{"nodes":[{"node_id":"n1","subject":"May","predicate":"precedes","obj":"June"}],"edges":[]},"accept_surfaces":["May precedes June"],"constraints":{"must_contain":["May","precedes","June"],"word_order":["May","precedes","June"],"max_words":8}} +{"id":"cal_p07","proposition_graph":{"nodes":[{"node_id":"n1","subject":"spring","predicate":"precedes","obj":"summer"}],"edges":[]},"accept_surfaces":["spring precedes summer"],"constraints":{"must_contain":["spring","precedes","summer"],"word_order":["spring","precedes","summer"],"max_words":8}} +{"id":"cal_p08","proposition_graph":{"nodes":[{"node_id":"n1","subject":"summer","predicate":"precedes","obj":"autumn"}],"edges":[]},"accept_surfaces":["summer precedes autumn"],"constraints":{"must_contain":["summer","precedes","autumn"],"word_order":["summer","precedes","autumn"],"max_words":8}} +{"id":"cal_p09","proposition_graph":{"nodes":[{"node_id":"n1","subject":"autumn","predicate":"precedes","obj":"winter"}],"edges":[]},"accept_surfaces":["autumn precedes winter"],"constraints":{"must_contain":["autumn","precedes","winter"],"word_order":["autumn","precedes","winter"],"max_words":8}} +{"id":"cal_p10","proposition_graph":{"nodes":[{"node_id":"n1","subject":"Monday","predicate":"occurs_in","obj":"week"}],"edges":[]},"accept_surfaces":["Monday occurs in week"],"constraints":{"must_contain":["Monday","occurs","week"],"word_order":["Monday","occurs","week"],"max_words":8}} +{"id":"col_p01","proposition_graph":{"nodes":[{"node_id":"n1","subject":"red","predicate":"is_warmer_than","obj":"blue"}],"edges":[]},"accept_surfaces":["red is warmer than blue"],"constraints":{"must_contain":["red","warmer","than","blue"],"word_order":["red","warmer","than","blue"],"max_words":8}} +{"id":"col_p02","proposition_graph":{"nodes":[{"node_id":"n1","subject":"orange","predicate":"is_warmer_than","obj":"green"}],"edges":[]},"accept_surfaces":["orange is warmer than green"],"constraints":{"must_contain":["orange","warmer","than","green"],"word_order":["orange","warmer","than","green"],"max_words":8}} +{"id":"col_p03","proposition_graph":{"nodes":[{"node_id":"n1","subject":"yellow","predicate":"is_warmer_than","obj":"purple"}],"edges":[]},"accept_surfaces":["yellow is warmer than purple"],"constraints":{"must_contain":["yellow","warmer","than","purple"],"word_order":["yellow","warmer","than","purple"],"max_words":8}} +{"id":"col_p04","proposition_graph":{"nodes":[{"node_id":"n1","subject":"blue","predicate":"is_cooler_than","obj":"red"}],"edges":[]},"accept_surfaces":["blue is cooler than red"],"constraints":{"must_contain":["blue","cooler","than","red"],"word_order":["blue","cooler","than","red"],"max_words":8}} +{"id":"col_p05","proposition_graph":{"nodes":[{"node_id":"n1","subject":"white","predicate":"is_lighter_than","obj":"gray"}],"edges":[]},"accept_surfaces":["white is lighter than gray"],"constraints":{"must_contain":["white","lighter","than","gray"],"word_order":["white","lighter","than","gray"],"max_words":8}} +{"id":"col_p06","proposition_graph":{"nodes":[{"node_id":"n1","subject":"gray","predicate":"is_lighter_than","obj":"black"}],"edges":[]},"accept_surfaces":["gray is lighter than black"],"constraints":{"must_contain":["gray","lighter","than","black"],"word_order":["gray","lighter","than","black"],"max_words":8}} +{"id":"col_p07","proposition_graph":{"nodes":[{"node_id":"n1","subject":"pink","predicate":"is_lighter_than","obj":"red"}],"edges":[]},"accept_surfaces":["pink is lighter than red"],"constraints":{"must_contain":["pink","lighter","than","red"],"word_order":["pink","lighter","than","red"],"max_words":8}} +{"id":"col_p08","proposition_graph":{"nodes":[{"node_id":"n1","subject":"red","predicate":"is_similar_to","obj":"orange"}],"edges":[]},"accept_surfaces":["red is similar to orange"],"constraints":{"must_contain":["red","similar","to","orange"],"word_order":["red","similar","to","orange"],"max_words":8}} +{"id":"col_p09","proposition_graph":{"nodes":[{"node_id":"n1","subject":"blue","predicate":"is_similar_to","obj":"purple"}],"edges":[]},"accept_surfaces":["blue is similar to purple"],"constraints":{"must_contain":["blue","similar","to","purple"],"word_order":["blue","similar","to","purple"],"max_words":8}} +{"id":"col_p10","proposition_graph":{"nodes":[{"node_id":"n1","subject":"red","predicate":"is_opposite_of","obj":"cyan"}],"edges":[]},"accept_surfaces":["red is opposite of cyan"],"constraints":{"must_contain":["red","opposite","of","cyan"],"word_order":["red","opposite","of","cyan"],"max_words":8}} diff --git a/evals/zero_code_domain_acquisition/gaps.md b/evals/zero_code_domain_acquisition/gaps.md new file mode 100644 index 00000000..0afc45f7 --- /dev/null +++ b/evals/zero_code_domain_acquisition/gaps.md @@ -0,0 +1,10 @@ +# Engineering Gap Log + +Tracks any Python edit required to pass zero-code domain acquisition eval cases. +An empty log at v1 pass means the extensibility contract is solid. + +## v1 + +No gaps logged. All three domains (kinship, calendar, color) pass through the +existing realizer fallback path (`predicate.replace("_", " ")`) without any +Python modifications. diff --git a/evals/zero_code_domain_acquisition/holdouts/v1/cases.jsonl b/evals/zero_code_domain_acquisition/holdouts/v1/cases.jsonl new file mode 100644 index 00000000..68150299 --- /dev/null +++ b/evals/zero_code_domain_acquisition/holdouts/v1/cases.jsonl @@ -0,0 +1,21 @@ +{"id": "kin_h07", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "uncle", "predicate": "is_relative_of", "obj": "cousin"}], "edges": []}, "accept_surfaces": ["uncle is relative of cousin"], "constraints": {"must_contain": ["uncle", "relative", "cousin"], "word_order": ["uncle", "relative", "cousin"], "max_words": 8}} +{"id": "kin_h08", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "son", "predicate": "is_child_of", "obj": "mother"}], "edges": []}, "accept_surfaces": ["son is child of mother"], "constraints": {"must_contain": ["son", "child", "mother"], "word_order": ["son", "child", "mother"], "max_words": 8}} +{"id": "kin_h09", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "grandson", "predicate": "is_descendant_of", "obj": "grandfather"}], "edges": []}, "accept_surfaces": ["grandson is descendant of grandfather"], "constraints": {"must_contain": ["grandson", "descendant", "grandfather"], "word_order": ["grandson", "descendant", "grandfather"], "max_words": 8}} +{"id": "kin_h10", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "sister", "predicate": "is_relative_of", "obj": "brother"}], "edges": []}, "accept_surfaces": ["sister is relative of brother"], "constraints": {"must_contain": ["sister", "relative", "brother"], "word_order": ["sister", "relative", "brother"], "max_words": 8}} +{"id": "cal_h07", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "spring", "predicate": "precedes", "obj": "summer"}], "edges": []}, "accept_surfaces": ["spring precedes summer"], "constraints": {"must_contain": ["spring", "precedes", "summer"], "word_order": ["spring", "precedes", "summer"], "max_words": 8}} +{"id": "cal_h08", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "summer", "predicate": "precedes", "obj": "autumn"}], "edges": []}, "accept_surfaces": ["summer precedes autumn"], "constraints": {"must_contain": ["summer", "precedes", "autumn"], "word_order": ["summer", "precedes", "autumn"], "max_words": 8}} +{"id": "cal_h09", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "autumn", "predicate": "precedes", "obj": "winter"}], "edges": []}, "accept_surfaces": ["autumn precedes winter"], "constraints": {"must_contain": ["autumn", "precedes", "winter"], "word_order": ["autumn", "precedes", "winter"], "max_words": 8}} +{"id": "cal_h10", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "Monday", "predicate": "occurs_in", "obj": "week"}], "edges": []}, "accept_surfaces": ["Monday occurs in week"], "constraints": {"must_contain": ["Monday", "occurs", "week"], "word_order": ["Monday", "occurs", "week"], "max_words": 8}} +{"id": "col_h07", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "pink", "predicate": "is_lighter_than", "obj": "red"}], "edges": []}, "accept_surfaces": ["pink is lighter than red"], "constraints": {"must_contain": ["pink", "lighter", "than", "red"], "word_order": ["pink", "lighter", "than", "red"], "max_words": 8}} +{"id": "col_h08", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "red", "predicate": "is_similar_to", "obj": "orange"}], "edges": []}, "accept_surfaces": ["red is similar to orange"], "constraints": {"must_contain": ["red", "similar", "to", "orange"], "word_order": ["red", "similar", "to", "orange"], "max_words": 8}} +{"id": "col_h09", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "blue", "predicate": "is_similar_to", "obj": "purple"}], "edges": []}, "accept_surfaces": ["blue is similar to purple"], "constraints": {"must_contain": ["blue", "similar", "to", "purple"], "word_order": ["blue", "similar", "to", "purple"], "max_words": 8}} +{"id": "col_h10", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "red", "predicate": "is_opposite_of", "obj": "cyan"}], "edges": []}, "accept_surfaces": ["red is opposite of cyan"], "constraints": {"must_contain": ["red", "opposite", "of", "cyan"], "word_order": ["red", "opposite", "of", "cyan"], "max_words": 8}} +{"id": "kin_h11", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "niece", "predicate": "is_child_of", "obj": "aunt"}], "edges": []}, "accept_surfaces": ["niece is child of aunt"], "constraints": {"must_contain": ["niece", "child", "aunt"], "word_order": ["niece", "child", "aunt"], "max_words": 8}} +{"id": "kin_h12", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "granddaughter", "predicate": "is_descendant_of", "obj": "grandmother"}], "edges": []}, "accept_surfaces": ["granddaughter is descendant of grandmother"], "constraints": {"must_contain": ["granddaughter", "descendant", "grandmother"], "word_order": ["granddaughter", "descendant", "grandmother"], "max_words": 8}} +{"id": "kin_h13", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wife", "predicate": "is_spouse_of", "obj": "husband"}], "edges": []}, "accept_surfaces": ["wife is spouse of husband"], "constraints": {"must_contain": ["wife", "spouse", "husband"], "word_order": ["wife", "spouse", "husband"], "max_words": 8}} +{"id": "cal_h11", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "Saturday", "predicate": "precedes", "obj": "Sunday"}], "edges": []}, "accept_surfaces": ["Saturday precedes Sunday"], "constraints": {"must_contain": ["Saturday", "precedes", "Sunday"], "word_order": ["Saturday", "precedes", "Sunday"], "max_words": 8}} +{"id": "cal_h12", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "July", "predicate": "precedes", "obj": "August"}], "edges": []}, "accept_surfaces": ["July precedes August"], "constraints": {"must_contain": ["July", "precedes", "August"], "word_order": ["July", "precedes", "August"], "max_words": 8}} +{"id": "cal_h13", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "winter", "predicate": "precedes", "obj": "spring"}], "edges": []}, "accept_surfaces": ["winter precedes spring"], "constraints": {"must_contain": ["winter", "precedes", "spring"], "word_order": ["winter", "precedes", "spring"], "max_words": 8}} +{"id": "col_h11", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "green", "predicate": "is_cooler_than", "obj": "orange"}], "edges": []}, "accept_surfaces": ["green is cooler than orange"], "constraints": {"must_contain": ["green", "cooler", "than", "orange"], "word_order": ["green", "cooler", "than", "orange"], "max_words": 8}} +{"id": "col_h12", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "purple", "predicate": "is_similar_to", "obj": "violet"}], "edges": []}, "accept_surfaces": ["purple is similar to violet"], "constraints": {"must_contain": ["purple", "similar", "to", "violet"], "word_order": ["purple", "similar", "to", "violet"], "max_words": 8}} +{"id": "col_h13", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "black", "predicate": "is_opposite_of", "obj": "white"}], "edges": []}, "accept_surfaces": ["black is opposite of white"], "constraints": {"must_contain": ["black", "opposite", "of", "white"], "word_order": ["black", "opposite", "of", "white"], "max_words": 8}} diff --git a/evals/zero_code_domain_acquisition/public/v1/cases.jsonl b/evals/zero_code_domain_acquisition/public/v1/cases.jsonl new file mode 100644 index 00000000..440ac502 --- /dev/null +++ b/evals/zero_code_domain_acquisition/public/v1/cases.jsonl @@ -0,0 +1,18 @@ +{"id": "kin_p01", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "mother", "predicate": "is_parent_of", "obj": "son"}], "edges": []}, "accept_surfaces": ["mother is parent of son"], "constraints": {"must_contain": ["mother", "parent", "son"], "word_order": ["mother", "parent", "son"], "max_words": 8}} +{"id": "kin_p02", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "father", "predicate": "is_parent_of", "obj": "daughter"}], "edges": []}, "accept_surfaces": ["father is parent of daughter"], "constraints": {"must_contain": ["father", "parent", "daughter"], "word_order": ["father", "parent", "daughter"], "max_words": 8}} +{"id": "kin_p03", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "brother", "predicate": "is_sibling_of", "obj": "sister"}], "edges": []}, "accept_surfaces": ["brother is sibling of sister"], "constraints": {"must_contain": ["brother", "sibling", "sister"], "word_order": ["brother", "sibling", "sister"], "max_words": 8}} +{"id": "kin_p04", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "grandmother", "predicate": "is_ancestor_of", "obj": "grandchild"}], "edges": []}, "accept_surfaces": ["grandmother is ancestor of grandchild"], "constraints": {"must_contain": ["grandmother", "ancestor", "grandchild"], "word_order": ["grandmother", "ancestor", "grandchild"], "max_words": 8}} +{"id": "kin_p05", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "husband", "predicate": "is_spouse_of", "obj": "wife"}], "edges": []}, "accept_surfaces": ["husband is spouse of wife"], "constraints": {"must_contain": ["husband", "spouse", "wife"], "word_order": ["husband", "spouse", "wife"], "max_words": 8}} +{"id": "kin_p06", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "aunt", "predicate": "is_relative_of", "obj": "nephew"}], "edges": []}, "accept_surfaces": ["aunt is relative of nephew"], "constraints": {"must_contain": ["aunt", "relative", "nephew"], "word_order": ["aunt", "relative", "nephew"], "max_words": 8}} +{"id": "cal_p01", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "Monday", "predicate": "precedes", "obj": "Tuesday"}], "edges": []}, "accept_surfaces": ["Monday precedes Tuesday"], "constraints": {"must_contain": ["Monday", "precedes", "Tuesday"], "word_order": ["Monday", "precedes", "Tuesday"], "max_words": 8}} +{"id": "cal_p02", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "Wednesday", "predicate": "precedes", "obj": "Thursday"}], "edges": []}, "accept_surfaces": ["Wednesday precedes Thursday"], "constraints": {"must_contain": ["Wednesday", "precedes", "Thursday"], "word_order": ["Wednesday", "precedes", "Thursday"], "max_words": 8}} +{"id": "cal_p03", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "Friday", "predicate": "precedes", "obj": "Saturday"}], "edges": []}, "accept_surfaces": ["Friday precedes Saturday"], "constraints": {"must_contain": ["Friday", "precedes", "Saturday"], "word_order": ["Friday", "precedes", "Saturday"], "max_words": 8}} +{"id": "cal_p04", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "January", "predicate": "precedes", "obj": "February"}], "edges": []}, "accept_surfaces": ["January precedes February"], "constraints": {"must_contain": ["January", "precedes", "February"], "word_order": ["January", "precedes", "February"], "max_words": 8}} +{"id": "cal_p05", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "March", "predicate": "precedes", "obj": "April"}], "edges": []}, "accept_surfaces": ["March precedes April"], "constraints": {"must_contain": ["March", "precedes", "April"], "word_order": ["March", "precedes", "April"], "max_words": 8}} +{"id": "cal_p06", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "May", "predicate": "precedes", "obj": "June"}], "edges": []}, "accept_surfaces": ["May precedes June"], "constraints": {"must_contain": ["May", "precedes", "June"], "word_order": ["May", "precedes", "June"], "max_words": 8}} +{"id": "col_p01", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "red", "predicate": "is_warmer_than", "obj": "blue"}], "edges": []}, "accept_surfaces": ["red is warmer than blue"], "constraints": {"must_contain": ["red", "warmer", "than", "blue"], "word_order": ["red", "warmer", "than", "blue"], "max_words": 8}} +{"id": "col_p02", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "orange", "predicate": "is_warmer_than", "obj": "green"}], "edges": []}, "accept_surfaces": ["orange is warmer than green"], "constraints": {"must_contain": ["orange", "warmer", "than", "green"], "word_order": ["orange", "warmer", "than", "green"], "max_words": 8}} +{"id": "col_p03", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "yellow", "predicate": "is_warmer_than", "obj": "purple"}], "edges": []}, "accept_surfaces": ["yellow is warmer than purple"], "constraints": {"must_contain": ["yellow", "warmer", "than", "purple"], "word_order": ["yellow", "warmer", "than", "purple"], "max_words": 8}} +{"id": "col_p04", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "blue", "predicate": "is_cooler_than", "obj": "red"}], "edges": []}, "accept_surfaces": ["blue is cooler than red"], "constraints": {"must_contain": ["blue", "cooler", "than", "red"], "word_order": ["blue", "cooler", "than", "red"], "max_words": 8}} +{"id": "col_p05", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "white", "predicate": "is_lighter_than", "obj": "gray"}], "edges": []}, "accept_surfaces": ["white is lighter than gray"], "constraints": {"must_contain": ["white", "lighter", "than", "gray"], "word_order": ["white", "lighter", "than", "gray"], "max_words": 8}} +{"id": "col_p06", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "gray", "predicate": "is_lighter_than", "obj": "black"}], "edges": []}, "accept_surfaces": ["gray is lighter than black"], "constraints": {"must_contain": ["gray", "lighter", "than", "black"], "word_order": ["gray", "lighter", "than", "black"], "max_words": 8}} diff --git a/evals/zero_code_domain_acquisition/runner.py b/evals/zero_code_domain_acquisition/runner.py new file mode 100644 index 00000000..a1a1c89d --- /dev/null +++ b/evals/zero_code_domain_acquisition/runner.py @@ -0,0 +1,228 @@ +"""Zero-code domain acquisition eval lane runner. + +Scores the deterministic realizer on novel-domain prompts authored entirely +through pack data (vocabulary, relations, axioms, teaching examples) with +no Python edits. Each case is a PropositionGraph prompt from a surprise +domain (kinship, calendar, color). + +Any Python edit required to pass is logged as an "engineering gap" — +an empty gap log at v1 pass proves the extensibility contract. + +Conforms to the framework interface: ``run_lane(cases, config=None) -> report``. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True, slots=True) +class CaseResult: + case_id: str + domain: str + passed: bool + surface: str + failure_reasons: tuple[str, ...] + + +@dataclass(slots=True) +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +def _check_word_order(order: list[str], surface_words: list[str]) -> bool: + positions: list[int] = [] + for word in order: + found = False + start = positions[-1] + 1 if positions else 0 + for i in range(start, len(surface_words)): + if surface_words[i].lower() == word.lower(): + positions.append(i) + found = True + break + if not found: + return False + return True + + +def _realize_from_graph(case: dict[str, Any]) -> str: + from generate.graph_planner import ( + ArticulationStep, + ArticulationTarget, + GraphEdge, + GraphNode, + PropositionGraph, + Relation, + RhetoricalMove, + ) + from generate.intent import IntentTag + from generate.realizer import realize_target + + graph_data = case["proposition_graph"] + nodes_data = graph_data["nodes"] + edges_data = graph_data.get("edges", []) + + _RELATION_MAP = { + "conjunction": Relation.CONJUNCTION, + "disjunction": Relation.DISJUNCTION, + "complement": Relation.COMPLEMENT, + "relative": Relation.RELATIVE, + "sequence": Relation.SEQUENCE, + "cause": Relation.CAUSE, + "contrast": Relation.CONTRAST, + "elaboration": Relation.ELABORATION, + "correction": Relation.CORRECTION, + } + + nodes = [] + for nd in nodes_data: + nodes.append(GraphNode( + node_id=nd["node_id"], + subject=nd["subject"], + predicate=nd["predicate"], + obj=nd["obj"], + source_intent=IntentTag.UNKNOWN, + )) + + edges = [] + for e in edges_data: + rel_str = e.get("relation", "sequence") + edges.append(GraphEdge( + source=e["source"], + target=e["target"], + relation=_RELATION_MAP.get(rel_str, Relation.SEQUENCE), + )) + + graph = PropositionGraph(nodes=tuple(nodes), edges=tuple(edges)) + + node_features = {nd["node_id"]: nd for nd in nodes_data} + steps = [] + for node in nodes: + nd = node_features[node.node_id] + steps.append(ArticulationStep( + node_id=node.node_id, + subject=node.subject, + predicate=node.predicate, + move=RhetoricalMove.ASSERT, + negated=nd.get("negated", False), + quantifier=nd.get("quantifier"), + tense=nd.get("tense"), + aspect=nd.get("aspect"), + )) + + target = ArticulationTarget(steps=tuple(steps), source_intent=IntentTag.UNKNOWN) + plan = realize_target(target, graph) + return plan.surface.rstrip(".") + + +def _extract_domain(case: dict[str, Any]) -> str: + case_id = case.get("id", "") + if case_id.startswith("kin_"): + return "kinship" + if case_id.startswith("cal_"): + return "calendar" + if case_id.startswith("col_"): + return "color" + return "unknown" + + +def _score_case(case: dict[str, Any]) -> CaseResult: + domain = _extract_domain(case) + + try: + surface = _realize_from_graph(case) + except Exception as exc: + return CaseResult( + case_id=case["id"], + domain=domain, + passed=False, + surface=f"ERROR: {exc}", + failure_reasons=(f"realizer error: {exc}",), + ) + + accept = case.get("accept_surfaces", []) + constraints = case.get("constraints", {}) + failures: list[str] = [] + + surface_lower = surface.lower().strip() + exact_match = any(s.lower().strip() == surface_lower for s in accept) + + if not exact_match and constraints: + surface_words = surface_lower.split() + + for word in constraints.get("must_contain", []): + if word.lower() not in surface_lower: + failures.append(f"missing required word: {word}") + + word_order = constraints.get("word_order", []) + if word_order and not _check_word_order(word_order, surface_words): + failures.append(f"word order violated: expected {word_order}") + + max_words = constraints.get("max_words") + if max_words is not None and len(surface_words) > max_words: + failures.append(f"too many words: {len(surface_words)} > {max_words}") + + reject = case.get("reject_surfaces", []) + if any(s.lower().strip() == surface_lower for s in reject): + failures.append("surface matched a reject pattern") + + passed = exact_match or (not failures and bool(constraints)) + + return CaseResult( + case_id=case["id"], + domain=domain, + passed=passed, + surface=surface, + failure_reasons=tuple(failures), + ) + + +def run_lane( + cases: list[dict[str, Any]], + *, + config: Any = None, +) -> LaneReport: + total = 0 + passed = 0 + by_domain: dict[str, dict[str, int]] = {} + case_details: list[dict[str, Any]] = [] + + for case in cases: + cr = _score_case(case) + total += 1 + if cr.passed: + passed += 1 + + if cr.domain not in by_domain: + by_domain[cr.domain] = {"total": 0, "passed": 0} + by_domain[cr.domain]["total"] += 1 + if cr.passed: + by_domain[cr.domain]["passed"] += 1 + + case_details.append({ + "case_id": cr.case_id, + "domain": cr.domain, + "passed": cr.passed, + "surface": cr.surface, + "failure_reasons": list(cr.failure_reasons), + }) + + domain_scores = { + k: round(v["passed"] / v["total"], 4) if v["total"] else 0.0 + for k, v in sorted(by_domain.items()) + } + + all_domains_pass = all( + score >= 0.80 for score in domain_scores.values() + ) + + metrics = { + "total": total, + "passed": passed, + "accuracy": round(passed / total, 4) if total else 0.0, + "by_domain": domain_scores, + "all_domains_pass": all_domains_pass, + } + + return LaneReport(metrics=metrics, case_details=case_details)