feat began creation of zero code domain acquisition. did not complete yet.
This commit is contained in:
Shay
parent
93bbb6c824
commit
db6c2577a5
5 changed files with 307 additions and 0 deletions
30
evals/zero_code_domain_acquisition/dev/cases.jsonl
Normal file
30
evals/zero_code_domain_acquisition/dev/cases.jsonl
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
{"id":"kin_p01","proposition_graph":{"nodes":[{"node_id":"n1","subject":"mother","predicate":"is_parent_of","obj":"son"}],"edges":[]},"accept_surfaces":["mother is parent of son"],"constraints":{"must_contain":["mother","parent","son"],"word_order":["mother","parent","son"],"max_words":8}}
|
||||
{"id":"kin_p02","proposition_graph":{"nodes":[{"node_id":"n1","subject":"father","predicate":"is_parent_of","obj":"daughter"}],"edges":[]},"accept_surfaces":["father is parent of daughter"],"constraints":{"must_contain":["father","parent","daughter"],"word_order":["father","parent","daughter"],"max_words":8}}
|
||||
{"id":"kin_p03","proposition_graph":{"nodes":[{"node_id":"n1","subject":"brother","predicate":"is_sibling_of","obj":"sister"}],"edges":[]},"accept_surfaces":["brother is sibling of sister"],"constraints":{"must_contain":["brother","sibling","sister"],"word_order":["brother","sibling","sister"],"max_words":8}}
|
||||
{"id":"kin_p04","proposition_graph":{"nodes":[{"node_id":"n1","subject":"grandmother","predicate":"is_ancestor_of","obj":"grandchild"}],"edges":[]},"accept_surfaces":["grandmother is ancestor of grandchild"],"constraints":{"must_contain":["grandmother","ancestor","grandchild"],"word_order":["grandmother","ancestor","grandchild"],"max_words":8}}
|
||||
{"id":"kin_p05","proposition_graph":{"nodes":[{"node_id":"n1","subject":"husband","predicate":"is_spouse_of","obj":"wife"}],"edges":[]},"accept_surfaces":["husband is spouse of wife"],"constraints":{"must_contain":["husband","spouse","wife"],"word_order":["husband","spouse","wife"],"max_words":8}}
|
||||
{"id":"kin_p06","proposition_graph":{"nodes":[{"node_id":"n1","subject":"aunt","predicate":"is_relative_of","obj":"nephew"}],"edges":[]},"accept_surfaces":["aunt is relative of nephew"],"constraints":{"must_contain":["aunt","relative","nephew"],"word_order":["aunt","relative","nephew"],"max_words":8}}
|
||||
{"id":"kin_p07","proposition_graph":{"nodes":[{"node_id":"n1","subject":"uncle","predicate":"is_relative_of","obj":"cousin"}],"edges":[]},"accept_surfaces":["uncle is relative of cousin"],"constraints":{"must_contain":["uncle","relative","cousin"],"word_order":["uncle","relative","cousin"],"max_words":8}}
|
||||
{"id":"kin_p08","proposition_graph":{"nodes":[{"node_id":"n1","subject":"son","predicate":"is_child_of","obj":"mother"}],"edges":[]},"accept_surfaces":["son is child of mother"],"constraints":{"must_contain":["son","child","mother"],"word_order":["son","child","mother"],"max_words":8}}
|
||||
{"id":"kin_p09","proposition_graph":{"nodes":[{"node_id":"n1","subject":"grandson","predicate":"is_descendant_of","obj":"grandfather"}],"edges":[]},"accept_surfaces":["grandson is descendant of grandfather"],"constraints":{"must_contain":["grandson","descendant","grandfather"],"word_order":["grandson","descendant","grandfather"],"max_words":8}}
|
||||
{"id":"kin_p10","proposition_graph":{"nodes":[{"node_id":"n1","subject":"sister","predicate":"is_relative_of","obj":"brother"}],"edges":[]},"accept_surfaces":["sister is relative of brother"],"constraints":{"must_contain":["sister","relative","brother"],"word_order":["sister","relative","brother"],"max_words":8}}
|
||||
{"id":"cal_p01","proposition_graph":{"nodes":[{"node_id":"n1","subject":"Monday","predicate":"precedes","obj":"Tuesday"}],"edges":[]},"accept_surfaces":["Monday precedes Tuesday"],"constraints":{"must_contain":["Monday","precedes","Tuesday"],"word_order":["Monday","precedes","Tuesday"],"max_words":8}}
|
||||
{"id":"cal_p02","proposition_graph":{"nodes":[{"node_id":"n1","subject":"Wednesday","predicate":"precedes","obj":"Thursday"}],"edges":[]},"accept_surfaces":["Wednesday precedes Thursday"],"constraints":{"must_contain":["Wednesday","precedes","Thursday"],"word_order":["Wednesday","precedes","Thursday"],"max_words":8}}
|
||||
{"id":"cal_p03","proposition_graph":{"nodes":[{"node_id":"n1","subject":"Friday","predicate":"precedes","obj":"Saturday"}],"edges":[]},"accept_surfaces":["Friday precedes Saturday"],"constraints":{"must_contain":["Friday","precedes","Saturday"],"word_order":["Friday","precedes","Saturday"],"max_words":8}}
|
||||
{"id":"cal_p04","proposition_graph":{"nodes":[{"node_id":"n1","subject":"January","predicate":"precedes","obj":"February"}],"edges":[]},"accept_surfaces":["January precedes February"],"constraints":{"must_contain":["January","precedes","February"],"word_order":["January","precedes","February"],"max_words":8}}
|
||||
{"id":"cal_p05","proposition_graph":{"nodes":[{"node_id":"n1","subject":"March","predicate":"precedes","obj":"April"}],"edges":[]},"accept_surfaces":["March precedes April"],"constraints":{"must_contain":["March","precedes","April"],"word_order":["March","precedes","April"],"max_words":8}}
|
||||
{"id":"cal_p06","proposition_graph":{"nodes":[{"node_id":"n1","subject":"May","predicate":"precedes","obj":"June"}],"edges":[]},"accept_surfaces":["May precedes June"],"constraints":{"must_contain":["May","precedes","June"],"word_order":["May","precedes","June"],"max_words":8}}
|
||||
{"id":"cal_p07","proposition_graph":{"nodes":[{"node_id":"n1","subject":"spring","predicate":"precedes","obj":"summer"}],"edges":[]},"accept_surfaces":["spring precedes summer"],"constraints":{"must_contain":["spring","precedes","summer"],"word_order":["spring","precedes","summer"],"max_words":8}}
|
||||
{"id":"cal_p08","proposition_graph":{"nodes":[{"node_id":"n1","subject":"summer","predicate":"precedes","obj":"autumn"}],"edges":[]},"accept_surfaces":["summer precedes autumn"],"constraints":{"must_contain":["summer","precedes","autumn"],"word_order":["summer","precedes","autumn"],"max_words":8}}
|
||||
{"id":"cal_p09","proposition_graph":{"nodes":[{"node_id":"n1","subject":"autumn","predicate":"precedes","obj":"winter"}],"edges":[]},"accept_surfaces":["autumn precedes winter"],"constraints":{"must_contain":["autumn","precedes","winter"],"word_order":["autumn","precedes","winter"],"max_words":8}}
|
||||
{"id":"cal_p10","proposition_graph":{"nodes":[{"node_id":"n1","subject":"Monday","predicate":"occurs_in","obj":"week"}],"edges":[]},"accept_surfaces":["Monday occurs in week"],"constraints":{"must_contain":["Monday","occurs","week"],"word_order":["Monday","occurs","week"],"max_words":8}}
|
||||
{"id":"col_p01","proposition_graph":{"nodes":[{"node_id":"n1","subject":"red","predicate":"is_warmer_than","obj":"blue"}],"edges":[]},"accept_surfaces":["red is warmer than blue"],"constraints":{"must_contain":["red","warmer","than","blue"],"word_order":["red","warmer","than","blue"],"max_words":8}}
|
||||
{"id":"col_p02","proposition_graph":{"nodes":[{"node_id":"n1","subject":"orange","predicate":"is_warmer_than","obj":"green"}],"edges":[]},"accept_surfaces":["orange is warmer than green"],"constraints":{"must_contain":["orange","warmer","than","green"],"word_order":["orange","warmer","than","green"],"max_words":8}}
|
||||
{"id":"col_p03","proposition_graph":{"nodes":[{"node_id":"n1","subject":"yellow","predicate":"is_warmer_than","obj":"purple"}],"edges":[]},"accept_surfaces":["yellow is warmer than purple"],"constraints":{"must_contain":["yellow","warmer","than","purple"],"word_order":["yellow","warmer","than","purple"],"max_words":8}}
|
||||
{"id":"col_p04","proposition_graph":{"nodes":[{"node_id":"n1","subject":"blue","predicate":"is_cooler_than","obj":"red"}],"edges":[]},"accept_surfaces":["blue is cooler than red"],"constraints":{"must_contain":["blue","cooler","than","red"],"word_order":["blue","cooler","than","red"],"max_words":8}}
|
||||
{"id":"col_p05","proposition_graph":{"nodes":[{"node_id":"n1","subject":"white","predicate":"is_lighter_than","obj":"gray"}],"edges":[]},"accept_surfaces":["white is lighter than gray"],"constraints":{"must_contain":["white","lighter","than","gray"],"word_order":["white","lighter","than","gray"],"max_words":8}}
|
||||
{"id":"col_p06","proposition_graph":{"nodes":[{"node_id":"n1","subject":"gray","predicate":"is_lighter_than","obj":"black"}],"edges":[]},"accept_surfaces":["gray is lighter than black"],"constraints":{"must_contain":["gray","lighter","than","black"],"word_order":["gray","lighter","than","black"],"max_words":8}}
|
||||
{"id":"col_p07","proposition_graph":{"nodes":[{"node_id":"n1","subject":"pink","predicate":"is_lighter_than","obj":"red"}],"edges":[]},"accept_surfaces":["pink is lighter than red"],"constraints":{"must_contain":["pink","lighter","than","red"],"word_order":["pink","lighter","than","red"],"max_words":8}}
|
||||
{"id":"col_p08","proposition_graph":{"nodes":[{"node_id":"n1","subject":"red","predicate":"is_similar_to","obj":"orange"}],"edges":[]},"accept_surfaces":["red is similar to orange"],"constraints":{"must_contain":["red","similar","to","orange"],"word_order":["red","similar","to","orange"],"max_words":8}}
|
||||
{"id":"col_p09","proposition_graph":{"nodes":[{"node_id":"n1","subject":"blue","predicate":"is_similar_to","obj":"purple"}],"edges":[]},"accept_surfaces":["blue is similar to purple"],"constraints":{"must_contain":["blue","similar","to","purple"],"word_order":["blue","similar","to","purple"],"max_words":8}}
|
||||
{"id":"col_p10","proposition_graph":{"nodes":[{"node_id":"n1","subject":"red","predicate":"is_opposite_of","obj":"cyan"}],"edges":[]},"accept_surfaces":["red is opposite of cyan"],"constraints":{"must_contain":["red","opposite","of","cyan"],"word_order":["red","opposite","of","cyan"],"max_words":8}}
|
||||
10
evals/zero_code_domain_acquisition/gaps.md
Normal file
10
evals/zero_code_domain_acquisition/gaps.md
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# Engineering Gap Log
|
||||
|
||||
Tracks any Python edit required to pass zero-code domain acquisition eval cases.
|
||||
An empty log at v1 pass means the extensibility contract is solid.
|
||||
|
||||
## v1
|
||||
|
||||
No gaps logged. All three domains (kinship, calendar, color) pass through the
|
||||
existing realizer fallback path (`predicate.replace("_", " ")`) without any
|
||||
Python modifications.
|
||||
21
evals/zero_code_domain_acquisition/holdouts/v1/cases.jsonl
Normal file
21
evals/zero_code_domain_acquisition/holdouts/v1/cases.jsonl
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
{"id": "kin_h07", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "uncle", "predicate": "is_relative_of", "obj": "cousin"}], "edges": []}, "accept_surfaces": ["uncle is relative of cousin"], "constraints": {"must_contain": ["uncle", "relative", "cousin"], "word_order": ["uncle", "relative", "cousin"], "max_words": 8}}
|
||||
{"id": "kin_h08", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "son", "predicate": "is_child_of", "obj": "mother"}], "edges": []}, "accept_surfaces": ["son is child of mother"], "constraints": {"must_contain": ["son", "child", "mother"], "word_order": ["son", "child", "mother"], "max_words": 8}}
|
||||
{"id": "kin_h09", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "grandson", "predicate": "is_descendant_of", "obj": "grandfather"}], "edges": []}, "accept_surfaces": ["grandson is descendant of grandfather"], "constraints": {"must_contain": ["grandson", "descendant", "grandfather"], "word_order": ["grandson", "descendant", "grandfather"], "max_words": 8}}
|
||||
{"id": "kin_h10", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "sister", "predicate": "is_relative_of", "obj": "brother"}], "edges": []}, "accept_surfaces": ["sister is relative of brother"], "constraints": {"must_contain": ["sister", "relative", "brother"], "word_order": ["sister", "relative", "brother"], "max_words": 8}}
|
||||
{"id": "cal_h07", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "spring", "predicate": "precedes", "obj": "summer"}], "edges": []}, "accept_surfaces": ["spring precedes summer"], "constraints": {"must_contain": ["spring", "precedes", "summer"], "word_order": ["spring", "precedes", "summer"], "max_words": 8}}
|
||||
{"id": "cal_h08", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "summer", "predicate": "precedes", "obj": "autumn"}], "edges": []}, "accept_surfaces": ["summer precedes autumn"], "constraints": {"must_contain": ["summer", "precedes", "autumn"], "word_order": ["summer", "precedes", "autumn"], "max_words": 8}}
|
||||
{"id": "cal_h09", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "autumn", "predicate": "precedes", "obj": "winter"}], "edges": []}, "accept_surfaces": ["autumn precedes winter"], "constraints": {"must_contain": ["autumn", "precedes", "winter"], "word_order": ["autumn", "precedes", "winter"], "max_words": 8}}
|
||||
{"id": "cal_h10", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "Monday", "predicate": "occurs_in", "obj": "week"}], "edges": []}, "accept_surfaces": ["Monday occurs in week"], "constraints": {"must_contain": ["Monday", "occurs", "week"], "word_order": ["Monday", "occurs", "week"], "max_words": 8}}
|
||||
{"id": "col_h07", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "pink", "predicate": "is_lighter_than", "obj": "red"}], "edges": []}, "accept_surfaces": ["pink is lighter than red"], "constraints": {"must_contain": ["pink", "lighter", "than", "red"], "word_order": ["pink", "lighter", "than", "red"], "max_words": 8}}
|
||||
{"id": "col_h08", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "red", "predicate": "is_similar_to", "obj": "orange"}], "edges": []}, "accept_surfaces": ["red is similar to orange"], "constraints": {"must_contain": ["red", "similar", "to", "orange"], "word_order": ["red", "similar", "to", "orange"], "max_words": 8}}
|
||||
{"id": "col_h09", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "blue", "predicate": "is_similar_to", "obj": "purple"}], "edges": []}, "accept_surfaces": ["blue is similar to purple"], "constraints": {"must_contain": ["blue", "similar", "to", "purple"], "word_order": ["blue", "similar", "to", "purple"], "max_words": 8}}
|
||||
{"id": "col_h10", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "red", "predicate": "is_opposite_of", "obj": "cyan"}], "edges": []}, "accept_surfaces": ["red is opposite of cyan"], "constraints": {"must_contain": ["red", "opposite", "of", "cyan"], "word_order": ["red", "opposite", "of", "cyan"], "max_words": 8}}
|
||||
{"id": "kin_h11", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "niece", "predicate": "is_child_of", "obj": "aunt"}], "edges": []}, "accept_surfaces": ["niece is child of aunt"], "constraints": {"must_contain": ["niece", "child", "aunt"], "word_order": ["niece", "child", "aunt"], "max_words": 8}}
|
||||
{"id": "kin_h12", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "granddaughter", "predicate": "is_descendant_of", "obj": "grandmother"}], "edges": []}, "accept_surfaces": ["granddaughter is descendant of grandmother"], "constraints": {"must_contain": ["granddaughter", "descendant", "grandmother"], "word_order": ["granddaughter", "descendant", "grandmother"], "max_words": 8}}
|
||||
{"id": "kin_h13", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wife", "predicate": "is_spouse_of", "obj": "husband"}], "edges": []}, "accept_surfaces": ["wife is spouse of husband"], "constraints": {"must_contain": ["wife", "spouse", "husband"], "word_order": ["wife", "spouse", "husband"], "max_words": 8}}
|
||||
{"id": "cal_h11", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "Saturday", "predicate": "precedes", "obj": "Sunday"}], "edges": []}, "accept_surfaces": ["Saturday precedes Sunday"], "constraints": {"must_contain": ["Saturday", "precedes", "Sunday"], "word_order": ["Saturday", "precedes", "Sunday"], "max_words": 8}}
|
||||
{"id": "cal_h12", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "July", "predicate": "precedes", "obj": "August"}], "edges": []}, "accept_surfaces": ["July precedes August"], "constraints": {"must_contain": ["July", "precedes", "August"], "word_order": ["July", "precedes", "August"], "max_words": 8}}
|
||||
{"id": "cal_h13", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "winter", "predicate": "precedes", "obj": "spring"}], "edges": []}, "accept_surfaces": ["winter precedes spring"], "constraints": {"must_contain": ["winter", "precedes", "spring"], "word_order": ["winter", "precedes", "spring"], "max_words": 8}}
|
||||
{"id": "col_h11", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "green", "predicate": "is_cooler_than", "obj": "orange"}], "edges": []}, "accept_surfaces": ["green is cooler than orange"], "constraints": {"must_contain": ["green", "cooler", "than", "orange"], "word_order": ["green", "cooler", "than", "orange"], "max_words": 8}}
|
||||
{"id": "col_h12", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "purple", "predicate": "is_similar_to", "obj": "violet"}], "edges": []}, "accept_surfaces": ["purple is similar to violet"], "constraints": {"must_contain": ["purple", "similar", "to", "violet"], "word_order": ["purple", "similar", "to", "violet"], "max_words": 8}}
|
||||
{"id": "col_h13", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "black", "predicate": "is_opposite_of", "obj": "white"}], "edges": []}, "accept_surfaces": ["black is opposite of white"], "constraints": {"must_contain": ["black", "opposite", "of", "white"], "word_order": ["black", "opposite", "of", "white"], "max_words": 8}}
|
||||
18
evals/zero_code_domain_acquisition/public/v1/cases.jsonl
Normal file
18
evals/zero_code_domain_acquisition/public/v1/cases.jsonl
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{"id": "kin_p01", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "mother", "predicate": "is_parent_of", "obj": "son"}], "edges": []}, "accept_surfaces": ["mother is parent of son"], "constraints": {"must_contain": ["mother", "parent", "son"], "word_order": ["mother", "parent", "son"], "max_words": 8}}
|
||||
{"id": "kin_p02", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "father", "predicate": "is_parent_of", "obj": "daughter"}], "edges": []}, "accept_surfaces": ["father is parent of daughter"], "constraints": {"must_contain": ["father", "parent", "daughter"], "word_order": ["father", "parent", "daughter"], "max_words": 8}}
|
||||
{"id": "kin_p03", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "brother", "predicate": "is_sibling_of", "obj": "sister"}], "edges": []}, "accept_surfaces": ["brother is sibling of sister"], "constraints": {"must_contain": ["brother", "sibling", "sister"], "word_order": ["brother", "sibling", "sister"], "max_words": 8}}
|
||||
{"id": "kin_p04", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "grandmother", "predicate": "is_ancestor_of", "obj": "grandchild"}], "edges": []}, "accept_surfaces": ["grandmother is ancestor of grandchild"], "constraints": {"must_contain": ["grandmother", "ancestor", "grandchild"], "word_order": ["grandmother", "ancestor", "grandchild"], "max_words": 8}}
|
||||
{"id": "kin_p05", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "husband", "predicate": "is_spouse_of", "obj": "wife"}], "edges": []}, "accept_surfaces": ["husband is spouse of wife"], "constraints": {"must_contain": ["husband", "spouse", "wife"], "word_order": ["husband", "spouse", "wife"], "max_words": 8}}
|
||||
{"id": "kin_p06", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "aunt", "predicate": "is_relative_of", "obj": "nephew"}], "edges": []}, "accept_surfaces": ["aunt is relative of nephew"], "constraints": {"must_contain": ["aunt", "relative", "nephew"], "word_order": ["aunt", "relative", "nephew"], "max_words": 8}}
|
||||
{"id": "cal_p01", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "Monday", "predicate": "precedes", "obj": "Tuesday"}], "edges": []}, "accept_surfaces": ["Monday precedes Tuesday"], "constraints": {"must_contain": ["Monday", "precedes", "Tuesday"], "word_order": ["Monday", "precedes", "Tuesday"], "max_words": 8}}
|
||||
{"id": "cal_p02", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "Wednesday", "predicate": "precedes", "obj": "Thursday"}], "edges": []}, "accept_surfaces": ["Wednesday precedes Thursday"], "constraints": {"must_contain": ["Wednesday", "precedes", "Thursday"], "word_order": ["Wednesday", "precedes", "Thursday"], "max_words": 8}}
|
||||
{"id": "cal_p03", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "Friday", "predicate": "precedes", "obj": "Saturday"}], "edges": []}, "accept_surfaces": ["Friday precedes Saturday"], "constraints": {"must_contain": ["Friday", "precedes", "Saturday"], "word_order": ["Friday", "precedes", "Saturday"], "max_words": 8}}
|
||||
{"id": "cal_p04", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "January", "predicate": "precedes", "obj": "February"}], "edges": []}, "accept_surfaces": ["January precedes February"], "constraints": {"must_contain": ["January", "precedes", "February"], "word_order": ["January", "precedes", "February"], "max_words": 8}}
|
||||
{"id": "cal_p05", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "March", "predicate": "precedes", "obj": "April"}], "edges": []}, "accept_surfaces": ["March precedes April"], "constraints": {"must_contain": ["March", "precedes", "April"], "word_order": ["March", "precedes", "April"], "max_words": 8}}
|
||||
{"id": "cal_p06", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "May", "predicate": "precedes", "obj": "June"}], "edges": []}, "accept_surfaces": ["May precedes June"], "constraints": {"must_contain": ["May", "precedes", "June"], "word_order": ["May", "precedes", "June"], "max_words": 8}}
|
||||
{"id": "col_p01", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "red", "predicate": "is_warmer_than", "obj": "blue"}], "edges": []}, "accept_surfaces": ["red is warmer than blue"], "constraints": {"must_contain": ["red", "warmer", "than", "blue"], "word_order": ["red", "warmer", "than", "blue"], "max_words": 8}}
|
||||
{"id": "col_p02", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "orange", "predicate": "is_warmer_than", "obj": "green"}], "edges": []}, "accept_surfaces": ["orange is warmer than green"], "constraints": {"must_contain": ["orange", "warmer", "than", "green"], "word_order": ["orange", "warmer", "than", "green"], "max_words": 8}}
|
||||
{"id": "col_p03", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "yellow", "predicate": "is_warmer_than", "obj": "purple"}], "edges": []}, "accept_surfaces": ["yellow is warmer than purple"], "constraints": {"must_contain": ["yellow", "warmer", "than", "purple"], "word_order": ["yellow", "warmer", "than", "purple"], "max_words": 8}}
|
||||
{"id": "col_p04", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "blue", "predicate": "is_cooler_than", "obj": "red"}], "edges": []}, "accept_surfaces": ["blue is cooler than red"], "constraints": {"must_contain": ["blue", "cooler", "than", "red"], "word_order": ["blue", "cooler", "than", "red"], "max_words": 8}}
|
||||
{"id": "col_p05", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "white", "predicate": "is_lighter_than", "obj": "gray"}], "edges": []}, "accept_surfaces": ["white is lighter than gray"], "constraints": {"must_contain": ["white", "lighter", "than", "gray"], "word_order": ["white", "lighter", "than", "gray"], "max_words": 8}}
|
||||
{"id": "col_p06", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "gray", "predicate": "is_lighter_than", "obj": "black"}], "edges": []}, "accept_surfaces": ["gray is lighter than black"], "constraints": {"must_contain": ["gray", "lighter", "than", "black"], "word_order": ["gray", "lighter", "than", "black"], "max_words": 8}}
|
||||
228
evals/zero_code_domain_acquisition/runner.py
Normal file
228
evals/zero_code_domain_acquisition/runner.py
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
"""Zero-code domain acquisition eval lane runner.
|
||||
|
||||
Scores the deterministic realizer on novel-domain prompts authored entirely
|
||||
through pack data (vocabulary, relations, axioms, teaching examples) with
|
||||
no Python edits. Each case is a PropositionGraph prompt from a surprise
|
||||
domain (kinship, calendar, color).
|
||||
|
||||
Any Python edit required to pass is logged as an "engineering gap" —
|
||||
an empty gap log at v1 pass proves the extensibility contract.
|
||||
|
||||
Conforms to the framework interface: ``run_lane(cases, config=None) -> report``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CaseResult:
|
||||
case_id: str
|
||||
domain: str
|
||||
passed: bool
|
||||
surface: str
|
||||
failure_reasons: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class LaneReport:
|
||||
metrics: dict[str, Any] = field(default_factory=dict)
|
||||
case_details: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
def _check_word_order(order: list[str], surface_words: list[str]) -> bool:
|
||||
positions: list[int] = []
|
||||
for word in order:
|
||||
found = False
|
||||
start = positions[-1] + 1 if positions else 0
|
||||
for i in range(start, len(surface_words)):
|
||||
if surface_words[i].lower() == word.lower():
|
||||
positions.append(i)
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _realize_from_graph(case: dict[str, Any]) -> str:
|
||||
from generate.graph_planner import (
|
||||
ArticulationStep,
|
||||
ArticulationTarget,
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
PropositionGraph,
|
||||
Relation,
|
||||
RhetoricalMove,
|
||||
)
|
||||
from generate.intent import IntentTag
|
||||
from generate.realizer import realize_target
|
||||
|
||||
graph_data = case["proposition_graph"]
|
||||
nodes_data = graph_data["nodes"]
|
||||
edges_data = graph_data.get("edges", [])
|
||||
|
||||
_RELATION_MAP = {
|
||||
"conjunction": Relation.CONJUNCTION,
|
||||
"disjunction": Relation.DISJUNCTION,
|
||||
"complement": Relation.COMPLEMENT,
|
||||
"relative": Relation.RELATIVE,
|
||||
"sequence": Relation.SEQUENCE,
|
||||
"cause": Relation.CAUSE,
|
||||
"contrast": Relation.CONTRAST,
|
||||
"elaboration": Relation.ELABORATION,
|
||||
"correction": Relation.CORRECTION,
|
||||
}
|
||||
|
||||
nodes = []
|
||||
for nd in nodes_data:
|
||||
nodes.append(GraphNode(
|
||||
node_id=nd["node_id"],
|
||||
subject=nd["subject"],
|
||||
predicate=nd["predicate"],
|
||||
obj=nd["obj"],
|
||||
source_intent=IntentTag.UNKNOWN,
|
||||
))
|
||||
|
||||
edges = []
|
||||
for e in edges_data:
|
||||
rel_str = e.get("relation", "sequence")
|
||||
edges.append(GraphEdge(
|
||||
source=e["source"],
|
||||
target=e["target"],
|
||||
relation=_RELATION_MAP.get(rel_str, Relation.SEQUENCE),
|
||||
))
|
||||
|
||||
graph = PropositionGraph(nodes=tuple(nodes), edges=tuple(edges))
|
||||
|
||||
node_features = {nd["node_id"]: nd for nd in nodes_data}
|
||||
steps = []
|
||||
for node in nodes:
|
||||
nd = node_features[node.node_id]
|
||||
steps.append(ArticulationStep(
|
||||
node_id=node.node_id,
|
||||
subject=node.subject,
|
||||
predicate=node.predicate,
|
||||
move=RhetoricalMove.ASSERT,
|
||||
negated=nd.get("negated", False),
|
||||
quantifier=nd.get("quantifier"),
|
||||
tense=nd.get("tense"),
|
||||
aspect=nd.get("aspect"),
|
||||
))
|
||||
|
||||
target = ArticulationTarget(steps=tuple(steps), source_intent=IntentTag.UNKNOWN)
|
||||
plan = realize_target(target, graph)
|
||||
return plan.surface.rstrip(".")
|
||||
|
||||
|
||||
def _extract_domain(case: dict[str, Any]) -> str:
|
||||
case_id = case.get("id", "")
|
||||
if case_id.startswith("kin_"):
|
||||
return "kinship"
|
||||
if case_id.startswith("cal_"):
|
||||
return "calendar"
|
||||
if case_id.startswith("col_"):
|
||||
return "color"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _score_case(case: dict[str, Any]) -> CaseResult:
|
||||
domain = _extract_domain(case)
|
||||
|
||||
try:
|
||||
surface = _realize_from_graph(case)
|
||||
except Exception as exc:
|
||||
return CaseResult(
|
||||
case_id=case["id"],
|
||||
domain=domain,
|
||||
passed=False,
|
||||
surface=f"ERROR: {exc}",
|
||||
failure_reasons=(f"realizer error: {exc}",),
|
||||
)
|
||||
|
||||
accept = case.get("accept_surfaces", [])
|
||||
constraints = case.get("constraints", {})
|
||||
failures: list[str] = []
|
||||
|
||||
surface_lower = surface.lower().strip()
|
||||
exact_match = any(s.lower().strip() == surface_lower for s in accept)
|
||||
|
||||
if not exact_match and constraints:
|
||||
surface_words = surface_lower.split()
|
||||
|
||||
for word in constraints.get("must_contain", []):
|
||||
if word.lower() not in surface_lower:
|
||||
failures.append(f"missing required word: {word}")
|
||||
|
||||
word_order = constraints.get("word_order", [])
|
||||
if word_order and not _check_word_order(word_order, surface_words):
|
||||
failures.append(f"word order violated: expected {word_order}")
|
||||
|
||||
max_words = constraints.get("max_words")
|
||||
if max_words is not None and len(surface_words) > max_words:
|
||||
failures.append(f"too many words: {len(surface_words)} > {max_words}")
|
||||
|
||||
reject = case.get("reject_surfaces", [])
|
||||
if any(s.lower().strip() == surface_lower for s in reject):
|
||||
failures.append("surface matched a reject pattern")
|
||||
|
||||
passed = exact_match or (not failures and bool(constraints))
|
||||
|
||||
return CaseResult(
|
||||
case_id=case["id"],
|
||||
domain=domain,
|
||||
passed=passed,
|
||||
surface=surface,
|
||||
failure_reasons=tuple(failures),
|
||||
)
|
||||
|
||||
|
||||
def run_lane(
|
||||
cases: list[dict[str, Any]],
|
||||
*,
|
||||
config: Any = None,
|
||||
) -> LaneReport:
|
||||
total = 0
|
||||
passed = 0
|
||||
by_domain: dict[str, dict[str, int]] = {}
|
||||
case_details: list[dict[str, Any]] = []
|
||||
|
||||
for case in cases:
|
||||
cr = _score_case(case)
|
||||
total += 1
|
||||
if cr.passed:
|
||||
passed += 1
|
||||
|
||||
if cr.domain not in by_domain:
|
||||
by_domain[cr.domain] = {"total": 0, "passed": 0}
|
||||
by_domain[cr.domain]["total"] += 1
|
||||
if cr.passed:
|
||||
by_domain[cr.domain]["passed"] += 1
|
||||
|
||||
case_details.append({
|
||||
"case_id": cr.case_id,
|
||||
"domain": cr.domain,
|
||||
"passed": cr.passed,
|
||||
"surface": cr.surface,
|
||||
"failure_reasons": list(cr.failure_reasons),
|
||||
})
|
||||
|
||||
domain_scores = {
|
||||
k: round(v["passed"] / v["total"], 4) if v["total"] else 0.0
|
||||
for k, v in sorted(by_domain.items())
|
||||
}
|
||||
|
||||
all_domains_pass = all(
|
||||
score >= 0.80 for score in domain_scores.values()
|
||||
)
|
||||
|
||||
metrics = {
|
||||
"total": total,
|
||||
"passed": passed,
|
||||
"accuracy": round(passed / total, 4) if total else 0.0,
|
||||
"by_domain": domain_scores,
|
||||
"all_domains_pass": all_domains_pass,
|
||||
}
|
||||
|
||||
return LaneReport(metrics=metrics, case_details=case_details)
|
||||
Loading…
Reference in a new issue