core/sme_graphs.jsonl
Shay 96e5f46866 Implement blind-test embedding and evaluation for SME v2
Protects structure-mapping invariant. S1-S4 labels isolated from graph embeddings. Procrustes tests verify geometric separability and invariance. Verdict: GO.
2026-07-19 16:37:32 -07:00

51 lines
20 KiB
JSON

{"id": "1", "graph": {"entities": ["Mouse", "Keyboard"], "initial_state": [{"entity": "Mouse", "quantity": {"unit": "dollars", "value": 16.0}}], "operations": [{"actor": "Keyboard", "kind": "compare_multiplicative", "operand": {"direction": "times", "reference_actor": "Mouse", "factor": 3.0}}], "unknown": {"entity": null, "unit": "dollars"}}}
{"id": "2", "graph": {"entities": ["Ladies", "Girls"], "initial_state": [{"entity": "Ladies", "quantity": {"unit": "people", "value": 100.0}}], "operations": [{"actor": "Girls", "kind": "compare_multiplicative", "operand": {"direction": "times", "reference_actor": "Ladies", "factor": 3.0}}], "unknown": {"entity": null, "unit": "people"}}}
{"id": "9", "graph": {"entities": ["Ivan", "Jerry"], "initial_state": [{"entity": "Ivan", "quantity": {"unit": "dice", "value": 20.0}}], "operations": [{"actor": "Jerry", "kind": "compare_multiplicative", "operand": {"direction": "times", "reference_actor": "Ivan", "factor": 2.0}}], "unknown": {"entity": null, "unit": "dice"}}}
{"id": "17", "graph": {"entities": ["Sidney", "Brooke"], "initial_state": [{"entity": "Sidney", "quantity": {"unit": "jumping jacks", "value": 146.0}}], "operations": [{"actor": "Brooke", "kind": "compare_multiplicative", "operand": {"direction": "times", "reference_actor": "Sidney", "factor": 3.0}}], "unknown": {"entity": "Brooke", "unit": "jumping jacks"}}}
{"id": "18", "graph": {"entities": ["Tom", "Jerry"], "initial_state": [{"entity": "Tom", "quantity": {"unit": "apples", "value": 7}}], "operations": [{"actor": "Jerry", "kind": "compare_multiplicative", "operand": {"direction": "times", "reference_actor": "Tom", "factor": 3.0}}], "unknown": {"entity": null, "unit": "apples"}}}
{"id": "24", "graph": {"entities": ["Jan"], "initial_state": [], "operations": [{"actor": "Jan", "kind": "add", "operand": {"unit": "feet", "value": 1000}}, {"actor": "Jan", "kind": "unit_partition", "operand": {"result_unit": "sections", "unit": "feet", "value": 25.0}}, {"actor": "Jan", "kind": "fraction_portion", "operand": {"denominator": 4, "numerator": 1, "referent": "that"}}, {"actor": "Jan", "kind": "fraction_portion", "operand": {"denominator": 2, "numerator": 1, "referent": "rest"}}], "unknown": {"entity": "Jan", "unit": "sections"}}}
{"id": "46", "graph": {"entities": ["Sidney", "Brooke"], "initial_state": [{"entity": "Sidney", "quantity": {"unit": "jumping jacks", "value": 146.0}}], "operations": [{"actor": "Brooke", "kind": "compare_multiplicative", "operand": {"direction": "times", "reference_actor": "Sidney", "factor": 3.0}}], "unknown": {"entity": "Brooke", "unit": "jumping jacks"}}}
{"id": "51", "graph": {"entities": ["Mouse", "Keyboard"], "initial_state": [{"entity": "Mouse", "quantity": {"unit": "dollars", "value": 16.0}}], "operations": [{"actor": "Keyboard", "kind": "compare_multiplicative", "operand": {"direction": "times", "reference_actor": "Mouse", "factor": 3.0}}], "unknown": {"entity": null, "unit": "dollars"}}}
{"id": "60", "graph": {"entities": ["Ladies", "Girls"], "initial_state": [{"entity": "Ladies", "quantity": {"unit": "people", "value": 100.0}}], "operations": [{"actor": "Girls", "kind": "compare_multiplicative", "operand": {"direction": "times", "reference_actor": "Ladies", "factor": 3.0}}], "unknown": {"entity": null, "unit": "people"}}}
{"id": "74", "graph": {"entities": ["Dan"], "initial_state": [{"entity": "Dan", "quantity": {"unit": "stamps", "value": 20}}], "operations": [{"actor": "Dan", "kind": "add", "operand": {"unit": "stamps", "value": 30}}], "unknown": {"entity": "Dan", "unit": "stamps"}}}
{"id": "87", "graph": {"entities": ["Kate"], "initial_state": [{"entity": "Kate", "quantity": {"unit": "pencils", "value": 20}}], "operations": [{"actor": "Kate", "kind": "add", "operand": {"unit": "pencils", "value": 5}}], "unknown": {"entity": "Kate", "unit": "pencils"}}}
{"id": "gsm8k-holdout-dev-v1-0101", "graph": {"entities": ["Eduardo", "Frankie"], "initial_state": [{"entity": "Eduardo", "quantity": {"unit": "lasts", "value": 3.0}}], "operations": [{"actor": "Frankie", "kind": "compare_multiplicative", "operand": {"direction": "times", "reference_actor": "Eduardo", "factor": 2.0}}], "unknown": {"entity": null, "unit": "lasts"}}}
{"id": "gsm8k-holdout-dev-v1-0108", "graph": {"entities": ["Newport Beach", "Dana Point Beach"], "initial_state": [{"entity": "Newport Beach", "quantity": {"unit": "sharks", "value": 22.0}}], "operations": [{"actor": "Dana Point Beach", "kind": "compare_multiplicative", "operand": {"direction": "times", "reference_actor": "Newport Beach", "factor": 4.0}}], "unknown": {"entity": null, "unit": "sharks"}}}
{"id": "gsm8k-holdout-dev-v1-0268", "graph": {"entities": ["Bruce", "Bruce Scored 4 Goals While Michael"], "initial_state": [{"entity": "Bruce", "quantity": {"unit": "whiles", "value": 4.0}}], "operations": [{"actor": "Bruce Scored 4 Goals While Michael", "kind": "compare_multiplicative", "operand": {"direction": "times", "reference_actor": "Bruce", "factor": 3.0}}], "unknown": {"entity": null, "unit": "whiles"}}}
{"id": "gsm8k-holdout-dev-v1-0411", "graph": {"entities": ["female students", "male students"], "initial_state": [{"entity": "female students", "quantity": {"unit": "students", "value": 13.0}}], "operations": [{"actor": "male students", "kind": "compare_multiplicative", "operand": {"direction": "times", "reference_actor": "female students", "factor": 3.0}}], "unknown": {"entity": null, "unit": "students"}}}
{"id": "gsm8k-holdout-dev-v1-0453", "graph": {"entities": ["dad", "Olaf"], "initial_state": [{"entity": "dad", "quantity": {"unit": "points", "value": 7.0}}], "operations": [{"actor": "Olaf", "kind": "compare_multiplicative", "operand": {"direction": "times", "reference_actor": "dad", "factor": 3.0}}], "unknown": {"entity": null, "unit": "points"}}}
{"id": "gma-102", "graph": {"entities": ["Eve", "David"], "initial_state": [{"entity": "Eve", "quantity": {"unit": "coins", "value": 15}}, {"entity": "David", "quantity": {"unit": "coins", "value": 47}}], "operations": [{"actor": "Eve", "kind": "transfer", "operand": {"unit": "coins", "value": 7}, "target": "David"}], "unknown": {"entity": "David", "unit": "coins"}}}
{"id": "gma-103", "graph": {"entities": ["Charlie", "Grace"], "initial_state": [{"entity": "Charlie", "quantity": {"unit": "oranges", "value": 42}}, {"entity": "Grace", "quantity": {"unit": "oranges", "value": 48}}], "operations": [{"actor": "Charlie", "kind": "transfer", "operand": {"unit": "oranges", "value": 2}, "target": "Grace"}], "unknown": {"entity": "Grace", "unit": "oranges"}}}
{"id": "gma-106", "graph": {"entities": ["Eve", "Grace", "Mallory", "David"], "initial_state": [{"entity": "Eve", "quantity": {"unit": "crayons", "value": 34}}, {"entity": "Grace", "quantity": {"unit": "crayons", "value": 16}}, {"entity": "Mallory", "quantity": {"unit": "crayons", "value": 32}}, {"entity": "David", "quantity": {"unit": "crayons", "value": 32}}], "operations": [{"actor": "Eve", "kind": "transfer", "operand": {"unit": "crayons", "value": 17}, "target": "Grace"}], "unknown": {"entity": "Grace", "unit": "crayons"}}}
{"id": "gma-108", "graph": {"entities": ["Victor", "Judy"], "initial_state": [{"entity": "Victor", "quantity": {"unit": "pencils", "value": 33}}, {"entity": "Judy", "quantity": {"unit": "pencils", "value": 46}}], "operations": [{"actor": "Victor", "kind": "transfer", "operand": {"unit": "pencils", "value": 13}, "target": "Judy"}], "unknown": {"entity": "Judy", "unit": "pencils"}}}
{"id": "gma-109", "graph": {"entities": ["Heidi", "Judy"], "initial_state": [{"entity": "Heidi", "quantity": {"unit": "crayons", "value": 24}}, {"entity": "Judy", "quantity": {"unit": "crayons", "value": 16}}], "operations": [{"actor": "Heidi", "kind": "transfer", "operand": {"unit": "crayons", "value": 13}, "target": "Judy"}], "unknown": {"entity": null, "unit": "crayons"}}}
{"id": "gma-110", "graph": {"entities": ["Nori", "Frank", "Walter", "Trent", "David"], "initial_state": [{"entity": "Nori", "quantity": {"unit": "stickers", "value": 14}}, {"entity": "Frank", "quantity": {"unit": "stickers", "value": 48}}, {"entity": "Walter", "quantity": {"unit": "stickers", "value": 50}}, {"entity": "Trent", "quantity": {"unit": "stickers", "value": 20}}, {"entity": "David", "quantity": {"unit": "stickers", "value": 44}}], "operations": [{"actor": "Nori", "kind": "transfer", "operand": {"unit": "stickers", "value": 12}, "target": "Frank"}], "unknown": {"entity": "Frank", "unit": "stickers"}}}
{"id": "gma-111", "graph": {"entities": ["Ruth", "Owen"], "initial_state": [{"entity": "Ruth", "quantity": {"unit": "stickers", "value": 50}}, {"entity": "Owen", "quantity": {"unit": "stickers", "value": 45}}], "operations": [{"actor": "Ruth", "kind": "transfer", "operand": {"unit": "stickers", "value": 15}, "target": "Owen"}], "unknown": {"entity": null, "unit": "stickers"}}}
{"id": "gma-112", "graph": {"entities": ["Heidi", "Bob"], "initial_state": [{"entity": "Heidi", "quantity": {"unit": "marbles", "value": 35}}, {"entity": "Bob", "quantity": {"unit": "marbles", "value": 27}}], "operations": [{"actor": "Heidi", "kind": "transfer", "operand": {"unit": "marbles", "value": 5}, "target": "Bob"}], "unknown": {"entity": "Bob", "unit": "marbles"}}}
{"id": "gma-115", "graph": {"entities": ["Victor", "Walter"], "initial_state": [{"entity": "Victor", "quantity": {"unit": "stickers", "value": 47}}, {"entity": "Walter", "quantity": {"unit": "stickers", "value": 37}}], "operations": [{"actor": "Victor", "kind": "transfer", "operand": {"unit": "stickers", "value": 38}, "target": "Walter"}], "unknown": {"entity": null, "unit": "stickers"}}}
{"id": "gma-116", "graph": {"entities": ["Eve", "Trent", "Sara"], "initial_state": [{"entity": "Eve", "quantity": {"unit": "crayons", "value": 13}}, {"entity": "Trent", "quantity": {"unit": "crayons", "value": 17}}, {"entity": "Sara", "quantity": {"unit": "crayons", "value": 19}}], "operations": [{"actor": "Eve", "kind": "transfer", "operand": {"unit": "crayons", "value": 11}, "target": "Trent"}], "unknown": {"entity": "Trent", "unit": "crayons"}}}
{"id": "gma-118", "graph": {"entities": ["Ruth", "Trent", "Ivan", "Alice", "Mallory"], "initial_state": [{"entity": "Ruth", "quantity": {"unit": "crayons", "value": 44}}, {"entity": "Trent", "quantity": {"unit": "crayons", "value": 27}}, {"entity": "Ivan", "quantity": {"unit": "crayons", "value": 31}}, {"entity": "Alice", "quantity": {"unit": "crayons", "value": 17}}, {"entity": "Mallory", "quantity": {"unit": "crayons", "value": 28}}], "operations": [{"actor": "Ruth", "kind": "transfer", "operand": {"unit": "crayons", "value": 28}, "target": "Trent"}], "unknown": {"entity": "Trent", "unit": "crayons"}}}
{"id": "gma-119", "graph": {"entities": ["Ivan", "Trent", "Frank"], "initial_state": [{"entity": "Ivan", "quantity": {"unit": "coins", "value": 16}}, {"entity": "Trent", "quantity": {"unit": "coins", "value": 50}}, {"entity": "Frank", "quantity": {"unit": "coins", "value": 29}}], "operations": [{"actor": "Ivan", "kind": "transfer", "operand": {"unit": "coins", "value": 14}, "target": "Trent"}], "unknown": {"entity": "Trent", "unit": "coins"}}}
{"id": "gma-120", "graph": {"entities": ["Nori", "Frank"], "initial_state": [{"entity": "Nori", "quantity": {"unit": "coins", "value": 43}}, {"entity": "Frank", "quantity": {"unit": "coins", "value": 10}}], "operations": [{"actor": "Nori", "kind": "transfer", "operand": {"unit": "coins", "value": 39}, "target": "Frank"}], "unknown": {"entity": null, "unit": "coins"}}}
{"id": "gma-121", "graph": {"entities": ["David", "Nori", "Judy"], "initial_state": [{"entity": "David", "quantity": {"unit": "oranges", "value": 13}}, {"entity": "Nori", "quantity": {"unit": "oranges", "value": 25}}, {"entity": "Judy", "quantity": {"unit": "oranges", "value": 46}}], "operations": [{"actor": "David", "kind": "transfer", "operand": {"unit": "oranges", "value": 2}, "target": "Nori"}], "unknown": {"entity": "Nori", "unit": "oranges"}}}
{"id": "gma-125", "graph": {"entities": ["Nori", "Ruth", "Trent", "Victor", "Bob"], "initial_state": [{"entity": "Nori", "quantity": {"unit": "oranges", "value": 24}}, {"entity": "Ruth", "quantity": {"unit": "oranges", "value": 14}}, {"entity": "Trent", "quantity": {"unit": "oranges", "value": 31}}, {"entity": "Victor", "quantity": {"unit": "oranges", "value": 11}}, {"entity": "Bob", "quantity": {"unit": "oranges", "value": 47}}], "operations": [{"actor": "Nori", "kind": "transfer", "operand": {"unit": "oranges", "value": 18}, "target": "Ruth"}], "unknown": {"entity": "Ruth", "unit": "oranges"}}}
{"id": "gma-127", "graph": {"entities": ["Charlie", "Bob"], "initial_state": [{"entity": "Charlie", "quantity": {"unit": "marbles", "value": 14}}, {"entity": "Bob", "quantity": {"unit": "marbles", "value": 42}}], "operations": [{"actor": "Charlie", "kind": "transfer", "operand": {"unit": "marbles", "value": 4}, "target": "Bob"}], "unknown": {"entity": null, "unit": "marbles"}}}
{"id": "gma-130", "graph": {"entities": ["David", "Walter"], "initial_state": [{"entity": "David", "quantity": {"unit": "cards", "value": 32}}, {"entity": "Walter", "quantity": {"unit": "cards", "value": 37}}], "operations": [{"actor": "David", "kind": "transfer", "operand": {"unit": "cards", "value": 14}, "target": "Walter"}], "unknown": {"entity": null, "unit": "cards"}}}
{"id": "gma-133", "graph": {"entities": ["Victor", "Ruth"], "initial_state": [{"entity": "Victor", "quantity": {"unit": "apples", "value": 37}}, {"entity": "Ruth", "quantity": {"unit": "apples", "value": 21}}], "operations": [{"actor": "Victor", "kind": "transfer", "operand": {"unit": "apples", "value": 18}, "target": "Ruth"}], "unknown": {"entity": null, "unit": "apples"}}}
{"id": "gma-134", "graph": {"entities": ["Charlie", "Ruth"], "initial_state": [{"entity": "Charlie", "quantity": {"unit": "coins", "value": 16}}, {"entity": "Ruth", "quantity": {"unit": "coins", "value": 13}}], "operations": [{"actor": "Charlie", "kind": "transfer", "operand": {"unit": "coins", "value": 11}, "target": "Ruth"}], "unknown": {"entity": "Ruth", "unit": "coins"}}}
{"id": "gma-135", "graph": {"entities": ["Heidi", "Frank"], "initial_state": [{"entity": "Heidi", "quantity": {"unit": "cards", "value": 41}}, {"entity": "Frank", "quantity": {"unit": "cards", "value": 40}}], "operations": [{"actor": "Heidi", "kind": "transfer", "operand": {"unit": "cards", "value": 14}, "target": "Frank"}], "unknown": {"entity": null, "unit": "cards"}}}
{"id": "gma-113", "graph": {"entities": ["Pia", "Hugo", "Zane", "Milo", "Jay"], "initial_state": [{"entity": "Pia", "quantity": {"unit": "apples", "value": 18}}, {"entity": "Hugo", "quantity": {"unit": "apples", "value": 36}}, {"entity": "Zane", "quantity": {"unit": "apples", "value": 33}}, {"entity": "Milo", "quantity": {"unit": "apples", "value": 23}}, {"entity": "Jay", "quantity": {"unit": "apples", "value": 23}}], "operations": [{"actor": "Milo", "kind": "add", "operand": {"unit": "apples", "value": 12}}], "unknown": {"entity": null, "unit": "apples"}}}
{"id": "gma-115", "graph": {"entities": ["Jade", "Peter"], "initial_state": [{"entity": "Jade", "quantity": {"unit": "cups", "value": 20}}, {"entity": "Peter", "quantity": {"unit": "cups", "value": 31}}], "operations": [{"actor": "Jade", "kind": "add", "operand": {"unit": "cups", "value": 11}}], "unknown": {"entity": "Jade", "unit": "cups"}}}
{"id": "gma-117", "graph": {"entities": ["Ruth"], "initial_state": [{"entity": "Ruth", "quantity": {"unit": "cards", "value": 32}}], "operations": [{"actor": "Ruth", "kind": "add", "operand": {"unit": "cards", "value": 9}}, {"actor": "Ruth", "kind": "add", "operand": {"unit": "cards", "value": 7}}], "unknown": {"entity": "Ruth", "unit": "cards"}}}
{"id": "gma-121", "graph": {"entities": ["Wren"], "initial_state": [{"entity": "Wren", "quantity": {"unit": "marbles", "value": 31}}], "operations": [{"actor": "Wren", "kind": "add", "operand": {"unit": "marbles", "value": 5}}, {"actor": "Wren", "kind": "subtract", "operand": {"unit": "marbles", "value": 34}}], "unknown": {"entity": "Wren", "unit": "marbles"}}}
{"id": "gma-124", "graph": {"entities": ["Aria"], "initial_state": [{"entity": "Aria", "quantity": {"unit": "pencils", "value": 39}}], "operations": [{"actor": "Aria", "kind": "add", "operand": {"unit": "pencils", "value": 12}}], "unknown": {"entity": "Aria", "unit": "pencils"}}}
{"id": "gma-135", "graph": {"entities": ["Elle", "Clara", "Ivan"], "initial_state": [{"entity": "Elle", "quantity": {"unit": "kites", "value": 12}}, {"entity": "Clara", "quantity": {"unit": "kites", "value": 11}}, {"entity": "Ivan", "quantity": {"unit": "kites", "value": 37}}], "operations": [{"actor": "Elle", "kind": "add", "operand": {"unit": "kites", "value": 5}}, {"actor": "Ivan", "kind": "add", "operand": {"unit": "kites", "value": 3}}], "unknown": {"entity": "Elle", "unit": "kites"}}}
{"id": "gma-138", "graph": {"entities": ["Kai", "Ina"], "initial_state": [{"entity": "Kai", "quantity": {"unit": "shells", "value": 39}}, {"entity": "Ina", "quantity": {"unit": "shells", "value": 32}}], "operations": [{"actor": "Kai", "kind": "subtract", "operand": {"unit": "shells", "value": 60}}], "unknown": {"entity": "Kai", "unit": "shells"}}}
{"id": "gma-147", "graph": {"entities": ["Seth", "Tom", "Jade", "Blake", "Rina"], "initial_state": [{"entity": "Seth", "quantity": {"unit": "pencils", "value": 15}}, {"entity": "Tom", "quantity": {"unit": "pencils", "value": 13}}, {"entity": "Jade", "quantity": {"unit": "pencils", "value": 22}}, {"entity": "Blake", "quantity": {"unit": "pencils", "value": 17}}, {"entity": "Rina", "quantity": {"unit": "pencils", "value": 40}}], "operations": [{"actor": "Seth", "kind": "add", "operand": {"unit": "pencils", "value": 11}}], "unknown": {"entity": null, "unit": "pencils"}}}
{"id": "gma-154", "graph": {"entities": ["Gina"], "initial_state": [{"entity": "Gina", "quantity": {"unit": "sheets", "value": 14}}], "operations": [{"actor": "Gina", "kind": "subtract", "operand": {"unit": "sheets", "value": 10}}, {"actor": "Gina", "kind": "add", "operand": {"unit": "sheets", "value": 6}}], "unknown": {"entity": "Gina", "unit": "sheets"}}}
{"id": "gma-156", "graph": {"entities": ["Finn"], "initial_state": [{"entity": "Finn", "quantity": {"unit": "letters", "value": 21}}], "operations": [{"actor": "Finn", "kind": "subtract", "operand": {"unit": "letters", "value": 4}}], "unknown": {"entity": "Finn", "unit": "letters"}}}
{"id": "gma-157", "graph": {"entities": ["Finn"], "initial_state": [{"entity": "Finn", "quantity": {"unit": "scarves", "value": 40}}], "operations": [{"actor": "Finn", "kind": "add", "operand": {"unit": "scarves", "value": 7}}, {"actor": "Finn", "kind": "subtract", "operand": {"unit": "scarves", "value": 1}}, {"actor": "Finn", "kind": "add", "operand": {"unit": "scarves", "value": 9}}], "unknown": {"entity": "Finn", "unit": "scarves"}}}
{"id": "gma-163", "graph": {"entities": ["Ruth"], "initial_state": [{"entity": "Ruth", "quantity": {"unit": "balls", "value": 24}}], "operations": [{"actor": "Ruth", "kind": "add", "operand": {"unit": "balls", "value": 8}}, {"actor": "Ruth", "kind": "add", "operand": {"unit": "balls", "value": 5}}, {"actor": "Ruth", "kind": "add", "operand": {"unit": "balls", "value": 11}}], "unknown": {"entity": "Ruth", "unit": "balls"}}}
{"id": "gma-195", "graph": {"entities": ["Ben"], "initial_state": [{"entity": "Ben", "quantity": {"unit": "buttons", "value": 26}}], "operations": [{"actor": "Ben", "kind": "add", "operand": {"unit": "buttons", "value": 6}}, {"actor": "Ben", "kind": "subtract", "operand": {"unit": "buttons", "value": 75}}], "unknown": {"entity": "Ben", "unit": "buttons"}}}
{"id": "gma-009", "graph": {"entities": ["Sam"], "initial_state": [{"entity": "Sam", "quantity": {"unit": "marbles", "value": 36}}], "operations": [{"actor": "Sam", "kind": "subtract", "operand": {"unit": "marbles", "value": 24}}, {"actor": "Sam", "kind": "subtract", "operand": {"unit": "marbles", "value": 5}}], "unknown": {"entity": "Sam", "unit": "marbles"}}}
{"id": "2087", "graph": {"entities": ["Tom"], "initial_state": [{"entity": "Tom", "quantity": {"unit": "stickers", "value": 58}}], "operations": [{"actor": "Tom", "kind": "subtract", "operand": {"unit": "aways", "value": 29}}], "unknown": {"entity": "Tom", "unit": "stickers"}}}