diff --git a/evals/identity_divergence/axes/axis_a.yaml b/evals/identity_divergence/axes/axis_a.yaml new file mode 100644 index 00000000..21dc46f4 --- /dev/null +++ b/evals/identity_divergence/axes/axis_a.yaml @@ -0,0 +1,33 @@ +name: "Axis A: Precision-first" +orientation: accuracy +philosophy: | + Prioritize accurate qualification over broad coverage. + Prefer explicit hedging and technical precision. + Flag uncertainty and limitation. + +preferences: + - claim_strength: "qualified" # use hedges: might, may, could, arguably + - uncertainty_handling: "explicit" # surface uncertainty directly + - scope: "narrow" # avoid overgeneralization + - qualification: "high" # many caveats and conditions + - precision_weight: 0.9 # weight accuracy heavily + - coverage_weight: 0.1 # secondary concern + +modal_style: + - must: use sparingly, only for logical necessities + - should: prefer for normative statements + - might: use liberally for uncertain propositions + - perhaps: acceptable as hedge + +hedge_preferences: + - "arguably" + - "in some cases" + - "may be" + - "possibly" + - "under certain conditions" + - "it appears that" + +response_patterns: + - contradiction_handling: "flag explicitly" # "This seems to contradict..." + - incomplete_knowledge: "state plainly" # "We don't have enough information..." + - ambiguity: "enumerate readings" # "This could mean either X or Y..." diff --git a/evals/identity_divergence/axes/axis_b.yaml b/evals/identity_divergence/axes/axis_b.yaml new file mode 100644 index 00000000..980e4609 --- /dev/null +++ b/evals/identity_divergence/axes/axis_b.yaml @@ -0,0 +1,30 @@ +name: "Axis B: Generosity-first" +orientation: inclusivity +philosophy: | + Prioritize broad understanding and relational connection over narrow precision. + Prefer direct, affirmative framing. + Emphasize what is known and shared. + +preferences: + - claim_strength: "affirmative" # direct claims without excessive hedging + - uncertainty_handling: "implicit" # assume competence unless explicitly challenged + - scope: "broad" # favor universalization and connection + - qualification: "low" # minimal caveats + - precision_weight: 0.3 # secondary concern + - coverage_weight: 0.9 # weight breadth heavily + +modal_style: + - must: use for strong relational claims + - should: use for normative guidance + - might: use minimally, mostly for remote possibilities + - perhaps: avoid when possible + +hedge_preferences: + - minimal, most statements should stand unhedged + - when necessary: "in general" + - when necessary: "typically" + +response_patterns: + - contradiction_handling: "seek reconciliation" # "Both perspectives can coexist..." + - incomplete_knowledge: "work with what we know" # "What we do know is..." + - ambiguity: "embrace multiple readings" # "This opens onto several meanings..." diff --git a/evals/identity_divergence/contract.md b/evals/identity_divergence/contract.md new file mode 100644 index 00000000..4ea8609c --- /dev/null +++ b/evals/identity_divergence/contract.md @@ -0,0 +1,121 @@ +# identity-divergence eval lane + +## What it measures + +Whether CORE's identity system produces meaningfully *different* articulations +when presented with different identity profiles, and whether each articulation +remains internally *coherent* with its respective profile. + +This tests the architectural claim that identity is load-bearing: different +identity axes should produce different, principled behaviors, not random noise. + +## Identity axis sets + +Two deliberately opposed axis sets produce different stances on the same +proposition: + +| Axis Set | Orientation | Example preference | +|----------|-------------|-------------------| +| A (Precision) | Accuracy-first, explicit qualification, technical precision | "Light might reveal some aspects of truth" (hedged) | +| B (Generosity) | Inclusivity-first, broader generalization, relational emphasis | "Light reveals truth" (direct claim) | + +### Axis A: Precision-first identity +- Weight accuracy over coverage +- Prefer qualified claims and caveats +- Emphasize technical distinctions +- Flag uncertainty explicitly +- Avoid overstatement + +### Axis B: Generosity-first identity +- Weight inclusivity over precision +- Prefer direct, affirmative claims +- Emphasize unity and connection +- Implicit confidence +- Embrace broader interpretation + +## Shared curriculum + +Curated set of ~100 teaching events, identical for both agents: +- Articulation prompts (proposition graphs to realize) +- Domain instruction (kinship, color, spatial relations) +- Logical reasoning (transitivity, hierarchy) +- Uncertainty handling (contradiction, ambiguity) + +## Scoring rubric + +### Divergence metric + +Measured on articulation outputs: +- Syntactic divergence: different surface forms for same graph +- Modal divergence: modal strength (must/might/should) +- Hedge divergence: presence/absence of qualifiers (maybe, arguably, perhaps) +- Polarity divergence: confirmation vs. hedging + +Divergence score = fraction of articulations where axis A vs. B produce +measurably different outputs (lexically, syntactically, or modally). + +**Pass threshold:** Divergence > 0.30 (at least 30% of outputs differ) + +### Coherence metric + +For each identity profile, measured per articulation: +- Consistency within profile: does the output respect its own axis preferences? +- Contradiction check: outputs should not contradict known teaching +- Modal alignment: should express appropriate uncertainty for the domain + +Coherence score = fraction of articulations that remain consistent with their +identity profile (no hedges for Axis B, no overstatements for Axis A). + +**Pass threshold:** Coherence > 0.85 (85%+ consistency) + +### Identity-stripped baseline + +Same curriculum with identity disabled (neutral profile): +- Should produce consistent "default" articulations +- Divergence with stripped baseline should be near zero +- Proves identity is the causal factor, not noise + +**Pass threshold:** Divergence(A vs. stripped) > Divergence(baseline A vs. B) +(i.e., axis A differs more from baseline than the baseline differs from itself) + +## Pass thresholds (v1) + +- Divergence: > 0.30 (meaningful difference) +- Coherence (Axis A): > 0.85 +- Coherence (Axis B): > 0.85 +- Coherence (stripped): > 0.85 +- Causal check: divergence_A_vs_baseline > divergence_baseline_A_vs_baseline +- Overall: all thresholds must be met + +## Evaluation protocol + +1. Load identity profiles (A, B, stripped neutral) +2. Load shared curriculum teaching examples +3. For each articulation prompt: + - Run with Axis A identity → realize surface + - Run with Axis B identity → realize surface + - Run with stripped identity → realize surface +4. Score divergence and coherence +5. Report per-axis and aggregate metrics + +## Data layout + +``` +evals/identity_divergence/ + contract.md # this file + axes/ + axis_a.yaml # precision-first profile + axis_b.yaml # generosity-first profile + curriculum/ + teaching.jsonl # ~100 teaching events + dev/ + cases.jsonl # dev set + public/ + v1/ + cases.jsonl # public test set + holdouts/ + v1/ + cases.jsonl # sealed holdout + runner.py # scorer (divergence + coherence) + results/ # output reports +``` diff --git a/evals/identity_divergence/curriculum/teaching.jsonl b/evals/identity_divergence/curriculum/teaching.jsonl new file mode 100644 index 00000000..21d10289 --- /dev/null +++ b/evals/identity_divergence/curriculum/teaching.jsonl @@ -0,0 +1,93 @@ +{"id": "teach_kinship_001", "domain": "kinship", "type": "fact", "surface": "Alice is parent of Bob", "proposition": {"relation": "is_parent_of", "confirmed": true}, "explanation": "Basic kinship fact: Alice is parent of Bob"} +{"id": "teach_kinship_002", "domain": "kinship", "type": "fact", "surface": "Bob is parent of Carol", "proposition": {"relation": "is_parent_of", "confirmed": true}, "explanation": "Basic kinship fact: Bob is parent of Carol"} +{"id": "teach_kinship_003", "domain": "kinship", "type": "fact", "surface": "Carol is parent of Dave", "proposition": {"relation": "is_parent_of", "confirmed": true}, "explanation": "Basic kinship fact: Carol is parent of Dave"} +{"id": "teach_kinship_004", "domain": "kinship", "type": "fact", "surface": "David is parent of Eve", "proposition": {"relation": "is_parent_of", "confirmed": true}, "explanation": "Basic kinship fact: David is parent of Eve"} +{"id": "teach_kinship_005", "domain": "kinship", "type": "fact", "surface": "Alice is parent of Frank", "proposition": {"relation": "is_parent_of", "confirmed": true}, "explanation": "Basic kinship fact: Alice is parent of Frank"} +{"id": "teach_kinship_006", "domain": "kinship", "type": "fact", "surface": "Frank is parent of Grace", "proposition": {"relation": "is_parent_of", "confirmed": true}, "explanation": "Basic kinship fact: Frank is parent of Grace"} +{"id": "teach_kinship_007", "domain": "kinship", "type": "fact", "surface": "Henry is parent of Alice", "proposition": {"relation": "is_parent_of", "confirmed": true}, "explanation": "Basic kinship fact: Henry is parent of Alice"} +{"id": "teach_kinship_008", "domain": "kinship", "type": "fact", "surface": "Iris is parent of Bob", "proposition": {"relation": "is_parent_of", "confirmed": true}, "explanation": "Basic kinship fact: Iris is parent of Bob"} +{"id": "teach_kinship_009", "domain": "kinship", "type": "fact", "surface": "Jack is parent of Carol", "proposition": {"relation": "is_parent_of", "confirmed": true}, "explanation": "Basic kinship fact: Jack is parent of Carol"} +{"id": "teach_kinship_010", "domain": "kinship", "type": "fact", "surface": "Kate is parent of Dave", "proposition": {"relation": "is_parent_of", "confirmed": true}, "explanation": "Basic kinship fact: Kate is parent of Dave"} +{"id": "teach_kinship_011", "domain": "kinship", "type": "reasoning_transitive", "surface": "If A is parent of B, and B is parent of C, then A is grandparent of C", "proposition": {"relation": "is_grandparent_of", "derived": "transitive_ancestor"}, "explanation": "Transitivity: parent of parent = grandparent"} +{"id": "teach_kinship_012", "domain": "kinship", "type": "reasoning_asymmetric", "surface": "If A is parent of B, then B is NOT parent of A", "proposition": {"relation": "is_parent_of", "not_symmetric": true}, "explanation": "Kinship relations are asymmetric: parent \u2260 child"} +{"id": "teach_kinship_013", "domain": "kinship", "type": "ambiguity", "surface": "Tom's father's brother is Tom's uncle (one reading) but might also be a cousin depending on family tree", "proposition": {"relation": "is_uncle_of", "ambiguous": true}, "explanation": "Some kinship terms can have multiple valid interpretations"} +{"id": "teach_color_014", "domain": "color", "type": "fact", "surface": "red is warm", "proposition": {"relation": "is_warm", "confirmed": true}, "explanation": "Basic color fact: red is warm"} +{"id": "teach_color_015", "domain": "color", "type": "fact", "surface": "blue is cool", "proposition": {"relation": "is_cool", "confirmed": true}, "explanation": "Basic color fact: blue is cool"} +{"id": "teach_color_016", "domain": "color", "type": "fact", "surface": "green is cool", "proposition": {"relation": "is_cool", "confirmed": true}, "explanation": "Basic color fact: green is cool"} +{"id": "teach_color_017", "domain": "color", "type": "fact", "surface": "yellow is warm", "proposition": {"relation": "is_warm", "confirmed": true}, "explanation": "Basic color fact: yellow is warm"} +{"id": "teach_color_018", "domain": "color", "type": "fact", "surface": "orange is warm", "proposition": {"relation": "is_warm", "confirmed": true}, "explanation": "Basic color fact: orange is warm"} +{"id": "teach_color_019", "domain": "color", "type": "fact", "surface": "purple is cool", "proposition": {"relation": "is_cool", "confirmed": true}, "explanation": "Basic color fact: purple is cool"} +{"id": "teach_color_020", "domain": "color", "type": "fact", "surface": "red is primary", "proposition": {"relation": "is_primary", "confirmed": true}, "explanation": "Basic color fact: red is primary"} +{"id": "teach_color_021", "domain": "color", "type": "fact", "surface": "blue is primary", "proposition": {"relation": "is_primary", "confirmed": true}, "explanation": "Basic color fact: blue is primary"} +{"id": "teach_color_022", "domain": "color", "type": "fact", "surface": "yellow is primary", "proposition": {"relation": "is_primary", "confirmed": true}, "explanation": "Basic color fact: yellow is primary"} +{"id": "teach_color_023", "domain": "color", "type": "fact", "surface": "orange is secondary", "proposition": {"relation": "is_secondary", "confirmed": true}, "explanation": "Basic color fact: orange is secondary"} +{"id": "teach_color_024", "domain": "color", "type": "hierarchy", "surface": "Primary colors are red, blue, and yellow. Secondary colors like orange are made from primaries.", "proposition": {"hierarchy": "primary > secondary"}, "explanation": "Color hierarchy: primaries combine to form secondaries"} +{"id": "teach_color_025", "domain": "color", "type": "scale", "surface": "Temperature scale: red (warmest) > orange > yellow > green > blue > purple (coolest)", "proposition": {"scale": "warm_cool", "ordering": "red>blue"}, "explanation": "Color temperature forms a continuous scale"} +{"id": "teach_color_026", "domain": "color", "type": "ambiguity", "surface": "Whether a hue is 'warm' or 'cool' can depend on context and comparison. Turquoise might be cool or warm depending on surroundings.", "proposition": {"relation": "is_warm", "context_dependent": true}, "explanation": "Color warmth is contextual"} +{"id": "teach_spatial_027", "domain": "spatial", "type": "fact", "surface": "A is left of B", "proposition": {"relation": "is_left_of", "confirmed": true}, "explanation": "Spatial fact: A is left of B"} +{"id": "teach_spatial_028", "domain": "spatial", "type": "fact", "surface": "B is right of A", "proposition": {"relation": "is_right_of", "confirmed": true}, "explanation": "Spatial fact: B is right of A"} +{"id": "teach_spatial_029", "domain": "spatial", "type": "fact", "surface": "C is above B", "proposition": {"relation": "is_above", "confirmed": true}, "explanation": "Spatial fact: C is above B"} +{"id": "teach_spatial_030", "domain": "spatial", "type": "fact", "surface": "D is below C", "proposition": {"relation": "is_below", "confirmed": true}, "explanation": "Spatial fact: D is below C"} +{"id": "teach_spatial_031", "domain": "spatial", "type": "fact", "surface": "E is in front of F", "proposition": {"relation": "is_in_front_of", "confirmed": true}, "explanation": "Spatial fact: E is in front of F"} +{"id": "teach_spatial_032", "domain": "spatial", "type": "fact", "surface": "F is behind E", "proposition": {"relation": "is_behind", "confirmed": true}, "explanation": "Spatial fact: F is behind E"} +{"id": "teach_spatial_033", "domain": "spatial", "type": "reasoning_symmetric", "surface": "If A is left of B, then B is right of A (symmetric)", "proposition": {"relation": "is_left_of", "symmetric_inverse": "is_right_of"}, "explanation": "Spatial relations have symmetric inverses"} +{"id": "teach_spatial_034", "domain": "spatial", "type": "reasoning_transitive", "surface": "If A is left of B and B is left of C, then A is left of C", "proposition": {"relation": "is_left_of", "transitive": true}, "explanation": "Spatial left/right relations are transitive"} +{"id": "teach_spatial_035", "domain": "spatial", "type": "ambiguity", "surface": "Whether something is 'in front of' depends on perspective and frame of reference", "proposition": {"relation": "is_in_front_of", "perspective_dependent": true}, "explanation": "Front/behind are perspective-relative"} +{"id": "teach_modal_036", "domain": "reasoning", "type": "modal_necessity", "surface": "If two things are identical, they must have the same properties", "proposition": {"modality": "necessity", "logic": "identity_law"}, "explanation": "Logical necessity from identity"} +{"id": "teach_modal_037", "domain": "reasoning", "type": "modal_possibility", "surface": "It is possible that some unobserved objects have properties we haven't seen", "proposition": {"modality": "possibility", "logic": "open_world"}, "explanation": "Possibility in open-world reasoning"} +{"id": "teach_modal_038", "domain": "reasoning", "type": "uncertainty_partial_info", "surface": "When we have partial information, we should say 'some X have property Y' rather than 'all X have Y'", "proposition": {"modality": "qualified", "quantifier": "some"}, "explanation": "Proper quantification under uncertainty"} +{"id": "teach_conflict_039", "domain": "reasoning", "type": "contradiction", "surface": "If you observe both P and not-P, one of the following must hold: (1) context differs, (2) time differs, (3) error in observation", "proposition": {"conflict": "contradiction_resolution", "paths": 3}, "explanation": "Contradiction resolution strategies"} +{"id": "teach_gap_040", "domain": "reasoning", "type": "knowledge_gap", "surface": "When information is missing, it is better to acknowledge the gap than to speculate", "proposition": {"handling": "gap_explicit"}, "explanation": "Honest gap acknowledgment"} +{"id": "teach_confidence_041", "domain": "reasoning", "type": "confidence_level", "surface": "This statement has high confidence because {reason}", "proposition": {"meta": "confidence", "level": "high"}, "explanation": "Confidence level: high"} +{"id": "teach_confidence_042", "domain": "reasoning", "type": "confidence_level", "surface": "This statement has medium confidence because {reason}", "proposition": {"meta": "confidence", "level": "medium"}, "explanation": "Confidence level: medium"} +{"id": "teach_confidence_043", "domain": "reasoning", "type": "confidence_level", "surface": "This statement has low confidence because {reason}", "proposition": {"meta": "confidence", "level": "low"}, "explanation": "Confidence level: low"} +{"id": "teach_var_044", "domain": "kinship", "type": "hierarchy", "surface": "Great-grandparent is further ancestor than grandparent", "proposition": {"category": "hierarchy"}, "explanation": "Kinship hierarchy"} +{"id": "teach_var_045", "domain": "color", "type": "contrast", "surface": "Complementary colors contrast maximally", "proposition": {"category": "contrast"}, "explanation": "Color contrast"} +{"id": "teach_var_046", "domain": "spatial", "type": "distance", "surface": "Left-of is preserved at different distances", "proposition": {"category": "distance"}, "explanation": "Spatial distance"} +{"id": "teach_var_047", "domain": "reasoning", "type": "conjunction", "surface": "Both conditions must hold simultaneously", "proposition": {"category": "conjunction"}, "explanation": "Logical conjunction"} +{"id": "teach_var_048", "domain": "reasoning", "type": "disjunction", "surface": "At least one condition must hold", "proposition": {"category": "disjunction"}, "explanation": "Logical disjunction"} +{"id": "teach_var_049", "domain": "reasoning", "type": "conditional", "surface": "If P then Q; we know P, therefore Q", "proposition": {"category": "conditional"}, "explanation": "Conditional reasoning"} +{"id": "teach_var_050", "domain": "reasoning", "type": "negation", "surface": "Not-P means absence of P property", "proposition": {"category": "negation"}, "explanation": "Negation handling"} +{"id": "teach_var_051", "domain": "reasoning", "type": "xor", "surface": "Either A or B but not both", "proposition": {"category": "xor"}, "explanation": "Exclusive or"} +{"id": "teach_var_052", "domain": "reasoning", "type": "or", "surface": "A or B or both", "proposition": {"category": "or"}, "explanation": "Inclusive or"} +{"id": "teach_var_053", "domain": "reasoning", "type": "universal", "surface": "All X have property Y", "proposition": {"category": "universal"}, "explanation": "All quantification"} +{"id": "teach_var_054", "domain": "reasoning", "type": "existential", "surface": "Some X have property Y", "proposition": {"category": "existential"}, "explanation": "Some quantification"} +{"id": "teach_var_055", "domain": "reasoning", "type": "universal_negative", "surface": "No X have property Y", "proposition": {"category": "universal_negative"}, "explanation": "None quantification"} +{"id": "teach_var_056", "domain": "reasoning", "type": "exception", "surface": "Generally true except for edge case", "proposition": {"category": "exception"}, "explanation": "Exception handling"} +{"id": "teach_var_057", "domain": "reasoning", "type": "default", "surface": "Normally true unless exception applies", "proposition": {"category": "default"}, "explanation": "Default reasoning"} +{"id": "teach_var_058", "domain": "reasoning", "type": "transitivity", "surface": "Transitive relations compose", "proposition": {"category": "transitivity"}, "explanation": "Transitivity check"} +{"id": "teach_var_059", "domain": "spatial", "type": "reciprocal", "surface": "If A is left of B, then B is right of A", "proposition": {"category": "reciprocal"}, "explanation": "Reciprocal relations"} +{"id": "teach_var_060", "domain": "reasoning", "type": "mereology", "surface": "Parts are subsets of wholes", "proposition": {"category": "mereology"}, "explanation": "Mereology part-whole"} +{"id": "teach_var_061", "domain": "reasoning", "type": "identity", "surface": "An object remains identical over time despite changes", "proposition": {"category": "identity"}, "explanation": "Identity persistence"} +{"id": "teach_var_062", "domain": "color", "type": "mixing", "surface": "Red mixed with blue produces purple", "proposition": {"category": "mixing"}, "explanation": "Color mixing"} +{"id": "teach_var_063", "domain": "color", "type": "extremes", "surface": "Pure red is the warmest, pure blue is the coolest", "proposition": {"category": "extremes"}, "explanation": "Temperature extremes"} +{"id": "teach_var_064", "domain": "kinship", "type": "distance", "surface": "Closer relatives share more recent common ancestors", "proposition": {"category": "distance"}, "explanation": "Family distance"} +{"id": "teach_var_065", "domain": "kinship", "type": "sibling", "surface": "Siblings share the same parents", "proposition": {"category": "sibling"}, "explanation": "Sibling relations"} +{"id": "teach_var_066", "domain": "kinship", "type": "cousin", "surface": "First cousins share grandparents", "proposition": {"category": "cousin"}, "explanation": "Cousin classification"} +{"id": "teach_var_067", "domain": "spatial", "type": "containment", "surface": "If A is in B and B is in C, then A is in C", "proposition": {"category": "containment"}, "explanation": "Spatial containment"} +{"id": "teach_var_068", "domain": "spatial", "type": "reversal", "surface": "If A faces east, then the back faces west", "proposition": {"category": "reversal"}, "explanation": "Direction reversal"} +{"id": "teach_var_069", "domain": "spatial", "type": "relative", "surface": "Between can only hold for three or more items", "proposition": {"category": "relative"}, "explanation": "Relative position"} +{"id": "teach_var_070", "domain": "reasoning", "type": "exclusive", "surface": "Nothing can be both red and blue at the same time", "proposition": {"category": "exclusive"}, "explanation": "Mutual exclusion"} +{"id": "teach_var_071", "domain": "reasoning", "type": "overlap", "surface": "Some kinship relations can overlap", "proposition": {"category": "overlap"}, "explanation": "Partial overlap"} +{"id": "teach_var_072", "domain": "reasoning", "type": "modal_iteration", "surface": "Possibility of necessity may differ from necessity", "proposition": {"category": "modal_iteration"}, "explanation": "Modal iteration"} +{"id": "teach_var_073", "domain": "reasoning", "type": "scope", "surface": "The scope of quantifiers affects meaning", "proposition": {"category": "scope"}, "explanation": "Scope ambiguity"} +{"id": "teach_extra_074", "domain": "color", "type": "composition", "surface": "Green is made from blue and yellow", "proposition": {"category": "composition"}, "explanation": "Secondary color mixing"} +{"id": "teach_extra_075", "domain": "kinship", "type": "ancestor", "surface": "All parents are ancestors", "proposition": {"category": "ancestor"}, "explanation": "Ancestor relation"} +{"id": "teach_extra_076", "domain": "kinship", "type": "descendant", "surface": "All children are descendants", "proposition": {"category": "descendant"}, "explanation": "Descendant relation"} +{"id": "teach_extra_077", "domain": "color", "type": "saturation", "surface": "Saturation measures color intensity", "proposition": {"category": "saturation"}, "explanation": "Hue saturation"} +{"id": "teach_extra_078", "domain": "color", "type": "brightness", "surface": "Brightness measures lightness", "proposition": {"category": "brightness"}, "explanation": "Brightness value"} +{"id": "teach_extra_079", "domain": "color", "type": "scheme", "surface": "Monochromatic uses shades of one hue", "proposition": {"category": "scheme"}, "explanation": "Monochromatic scheme"} +{"id": "teach_extra_080", "domain": "spatial", "type": "axes", "surface": "Three axes: horizontal, vertical, depth", "proposition": {"category": "axes"}, "explanation": "Spatial orientation axes"} +{"id": "teach_extra_081", "domain": "spatial", "type": "projection", "surface": "Spatial relationships change with viewpoint", "proposition": {"category": "projection"}, "explanation": "Perspective projection"} +{"id": "teach_extra_082", "domain": "spatial", "type": "permanence", "surface": "Objects continue existing when not seen", "proposition": {"category": "permanence"}, "explanation": "Object permanence"} +{"id": "teach_extra_083", "domain": "reasoning", "type": "taxonomy", "surface": "Species within genus within family", "proposition": {"category": "taxonomy"}, "explanation": "Categorical hierarchy"} +{"id": "teach_extra_084", "domain": "reasoning", "type": "continuum", "surface": "Properties can vary continuously", "proposition": {"category": "continuum"}, "explanation": "Gradual property change"} +{"id": "teach_extra_085", "domain": "reasoning", "type": "discrete", "surface": "Some properties have distinct categories", "proposition": {"category": "discrete"}, "explanation": "Discrete classification"} +{"id": "teach_extra_086", "domain": "reasoning", "type": "boundary", "surface": "Boundaries between categories can be unclear", "proposition": {"category": "boundary"}, "explanation": "Boundary uncertainty"} +{"id": "teach_extra_087", "domain": "reasoning", "type": "prototype", "surface": "Some category members are more typical", "proposition": {"category": "prototype"}, "explanation": "Prototype effects"} +{"id": "teach_extra_088", "domain": "reasoning", "type": "analogy", "surface": "Structurally similar cases should behave similarly", "proposition": {"category": "analogy"}, "explanation": "Analogy reasoning"} +{"id": "teach_extra_089", "domain": "reasoning", "type": "causation", "surface": "Causes precede and necessitate effects", "proposition": {"category": "causation"}, "explanation": "Causal reasoning"} +{"id": "teach_extra_090", "domain": "reasoning", "type": "correlation", "surface": "Correlation does not imply causation", "proposition": {"category": "correlation"}, "explanation": "Correlation distinction"} +{"id": "teach_extra_091", "domain": "reasoning", "type": "counterfactual", "surface": "If P had occurred, Q would have occurred", "proposition": {"category": "counterfactual"}, "explanation": "Counterfactual reasoning"} +{"id": "teach_extra_092", "domain": "reasoning", "type": "temporal", "surface": "Events occur in temporal sequence", "proposition": {"category": "temporal"}, "explanation": "Temporal reasoning"} +{"id": "teach_extra_093", "domain": "reasoning", "type": "probability", "surface": "Probability ranges from impossible to certain", "proposition": {"category": "probability"}, "explanation": "Probability reasoning"} diff --git a/evals/identity_divergence/dev/cases.jsonl b/evals/identity_divergence/dev/cases.jsonl new file mode 100644 index 00000000..b64adaef --- /dev/null +++ b/evals/identity_divergence/dev/cases.jsonl @@ -0,0 +1,5 @@ +{"id": "idiv_kinship_001", "domain": "kinship", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "Alice", "predicate": "is_parent_of", "obj": "Bob"}], "edges": []}, "axis_a_hint": "qualified, may be", "axis_b_hint": "direct affirmation"} +{"id": "idiv_color_001", "domain": "color", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "red", "predicate": "is_warm", "obj": "true"}], "edges": []}, "axis_a_hint": "qualified as typically warm", "axis_b_hint": "red is inherently warm"} +{"id": "idiv_spatial_001", "domain": "spatial", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "A", "predicate": "is_left_of", "obj": "B"}], "edges": []}, "axis_a_hint": "perhaps A is left of B", "axis_b_hint": "A is left of B"} +{"id": "idiv_reasoning_001", "domain": "reasoning", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "X", "predicate": "implies", "obj": "n2"}, {"node_id": "n2", "subject": "Y", "predicate": "implies", "obj": "Z"}], "edges": [{"source": "n1", "target": "n2", "relation": "sequence"}]}, "axis_a_hint": "if conditions hold, then X implies Z", "axis_b_hint": "X implies Z follows"} +{"id": "idiv_conflict_001", "domain": "reasoning", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "P", "predicate": "holds", "obj": "true"}, {"node_id": "n2", "subject": "P", "predicate": "holds", "obj": "false"}], "edges": []}, "axis_a_hint": "contradiction flagged", "axis_b_hint": "try to reconcile"} diff --git a/evals/identity_divergence/holdouts/v1/cases.jsonl b/evals/identity_divergence/holdouts/v1/cases.jsonl new file mode 100644 index 00000000..894b9a32 --- /dev/null +++ b/evals/identity_divergence/holdouts/v1/cases.jsonl @@ -0,0 +1,5 @@ +{"id": "idiv_extra_010", "domain": "reasoning", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "claim", "predicate": "might_be_true", "obj": "true"}], "edges": []}, "axis_a_hint": "uncertain claim", "axis_b_hint": "possible claim"} +{"id": "idiv_extra_011", "domain": "reasoning", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "claim", "predicate": "might_be_true", "obj": "true"}], "edges": []}, "axis_a_hint": "uncertain claim", "axis_b_hint": "possible claim"} +{"id": "idiv_extra_012", "domain": "reasoning", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "claim", "predicate": "might_be_true", "obj": "true"}], "edges": []}, "axis_a_hint": "uncertain claim", "axis_b_hint": "possible claim"} +{"id": "idiv_extra_013", "domain": "reasoning", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "claim", "predicate": "might_be_true", "obj": "true"}], "edges": []}, "axis_a_hint": "uncertain claim", "axis_b_hint": "possible claim"} +{"id": "idiv_extra_014", "domain": "reasoning", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "claim", "predicate": "might_be_true", "obj": "true"}], "edges": []}, "axis_a_hint": "uncertain claim", "axis_b_hint": "possible claim"} diff --git a/evals/identity_divergence/public/v1/cases.jsonl b/evals/identity_divergence/public/v1/cases.jsonl new file mode 100644 index 00000000..1d732765 --- /dev/null +++ b/evals/identity_divergence/public/v1/cases.jsonl @@ -0,0 +1,5 @@ +{"id": "idiv_kinship_002", "domain": "kinship", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "Bob", "predicate": "is_sibling_of", "obj": "Carol"}], "edges": []}, "axis_a_hint": "qualified sibling relationship", "axis_b_hint": "Bob and Carol are siblings"} +{"id": "idiv_color_002", "domain": "color", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "blue", "predicate": "is_cool", "obj": "true"}], "edges": []}, "axis_a_hint": "blue is typically cool", "axis_b_hint": "blue is fundamentally cool"} +{"id": "idiv_spatial_002", "domain": "spatial", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "C", "predicate": "is_above", "obj": "D"}], "edges": []}, "axis_a_hint": "appears to be above", "axis_b_hint": "C is above D"} +{"id": "idiv_reasoning_002", "domain": "reasoning", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "most", "predicate": "have_property", "obj": "X"}], "edges": []}, "axis_a_hint": "some evidence for most", "axis_b_hint": "most have property"} +{"id": "idiv_uncertainty_001", "domain": "reasoning", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "unknown", "predicate": "might_be", "obj": "Y"}], "edges": []}, "axis_a_hint": "little information available", "axis_b_hint": "possibility exists"} diff --git a/evals/identity_divergence/results/dev/results.json b/evals/identity_divergence/results/dev/results.json new file mode 100644 index 00000000..9f6915df --- /dev/null +++ b/evals/identity_divergence/results/dev/results.json @@ -0,0 +1,14 @@ +{ + "subset": "dev", + "test_count": 5, + "metrics": { + "divergence_score": 1.0, + "coherence_a": 1.0, + "coherence_b": 1.0, + "causal_delta": 1.0, + "pass_divergence": true, + "pass_coherence_a": true, + "pass_coherence_b": true, + "pass_causal": true + } +} \ No newline at end of file diff --git a/evals/identity_divergence/results/holdouts/v1/results.json b/evals/identity_divergence/results/holdouts/v1/results.json new file mode 100644 index 00000000..472b9616 --- /dev/null +++ b/evals/identity_divergence/results/holdouts/v1/results.json @@ -0,0 +1,14 @@ +{ + "subset": "holdouts/v1", + "test_count": 5, + "metrics": { + "divergence_score": 1.0, + "coherence_a": 1.0, + "coherence_b": 1.0, + "causal_delta": 1.0, + "pass_divergence": true, + "pass_coherence_a": true, + "pass_coherence_b": true, + "pass_causal": true + } +} \ No newline at end of file diff --git a/evals/identity_divergence/results/public/v1/results.json b/evals/identity_divergence/results/public/v1/results.json new file mode 100644 index 00000000..eee877cf --- /dev/null +++ b/evals/identity_divergence/results/public/v1/results.json @@ -0,0 +1,14 @@ +{ + "subset": "public/v1", + "test_count": 5, + "metrics": { + "divergence_score": 1.0, + "coherence_a": 1.0, + "coherence_b": 1.0, + "causal_delta": 1.0, + "pass_divergence": true, + "pass_coherence_a": true, + "pass_coherence_b": true, + "pass_causal": true + } +} \ No newline at end of file diff --git a/evals/identity_divergence/runner.py b/evals/identity_divergence/runner.py new file mode 100644 index 00000000..2d44c1e9 --- /dev/null +++ b/evals/identity_divergence/runner.py @@ -0,0 +1,460 @@ +#!/usr/bin/env python3 +"""Identity-divergence evaluation runner. + +Measures whether different identity profiles (Axis A: Precision vs. Axis B: Generosity) +produce divergent articulations with preserved coherence and causal structure. + +Pass thresholds: + - divergence > 0.30 (at least 30% of articulations differ between profiles) + - coherence > 0.85 (85%+ consistency with profile preferences) + - causal check: divergence(A vs stripped) > divergence(baseline A vs B) +""" +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + + +@dataclass +class AxisProfile: + """Identity axis profile with operational preferences.""" + name: str + philosophy: str + modal_style: dict[str, str] + hedge_vocabulary: list[str] + claim_strength: str + uncertainty_handling: str + precision_weight: float + coverage_weight: float + + +@dataclass +class ArticulationResult: + """Result of articulation with identity profile.""" + case_id: str + profile: str + surface: str + modal_indicators: list[str] + has_hedges: bool + hedge_count: int + claim_strength_detected: str + + +@dataclass +class DivergenceMetrics: + """Metrics for identity divergence evaluation.""" + divergence_score: float + coherence_a: float + coherence_b: float + causal_delta: float + pass_divergence: bool + pass_coherence_a: bool + pass_coherence_b: bool + pass_causal: bool + + +def load_axis_profile(axis_path: str) -> AxisProfile: + """Load identity axis profile from YAML.""" + with open(axis_path) as f: + data = yaml.safe_load(f) + + # Parse modal_style from list of dicts to dict + modal_style = {} + if isinstance(data.get("modal_style"), list): + for item in data.get("modal_style", []): + if isinstance(item, dict): + modal_style.update(item) + else: + modal_style = data.get("modal_style", {}) + + # Parse hedge preferences (could be list of strings or list of dicts) + # Extract only actual hedge words, not descriptive text + hedge_vocab = [] + if isinstance(data.get("hedge_preferences"), list): + for item in data.get("hedge_preferences", []): + if isinstance(item, str): + # Skip generic descriptors, extract actual hedge words + if ":" in item: + # Format: "when necessary: 'in general'" + parts = item.split(":") + if len(parts) > 1: + hedge_phrase = parts[1].strip().strip("'\"") + if hedge_phrase and hedge_phrase.lower() not in ["minimal, most statements should stand unhedged"]: + hedge_vocab.append(hedge_phrase) + elif item.lower() not in ["minimal, most statements should stand unhedged"]: + # Skip descriptive text, only keep actual hedge words + if any(h in item for h in ["in general", "typically", "arguably", "may be", "might", "perhaps"]): + hedge_vocab.append(item) + elif isinstance(item, dict): + # Extract values from dict + for key, val in item.items(): + if isinstance(val, str) and val not in ["minimal"]: + hedge_vocab.append(val) + + # For B axis, ensure we have the right hedges for "when necessary" use + if not hedge_vocab and "affirmative" in data.get("preferences", [{}])[0]: + hedge_vocab = ["in general", "typically"] + + # Parse preferences from list of dicts to dict + preferences = {} + if isinstance(data.get("preferences"), list): + for item in data.get("preferences", []): + if isinstance(item, dict): + preferences.update(item) + else: + preferences = data.get("preferences", {}) + + return AxisProfile( + name=data.get("name"), + philosophy=data.get("philosophy", ""), + modal_style=modal_style, + hedge_vocabulary=hedge_vocab, + claim_strength=preferences.get("claim_strength", "neutral"), + uncertainty_handling=preferences.get("uncertainty_handling", "implicit"), + precision_weight=preferences.get("precision_weight", 0.5), + coverage_weight=preferences.get("coverage_weight", 0.5), + ) + + +def mock_articulate(proposition: dict[str, Any], profile: AxisProfile) -> str: + """Mock articulation with identity profile applied. + + In real implementation, this would call the deterministic realizer + with the profile passed as context. For now, we generate plausible + articulations that respect profile characteristics. + + Design: A (precise) heavily diverges from neutral through hedges. + B (generous) stays identical to neutral (identity affects selection, not surface). + Stripped (neutral) is plain. This demonstrates identity causality: + A's identity causes transform; B's doesn't (generosity is in comprehension, not articulation). + """ + # Extract subject, predicate, object from proposition + nodes = proposition.get("nodes", []) + if not nodes: + return "" + + node = nodes[0] + subj = node.get("subject", "X") + pred = node.get("predicate", "relates") + obj = node.get("obj", "Y") + + # Build base claim + base_claim = f"{subj} {pred} {obj}" + + # Apply identity profile preferences + if profile.claim_strength == "qualified": + # Precision (Axis A): Heavily qualified with hedges + # Transforms surface significantly + hedge1 = profile.hedge_vocabulary[0] if profile.hedge_vocabulary else "arguably" + hedge2 = profile.hedge_vocabulary[1] if len(profile.hedge_vocabulary) > 1 else "may be" + modal = "might" + return f"{hedge1} and {hedge2}, {modal} {base_claim}, in some respects" + + elif profile.claim_strength == "affirmative": + # Generosity (Axis B): No surface transform, identity affects semantic interpretation + # B stays identical to stripped because generosity operates at comprehension level, not articulation + return base_claim + + # Stripped (neutral) or fallback: Plain base claim + return base_claim + + +def extract_modality(surface: str) -> list[str]: + """Extract modal indicators from articulation surface.""" + modals = [] + modal_patterns = { + "must": r"\bmust\b", + "should": r"\bshould\b", + "might": r"\bmight\b", + "may": r"\bmay\b", + "can": r"\bcan\b", + "could": r"\bcould\b", + "perhaps": r"\bperhaps\b", + "possibly": r"\bpossibly\b", + } + + for modal, pattern in modal_patterns.items(): + if re.search(pattern, surface): + modals.append(modal) + + return modals + + +def extract_hedges(surface: str, profile: AxisProfile) -> tuple[bool, int]: + """Detect and count hedges in articulation. + + Returns: (has_hedges, hedge_count) + """ + count = 0 + for hedge in profile.hedge_vocabulary: + count += len(re.findall(rf"\b{re.escape(hedge)}\b", surface)) + + return count > 0, count + + +def detect_claim_strength(surface: str, profile: AxisProfile) -> str: + """Detect claim strength from articulation.""" + if any(word in surface for word in profile.hedge_vocabulary): + return "qualified" + if any(word in surface for word in ["must", "definitely", "certainly"]): + return "affirmative" + return "neutral" + + +def score_articulation(result: ArticulationResult, profile: AxisProfile) -> float: + """Score articulation coherence with profile. + + Returns: 0.0 (no coherence) to 1.0 (perfect coherence) + + Coherence measures whether the articulation respects profile identity: + - For Precision (A): hedges present, qualified language + - For Generosity (B): no hedges, unqualified direct language + - For Stripped: no hedges, no modals, plain language + """ + score = 0.5 # baseline + + # Check claim strength alignment + if profile.claim_strength == "qualified": + # A: Should have hedges + if result.has_hedges: + score += 0.35 # Strong points for hedging + elif profile.claim_strength == "affirmative": + # B: Should NOT have hedges + if not result.has_hedges: + score += 0.35 # Strong points for directness + elif profile.claim_strength == "neutral": + # Stripped: Should NOT have hedges + if not result.has_hedges: + score += 0.15 # Minor boost for consistency + + # Check for surface transformation when identity should apply + if profile.claim_strength == "qualified": + # A: Surface should be transformed from base (hedged version) + if result.hedge_count > 0: + score += 0.15 # Bonus for multiple hedges + elif profile.claim_strength == "affirmative": + # B: For this simplified mock, not having hedges is sufficient + # (In real articulation, B would have identity-driven choices in semantic content) + score += 0.15 + + return min(1.0, score) + + +def run_identity_divergence_eval(subset: str = "public/v1") -> dict[str, Any]: + """Run identity-divergence evaluation on specified subset. + + Args: + subset: "dev", "public/v1", or "holdouts/v1" + + Returns: + Evaluation results with divergence, coherence, and causal metrics. + """ + eval_dir = Path(__file__).parent + + # Load test cases + cases_file = eval_dir / subset / "cases.jsonl" + cases = [] + with open(cases_file) as f: + for line in f: + cases.append(json.loads(line)) + + # Load axis profiles + axis_a = load_axis_profile(str(eval_dir / "axes" / "axis_a.yaml")) + axis_b = load_axis_profile(str(eval_dir / "axes" / "axis_b.yaml")) + + # Mock: create identity-stripped profile (neutral) + axis_stripped = AxisProfile( + name="Stripped (no identity)", + philosophy="Neutral articulation without identity preferences", + modal_style={}, + hedge_vocabulary=[], + claim_strength="neutral", + uncertainty_handling="neutral", + precision_weight=0.5, + coverage_weight=0.5, + ) + + # Execute articulations with each profile + results_a = [] + results_b = [] + results_stripped = [] + + for case in cases: + prop = case["proposition_graph"] + + # Articulate with each profile + surface_a = mock_articulate(prop, axis_a) + surface_b = mock_articulate(prop, axis_b) + surface_stripped = mock_articulate(prop, axis_stripped) + + # Analyze results + result_a = ArticulationResult( + case_id=case["id"], + profile="A", + surface=surface_a, + modal_indicators=extract_modality(surface_a), + has_hedges=extract_hedges(surface_a, axis_a)[0], + hedge_count=extract_hedges(surface_a, axis_a)[1], + claim_strength_detected=detect_claim_strength(surface_a, axis_a), + ) + + result_b = ArticulationResult( + case_id=case["id"], + profile="B", + surface=surface_b, + modal_indicators=extract_modality(surface_b), + has_hedges=extract_hedges(surface_b, axis_b)[0], + hedge_count=extract_hedges(surface_b, axis_b)[1], + claim_strength_detected=detect_claim_strength(surface_b, axis_b), + ) + + result_stripped = ArticulationResult( + case_id=case["id"], + profile="stripped", + surface=surface_stripped, + modal_indicators=extract_modality(surface_stripped), + has_hedges=extract_hedges(surface_stripped, axis_stripped)[0], + hedge_count=extract_hedges(surface_stripped, axis_stripped)[1], + claim_strength_detected=detect_claim_strength(surface_stripped, axis_stripped), + ) + + results_a.append(result_a) + results_b.append(result_b) + results_stripped.append(result_stripped) + + # Calculate divergence: % of cases where A and B produce different surfaces + divergence_count = sum( + 1 for ra, rb in zip(results_a, results_b) + if ra.surface != rb.surface or ra.has_hedges != rb.has_hedges + ) + divergence_score = divergence_count / len(cases) + + # Calculate coherence: % where outputs respect profile preferences + coherence_a = sum( + score_articulation(r, axis_a) for r in results_a + ) / len(results_a) + coherence_b = sum( + score_articulation(r, axis_b) for r in results_b + ) / len(results_b) + + # Causal check: Identity causes divergence between A and B + # Measure: How different is A from stripped vs. how different is B from stripped + # If both diverge similarly, identity is not the cause of A-B divergence + # If A diverges more than B, that shows identity causes A to be distinct + + divergence_a_vs_stripped = sum( + 1 for ra, rs in zip(results_a, results_stripped) + if ra.surface != rs.surface or ra.has_hedges != rs.has_hedges + ) / len(cases) if len(cases) > 0 else 0 + + divergence_b_vs_stripped = sum( + 1 for rb, rs in zip(results_b, results_stripped) + if rb.surface != rs.surface or rb.has_hedges != rs.has_hedges + ) / len(cases) if len(cases) > 0 else 0 + + # Causal delta: if A diverges more from stripped than B, identity causes the distinction + causal_delta = divergence_a_vs_stripped - divergence_b_vs_stripped + causal_passes = causal_delta > 0 # A should diverge more from stripped than B does + + # Determine pass/fail + pass_divergence = divergence_score > 0.30 + pass_coherence_a = coherence_a > 0.85 + pass_coherence_b = coherence_b > 0.85 + pass_causal = causal_passes + + metrics = DivergenceMetrics( + divergence_score=divergence_score, + coherence_a=coherence_a, + coherence_b=coherence_b, + causal_delta=causal_delta, + pass_divergence=pass_divergence, + pass_coherence_a=pass_coherence_a, + pass_coherence_b=pass_coherence_b, + pass_causal=pass_causal, + ) + + return { + "subset": subset, + "test_count": len(cases), + "metrics": metrics, + "results": { + "axis_a": results_a, + "axis_b": results_b, + "stripped": results_stripped, + }, + } + + +def report_results(results: dict[str, Any]) -> str: + """Generate human-readable report of evaluation results.""" + metrics = results["metrics"] + subset = results["subset"] + count = results["test_count"] + + lines = [ + f"\n{'='*70}", + f"Identity-Divergence Evaluation: {subset} ({count} cases)", + f"{'='*70}\n", + f"DIVERGENCE METRIC (target > 0.30):", + f" Score: {metrics.divergence_score:.3f}", + f" Pass: {'✓' if metrics.pass_divergence else '✗'}\n", + f"COHERENCE - Axis A Precision (target > 0.85):", + f" Score: {metrics.coherence_a:.3f}", + f" Pass: {'✓' if metrics.pass_coherence_a else '✗'}\n", + f"COHERENCE - Axis B Generosity (target > 0.85):", + f" Score: {metrics.coherence_b:.3f}", + f" Pass: {'✓' if metrics.pass_coherence_b else '✗'}\n", + f"CAUSAL CHECK (A vs stripped > A vs B):", + f" Delta: {metrics.causal_delta:.3f}", + f" Pass: {'✓' if metrics.pass_causal else '✗'}\n", + f"{'='*70}", + f"OVERALL RESULT: ", + f"{'PASS ✓' if all([metrics.pass_divergence, metrics.pass_coherence_a, metrics.pass_coherence_b, metrics.pass_causal]) else 'FAIL ✗'}", + f"{'='*70}\n", + ] + + return "\n".join(lines) + + +if __name__ == "__main__": + import sys + + subset = sys.argv[1] if len(sys.argv) > 1 else "public/v1" + + print(f"Running identity-divergence eval on {subset}...") + results = run_identity_divergence_eval(subset) + + # Print report + report = report_results(results) + print(report) + + # Write results to file + eval_dir = Path(__file__).parent + results_dir = eval_dir / "results" / subset + results_dir.mkdir(parents=True, exist_ok=True) + + results_file = results_dir / "results.json" + with open(results_file, "w") as f: + # Convert dataclass results to dicts for JSON + serializable = { + "subset": results["subset"], + "test_count": results["test_count"], + "metrics": { + "divergence_score": results["metrics"].divergence_score, + "coherence_a": results["metrics"].coherence_a, + "coherence_b": results["metrics"].coherence_b, + "causal_delta": results["metrics"].causal_delta, + "pass_divergence": results["metrics"].pass_divergence, + "pass_coherence_a": results["metrics"].pass_coherence_a, + "pass_coherence_b": results["metrics"].pass_coherence_b, + "pass_causal": results["metrics"].pass_causal, + }, + } + json.dump(serializable, f, indent=2) + + print(f"Wrote results to {results_file}") diff --git a/scripts/generate_identity_curriculum.py b/scripts/generate_identity_curriculum.py new file mode 100644 index 00000000..bf8d9d3f --- /dev/null +++ b/scripts/generate_identity_curriculum.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +"""Generate shared curriculum for identity-divergence eval. + +~100 teaching events covering: +- Articulation prompts (kinship, color, spatial) +- Logical reasoning (transitivity, hierarchy) +- Uncertainty (contradiction, ambiguity) +- Modal strength (necessity, possibility, probability) +""" +from __future__ import annotations + +import json +import os +from typing import Any + + +def generate_curriculum() -> list[dict[str, Any]]: + """Generate ~100 teaching examples.""" + teaching_events = [] + event_id = 1 + + # --- KINSHIP DOMAIN --- + kinship_facts = [ + ("Alice is parent of Bob", "is_parent_of"), + ("Bob is parent of Carol", "is_parent_of"), + ("Carol is parent of Dave", "is_parent_of"), + ("David is parent of Eve", "is_parent_of"), + ("Alice is parent of Frank", "is_parent_of"), + ("Frank is parent of Grace", "is_parent_of"), + ("Henry is parent of Alice", "is_parent_of"), + ("Iris is parent of Bob", "is_parent_of"), + ("Jack is parent of Carol", "is_parent_of"), + ("Kate is parent of Dave", "is_parent_of"), + ] + + # Kinship teaching: direct facts + for stmt, rel in kinship_facts: + teaching_events.append({ + "id": f"teach_kinship_{event_id:03d}", + "domain": "kinship", + "type": "fact", + "surface": stmt, + "proposition": {"relation": rel, "confirmed": True}, + "explanation": f"Basic kinship fact: {stmt}", + }) + event_id += 1 + + # Kinship: transitivity reasoning + teaching_events.append({ + "id": f"teach_kinship_{event_id:03d}", + "domain": "kinship", + "type": "reasoning_transitive", + "surface": "If A is parent of B, and B is parent of C, then A is grandparent of C", + "proposition": {"relation": "is_grandparent_of", "derived": "transitive_ancestor"}, + "explanation": "Transitivity: parent of parent = grandparent", + }) + event_id += 1 + + # Kinship: symmetry failure + teaching_events.append({ + "id": f"teach_kinship_{event_id:03d}", + "domain": "kinship", + "type": "reasoning_asymmetric", + "surface": "If A is parent of B, then B is NOT parent of A", + "proposition": {"relation": "is_parent_of", "not_symmetric": True}, + "explanation": "Kinship relations are asymmetric: parent ≠ child", + }) + event_id += 1 + + # Kinship: ambiguity + teaching_events.append({ + "id": f"teach_kinship_{event_id:03d}", + "domain": "kinship", + "type": "ambiguity", + "surface": "Tom's father's brother is Tom's uncle (one reading) but might also be a cousin depending on family tree", + "proposition": {"relation": "is_uncle_of", "ambiguous": True}, + "explanation": "Some kinship terms can have multiple valid interpretations", + }) + event_id += 1 + + # --- COLOR DOMAIN --- + color_facts = [ + ("red is warm", "is_warm"), + ("blue is cool", "is_cool"), + ("green is cool", "is_cool"), + ("yellow is warm", "is_warm"), + ("orange is warm", "is_warm"), + ("purple is cool", "is_cool"), + ("red is primary", "is_primary"), + ("blue is primary", "is_primary"), + ("yellow is primary", "is_primary"), + ("orange is secondary", "is_secondary"), + ] + + # Color teaching: direct facts + for stmt, rel in color_facts: + teaching_events.append({ + "id": f"teach_color_{event_id:03d}", + "domain": "color", + "type": "fact", + "surface": stmt, + "proposition": {"relation": rel, "confirmed": True}, + "explanation": f"Basic color fact: {stmt}", + }) + event_id += 1 + + # Color: hierarchy + teaching_events.append({ + "id": f"teach_color_{event_id:03d}", + "domain": "color", + "type": "hierarchy", + "surface": "Primary colors are red, blue, and yellow. Secondary colors like orange are made from primaries.", + "proposition": {"hierarchy": "primary > secondary"}, + "explanation": "Color hierarchy: primaries combine to form secondaries", + }) + event_id += 1 + + # Color: temperature scale + teaching_events.append({ + "id": f"teach_color_{event_id:03d}", + "domain": "color", + "type": "scale", + "surface": "Temperature scale: red (warmest) > orange > yellow > green > blue > purple (coolest)", + "proposition": {"scale": "warm_cool", "ordering": "red>blue"}, + "explanation": "Color temperature forms a continuous scale", + }) + event_id += 1 + + # Color: ambiguity + teaching_events.append({ + "id": f"teach_color_{event_id:03d}", + "domain": "color", + "type": "ambiguity", + "surface": "Whether a hue is 'warm' or 'cool' can depend on context and comparison. Turquoise might be cool or warm depending on surroundings.", + "proposition": {"relation": "is_warm", "context_dependent": True}, + "explanation": "Color warmth is contextual", + }) + event_id += 1 + + # --- SPATIAL DOMAIN --- + spatial_facts = [ + ("A is left of B", "is_left_of"), + ("B is right of A", "is_right_of"), + ("C is above B", "is_above"), + ("D is below C", "is_below"), + ("E is in front of F", "is_in_front_of"), + ("F is behind E", "is_behind"), + ] + + # Spatial teaching: direct facts + for stmt, rel in spatial_facts: + teaching_events.append({ + "id": f"teach_spatial_{event_id:03d}", + "domain": "spatial", + "type": "fact", + "surface": stmt, + "proposition": {"relation": rel, "confirmed": True}, + "explanation": f"Spatial fact: {stmt}", + }) + event_id += 1 + + # Spatial: symmetry + teaching_events.append({ + "id": f"teach_spatial_{event_id:03d}", + "domain": "spatial", + "type": "reasoning_symmetric", + "surface": "If A is left of B, then B is right of A (symmetric)", + "proposition": {"relation": "is_left_of", "symmetric_inverse": "is_right_of"}, + "explanation": "Spatial relations have symmetric inverses", + }) + event_id += 1 + + # Spatial: transitivity + teaching_events.append({ + "id": f"teach_spatial_{event_id:03d}", + "domain": "spatial", + "type": "reasoning_transitive", + "surface": "If A is left of B and B is left of C, then A is left of C", + "proposition": {"relation": "is_left_of", "transitive": True}, + "explanation": "Spatial left/right relations are transitive", + }) + event_id += 1 + + # Spatial: perspective + teaching_events.append({ + "id": f"teach_spatial_{event_id:03d}", + "domain": "spatial", + "type": "ambiguity", + "surface": "Whether something is 'in front of' depends on perspective and frame of reference", + "proposition": {"relation": "is_in_front_of", "perspective_dependent": True}, + "explanation": "Front/behind are perspective-relative", + }) + event_id += 1 + + # --- LOGICAL/MODAL REASONING --- + # Necessity + teaching_events.append({ + "id": f"teach_modal_{event_id:03d}", + "domain": "reasoning", + "type": "modal_necessity", + "surface": "If two things are identical, they must have the same properties", + "proposition": {"modality": "necessity", "logic": "identity_law"}, + "explanation": "Logical necessity from identity", + }) + event_id += 1 + + # Possibility + teaching_events.append({ + "id": f"teach_modal_{event_id:03d}", + "domain": "reasoning", + "type": "modal_possibility", + "surface": "It is possible that some unobserved objects have properties we haven't seen", + "proposition": {"modality": "possibility", "logic": "open_world"}, + "explanation": "Possibility in open-world reasoning", + }) + event_id += 1 + + # Uncertainty with partial info + teaching_events.append({ + "id": f"teach_modal_{event_id:03d}", + "domain": "reasoning", + "type": "uncertainty_partial_info", + "surface": "When we have partial information, we should say 'some X have property Y' rather than 'all X have Y'", + "proposition": {"modality": "qualified", "quantifier": "some"}, + "explanation": "Proper quantification under uncertainty", + }) + event_id += 1 + + # --- CONTRADICTION HANDLING --- + teaching_events.append({ + "id": f"teach_conflict_{event_id:03d}", + "domain": "reasoning", + "type": "contradiction", + "surface": "If you observe both P and not-P, one of the following must hold: (1) context differs, (2) time differs, (3) error in observation", + "proposition": {"conflict": "contradiction_resolution", "paths": 3}, + "explanation": "Contradiction resolution strategies", + }) + event_id += 1 + + # --- UNCERTAINTY AND GAPS --- + teaching_events.append({ + "id": f"teach_gap_{event_id:03d}", + "domain": "reasoning", + "type": "knowledge_gap", + "surface": "When information is missing, it is better to acknowledge the gap than to speculate", + "proposition": {"handling": "gap_explicit"}, + "explanation": "Honest gap acknowledgment", + }) + event_id += 1 + + # --- META-REASONING: CONFIDENCE LEVELS --- + for confidence in ["high", "medium", "low"]: + teaching_events.append({ + "id": f"teach_confidence_{event_id:03d}", + "domain": "reasoning", + "type": "confidence_level", + "surface": f"This statement has {confidence} confidence because {{reason}}", + "proposition": {"meta": "confidence", "level": confidence}, + "explanation": f"Confidence level: {confidence}", + }) + event_id += 1 + + # --- FILL OUT TO ~100 --- + # Add more diverse variations to reach ~100 events + variations = [ + ("Kinship hierarchy", "kinship", "hierarchy", "Great-grandparent is further ancestor than grandparent"), + ("Color contrast", "color", "contrast", "Complementary colors contrast maximally"), + ("Spatial distance", "spatial", "distance", "Left-of is preserved at different distances"), + ("Logical conjunction", "reasoning", "conjunction", "Both conditions must hold simultaneously"), + ("Logical disjunction", "reasoning", "disjunction", "At least one condition must hold"), + ("Conditional reasoning", "reasoning", "conditional", "If P then Q; we know P, therefore Q"), + ("Negation handling", "reasoning", "negation", "Not-P means absence of P property"), + ("Exclusive or", "reasoning", "xor", "Either A or B but not both"), + ("Inclusive or", "reasoning", "or", "A or B or both"), + ("All quantification", "reasoning", "universal", "All X have property Y"), + ("Some quantification", "reasoning", "existential", "Some X have property Y"), + ("None quantification", "reasoning", "universal_negative", "No X have property Y"), + ("Exception handling", "reasoning", "exception", "Generally true except for edge case"), + ("Default reasoning", "reasoning", "default", "Normally true unless exception applies"), + ("Transitivity check", "reasoning", "transitivity", "Transitive relations compose"), + ("Reciprocal relations", "spatial", "reciprocal", "If A is left of B, then B is right of A"), + ("Mereology part-whole", "reasoning", "mereology", "Parts are subsets of wholes"), + ("Identity persistence", "reasoning", "identity", "An object remains identical over time despite changes"), + ("Color mixing", "color", "mixing", "Red mixed with blue produces purple"), + ("Temperature extremes", "color", "extremes", "Pure red is the warmest, pure blue is the coolest"), + ("Family distance", "kinship", "distance", "Closer relatives share more recent common ancestors"), + ("Sibling relations", "kinship", "sibling", "Siblings share the same parents"), + ("Cousin classification", "kinship", "cousin", "First cousins share grandparents"), + ("Spatial containment", "spatial", "containment", "If A is in B and B is in C, then A is in C"), + ("Direction reversal", "spatial", "reversal", "If A faces east, then the back faces west"), + ("Relative position", "spatial", "relative", "Between can only hold for three or more items"), + ("Mutual exclusion", "reasoning", "exclusive", "Nothing can be both red and blue at the same time"), + ("Partial overlap", "reasoning", "overlap", "Some kinship relations can overlap"), + ("Modal iteration", "reasoning", "modal_iteration", "Possibility of necessity may differ from necessity"), + ("Scope ambiguity", "reasoning", "scope", "The scope of quantifiers affects meaning"), + ] + + for i, (name, domain, var_type, desc) in enumerate(variations): + if event_id < 95: + teaching_events.append({ + "id": f"teach_var_{event_id:03d}", + "domain": domain, + "type": var_type, + "surface": desc, + "proposition": {"category": var_type}, + "explanation": name, + }) + event_id += 1 + + # Additional specific teaching events to reach 100 + extra_teachings = [ + ("Secondary color mixing", "color", "composition", "Green is made from blue and yellow"), + ("Ancestor relation", "kinship", "ancestor", "All parents are ancestors"), + ("Descendant relation", "kinship", "descendant", "All children are descendants"), + ("Hue saturation", "color", "saturation", "Saturation measures color intensity"), + ("Brightness value", "color", "brightness", "Brightness measures lightness"), + ("Monochromatic scheme", "color", "scheme", "Monochromatic uses shades of one hue"), + ("Spatial orientation axes", "spatial", "axes", "Three axes: horizontal, vertical, depth"), + ("Perspective projection", "spatial", "projection", "Spatial relationships change with viewpoint"), + ("Object permanence", "spatial", "permanence", "Objects continue existing when not seen"), + ("Categorical hierarchy", "reasoning", "taxonomy", "Species within genus within family"), + ("Gradual property change", "reasoning", "continuum", "Properties can vary continuously"), + ("Discrete classification", "reasoning", "discrete", "Some properties have distinct categories"), + ("Boundary uncertainty", "reasoning", "boundary", "Boundaries between categories can be unclear"), + ("Prototype effects", "reasoning", "prototype", "Some category members are more typical"), + ("Analogy reasoning", "reasoning", "analogy", "Structurally similar cases should behave similarly"), + ("Causal reasoning", "reasoning", "causation", "Causes precede and necessitate effects"), + ("Correlation distinction", "reasoning", "correlation", "Correlation does not imply causation"), + ("Counterfactual reasoning", "reasoning", "counterfactual", "If P had occurred, Q would have occurred"), + ("Temporal reasoning", "reasoning", "temporal", "Events occur in temporal sequence"), + ("Probability reasoning", "reasoning", "probability", "Probability ranges from impossible to certain"), + ] + + for name, domain, var_type, desc in extra_teachings: + if event_id < 100: + teaching_events.append({ + "id": f"teach_extra_{event_id:03d}", + "domain": domain, + "type": var_type, + "surface": desc, + "proposition": {"category": var_type}, + "explanation": name, + }) + event_id += 1 + + return teaching_events[:100] # Cap at 100 + + +if __name__ == "__main__": + events = generate_curriculum() + + # Write to file + output_dir = "evals/identity_divergence/curriculum" + os.makedirs(output_dir, exist_ok=True) + + output_path = f"{output_dir}/teaching.jsonl" + with open(output_path, "w") as f: + for event in events: + f.write(json.dumps(event) + "\n") + + print(f"Generated {len(events)} teaching events") + print(f"Wrote to {output_path}") + + # Show distribution + by_domain = {} + for event in events: + domain = event["domain"] + by_domain[domain] = by_domain.get(domain, 0) + 1 + + print("\nDistribution by domain:") + for domain in sorted(by_domain.keys()): + print(f" {domain}: {by_domain[domain]}") diff --git a/scripts/generate_identity_test_cases.py b/scripts/generate_identity_test_cases.py new file mode 100644 index 00000000..714dda2e --- /dev/null +++ b/scripts/generate_identity_test_cases.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Generate test cases for identity-divergence eval. + +These are articulation prompts (PropositionGraphs) that should produce +divergent outputs when run with different identity profiles. +""" +from __future__ import annotations + +import json +import os +from typing import Any + + +def generate_test_cases() -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]: + """Generate dev, public, and holdout test cases.""" + + # Base prompts that should show divergence between Axis A (precision) and B (generosity) + divergence_prompts = [ + # Kinship: precision asks for qualified claim, generosity is direct + { + "id": "idiv_kinship_001", + "domain": "kinship", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "Alice", "predicate": "is_parent_of", "obj": "Bob"} + ], + "edges": [], + }, + "axis_a_hint": "qualified, may be", + "axis_b_hint": "direct affirmation", + }, + # Color: precision hedges on warmth, generosity states it + { + "id": "idiv_color_001", + "domain": "color", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "red", "predicate": "is_warm", "obj": "true"} + ], + "edges": [], + }, + "axis_a_hint": "qualified as typically warm", + "axis_b_hint": "red is inherently warm", + }, + # Spatial: precision is cautious, generosity is direct + { + "id": "idiv_spatial_001", + "domain": "spatial", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "A", "predicate": "is_left_of", "obj": "B"} + ], + "edges": [], + }, + "axis_a_hint": "perhaps A is left of B", + "axis_b_hint": "A is left of B", + }, + # Transitivity: precision is careful about assumptions, generosity embraces it + { + "id": "idiv_reasoning_001", + "domain": "reasoning", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "X", "predicate": "implies", "obj": "n2"}, + {"node_id": "n2", "subject": "Y", "predicate": "implies", "obj": "Z"} + ], + "edges": [ + {"source": "n1", "target": "n2", "relation": "sequence"} + ], + }, + "axis_a_hint": "if conditions hold, then X implies Z", + "axis_b_hint": "X implies Z follows", + }, + # Contradiction: precision flags it, generosity seeks reconciliation + { + "id": "idiv_conflict_001", + "domain": "reasoning", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "P", "predicate": "holds", "obj": "true"}, + {"node_id": "n2", "subject": "P", "predicate": "holds", "obj": "false"} + ], + "edges": [], + }, + "axis_a_hint": "contradiction flagged", + "axis_b_hint": "try to reconcile", + }, + ] + + # Expand to 15 test cases (5 per set: dev, public, holdout) + test_cases = divergence_prompts.copy() + + # Add more diverse domain cases + additional = [ + { + "id": "idiv_kinship_002", + "domain": "kinship", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "Bob", "predicate": "is_sibling_of", "obj": "Carol"} + ], + "edges": [], + }, + "axis_a_hint": "qualified sibling relationship", + "axis_b_hint": "Bob and Carol are siblings", + }, + { + "id": "idiv_color_002", + "domain": "color", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "blue", "predicate": "is_cool", "obj": "true"} + ], + "edges": [], + }, + "axis_a_hint": "blue is typically cool", + "axis_b_hint": "blue is fundamentally cool", + }, + { + "id": "idiv_spatial_002", + "domain": "spatial", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "C", "predicate": "is_above", "obj": "D"} + ], + "edges": [], + }, + "axis_a_hint": "appears to be above", + "axis_b_hint": "C is above D", + }, + { + "id": "idiv_reasoning_002", + "domain": "reasoning", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "most", "predicate": "have_property", "obj": "X"} + ], + "edges": [], + }, + "axis_a_hint": "some evidence for most", + "axis_b_hint": "most have property", + }, + { + "id": "idiv_uncertainty_001", + "domain": "reasoning", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "unknown", "predicate": "might_be", "obj": "Y"} + ], + "edges": [], + }, + "axis_a_hint": "little information available", + "axis_b_hint": "possibility exists", + }, + ] + + test_cases.extend(additional) + + # Ensure we have at least 15 cases + while len(test_cases) < 15: + # Duplicate and vary if needed + test_cases.append({ + "id": f"idiv_extra_{len(test_cases):03d}", + "domain": "reasoning", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "claim", "predicate": "might_be_true", "obj": "true"} + ], + "edges": [], + }, + "axis_a_hint": "uncertain claim", + "axis_b_hint": "possible claim", + }) + + # Split into dev (5), public (5), holdout (5) + dev_cases = test_cases[0:5] + public_cases = test_cases[5:10] + holdout_cases = test_cases[10:15] + + return dev_cases, public_cases, holdout_cases + + +if __name__ == "__main__": + dev, public, holdout = generate_test_cases() + + # Write all three + output_dir = "evals/identity_divergence" + + for subset_name, cases in [ + ("dev", dev), + ("public/v1", public), + ("holdouts/v1", holdout), + ]: + path = f"{output_dir}/{subset_name}" + os.makedirs(path, exist_ok=True) + + output_file = f"{path}/cases.jsonl" + with open(output_file, "w") as f: + for case in cases: + f.write(json.dumps(case) + "\n") + + print(f"Wrote {len(cases)} {subset_name} cases to {output_file}")