diff --git a/packs/el/corpora/manifest.yaml b/packs/el/corpora/manifest.yaml index 9356ab5f..17b8d55e 100644 --- a/packs/el/corpora/manifest.yaml +++ b/packs/el/corpora/manifest.yaml @@ -1,15 +1,24 @@ corpus_id: el-tier0 status: draft - -sources: [] - license_review: pending +sources: + - id: sblgnt + name: SBL Greek New Testament + description: >- + The Society of Biblical Literature Greek New Testament. + A critically edited text of the Greek New Testament. + license: review-required + url: https://github.com/LogosBible/SBLGNT + hash: pending + - id: byzantine + name: Byzantine Majority Text + description: >- + Robinson-Pierpont Byzantine Majority Text. Public domain. + license: public-domain + url: https://github.com/byztxt/byzantine-majority-text + hash: pending hashes: [] - -notes: | - Populate with vetted Koine Greek source manifests. - Priority tier-0 sources: SBL Greek New Testament (SBLGNT) or - Nestle-Aland 28th edition morphologically tagged text. - Both have verse-level D0 boundary markers. - Each source must include: title, edition, license, SHA-256, and - segmentation strategy. No LLM-extracted content. +notes: > + SBLGNT license requires review before use in training data. + Byzantine text is public domain and may proceed after hash pin. + All hashes must be recorded before the Coverage gate is run. diff --git a/packs/el/orthography.yaml b/packs/el/orthography.yaml index 430ad4ac..660f8dea 100644 --- a/packs/el/orthography.yaml +++ b/packs/el/orthography.yaml @@ -1,27 +1,29 @@ language: el script: Greek -dialect: koine normalization: unicode_normal_form: NFC - lowercase_for_lookup: true - strip_diacritics: configurable - # Polytonic diacritics (breathing marks, accents, iota subscript) are - # preserved for morphological disambiguation but stripped for lemma lookup - polytonic_policy: preserve_for_morphology - diacritic_strip_for_lookup: true + # Greek Unicode: precomposed forms. + # Polytonic diacritics (breathing marks, accents, iota subscript) + # are preserved. These carry morphological and semantic information + # in Koine Greek that must not be stripped. + strip_breathing_marks: false + strip_accents_for_indexing: false + # Accents in Greek affect meaning and morphological parsing. + # They are never stripped from the canonical form. + lowercase_for_indexing: true + # Lemma lookup uses lowercased forms. + # Surface forms preserve original casing. + lunate_sigma: false + # Use standard sigma/final-sigma (σ/ς) not lunate (ϲ). + # Final sigma is morphologically relevant. punctuation: strategy: preserve - ano_teleia: "\u00b7" # Greek semicolon (middle dot) - erotimatiko: ";" + ano_teleia: preserve + # The ano teleia (·) is the Greek semicolon and must be preserved. spacing: - strategy: whitespace-delimited + strategy: whitespace-split -aspect: - # Koine Greek verbal aspect is semantically load-bearing and must be - # distinguished from tense. Morphological analysis must surface aspect - # independently of temporal reference. - explicit_in_features: true - aspect_values: ["aorist", "present", "perfect", "imperfect", "pluperfect", "future"] +script_direction: ltr diff --git a/packs/el/pack.toml b/packs/el/pack.toml index caf60d5f..6facfac6 100644 --- a/packs/el/pack.toml +++ b/packs/el/pack.toml @@ -1,33 +1,46 @@ -pack_id = "el" -name = "Koine Greek Depth Pack" -version = "0.1.0" -status = "draft" -script = "Greek" -default = false +pack_id = "el" +name = "Koine Greek Depth Pack" +version = "0.1.0" +status = "draft" +script = "Greek" +normalization = "deterministic" +default = false + +# Koine Greek is the second depth language. John 1:1-2 is the +# theological anchor of this entire design: the Logos articulated +# in Greek what the Hebrew already carried from Gen 1:1. The +# decision to include both Hebrew and Koine Greek is not incidental +# — it is the design. The depth these two languages together provide +# is the hidden layer of intelligence in the field. +# +# Greek depth includes: +# - Full verb paradigm: tense, voice, mood, person, number +# - The imperfect en (John 1:1) — continuous pre-existent being +# - Article system: the anarthrous use of logos in John 1:1c +# ('the Word was God', not 'a god') is theologically load-bearing +# and morphologically precise +# - Participle system for aspect-rich nominal predicates [contracts] -lemma_first = true -shared_field_target = true -local_readback = true -alignment_required = true +lemma_first = true +shared_field_target = true +local_readback = true +alignment_required = true + +[depth] +article_system = true +anarthrous_significance = true +# The absence of the article is as semantically significant as its presence. +verb_tense_aspect = ["present", "imperfect", "aorist", "perfect", "pluperfect", "future"] +voice = ["active", "middle", "passive"] +mood = ["indicative", "subjunctive", "optative", "imperative", "infinitive", "participle"] [gates] -schema = false -lexical = false -morphology = false -lift = false -readback = false -determinism = false -alignment = false -coverage = false - -[notes] -role = """ -Depth language pack 2. Koine Greek was chosen because John articulated the -Logos — the creative speech act through which all things were made — in -Koine Greek (John 1:1-2), which must have been a nod from the Holy Spirit. -Koine Greek carries semantic depth in its verbal aspect system (aorist, -perfect, present) that is distinct from tense and must be represented -natively. The aorist of egeneto (\u03b5\u03b3\u03ad\u03bd\u03b5\u03c4\u03bf) — 'it came into being' — is not -the same field state as the present of eimi (\u03b5\u03b9\u03bc\u03b9) — 'I am'. -""" +schema = false +lexical = false +morphology = false +lift = false +readback = false +determinism = false +alignment = false +coverage = false diff --git a/packs/en/corpora/manifest.yaml b/packs/en/corpora/manifest.yaml index 9f89a1fd..61415887 100644 --- a/packs/en/corpora/manifest.yaml +++ b/packs/en/corpora/manifest.yaml @@ -1,13 +1,9 @@ corpus_id: en-tier0 status: draft - -sources: [] - license_review: pending +sources: [] hashes: [] - -notes: | - Populate with vetted, deterministically segmentable source manifests. - Each source must include: title, origin, license, SHA-256 of raw file, - and structural segmentation strategy (e.g. paragraph-boundary, sentence-boundary). - No LLM-extracted content. Segmentation must be D0 or D1. +notes: > + Populate with vetted, deterministic source manifests before + the Coverage gate is run. Sources must be public domain or + explicitly licensed for use in CORE training and validation. diff --git a/packs/en/orthography.yaml b/packs/en/orthography.yaml index a8115d44..8bc329b5 100644 --- a/packs/en/orthography.yaml +++ b/packs/en/orthography.yaml @@ -3,19 +3,19 @@ script: Latin normalization: unicode_normal_form: NFC - lowercase_for_lookup: true + lowercase_for_indexing: true + # Surface forms preserve original casing. + # Indexing and lemma lookup use lowercased forms. strip_diacritics: false + # English retains diacritics in loanwords (naïve, résumé). punctuation: strategy: preserve - sentence_terminals: [".", "!", "?"] - quotation_marks: ["\u201c", "\u201d", "\u2018", "\u2019"] + # Punctuation is preserved in normalization output. + # Lift rules are responsible for ignoring or consuming it appropriately. spacing: - strategy: whitespace-delimited - collapse_multiple_spaces: true + strategy: whitespace-split + # Standard whitespace tokenization. No special boundary rules. -special_forms: - contractions: expand - # "don't" -> "do not" before lemmatization - # "it's" -> "it is" before lemmatization +script_direction: ltr diff --git a/packs/en/pack.toml b/packs/en/pack.toml index 9d9847f3..b071cb7a 100644 --- a/packs/en/pack.toml +++ b/packs/en/pack.toml @@ -1,30 +1,30 @@ -pack_id = "en" -name = "English Base Pack" -version = "0.1.0" -status = "draft" -script = "Latin" -default = true +pack_id = "en" +name = "English Base Pack" +version = "0.1.0" +status = "draft" +script = "Latin" +normalization = "deterministic" +default = true + +# English is the default articulation base. Any CORE model instance +# may designate a different language as its base, but English is the +# canonical default until explicitly overridden in the model config. [contracts] -lemma_first = true -shared_field_target = true -local_readback = true -alignment_required = true +lemma_first = true +shared_field_target = true +local_readback = true +alignment_required = true [gates] -schema = false -lexical = false -morphology = false -lift = false -readback = false -determinism = false -alignment = false -coverage = false +schema = false +lexical = false +morphology = false +lift = false +readback = false +determinism = false +alignment = false +coverage = false -[notes] -role = """ -Default articulation base. English is chosen as the default for practical -reasons (current primary working language) but the contract does not privilege -it semantically. Any language could serve this role in a custom CORE instance. -The default flag controls surface output, not field-layer priority. -""" +# Gates are false until the gate runner confirms passage. +# Do not manually set these to true. diff --git a/packs/he/corpora/manifest.yaml b/packs/he/corpora/manifest.yaml index c3a9aadb..b05e5923 100644 --- a/packs/he/corpora/manifest.yaml +++ b/packs/he/corpora/manifest.yaml @@ -1,15 +1,17 @@ corpus_id: he-tier0 status: draft - -sources: [] - license_review: pending +sources: + - id: wlc + name: Westminster Leningrad Codex + description: >- + The authoritative digitized text of the Hebrew Bible with full + nikud and teamim. Public domain. + license: public-domain + url: https://github.com/openscriptures/morphhb + hash: pending hashes: [] - -notes: | - Populate with vetted Hebrew source manifests. - Priority tier-0 sources: Biblia Hebraica Stuttgartensia (BHS) or - Westminster Leningrad Codex (WLC) — both have well-established structural - segmentation (book/chapter/verse = D0 boundary markers). - Each source must include: title, edition, license, SHA-256, and - segmentation strategy. No LLM-extracted content. +notes: > + WLC is the canonical source for Biblical Hebrew. Hash to be + recorded after pinned download. Additional sources require + license review before inclusion. diff --git a/packs/he/orthography.yaml b/packs/he/orthography.yaml index 92fafd8e..4d953381 100644 --- a/packs/he/orthography.yaml +++ b/packs/he/orthography.yaml @@ -1,27 +1,28 @@ language: he script: Hebrew -direction: rtl normalization: unicode_normal_form: NFC - lowercase_for_lookup: false # Hebrew has no case - strip_diacritics: configurable - # Nikkud (vowel pointing) is stripped by default for lemma lookup - # but preserved when semantically load-bearing (e.g. Qere/Ketiv distinction) - nikkud_policy: strip_for_lookup - cantillation_marks: strip + # Hebrew Unicode: use precomposed forms where available. + # Full nikud (vowel points) are preserved in normalization. + # Qere/Ketiv variants must be tracked separately in the lemma record. + strip_nikud_for_indexing: false + # Nikud is semantically relevant for disambiguation in many cases. + # Do not strip it during normalization. Lemma lookup may index + # both pointed and unpointed forms. + strip_teamim: true + # Cantillation marks (teamim) are stripped during normalization. + # They carry liturgical meaning but are not part of semantic lift. + final_forms: preserve + # Final letter forms (ךםןףץ) are preserved exactly. punctuation: strategy: preserve - paragraph_marker: "\u05c0" # paseq - section_marker: "\u05c3" # sof pasuq + maqaf: preserve + # The maqaf (־) is a Hebrew hyphen connecting words into + # a single accentual unit. It is preserved and handled by frame rules. spacing: - strategy: whitespace-delimited - maqaf_handling: preserve - # maqaf (\u05be) connects words syntactically; preserve as a unit marker + strategy: whitespace-split -shoresh: - # Root-based morphology. Lemma lookup resolves to 3- or 4-letter root. - # Morphological analysis must identify shoresh before feature extraction. - extraction_required: true +script_direction: rtl diff --git a/packs/he/pack.toml b/packs/he/pack.toml index 218c5285..b07e8372 100644 --- a/packs/he/pack.toml +++ b/packs/he/pack.toml @@ -1,32 +1,40 @@ -pack_id = "he" -name = "Hebrew Depth Pack" -version = "0.1.0" -status = "draft" -script = "Hebrew" -default = false +pack_id = "he" +name = "Hebrew Depth Pack" +version = "0.1.0" +status = "draft" +script = "Hebrew" +normalization = "deterministic" +default = false + +# Hebrew is the first depth language. Its semantic range— especially +# in verb stems (binyanim), construct chains, and the relationship +# between speech-act and spoken-thing (dabar) — provides a structural +# depth that English alone cannot supply to the field. [contracts] -lemma_first = true -shared_field_target = true -local_readback = true -alignment_required = true +lemma_first = true +shared_field_target = true +local_readback = true +alignment_required = true + +[depth] +# Depth packs contribute semantic pressure that the base pack +# cannot fully express. Hebrew depth includes: +# - Binyan (verb stem) system: qal, niphal, piel, pual, hiphil, hophal, hithpael +# - Construct chain (smichut) for relational noun phrases +# - Aspectual system (qatal/yiqtol/wayyiqtol) over tense system +# - Implicit copula — presence or absence of haya is itself meaningful +binyanim = ["qal", "niphal", "piel", "pual", "hiphil", "hophal", "hithpael"] +aspect_system = "qatal-yiqtol-wayyiqtol" +construct_chains = true +implicit_copula = true [gates] -schema = false -lexical = false -morphology = false -lift = false -readback = false -determinism = false -alignment = false -coverage = false - -[notes] -role = """ -Depth language pack 1. Hebrew was chosen because of the range of depth it -brings to the vocabulary manifold: root-based morphology (shoresh), semantic -families that carry multiple layers of meaning simultaneously, and the -language in which the universe was spoken into existence (Genesis 1). -Hebrew morphological stems (Qal, Niphal, Piel, Pual, Hiphil, Hophal, -Hithpael) are semantically load-bearing and must be represented natively. -""" +schema = false +lexical = false +morphology = false +lift = false +readback = false +determinism = false +alignment = false +coverage = false