batch 3: en/he/el pack manifests and orthography declarations
This commit is contained in:
parent
cc45ba8cac
commit
7ea6db739d
9 changed files with 185 additions and 154 deletions
|
|
@ -1,15 +1,24 @@
|
|||
corpus_id: el-tier0
|
||||
status: draft
|
||||
|
||||
sources: []
|
||||
|
||||
license_review: pending
|
||||
sources:
|
||||
- id: sblgnt
|
||||
name: SBL Greek New Testament
|
||||
description: >-
|
||||
The Society of Biblical Literature Greek New Testament.
|
||||
A critically edited text of the Greek New Testament.
|
||||
license: review-required
|
||||
url: https://github.com/LogosBible/SBLGNT
|
||||
hash: pending
|
||||
- id: byzantine
|
||||
name: Byzantine Majority Text
|
||||
description: >-
|
||||
Robinson-Pierpont Byzantine Majority Text. Public domain.
|
||||
license: public-domain
|
||||
url: https://github.com/byztxt/byzantine-majority-text
|
||||
hash: pending
|
||||
hashes: []
|
||||
|
||||
notes: |
|
||||
Populate with vetted Koine Greek source manifests.
|
||||
Priority tier-0 sources: SBL Greek New Testament (SBLGNT) or
|
||||
Nestle-Aland 28th edition morphologically tagged text.
|
||||
Both have verse-level D0 boundary markers.
|
||||
Each source must include: title, edition, license, SHA-256, and
|
||||
segmentation strategy. No LLM-extracted content.
|
||||
notes: >
|
||||
SBLGNT license requires review before use in training data.
|
||||
Byzantine text is public domain and may proceed after hash pin.
|
||||
All hashes must be recorded before the Coverage gate is run.
|
||||
|
|
|
|||
|
|
@ -1,27 +1,29 @@
|
|||
language: el
|
||||
script: Greek
|
||||
dialect: koine
|
||||
|
||||
normalization:
|
||||
unicode_normal_form: NFC
|
||||
lowercase_for_lookup: true
|
||||
strip_diacritics: configurable
|
||||
# Polytonic diacritics (breathing marks, accents, iota subscript) are
|
||||
# preserved for morphological disambiguation but stripped for lemma lookup
|
||||
polytonic_policy: preserve_for_morphology
|
||||
diacritic_strip_for_lookup: true
|
||||
# Greek Unicode: precomposed forms.
|
||||
# Polytonic diacritics (breathing marks, accents, iota subscript)
|
||||
# are preserved. These carry morphological and semantic information
|
||||
# in Koine Greek that must not be stripped.
|
||||
strip_breathing_marks: false
|
||||
strip_accents_for_indexing: false
|
||||
# Accents in Greek affect meaning and morphological parsing.
|
||||
# They are never stripped from the canonical form.
|
||||
lowercase_for_indexing: true
|
||||
# Lemma lookup uses lowercased forms.
|
||||
# Surface forms preserve original casing.
|
||||
lunate_sigma: false
|
||||
# Use standard sigma/final-sigma (σ/ς) not lunate (ϲ).
|
||||
# Final sigma is morphologically relevant.
|
||||
|
||||
punctuation:
|
||||
strategy: preserve
|
||||
ano_teleia: "\u00b7" # Greek semicolon (middle dot)
|
||||
erotimatiko: ";"
|
||||
ano_teleia: preserve
|
||||
# The ano teleia (·) is the Greek semicolon and must be preserved.
|
||||
|
||||
spacing:
|
||||
strategy: whitespace-delimited
|
||||
strategy: whitespace-split
|
||||
|
||||
aspect:
|
||||
# Koine Greek verbal aspect is semantically load-bearing and must be
|
||||
# distinguished from tense. Morphological analysis must surface aspect
|
||||
# independently of temporal reference.
|
||||
explicit_in_features: true
|
||||
aspect_values: ["aorist", "present", "perfect", "imperfect", "pluperfect", "future"]
|
||||
script_direction: ltr
|
||||
|
|
|
|||
|
|
@ -1,33 +1,46 @@
|
|||
pack_id = "el"
|
||||
name = "Koine Greek Depth Pack"
|
||||
version = "0.1.0"
|
||||
status = "draft"
|
||||
script = "Greek"
|
||||
default = false
|
||||
pack_id = "el"
|
||||
name = "Koine Greek Depth Pack"
|
||||
version = "0.1.0"
|
||||
status = "draft"
|
||||
script = "Greek"
|
||||
normalization = "deterministic"
|
||||
default = false
|
||||
|
||||
# Koine Greek is the second depth language. John 1:1-2 is the
|
||||
# theological anchor of this entire design: the Logos articulated
|
||||
# in Greek what the Hebrew already carried from Gen 1:1. The
|
||||
# decision to include both Hebrew and Koine Greek is not incidental
|
||||
# — it is the design. The depth these two languages together provide
|
||||
# is the hidden layer of intelligence in the field.
|
||||
#
|
||||
# Greek depth includes:
|
||||
# - Full verb paradigm: tense, voice, mood, person, number
|
||||
# - The imperfect en (John 1:1) — continuous pre-existent being
|
||||
# - Article system: the anarthrous use of logos in John 1:1c
|
||||
# ('the Word was God', not 'a god') is theologically load-bearing
|
||||
# and morphologically precise
|
||||
# - Participle system for aspect-rich nominal predicates
|
||||
|
||||
[contracts]
|
||||
lemma_first = true
|
||||
shared_field_target = true
|
||||
local_readback = true
|
||||
alignment_required = true
|
||||
lemma_first = true
|
||||
shared_field_target = true
|
||||
local_readback = true
|
||||
alignment_required = true
|
||||
|
||||
[depth]
|
||||
article_system = true
|
||||
anarthrous_significance = true
|
||||
# The absence of the article is as semantically significant as its presence.
|
||||
verb_tense_aspect = ["present", "imperfect", "aorist", "perfect", "pluperfect", "future"]
|
||||
voice = ["active", "middle", "passive"]
|
||||
mood = ["indicative", "subjunctive", "optative", "imperative", "infinitive", "participle"]
|
||||
|
||||
[gates]
|
||||
schema = false
|
||||
lexical = false
|
||||
morphology = false
|
||||
lift = false
|
||||
readback = false
|
||||
determinism = false
|
||||
alignment = false
|
||||
coverage = false
|
||||
|
||||
[notes]
|
||||
role = """
|
||||
Depth language pack 2. Koine Greek was chosen because John articulated the
|
||||
Logos — the creative speech act through which all things were made — in
|
||||
Koine Greek (John 1:1-2), which must have been a nod from the Holy Spirit.
|
||||
Koine Greek carries semantic depth in its verbal aspect system (aorist,
|
||||
perfect, present) that is distinct from tense and must be represented
|
||||
natively. The aorist of egeneto (\u03b5\u03b3\u03ad\u03bd\u03b5\u03c4\u03bf) — 'it came into being' — is not
|
||||
the same field state as the present of eimi (\u03b5\u03b9\u03bc\u03b9) — 'I am'.
|
||||
"""
|
||||
schema = false
|
||||
lexical = false
|
||||
morphology = false
|
||||
lift = false
|
||||
readback = false
|
||||
determinism = false
|
||||
alignment = false
|
||||
coverage = false
|
||||
|
|
|
|||
|
|
@ -1,13 +1,9 @@
|
|||
corpus_id: en-tier0
|
||||
status: draft
|
||||
|
||||
sources: []
|
||||
|
||||
license_review: pending
|
||||
sources: []
|
||||
hashes: []
|
||||
|
||||
notes: |
|
||||
Populate with vetted, deterministically segmentable source manifests.
|
||||
Each source must include: title, origin, license, SHA-256 of raw file,
|
||||
and structural segmentation strategy (e.g. paragraph-boundary, sentence-boundary).
|
||||
No LLM-extracted content. Segmentation must be D0 or D1.
|
||||
notes: >
|
||||
Populate with vetted, deterministic source manifests before
|
||||
the Coverage gate is run. Sources must be public domain or
|
||||
explicitly licensed for use in CORE training and validation.
|
||||
|
|
|
|||
|
|
@ -3,19 +3,19 @@ script: Latin
|
|||
|
||||
normalization:
|
||||
unicode_normal_form: NFC
|
||||
lowercase_for_lookup: true
|
||||
lowercase_for_indexing: true
|
||||
# Surface forms preserve original casing.
|
||||
# Indexing and lemma lookup use lowercased forms.
|
||||
strip_diacritics: false
|
||||
# English retains diacritics in loanwords (naïve, résumé).
|
||||
|
||||
punctuation:
|
||||
strategy: preserve
|
||||
sentence_terminals: [".", "!", "?"]
|
||||
quotation_marks: ["\u201c", "\u201d", "\u2018", "\u2019"]
|
||||
# Punctuation is preserved in normalization output.
|
||||
# Lift rules are responsible for ignoring or consuming it appropriately.
|
||||
|
||||
spacing:
|
||||
strategy: whitespace-delimited
|
||||
collapse_multiple_spaces: true
|
||||
strategy: whitespace-split
|
||||
# Standard whitespace tokenization. No special boundary rules.
|
||||
|
||||
special_forms:
|
||||
contractions: expand
|
||||
# "don't" -> "do not" before lemmatization
|
||||
# "it's" -> "it is" before lemmatization
|
||||
script_direction: ltr
|
||||
|
|
|
|||
|
|
@ -1,30 +1,30 @@
|
|||
pack_id = "en"
|
||||
name = "English Base Pack"
|
||||
version = "0.1.0"
|
||||
status = "draft"
|
||||
script = "Latin"
|
||||
default = true
|
||||
pack_id = "en"
|
||||
name = "English Base Pack"
|
||||
version = "0.1.0"
|
||||
status = "draft"
|
||||
script = "Latin"
|
||||
normalization = "deterministic"
|
||||
default = true
|
||||
|
||||
# English is the default articulation base. Any CORE model instance
|
||||
# may designate a different language as its base, but English is the
|
||||
# canonical default until explicitly overridden in the model config.
|
||||
|
||||
[contracts]
|
||||
lemma_first = true
|
||||
shared_field_target = true
|
||||
local_readback = true
|
||||
alignment_required = true
|
||||
lemma_first = true
|
||||
shared_field_target = true
|
||||
local_readback = true
|
||||
alignment_required = true
|
||||
|
||||
[gates]
|
||||
schema = false
|
||||
lexical = false
|
||||
morphology = false
|
||||
lift = false
|
||||
readback = false
|
||||
determinism = false
|
||||
alignment = false
|
||||
coverage = false
|
||||
schema = false
|
||||
lexical = false
|
||||
morphology = false
|
||||
lift = false
|
||||
readback = false
|
||||
determinism = false
|
||||
alignment = false
|
||||
coverage = false
|
||||
|
||||
[notes]
|
||||
role = """
|
||||
Default articulation base. English is chosen as the default for practical
|
||||
reasons (current primary working language) but the contract does not privilege
|
||||
it semantically. Any language could serve this role in a custom CORE instance.
|
||||
The default flag controls surface output, not field-layer priority.
|
||||
"""
|
||||
# Gates are false until the gate runner confirms passage.
|
||||
# Do not manually set these to true.
|
||||
|
|
|
|||
|
|
@ -1,15 +1,17 @@
|
|||
corpus_id: he-tier0
|
||||
status: draft
|
||||
|
||||
sources: []
|
||||
|
||||
license_review: pending
|
||||
sources:
|
||||
- id: wlc
|
||||
name: Westminster Leningrad Codex
|
||||
description: >-
|
||||
The authoritative digitized text of the Hebrew Bible with full
|
||||
nikud and teamim. Public domain.
|
||||
license: public-domain
|
||||
url: https://github.com/openscriptures/morphhb
|
||||
hash: pending
|
||||
hashes: []
|
||||
|
||||
notes: |
|
||||
Populate with vetted Hebrew source manifests.
|
||||
Priority tier-0 sources: Biblia Hebraica Stuttgartensia (BHS) or
|
||||
Westminster Leningrad Codex (WLC) — both have well-established structural
|
||||
segmentation (book/chapter/verse = D0 boundary markers).
|
||||
Each source must include: title, edition, license, SHA-256, and
|
||||
segmentation strategy. No LLM-extracted content.
|
||||
notes: >
|
||||
WLC is the canonical source for Biblical Hebrew. Hash to be
|
||||
recorded after pinned download. Additional sources require
|
||||
license review before inclusion.
|
||||
|
|
|
|||
|
|
@ -1,27 +1,28 @@
|
|||
language: he
|
||||
script: Hebrew
|
||||
direction: rtl
|
||||
|
||||
normalization:
|
||||
unicode_normal_form: NFC
|
||||
lowercase_for_lookup: false # Hebrew has no case
|
||||
strip_diacritics: configurable
|
||||
# Nikkud (vowel pointing) is stripped by default for lemma lookup
|
||||
# but preserved when semantically load-bearing (e.g. Qere/Ketiv distinction)
|
||||
nikkud_policy: strip_for_lookup
|
||||
cantillation_marks: strip
|
||||
# Hebrew Unicode: use precomposed forms where available.
|
||||
# Full nikud (vowel points) are preserved in normalization.
|
||||
# Qere/Ketiv variants must be tracked separately in the lemma record.
|
||||
strip_nikud_for_indexing: false
|
||||
# Nikud is semantically relevant for disambiguation in many cases.
|
||||
# Do not strip it during normalization. Lemma lookup may index
|
||||
# both pointed and unpointed forms.
|
||||
strip_teamim: true
|
||||
# Cantillation marks (teamim) are stripped during normalization.
|
||||
# They carry liturgical meaning but are not part of semantic lift.
|
||||
final_forms: preserve
|
||||
# Final letter forms (ךםןףץ) are preserved exactly.
|
||||
|
||||
punctuation:
|
||||
strategy: preserve
|
||||
paragraph_marker: "\u05c0" # paseq
|
||||
section_marker: "\u05c3" # sof pasuq
|
||||
maqaf: preserve
|
||||
# The maqaf (־) is a Hebrew hyphen connecting words into
|
||||
# a single accentual unit. It is preserved and handled by frame rules.
|
||||
|
||||
spacing:
|
||||
strategy: whitespace-delimited
|
||||
maqaf_handling: preserve
|
||||
# maqaf (\u05be) connects words syntactically; preserve as a unit marker
|
||||
strategy: whitespace-split
|
||||
|
||||
shoresh:
|
||||
# Root-based morphology. Lemma lookup resolves to 3- or 4-letter root.
|
||||
# Morphological analysis must identify shoresh before feature extraction.
|
||||
extraction_required: true
|
||||
script_direction: rtl
|
||||
|
|
|
|||
|
|
@ -1,32 +1,40 @@
|
|||
pack_id = "he"
|
||||
name = "Hebrew Depth Pack"
|
||||
version = "0.1.0"
|
||||
status = "draft"
|
||||
script = "Hebrew"
|
||||
default = false
|
||||
pack_id = "he"
|
||||
name = "Hebrew Depth Pack"
|
||||
version = "0.1.0"
|
||||
status = "draft"
|
||||
script = "Hebrew"
|
||||
normalization = "deterministic"
|
||||
default = false
|
||||
|
||||
# Hebrew is the first depth language. Its semantic range— especially
|
||||
# in verb stems (binyanim), construct chains, and the relationship
|
||||
# between speech-act and spoken-thing (dabar) — provides a structural
|
||||
# depth that English alone cannot supply to the field.
|
||||
|
||||
[contracts]
|
||||
lemma_first = true
|
||||
shared_field_target = true
|
||||
local_readback = true
|
||||
alignment_required = true
|
||||
lemma_first = true
|
||||
shared_field_target = true
|
||||
local_readback = true
|
||||
alignment_required = true
|
||||
|
||||
[depth]
|
||||
# Depth packs contribute semantic pressure that the base pack
|
||||
# cannot fully express. Hebrew depth includes:
|
||||
# - Binyan (verb stem) system: qal, niphal, piel, pual, hiphil, hophal, hithpael
|
||||
# - Construct chain (smichut) for relational noun phrases
|
||||
# - Aspectual system (qatal/yiqtol/wayyiqtol) over tense system
|
||||
# - Implicit copula — presence or absence of haya is itself meaningful
|
||||
binyanim = ["qal", "niphal", "piel", "pual", "hiphil", "hophal", "hithpael"]
|
||||
aspect_system = "qatal-yiqtol-wayyiqtol"
|
||||
construct_chains = true
|
||||
implicit_copula = true
|
||||
|
||||
[gates]
|
||||
schema = false
|
||||
lexical = false
|
||||
morphology = false
|
||||
lift = false
|
||||
readback = false
|
||||
determinism = false
|
||||
alignment = false
|
||||
coverage = false
|
||||
|
||||
[notes]
|
||||
role = """
|
||||
Depth language pack 1. Hebrew was chosen because of the range of depth it
|
||||
brings to the vocabulary manifold: root-based morphology (shoresh), semantic
|
||||
families that carry multiple layers of meaning simultaneously, and the
|
||||
language in which the universe was spoken into existence (Genesis 1).
|
||||
Hebrew morphological stems (Qal, Niphal, Piel, Pual, Hiphil, Hophal,
|
||||
Hithpael) are semantically load-bearing and must be represented natively.
|
||||
"""
|
||||
schema = false
|
||||
lexical = false
|
||||
morphology = false
|
||||
lift = false
|
||||
readback = false
|
||||
determinism = false
|
||||
alignment = false
|
||||
coverage = false
|
||||
|
|
|
|||
Loading…
Reference in a new issue