fix(ai,reader): robust sentence segmentation without abbreviation cutoffs & context-aware literary translation
- Implement robustSentenceSplit across reader.js and backend (miner.py, cefr.py) to prevent false cutoffs on honorifics (Mr., Mrs., Dr.), single-letter initials (John F. Kennedy), decimals, acronyms (U.S., U.K.), and abbreviations (No. 4, etc.) - Upgrade translation prompt to professional literary Russian, eliminating literal word-for-word translation and preserving titles/honorifics - Inject surrounding paragraph context into translation API to avoid out-of-context translations - Inject offline dictionary hints into LLM prompt for accurate contextual word meaning - Prefer qwen2.5:3b for higher translation fluency and fidelity - Purge truncated cache entries from sentence_translations and restore Card 30 (briefcase) with authentic full sentence - Bump Service Worker cache to notes-v6.0 - Add unit tests for robust sentence splitting, prompt formatting, and translation API
This commit is contained in:
1 parent
3098d3016d
commit
532e2ec3b2
11 files changed
+313
-89
No files matched your search
+47
-13
@@ -102,31 +102,65 @@ for _lvl, _pos_map in CEFR_LEXICON.items():
|
||||
_WORD_INDEX[_w_low] = []
|
||||
_WORD_INDEX[_w_low].append((_lvl, _pos))
|
||||
|
||||
# Common abbreviations to prevent false splits
|
||||
_HONORIFICS = ["Mr.", "Mrs.", "Ms.", "Dr.", "Prof.", "St.", "vs.", "etc.", "e.g.", "i.e.", "Jr.", "Sr."]
|
||||
def robust_sentence_split(text: str) -> List[str]:
|
||||
"""
|
||||
Splits English narrative prose into clean, complete sentences.
|
||||
Protects decimals, acronyms, single-letter initials, honorifics, titles, and abbreviations.
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
|
||||
# Step 1: Protect decimal numbers (e.g. 3.14, 1.5)
|
||||
s = re.sub(r"(\d+)\.(\d+)", r"\1§DOT§\2", text)
|
||||
|
||||
# Step 2: Protect acronyms with dots like U.S., U.K., e.g., i.e., a.m., p.m., Ph.D.
|
||||
s = re.sub(r"\b((?:[A-Za-z]\.){2,})", lambda m: m.group(1).replace(".", "§DOT§"), s)
|
||||
|
||||
# Step 3: Protect single-letter initials like 'John F. Kennedy', 'J. K. Rowling'
|
||||
s = re.sub(r'\b([A-Za-z])\.(?=\s+[A-Za-z0-9"“\'‘«—])', r"\1§DOT§", s)
|
||||
|
||||
# Step 4: Protect common honorifics, titles, and abbreviations
|
||||
abbr_pattern = (
|
||||
r"\b("
|
||||
r"Mr|Mrs|Ms|Miss|Dr|Prof|Rev|Hon|Capt|Col|Gen|Lt|Cmdr|Sgt|Cpl|Pvt|Maj|Adm|"
|
||||
r"Gov|Sen|Rep|Pres|Sec|Treas|Insp|Det|Supt|Messrs|Mmes|St|Mt|Ft|"
|
||||
r"etc|vs|v|Jr|Sr|approx|app|apt|dept|univ|assn|assoc|bldg|co|corp|inc|ltd|"
|
||||
r"est|misc|al|ibid|op|cit|ed|eds|trans|vol|vols|no|nos|sec|secs|ch|chs|"
|
||||
r"fig|figs|p|pp|ave|blvd|rd|st|ct|ln|dr|hwy|"
|
||||
r"jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec"
|
||||
r")\."
|
||||
)
|
||||
s = re.sub(abbr_pattern, r"\1§DOT§", s, flags=re.IGNORECASE)
|
||||
|
||||
# Step 5: Protect ellipsis (... or ....)
|
||||
s = re.sub(r"\.{2,}", lambda m: "§ELLIPSIS§" + ("§DOT§" * len(m.group(0))), s)
|
||||
|
||||
# Step 6: Protect any period followed by lowercase letter
|
||||
s = re.sub(r"\.([\'\"”’»)]*\s+[a-zа-яё])", r"§DOT§\1", s)
|
||||
|
||||
# Step 7: Split on sentence terminators [.!?] followed by whitespace and uppercase/number/quote
|
||||
marked = re.sub(r'([.!?][\'\"”’»)]*)(\s+)(?=[A-Z0-9"“\'‘«—]|$)', r"\1§SEP§", s)
|
||||
|
||||
raw_sentences = [
|
||||
re.sub(r"\s+", " ", part.replace("§DOT§", ".").replace("§ELLIPSIS§", "")).strip()
|
||||
for part in marked.split("§SEP§")
|
||||
]
|
||||
return [sent for sent in raw_sentences if sent]
|
||||
|
||||
|
||||
def split_into_clean_sentences(text: str) -> List[str]:
|
||||
"""
|
||||
Safely splits narrative prose into clean, grammatically complete sentences.
|
||||
Protects honorifics and abbreviations from being split incorrectly.
|
||||
Protects honorifics, initials, and abbreviations from being split incorrectly.
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
|
||||
# Protect common abbreviations and titles
|
||||
protected = text
|
||||
for abbr in _HONORIFICS:
|
||||
protected = protected.replace(abbr, abbr.replace(".", "§DOT§"))
|
||||
|
||||
# Split on sentence-ending punctuation (.!?) followed by space or newline
|
||||
raw_splits = re.split(r"(?<=[.!?])\s+", protected)
|
||||
raw_splits = robust_sentence_split(text)
|
||||
|
||||
clean_sentences = []
|
||||
for s in raw_splits:
|
||||
restored = s.replace("§DOT§", ".").strip()
|
||||
# Flatten multiple spaces and line breaks
|
||||
flattened = re.sub(r"\s+", " ", restored)
|
||||
flattened = re.sub(r"\s+", " ", s).strip()
|
||||
|
||||
# Basic validity filters:
|
||||
# 1. Skip dialogue lines starting with quotation marks or dashes
|
||||
|
||||
Reference in new issue
Block a user