fix(ai,reader): robust sentence segmentation without abbreviation cutoffs & context-aware literary translation

- Implement robustSentenceSplit across reader.js and backend (miner.py, cefr.py) to prevent false cutoffs on honorifics (Mr., Mrs., Dr.), single-letter initials (John F. Kennedy), decimals, acronyms (U.S., U.K.), and abbreviations (No. 4, etc.)
- Upgrade translation prompt to professional literary Russian, eliminating literal word-for-word translation and preserving titles/honorifics
- Inject surrounding paragraph context into translation API to avoid out-of-context translations
- Inject offline dictionary hints into LLM prompt for accurate contextual word meaning
- Prefer qwen2.5:3b for higher translation fluency and fidelity
- Purge truncated cache entries from sentence_translations and restore Card 30 (briefcase) with authentic full sentence
- Bump Service Worker cache to notes-v6.0
- Add unit tests for robust sentence splitting, prompt formatting, and translation API
This commit is contained in:
Antigravity committed 2026-10-06 22:32:36 +03:00
1 parent 3098d3016d
commit 532e2ec3b2
11 files changed
+313 -89

No files matched your search

+47 -13
View File
@@ -102,31 +102,65 @@ for _lvl, _pos_map in CEFR_LEXICON.items():
_WORD_INDEX[_w_low] = []
_WORD_INDEX[_w_low].append((_lvl, _pos))
# Common abbreviations to prevent false splits
_HONORIFICS = ["Mr.", "Mrs.", "Ms.", "Dr.", "Prof.", "St.", "vs.", "etc.", "e.g.", "i.e.", "Jr.", "Sr."]
def robust_sentence_split(text: str) -> List[str]:
"""
Splits English narrative prose into clean, complete sentences.
Protects decimals, acronyms, single-letter initials, honorifics, titles, and abbreviations.
"""
if not text:
return []
# Step 1: Protect decimal numbers (e.g. 3.14, 1.5)
s = re.sub(r"(\d+)\.(\d+)", r"\1§DOT§\2", text)
# Step 2: Protect acronyms with dots like U.S., U.K., e.g., i.e., a.m., p.m., Ph.D.
s = re.sub(r"\b((?:[A-Za-z]\.){2,})", lambda m: m.group(1).replace(".", "§DOT§"), s)
# Step 3: Protect single-letter initials like 'John F. Kennedy', 'J. K. Rowling'
s = re.sub(r'\b([A-Za-z])\.(?=\s+[A-Za-z0-9"“\'‘«—])', r"\1§DOT§", s)
# Step 4: Protect common honorifics, titles, and abbreviations
abbr_pattern = (
r"\b("
r"Mr|Mrs|Ms|Miss|Dr|Prof|Rev|Hon|Capt|Col|Gen|Lt|Cmdr|Sgt|Cpl|Pvt|Maj|Adm|"
r"Gov|Sen|Rep|Pres|Sec|Treas|Insp|Det|Supt|Messrs|Mmes|St|Mt|Ft|"
r"etc|vs|v|Jr|Sr|approx|app|apt|dept|univ|assn|assoc|bldg|co|corp|inc|ltd|"
r"est|misc|al|ibid|op|cit|ed|eds|trans|vol|vols|no|nos|sec|secs|ch|chs|"
r"fig|figs|p|pp|ave|blvd|rd|st|ct|ln|dr|hwy|"
r"jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec"
r")\."
)
s = re.sub(abbr_pattern, r"\1§DOT§", s, flags=re.IGNORECASE)
# Step 5: Protect ellipsis (... or ....)
s = re.sub(r"\.{2,}", lambda m: "§ELLIPSIS§" + ("§DOT§" * len(m.group(0))), s)
# Step 6: Protect any period followed by lowercase letter
s = re.sub(r"\.([\'\"”’»)]*\s+[a-zа-яё])", r"§DOT§\1", s)
# Step 7: Split on sentence terminators [.!?] followed by whitespace and uppercase/number/quote
marked = re.sub(r'([.!?][\'\"”’»)]*)(\s+)(?=[A-Z0-9"“\'‘«—]|$)', r"\1§SEP§", s)
raw_sentences = [
re.sub(r"\s+", " ", part.replace("§DOT§", ".").replace("§ELLIPSIS§", "")).strip()
for part in marked.split("§SEP§")
]
return [sent for sent in raw_sentences if sent]
def split_into_clean_sentences(text: str) -> List[str]:
"""
Safely splits narrative prose into clean, grammatically complete sentences.
Protects honorifics and abbreviations from being split incorrectly.
Protects honorifics, initials, and abbreviations from being split incorrectly.
"""
if not text:
return []
# Protect common abbreviations and titles
protected = text
for abbr in _HONORIFICS:
protected = protected.replace(abbr, abbr.replace(".", "§DOT§"))
# Split on sentence-ending punctuation (.!?) followed by space or newline
raw_splits = re.split(r"(?<=[.!?])\s+", protected)
raw_splits = robust_sentence_split(text)
clean_sentences = []
for s in raw_splits:
restored = s.replace("§DOT§", ".").strip()
# Flatten multiple spaces and line breaks
flattened = re.sub(r"\s+", " ", restored)
flattened = re.sub(r"\s+", " ", s).strip()
# Basic validity filters:
# 1. Skip dialogue lines starting with quotation marks or dashes