fix(ai,reader): robust sentence segmentation without abbreviation cutoffs & context-aware literary translation

- Implement robustSentenceSplit across reader.js and backend (miner.py, cefr.py) to prevent false cutoffs on honorifics (Mr., Mrs., Dr.), single-letter initials (John F. Kennedy), decimals, acronyms (U.S., U.K.), and abbreviations (No. 4, etc.)
- Upgrade translation prompt to professional literary Russian, eliminating literal word-for-word translation and preserving titles/honorifics
- Inject surrounding paragraph context into translation API to avoid out-of-context translations
- Inject offline dictionary hints into LLM prompt for accurate contextual word meaning
- Prefer qwen2.5:3b for higher translation fluency and fidelity
- Purge truncated cache entries from sentence_translations and restore Card 30 (briefcase) with authentic full sentence
- Bump Service Worker cache to notes-v6.0
- Add unit tests for robust sentence splitting, prompt formatting, and translation API
This commit is contained in:
Antigravity committed 2026-10-06 22:32:36 +03:00
1 parent 3098d3016d
commit 532e2ec3b2
11 files changed
+313 -89

No files matched your search

+10 -7
View File
@@ -2056,10 +2056,13 @@ def resolve_context_sentence_for_term(
if not clean_term:
return ""
# 1. If candidate_sentence already contains this term (case-insensitive word boundary)
# 1. If candidate_sentence already contains this term (and is a complete sentence)
if candidate_sentence:
cand_clean = candidate_sentence.strip()
if re.search(rf"\b{re.escape(clean_term)}\b", cand_clean, re.IGNORECASE):
is_complete = bool(re.search(r"[.!?]['\"”’»)]*$", cand_clean)) and not bool(
re.search(r"\b(?:Mr|Mrs|Ms|Miss|Dr|Prof|St|Jr|Sr|[A-Za-z])\.['\"”’»)]*$", cand_clean, re.IGNORECASE)
)
if is_complete and re.search(rf"\b{re.escape(clean_term)}\b", cand_clean, re.IGNORECASE):
return re.sub(r"\s+", " ", cand_clean.replace("___", clean_term)).strip()
# 2. Look up in cefr_questions specifically for this term
@@ -2075,7 +2078,7 @@ def resolve_context_sentence_for_term(
# 3. Look up in books table
try:
books = conn.execute("SELECT chapters_json FROM books ORDER BY id DESC LIMIT 10").fetchall()
books = conn.execute("SELECT chapters_json FROM books ORDER BY id DESC LIMIT 50").fetchall()
for b in books:
try:
chapters = json.loads(b["chapters_json"])
@@ -2084,10 +2087,10 @@ def resolve_context_sentence_for_term(
for ch in chapters:
content = ch.get("content", "")
if clean_term.lower() in content.lower():
for s in re.split(r"(?<=[.!?])\s+", content):
s_clean = s.strip()
if 20 <= len(s_clean) <= 180 and re.search(rf"\b{re.escape(clean_term)}\b", s_clean, re.IGNORECASE):
return re.sub(r"\s+", " ", s_clean).strip()
from app.miner import robust_sentence_split
for s_clean in robust_sentence_split(content):
if 20 <= len(s_clean) <= 240 and re.search(rf"\b{re.escape(clean_term)}\b", s_clean, re.IGNORECASE):
return s_clean.strip()
except Exception:
pass