fix(ai,reader): robust sentence segmentation without abbreviation cutoffs & context-aware literary translation
- Implement robustSentenceSplit across reader.js and backend (miner.py, cefr.py) to prevent false cutoffs on honorifics (Mr., Mrs., Dr.), single-letter initials (John F. Kennedy), decimals, acronyms (U.S., U.K.), and abbreviations (No. 4, etc.) - Upgrade translation prompt to professional literary Russian, eliminating literal word-for-word translation and preserving titles/honorifics - Inject surrounding paragraph context into translation API to avoid out-of-context translations - Inject offline dictionary hints into LLM prompt for accurate contextual word meaning - Prefer qwen2.5:3b for higher translation fluency and fidelity - Purge truncated cache entries from sentence_translations and restore Card 30 (briefcase) with authentic full sentence - Bump Service Worker cache to notes-v6.0 - Add unit tests for robust sentence splitting, prompt formatting, and translation API
This commit is contained in:
1 parent
3098d3016d
commit
532e2ec3b2
11 files changed
+313
-89
No files matched your search
+10
-7
@@ -2056,10 +2056,13 @@ def resolve_context_sentence_for_term(
|
||||
if not clean_term:
|
||||
return ""
|
||||
|
||||
# 1. If candidate_sentence already contains this term (case-insensitive word boundary)
|
||||
# 1. If candidate_sentence already contains this term (and is a complete sentence)
|
||||
if candidate_sentence:
|
||||
cand_clean = candidate_sentence.strip()
|
||||
if re.search(rf"\b{re.escape(clean_term)}\b", cand_clean, re.IGNORECASE):
|
||||
is_complete = bool(re.search(r"[.!?]['\"”’»)]*$", cand_clean)) and not bool(
|
||||
re.search(r"\b(?:Mr|Mrs|Ms|Miss|Dr|Prof|St|Jr|Sr|[A-Za-z])\.['\"”’»)]*$", cand_clean, re.IGNORECASE)
|
||||
)
|
||||
if is_complete and re.search(rf"\b{re.escape(clean_term)}\b", cand_clean, re.IGNORECASE):
|
||||
return re.sub(r"\s+", " ", cand_clean.replace("___", clean_term)).strip()
|
||||
|
||||
# 2. Look up in cefr_questions specifically for this term
|
||||
@@ -2075,7 +2078,7 @@ def resolve_context_sentence_for_term(
|
||||
|
||||
# 3. Look up in books table
|
||||
try:
|
||||
books = conn.execute("SELECT chapters_json FROM books ORDER BY id DESC LIMIT 10").fetchall()
|
||||
books = conn.execute("SELECT chapters_json FROM books ORDER BY id DESC LIMIT 50").fetchall()
|
||||
for b in books:
|
||||
try:
|
||||
chapters = json.loads(b["chapters_json"])
|
||||
@@ -2084,10 +2087,10 @@ def resolve_context_sentence_for_term(
|
||||
for ch in chapters:
|
||||
content = ch.get("content", "")
|
||||
if clean_term.lower() in content.lower():
|
||||
for s in re.split(r"(?<=[.!?])\s+", content):
|
||||
s_clean = s.strip()
|
||||
if 20 <= len(s_clean) <= 180 and re.search(rf"\b{re.escape(clean_term)}\b", s_clean, re.IGNORECASE):
|
||||
return re.sub(r"\s+", " ", s_clean).strip()
|
||||
from app.miner import robust_sentence_split
|
||||
for s_clean in robust_sentence_split(content):
|
||||
if 20 <= len(s_clean) <= 240 and re.search(rf"\b{re.escape(clean_term)}\b", s_clean, re.IGNORECASE):
|
||||
return s_clean.strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
Reference in new issue
Block a user