fix(dictionary): strip Latin romanization debris from Russian translations and sanitize dictionary cache

app/ai.py

@@ -26,17 +26,10 @@ def clean_dict_translation(raw: str) -> str:
     """Extracts first 1-2 clean Cyrillic meanings, stripping phonetic / Latin noise."""
     if not raw:
         return ""
-    clean = re.sub(r'/[^/]*$', '', raw)
+    from app.dictionary import clean_russian_translation
-    parts = re.split(r'[,;]\s*', clean)
+    cleaned = clean_russian_translation(raw)
-    cyrillic_parts = []
+    parts = [p.strip() for p in cleaned.split(",") if p.strip()]
-    for p in parts:
+    return ", ".join(parts[:2]) if parts else raw[:50].strip()
-        p_clean = re.sub(r"[a-zA-Záéíóúýšžčñ']+", '', p).strip()
-        p_clean = re.sub(r'\s+', ' ', p_clean).strip(' /\\,;-()[]')
-        if p_clean and re.search(r'[\u0400-\u04FF]', p_clean) and p_clean not in cyrillic_parts:
-            cyrillic_parts.append(p_clean)
-        if len(cyrillic_parts) >= 2:
-            break
-    return ", ".join(cyrillic_parts) if cyrillic_parts else raw[:50].strip()
 
 
 def get_dictionary_hint(term: str) -> str:

app/cefr.py

@@ -15,6 +15,7 @@ import sqlite3
 import random
 from typing import Dict, Any, List, Optional
 from datetime import datetime
+from app.dictionary import clean_russian_translation
 
 CEFR_LEVELS = ["A1", "A2", "B1", "B2"]
 
@@ -2190,7 +2191,7 @@ def process_cefr_answer(
             "SELECT term, translation, phonetic, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE",
             (selected_clean,),
         ).fetchone()
-        sel_trans = sel_row["translation"] if sel_row else selected_clean
+        sel_trans = clean_russian_translation(sel_row["translation"]) if sel_row else selected_clean
         sel_phon = sel_row["phonetic"] if sel_row else ""
         sel_pos = sel_row["pos"] if sel_row else ""
         sel_ctx = resolve_context_sentence_for_term(conn, selected_clean)
@@ -2244,7 +2245,7 @@ def process_cefr_answer(
                 "SELECT term, translation, phonetic, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE",
                 (opt_clean,),
             ).fetchone()
-            opt_trans = opt_row["translation"] if opt_row else opt_clean
+            opt_trans = clean_russian_translation(opt_row["translation"]) if opt_row else opt_clean
             opt_phon = opt_row["phonetic"] if opt_row else ""
             opt_pos = opt_row["pos"] if opt_row else ""
 
@@ -2413,7 +2414,7 @@ def ingest_cefr_word(
         "SELECT term, translation, phonetic, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE",
         (clean_term,),
     ).fetchone()
-    trans = row["translation"] if row else clean_term
+    trans = clean_russian_translation(row["translation"]) if row else clean_term
     phon = row["phonetic"] if row else ""
     pos = row["pos"] if row else ""
 

app/database.py

@@ -432,6 +432,22 @@ def repair_mismatched_cefr_contexts(conn: sqlite3.Connection):
     except Exception:
         pass
 
+    # Clean up any leftover Latin transliteration artifacts from offline_dictionary
+    try:
+        dirty_check = conn.execute(
+            "SELECT 1 FROM offline_dictionary WHERE translation GLOB '*[a-zA-Záéíóúýšžčñ/\\\\]*' LIMIT 1"
+        ).fetchone()
+        if dirty_check:
+            rows = conn.execute(
+                "SELECT rowid, translation FROM offline_dictionary WHERE translation GLOB '*[a-zA-Záéíóúýšžčñ/\\\\]*'"
+            ).fetchall()
+            from app.dictionary import clean_russian_translation
+            cleaned = [(clean_russian_translation(r[1]), r[0]) for r in rows]
+            conn.executemany("UPDATE offline_dictionary SET translation = ? WHERE rowid = ?", cleaned)
+            conn.execute("DELETE FROM dictionary_cache")
+    except Exception:
+        pass
+
 
 def _seed_offline_dictionary(conn: sqlite3.Connection):
     """Seeds the offline dictionary from data/offline_dict.tsv.gz if the table is empty."""
@@ -452,12 +468,14 @@ def _seed_offline_dictionary(conn: sqlite3.Connection):
         return
 
     import gzip
+    from app.dictionary import clean_russian_translation
 
     batch = []
     with gzip.open(archive_path, "rt", encoding="utf-8") as f:
         for line in f:
             parts = line.rstrip("\r\n").split("\t")
             if len(parts) == 4:
+                parts[1] = clean_russian_translation(parts[1])
                 batch.append(parts)
                 if len(batch) >= 10000:
                     conn.executemany(

app/dictionary.py

@@ -223,6 +223,60 @@ async def fetch_free_dictionary_api(word: str, client: Optional[httpx.AsyncClien
         pass
     return {"phonetic": None, "audio_url": None}
 
+def clean_russian_translation(text: str) -> str:
+    """
+    Cleans up Russian translations by stripping Latin romanization / phonetic debris
+    (e.g. 'át’sjaпорхать át’/', 'krýl’jami/', 'ževáto-koríčnevyj/', '] ]/]')
+    and restoring clean, deduplicated Cyrillic phrases, while preserving grammatical notes.
+    """
+    if not text:
+        return ""
+
+    # Protect grammatical notes like (прош. вр. от go), (3-е л. ед. ч. от flutter)
+    notes = []
+    def save_note(m):
+        notes.append(m.group(0))
+        return f"«ПРИМЕЧАНИЕ_{len(notes)-1}»"
+
+    s = re.sub(r"\([а-яА-ЯёЁ0-9\s.,-]*\bот\s+[a-zA-Z\x27-]+\)", save_note, text)
+
+    # 1. Clean out brackets junk and stray backslashes
+    s = re.sub(r"\[+[\s\]/\\,;]*\]*", " ", s)
+    s = re.sub(r"[\]\[\\]+", " ", s)
+
+    # 2. Match Latin romanization fragments with accents/slashes/apostrophes
+    latin_rom = r"[a-zA-Z\u00C0-\u024F\u1E00-\u1EFF\x27\u2019/]+"
+    s = re.sub(rf"\s*{latin_rom}\s*", ", ", s)
+
+    # 3. Clean up multiple commas/semicolons
+    s = re.sub(r"[,;\s]*,[,;\s]*", ", ", s)
+
+    # 4. Clean and deduplicate phrases
+    items = []
+    seen = set()
+    for part in s.split(","):
+        p = part.strip(" /\\,;-\x27\u2019")
+        if p.count("(") > p.count(")"):
+            p += ")"
+        elif p.count(")") > p.count("("):
+            p = re.sub(r"\)+$", "", p).strip()
+        p = re.sub(r"\s+", " ", p)
+        for idx, n in enumerate(notes):
+            token = f"«ПРИМЕЧАНИЕ_{idx}»"
+            if token in p:
+                p = p.replace(token, n).strip()
+        if p and re.search(r"[\u0400-\u04FF0-9]", p):
+            p_lower = p.lower()
+            if p_lower not in seen:
+                seen.add(p_lower)
+                items.append(p)
+
+    res = ", ".join(items) if items else text.strip()
+    for idx, n in enumerate(notes):
+        res = res.replace(f"«ПРИМЕЧАНИЕ_{idx}»", n)
+    return res
+
+
 async def lookup_term(query: str) -> Dict[str, Any]:
     query_clean = query.strip()
     if not query_clean:
@@ -251,13 +305,14 @@ async def lookup_term(query: str) -> Dict[str, Any]:
                 ).fetchone()
                 if off_row:
                     raw_trans = off_row["translation"] or ""
-                    tr_list = [t.strip() for t in raw_trans.split(",") if t.strip()]
+                    clean_trans = clean_russian_translation(raw_trans)
+                    tr_list = [t.strip() for t in clean_trans.split(",") if t.strip()]
                     res = {
                         "query": query_clean,
                         "is_sentence": False,
-                        "translation": raw_trans,
+                        "translation": clean_trans,
                         "translations_list": tr_list,
-                        "translation_ru": raw_trans,
+                        "translation_ru": clean_trans,
                         "phonetic": off_row["phonetic"] or "",
                         "audio_file": "",
                         "audio_url": "",

app/miner.py

@@ -240,15 +240,10 @@ def clean_translation(trans: str) -> str:
     """Cleans up raw translation strings, removes slash phonetic residues and romanizations."""
     if not trans:
         return ""
-    t = re.sub(r"/[^/]+/?", "", trans)
+    from app.dictionary import clean_russian_translation
-    parts = [p.strip() for p in re.split(r"[,;]\s*", t) if p.strip()]
+    cleaned = clean_russian_translation(trans)
-    clean_parts = []
+    parts = [p.strip() for p in cleaned.split(",") if p.strip()]
-    for p in parts:
+    return ", ".join(parts[:3]) if parts else trans.split(",")[0].strip()
-        if re.search(r"[а-яА-ЯёЁ]", p):
-            clean_p = re.sub(r"[a-zA-ZáéíóúÁÉÍÓÚ]", "", p).strip()
-            if clean_p:
-                clean_parts.append(clean_p)
-    return ", ".join(clean_parts[:3]) if clean_parts else trans.split(",")[0].strip()
 
 
 async def mine_book_questions(

app/static/sw.js

@@ -1,7 +1,7 @@
 // Service Worker for Notes English PWA (Stage 2)
 // Strategy: Cache-first for static assets, Network-first for API calls.
 
-const CACHE_NAME = 'notes-v6.1';
+const CACHE_NAME = 'notes-v6.2';
 const OFFLINE_URL = '/';
 
 const PRECACHE_URLS = [

tests/test_api.py

@@ -2027,6 +2027,26 @@ def test_sentence_dictionary_hints_and_fauna_precision():
     assert "large" in prompt
 
 
+def test_clean_russian_translation_removes_romanization():
+    from app.dictionary import clean_russian_translation
+
+    raw_flutter = "трепет, порхание, развеваться át’sjaпорхать át’/, бить крыльями ’ krýl’jamiмахать крыльями át’ krýl’jami/"
+    clean_flutter = clean_russian_translation(raw_flutter)
+    assert clean_flutter == "трепет, порхание, развеваться, порхать, бить крыльями, махать крыльями"
+    assert "át’sja" not in clean_flutter
+    assert "krýl’jami" not in clean_flutter
+    assert "/" not in clean_flutter
+
+    raw_tawny = "рыжевато-коричневый, рыжевато-коричневый ževáto-koríčnevyj/"
+    clean_tawny = clean_russian_translation(raw_tawny)
+    assert clean_tawny == "рыжевато-коричневый"
+
+    raw_like = "подобный, похожий óžij/, как, типа ípa/"
+    clean_like = clean_russian_translation(raw_like)
+    assert clean_like == "подобный, похожий, как, типа"
+
+
+