fix(dictionary): strip Latin romanization debris from Russian translations and sanitize dictionary cache

This commit is contained in:
Antigravity committed 2026-10-06 23:08:54 +03:00
1 parent e371cdd4e5
commit 52db766b23
7 files changed
+109 -27

No files matched your search

+18
View File
@@ -432,6 +432,22 @@ def repair_mismatched_cefr_contexts(conn: sqlite3.Connection):
except Exception:
pass
# Clean up any leftover Latin transliteration artifacts from offline_dictionary
try:
dirty_check = conn.execute(
"SELECT 1 FROM offline_dictionary WHERE translation GLOB '*[a-zA-Záéíóúýšžčñ/\\\\]*' LIMIT 1"
).fetchone()
if dirty_check:
rows = conn.execute(
"SELECT rowid, translation FROM offline_dictionary WHERE translation GLOB '*[a-zA-Záéíóúýšžčñ/\\\\]*'"
).fetchall()
from app.dictionary import clean_russian_translation
cleaned = [(clean_russian_translation(r[1]), r[0]) for r in rows]
conn.executemany("UPDATE offline_dictionary SET translation = ? WHERE rowid = ?", cleaned)
conn.execute("DELETE FROM dictionary_cache")
except Exception:
pass
def _seed_offline_dictionary(conn: sqlite3.Connection):
"""Seeds the offline dictionary from data/offline_dict.tsv.gz if the table is empty."""
@@ -452,12 +468,14 @@ def _seed_offline_dictionary(conn: sqlite3.Connection):
return
import gzip
from app.dictionary import clean_russian_translation
batch = []
with gzip.open(archive_path, "rt", encoding="utf-8") as f:
for line in f:
parts = line.rstrip("\r\n").split("\t")
if len(parts) == 4:
parts[1] = clean_russian_translation(parts[1])
batch.append(parts)
if len(batch) >= 10000:
conn.executemany(