feat(dict): high-speed offline dictionary with wordforms and zero daemon SQLite mmap

scripts/build_offline_dict.py

@@ -0,0 +1,431 @@
+#!/usr/bin/env python3
+"""
+Offline Dictionary Builder for study.qpid.pro
+Builds a high-coverage (120,000–180,000 entries) English-Russian offline dictionary
+with all wordforms (inflections, plurals, past tense, participles, IT/tech terms).
+Outputs a compact, compressed TSV archive at data/offline_dict.tsv.gz (~3 MB).
+"""
+
+import gzip
+import json
+import os
+from pathlib import Path
+import re
+import sys
+import tarfile
+import time
+import urllib.request
+import xml.etree.ElementTree as ET
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+DATA_DIR = BASE_DIR / "data"
+SCRATCH_DIR = Path("/root/snap/antigravity-cli/common/.gemini/antigravity-cli/brain/ace1876b-1afa-47f4-97e0-4c4b0cad3879/scratch")
+OUT_FILE = DATA_DIR / "offline_dict.tsv.gz"
+
+POS_MAP = {
+    'n': 'Noun', 'v': 'Verb', 'adj': 'Adjective', 'adv': 'Adverb',
+    'pn': 'Proper Noun', 'prop': 'Proper Noun', 'prep': 'Preposition',
+    'conj': 'Conjunction', 'pron': 'Pronoun', 'interj': 'Interjection',
+    'suffix': 'Suffix', 'prefix': 'Prefix', 'article': 'Article', 'num': 'Numeral'
+}
+
+
+def clean_translation(text: str) -> str:
+    """Cleans raw dictionary markup into clean, human-readable Russian translations."""
+    # 1. Strip DSL markup
+    text = re.sub(r'\[p\]<[^>]+>\[/p\]', '', text)
+    text = re.sub(r'\[c [^\]]+\][^\[]*\[/c\]', '', text)
+    text = re.sub(r'\[[^\]]+\]', '', text)
+    text = re.sub(r'<<[^>]+>>', '', text)
+    text = re.sub(r'SEE:\s*', '', text)
+    text = re.sub(r'\\[^\\]+\\', '', text)
+
+    # 2. Strip Wikitext links [[target|display]] -> display, [[target]] -> target
+    text = re.sub(r'\[\[(?:[^|\]]*\|)?([^\]]+)\]\]', r'\1', text)
+
+    # 3. Strip acute stress marks
+    text = text.replace('\u0301', '')
+
+    # 4. Remove Latin transliterations like /neoxotnyj/
+    text = re.sub(r'/[a-zA-Z\s\',._-]+/?', '', text)
+
+    # 5. Split by comma or semicolon, trim and deduplicate
+    parts = [p.strip() for p in re.split(r'[,;]+', text) if p.strip()]
+    seen = set()
+    cleaned = []
+    for p in parts:
+        p = re.sub(r'\s+', ' ', p).strip()
+        # skip empty or stray symbols
+        if not p or len(p) < 2 or p in ('-', '—'):
+            continue
+        low = p.lower()
+        if low not in seen:
+            seen.add(low)
+            cleaned.append(p)
+
+    return ', '.join(cleaned[:4])
+
+
+def build():
+    t0 = time.time()
+    DATA_DIR.mkdir(parents=True, exist_ok=True)
+    SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
+
+    db_dict = {}  # term_lower -> {'term': str, 'translation': str, 'phonetic': str, 'pos': str}
+
+    # -------------------------------------------------------------------------
+    # 1. FreeDict TEI XML
+    # -------------------------------------------------------------------------
+    freedict_path = SCRATCH_DIR / "freedict.src.tar.xz"
+    if not freedict_path.exists():
+        print("Downloading FreeDict English-Russian dataset...")
+        url = "https://download.freedict.org/dictionaries/eng-rus/2024.10.10/freedict-eng-rus-2024.10.10.src.tar.xz"
+        urllib.request.urlretrieve(url, freedict_path)
+
+    print("Parsing FreeDict TEI XML...")
+    with tarfile.open(freedict_path, "r:xz") as tar:
+        f = tar.extractfile("eng-rus/eng-rus.tei")
+        for event, elem in ET.iterparse(f, events=("end",)):
+            tag = elem.tag.split("}")[-1]
+            if tag == "entry":
+                orth = elem.find(".//{*}orth")
+                if orth is not None and orth.text:
+                    term = orth.text.strip()
+                    t_lower = term.lower()
+                    pron = elem.find(".//{*}pron")
+                    phonetic = pron.text.strip() if pron is not None and pron.text else ""
+                    pos_elem = elem.find(".//{*}pos")
+                    raw_pos = pos_elem.text.strip() if pos_elem is not None and pos_elem.text else ""
+                    pos = POS_MAP.get(raw_pos.lower(), raw_pos)
+                    quotes = [q.text.strip() for q in elem.findall(".//{*}quote") if q.text]
+                    tr = clean_translation(", ".join(quotes))
+                    if tr:
+                        if t_lower not in db_dict:
+                            db_dict[t_lower] = {
+                                "term": term,
+                                "translation": tr,
+                                "phonetic": phonetic,
+                                "pos": pos,
+                            }
+                        else:
+                            cur = db_dict[t_lower]
+                            if not cur["phonetic"] and phonetic:
+                                cur["phonetic"] = phonetic
+                            if not cur["pos"] and pos:
+                                cur["pos"] = pos
+                elem.clear()
+
+    print(f"FreeDict loaded: {len(db_dict)} entries")
+
+    # -------------------------------------------------------------------------
+    # 2. Wiktionary DSL
+    # -------------------------------------------------------------------------
+    dsl_path = SCRATCH_DIR / "en-ru.dsl.dz"
+    if not dsl_path.exists():
+        print("Downloading Wiktionary DSL dataset...")
+        url = "https://raw.githubusercontent.com/open-dsl-dict/wiktionary-dict/master/dsl/oneway/en-ru-enwiktionary.dsl.dz"
+        urllib.request.urlretrieve(url, dsl_path)
+
+    print("Parsing Wiktionary DSL...")
+    with gzip.open(dsl_path, "rt", encoding="utf-16-le") as f:
+        current_term = None
+        current_ipa = ""
+        current_pos = ""
+        current_translations = []
+
+        for line in f:
+            line = line.rstrip("\r\n")
+            if not line or line.startswith("#"):
+                continue
+            if not line.startswith("\t"):
+                if current_term and current_translations:
+                    t_lower = current_term.lower()
+                    tr = clean_translation(", ".join(current_translations))
+                    if tr:
+                        if t_lower not in db_dict:
+                            db_dict[t_lower] = {
+                                "term": current_term,
+                                "translation": tr,
+                                "phonetic": current_ipa,
+                                "pos": current_pos,
+                            }
+                        else:
+                            cur = db_dict[t_lower]
+                            if not cur["phonetic"] and current_ipa:
+                                cur["phonetic"] = current_ipa
+                            if not cur["pos"] and current_pos:
+                                cur["pos"] = current_pos
+                            # Merge translations if short
+                            cur_parts = [p.strip() for p in cur["translation"].split(",") if p.strip()]
+                            for p in tr.split(","):
+                                p = p.strip()
+                                if p and p.lower() not in [x.lower() for x in cur_parts]:
+                                    cur_parts.append(p)
+                            cur["translation"] = ", ".join(cur_parts[:4])
+                current_term = line.strip()
+                current_ipa = ""
+                current_pos = ""
+                current_translations = []
+            else:
+                ipa_m = re.search(r"\[c gray\](/[^/]+/)\[/c\]", line)
+                if ipa_m and not current_ipa:
+                    current_ipa = ipa_m.group(1)
+                pos_m = re.search(r"\[p\]<([^>]+)>\[/p\]", line)
+                if pos_m and not current_pos:
+                    current_pos = POS_MAP.get(pos_m.group(1).lower(), pos_m.group(1))
+                if "[m1]" in line:
+                    tr_part = re.sub(r"\[m\d\]", "", line)
+                    tr_part = re.sub(r"\[/m\]", "", tr_part)
+                    current_translations.append(tr_part)
+
+    print(f"Combined base lemmas: {len(db_dict)} entries")
+
+    # -------------------------------------------------------------------------
+    # 3. Comprehensive Verb Forms
+    # -------------------------------------------------------------------------
+    verbs_path = SCRATCH_DIR / "verbs.json"
+    if not verbs_path.exists():
+        url = "https://raw.githubusercontent.com/monolithpl/verb.forms.dictionary/master/json/verbs-all.json"
+        urllib.request.urlretrieve(url, verbs_path)
+
+    with open(verbs_path, "r", encoding="utf-8", errors="replace") as f:
+        verb_data = json.load(f)
+
+    added_verbs = 0
+    for row in verb_data:
+        if not row or not isinstance(row, list):
+            continue
+        lemma = row[0].strip()
+        base_info = db_dict.get(lemma.lower())
+        if not base_info:
+            continue
+        base_trans = base_info["translation"]
+        forms = [
+            (row[1], f"{base_trans} (3-е л. ед. ч. от {lemma})"),
+            (row[2], f"{base_trans} (прош. вр. от {lemma})"),
+            (row[3], f"{base_trans} (причастие прош. вр. от {lemma})"),
+            (row[4], f"{base_trans} (причастие наст. вр. от {lemma})") if len(row) > 4 else ("", "")
+        ]
+        for form, desc in forms:
+            form = form.strip()
+            if not form or form == lemma:
+                continue
+            f_lower = form.lower()
+            if f_lower not in db_dict:
+                db_dict[f_lower] = {
+                    "term": form,
+                    "translation": desc,
+                    "phonetic": "",
+                    "pos": "Verb",
+                }
+                added_verbs += 1
+
+    print(f"Added {added_verbs} irregular/standard verb forms (total: {len(db_dict)})")
+
+    # -------------------------------------------------------------------------
+    # 4. Irregular Nouns & General Plurals
+    # -------------------------------------------------------------------------
+    irregular_nouns = {
+        'man': 'men', 'woman': 'women', 'child': 'children', 'tooth': 'teeth',
+        'foot': 'feet', 'mouse': 'mice', 'goose': 'geese', 'person': 'people',
+        'ox': 'oxen', 'leaf': 'leaves', 'half': 'halves', 'knife': 'knives',
+        'wife': 'wives', 'life': 'lives', 'wolf': 'wolves', 'thief': 'thieves',
+        'shelf': 'shelves', 'scarf': 'scarves', 'calf': 'calves', 'elf': 'elves',
+        'loaf': 'loaves', 'sheaf': 'sheaves', 'crisis': 'crises', 'thesis': 'theses',
+        'basis': 'bases', 'axis': 'axes', 'analysis': 'analyses', 'hypothesis': 'hypotheses',
+        'phenomenon': 'phenomena', 'criterion': 'criteria', 'datum': 'data',
+        'medium': 'media', 'index': 'indices', 'matrix': 'matrices', 'appendix': 'appendices',
+        'cactus': 'cacti', 'focus': 'foci', 'nucleus': 'nuclei', 'syllabus': 'syllabi'
+    }
+
+    added_plurals = 0
+    for sing, pl in irregular_nouns.items():
+        sing_info = db_dict.get(sing)
+        if sing_info and pl not in db_dict:
+            db_dict[pl] = {
+                'term': pl,
+                'translation': f"{sing_info['translation']} (мн. ч. от {sing})",
+                'phonetic': '',
+                'pos': 'Noun'
+            }
+            added_plurals += 1
+
+    # Regular noun plurals (-s, -es, -ies)
+    for t_lower, info in list(db_dict.items()):
+        if info.get('pos') == 'Noun' and len(t_lower) > 2 and ' ' not in t_lower and t_lower.isalpha():
+            if t_lower.endswith(('s', 'x', 'z', 'ch', 'sh')):
+                pl = t_lower + 'es'
+            elif t_lower.endswith('y') and t_lower[-2] not in 'aeiou':
+                pl = t_lower[:-1] + 'ies'
+            elif t_lower.endswith('o') and t_lower[-2] not in 'aeiou':
+                pl = t_lower + 'es'
+            else:
+                pl = t_lower + 's'
+
+            if pl not in db_dict:
+                db_dict[pl] = {
+                    'term': pl,
+                    'translation': f"{info['translation']} (мн. ч. от {info['term']})",
+                    'phonetic': '',
+                    'pos': 'Noun'
+                }
+                added_plurals += 1
+
+    print(f"Added {added_plurals} noun plurals (total: {len(db_dict)})")
+
+    # -------------------------------------------------------------------------
+    # 5. Regular Verb Inflections (-ed, -ing, -s) for remaining verbs
+    # -------------------------------------------------------------------------
+    added_reg_verbs = 0
+    for t_lower, info in list(db_dict.items()):
+        if info.get('pos') == 'Verb' and len(t_lower) > 2 and ' ' not in t_lower and t_lower.isalpha():
+            lemma = info['term']
+            # -s / -es
+            if t_lower.endswith(('s', 'x', 'z', 'ch', 'sh')):
+                v_3rd = t_lower + 'es'
+            elif t_lower.endswith('y') and t_lower[-2] not in 'aeiou':
+                v_3rd = t_lower[:-1] + 'ies'
+            else:
+                v_3rd = t_lower + 's'
+            if v_3rd not in db_dict:
+                db_dict[v_3rd] = {'term': v_3rd, 'translation': f"{info['translation']} (3-е л. ед. ч. от {lemma})", 'phonetic': '', 'pos': 'Verb'}
+                added_reg_verbs += 1
+
+            # -ed
+            if t_lower.endswith('e'):
+                v_ed = t_lower + 'd'
+            elif t_lower.endswith('y') and t_lower[-2] not in 'aeiou':
+                v_ed = t_lower[:-1] + 'ied'
+            else:
+                v_ed = t_lower + 'ed'
+            if v_ed not in db_dict:
+                db_dict[v_ed] = {'term': v_ed, 'translation': f"{info['translation']} (прош. вр. от {lemma})", 'phonetic': '', 'pos': 'Verb'}
+                added_reg_verbs += 1
+
+            # -ing
+            if t_lower.endswith('ie'):
+                v_ing = t_lower[:-2] + 'ying'
+            elif t_lower.endswith('e') and not t_lower.endswith(('ee', 'oe', 'ye')):
+                v_ing = t_lower[:-1] + 'ing'
+            else:
+                v_ing = t_lower + 'ing'
+            if v_ing not in db_dict:
+                db_dict[v_ing] = {'term': v_ing, 'translation': f"{info['translation']} (причастие от {lemma})", 'phonetic': '', 'pos': 'Verb'}
+                added_reg_verbs += 1
+
+    print(f"Added {added_reg_verbs} regular verb inflections (total: {len(db_dict)})")
+
+    # -------------------------------------------------------------------------
+    # 6. IT / Tech / Networking Terminology
+    # -------------------------------------------------------------------------
+    tech_glossary = {
+        'packet': ('пакет, блок данных', '/ˈpæk.ɪt/', 'Noun'),
+        'packets': ('пакеты данных', '', 'Noun'),
+        'router': ('маршрутизатор, роутер', '/ˈɹuː.tə/', 'Noun'),
+        'routers': ('маршрутизаторы, роутеры', '', 'Noun'),
+        'switch': ('коммутатор, свитч, переключатель', '/swɪtʃ/', 'Noun'),
+        'switches': ('коммутаторы, свитчи', '', 'Noun'),
+        'switching': ('коммутация (пакетов)', '', 'Noun'),
+        'subnet': ('подсеть', '/ˈsʌb.nɛt/', 'Noun'),
+        'subnets': ('подсети', '', 'Noun'),
+        'subnetting': ('разбиение на подсети, субнеттинг', '', 'Noun'),
+        'subnetted': ('разделенный на подсети', '', 'Adjective'),
+        'bandwidth': ('пропускная способность, ширина полосы пропускания', '/ˈbænd.wɪdθ/', 'Noun'),
+        'latency': ('задержка, время ожидания, латентность', '/ˈleɪ.tən.si/', 'Noun'),
+        'throughput': ('пропускная способность, производительность передачи', '/ˈθɹuː.pʊt/', 'Noun'),
+        'payload': ('полезная нагрузка (пакета данных)', '/ˈpeɪ.loʊd/', 'Noun'),
+        'handshake': ('рукопожатие, согласование связи (TCP/TLS)', '/ˈhænd.ʃeɪk/', 'Noun'),
+        'encapsulation': ('инкапсуляция данных', '/ɪnˌkæp.sjʊˈleɪ.ʃən/', 'Noun'),
+        'decapsulation': ('декапсуляция данных', '', 'Noun'),
+        'gateway': ('сетевой шлюз', '/ˈɡeɪt.weɪ/', 'Noun'),
+        'gateways': ('сетевые шлюзы', '', 'Noun'),
+        'firewall': ('межсетевой экран, файрвол, брандмауэр', '/ˈfaɪə.wɔːl/', 'Noun'),
+        'firewalls': ('межсетевые экраны, файрволы', '', 'Noun'),
+        'broadcast': ('широковещательный, широковещание', '/ˈbɹɔːd.kɑːst/', 'Noun'),
+        'multicast': ('многоадресная рассылка, мультикаст', '/ˈmʌl.ti.kæst/', 'Noun'),
+        'unicast': ('одноадресная рассылка, уникаст', '/ˈjuː.nɪ.kæst/', 'Noun'),
+        'loopback': ('петлевой интерфейс (loopback)', '/ˈluːp.bæk/', 'Noun'),
+        'topology': ('топология сети', '/təˈpɒl.ə.dʒi/', 'Noun'),
+        'topologies': ('сетевые топологии', '', 'Noun'),
+        'interface': ('интерфейс, сетевой порт', '/ˈɪn.tə.feɪs/', 'Noun'),
+        'interfaces': ('интерфейсы, сетевые порты', '', 'Noun'),
+        'vlan': ('виртуальная локальная сеть (VLAN)', '/viː.læn/', 'Noun'),
+        'vlans': ('виртуальные локальные сети (VLAN)', '', 'Noun'),
+        'trunk': ('магистральный канал, транк', '/tɹʌŋk/', 'Noun'),
+        'trunking': ('транкинг (объединение каналов/VLAN)', '', 'Noun'),
+        'dhcp': ('протокол динамической настройки узла (DHCP)', '', 'Noun'),
+        'dns': ('система доменных имен (DNS)', '', 'Noun'),
+        'arp': ('протокол разрешения адресов (ARP)', '', 'Noun'),
+        'icmp': ('протокол управляющих сообщений сети (ICMP)', '', 'Noun'),
+        'tcp': ('протокол управления передачей (TCP)', '', 'Noun'),
+        'udp': ('протокол пользовательских датаграмм (UDP)', '', 'Noun'),
+        'ipv4': ('протокол IPv4 (32-битная адресация)', '', 'Noun'),
+        'ipv6': ('протокол IPv6 (128-битная адресация)', '', 'Noun'),
+        'socket': ('сокет, программный интерфейс связи', '/ˈsɒk.ɪt/', 'Noun'),
+        'sockets': ('сокеты', '', 'Noun'),
+        'endpoint': ('конечная точка, эндпоинт', '/ˈɛndˌpɔɪnt/', 'Noun'),
+        'endpoints': ('конечные точки, эндпоинты', '', 'Noun'),
+        'daemon': ('фоновый процесс, демон', '/ˈdiː.mən/', 'Noun'),
+        'daemons': ('фоновые процессы, демоны', '', 'Noun'),
+        'buffer': ('буфер, промежуточная память', '/ˈbʌf.ə/', 'Noun'),
+        'buffering': ('буферизация', '', 'Noun'),
+        'buffers': ('буферы', '', 'Noun'),
+        'overflow': ('переполнение (буфера)', '/ˌoʊ.vɚˈfloʊ/', 'Noun'),
+        'underflow': ('исчерпание буфера', '', 'Noun'),
+        'checksum': ('контрольная сумма', '/ˈtʃɛk.sʌm/', 'Noun'),
+        'checksums': ('контрольные суммы', '', 'Noun'),
+        'hop': ('хоп, промежуточный узел маршрута', '/hɒp/', 'Noun'),
+        'hops': ('хопы, узлы маршрута', '', 'Noun'),
+        'duplex': ('дуплексный режим связи', '/ˈdjuː.plɛks/', 'Noun'),
+        'half-duplex': ('полудуплексный режим', '', 'Noun'),
+        'full-duplex': ('полнодуплексный режим', '', 'Noun'),
+        'attenuation': ('затухание сигнала', '/əˌtɛn.juˈeɪ.ʃən/', 'Noun'),
+        'crosstalk': ('перекрестные помехи', '/ˈkɹɔːs.tɔːk/', 'Noun'),
+        'jitter': ('джиттер, фазовое дрожание задержки', '/ˈdʒɪt.ə/', 'Noun'),
+        'collision': ('коллизия, столкновение пакетов', '/kəˈlɪʒ.ən/', 'Noun'),
+        'collisions': ('коллизии пакетов', '', 'Noun'),
+        'forwarding': ('пересылка, продвижение пакетов', '', 'Noun'),
+        'default-gateway': ('основной шлюз', '', 'Noun'),
+        'keepalive': ('поддержание соединения, keep-alive', '', 'Noun'),
+        'keep-alive': ('сигнал проверки активности соединения', '', 'Noun'),
+        'wildcard': ('маска подсети, групповой символ', '/ˈwaɪldˌkɑːd/', 'Noun'),
+        'acknowledgment': ('подтверждение приема (ACK)', '/əkˈnɒl.ɪdʒ.mənt/', 'Noun'),
+        'acknowledgements': ('подтверждения приема (ACK)', '', 'Noun'),
+        'fragmentation': ('фрагментация пакетов', '/ˌfɹæɡ.mənˈteɪ.ʃən/', 'Noun'),
+        'reassembly': ('сборка пакетов из фрагментов', '', 'Noun'),
+    }
+
+    added_tech = 0
+    for term, (tr, phon, pos) in tech_glossary.items():
+        t_lower = term.lower()
+        if t_lower not in db_dict or not db_dict[t_lower]["phonetic"]:
+            db_dict[t_lower] = {
+                "term": term,
+                "translation": tr,
+                "phonetic": phon or (db_dict.get(t_lower, {}).get("phonetic", "")),
+                "pos": pos,
+            }
+            added_tech += 1
+
+    print(f"Added {added_tech} IT/tech terms. Final database size: {len(db_dict)} entries")
+
+    # -------------------------------------------------------------------------
+    # 7. Write out compressed TSV
+    # -------------------------------------------------------------------------
+    print(f"Writing compressed dataset to {OUT_FILE}...")
+    with gzip.open(OUT_FILE, "wt", encoding="utf-8") as f:
+        for info in db_dict.values():
+            term = info["term"].replace("\t", " ").replace("\n", " ").strip()
+            tr = info["translation"].replace("\t", " ").replace("\n", " ").strip()
+            phon = info.get("phonetic", "").replace("\t", " ").replace("\n", " ").strip()
+            pos = info.get("pos", "").replace("\t", " ").replace("\n", " ").strip()
+            if term and tr:
+                f.write(f"{term}\t{tr}\t{phon}\t{pos}\n")
+
+    size_mb = os.path.getsize(OUT_FILE) / (1024 * 1024)
+    t1 = time.time()
+    print(f"✅ Generated {len(db_dict)} articles in {OUT_FILE} ({size_mb:.2f} MB) in {t1-t0:.2f}s")
+
+
+if __name__ == "__main__":
+    build()