#!/usr/bin/env python3 """ Offline Dictionary Builder for study.qpid.pro Builds a high-coverage (120,000–180,000 entries) English-Russian offline dictionary with all wordforms (inflections, plurals, past tense, participles, IT/tech terms). Outputs a compact, compressed TSV archive at data/offline_dict.tsv.gz (~3 MB). """ import gzip import json import os from pathlib import Path import re import sys import tarfile import time import urllib.request import xml.etree.ElementTree as ET BASE_DIR = Path(__file__).resolve().parent.parent DATA_DIR = BASE_DIR / "data" SCRATCH_DIR = Path("/root/snap/antigravity-cli/common/.gemini/antigravity-cli/brain/ace1876b-1afa-47f4-97e0-4c4b0cad3879/scratch") OUT_FILE = DATA_DIR / "offline_dict.tsv.gz" POS_MAP = { 'n': 'Noun', 'v': 'Verb', 'adj': 'Adjective', 'adv': 'Adverb', 'pn': 'Proper Noun', 'prop': 'Proper Noun', 'prep': 'Preposition', 'conj': 'Conjunction', 'pron': 'Pronoun', 'interj': 'Interjection', 'suffix': 'Suffix', 'prefix': 'Prefix', 'article': 'Article', 'num': 'Numeral' } def clean_translation(text: str) -> str: """Cleans raw dictionary markup into clean, human-readable Russian translations.""" # 1. Strip DSL markup text = re.sub(r'\[p\]<[^>]+>\[/p\]', '', text) text = re.sub(r'\[c [^\]]+\][^\[]*\[/c\]', '', text) text = re.sub(r'\[[^\]]+\]', '', text) text = re.sub(r'<<[^>]+>>', '', text) text = re.sub(r'SEE:\s*', '', text) text = re.sub(r'\\[^\\]+\\', '', text) # 2. Strip Wikitext links [[target|display]] -> display, [[target]] -> target text = re.sub(r'\[\[(?:[^|\]]*\|)?([^\]]+)\]\]', r'\1', text) # 3. Strip acute stress marks text = text.replace('\u0301', '') # 4. Remove Latin transliterations like /neoxotnyj/ text = re.sub(r'/[a-zA-Z\s\',._-]+/?', '', text) # 5. Split by comma or semicolon, trim and deduplicate parts = [p.strip() for p in re.split(r'[,;]+', text) if p.strip()] seen = set() cleaned = [] for p in parts: p = re.sub(r'\s+', ' ', p).strip() # skip empty or stray symbols if not p or len(p) < 2 or p in ('-', '—'): continue low = p.lower() if low not in seen: seen.add(low) cleaned.append(p) return ', '.join(cleaned[:4]) def build(): t0 = time.time() DATA_DIR.mkdir(parents=True, exist_ok=True) SCRATCH_DIR.mkdir(parents=True, exist_ok=True) db_dict = {} # term_lower -> {'term': str, 'translation': str, 'phonetic': str, 'pos': str} # ------------------------------------------------------------------------- # 1. FreeDict TEI XML # ------------------------------------------------------------------------- freedict_path = SCRATCH_DIR / "freedict.src.tar.xz" if not freedict_path.exists(): print("Downloading FreeDict English-Russian dataset...") url = "https://download.freedict.org/dictionaries/eng-rus/2024.10.10/freedict-eng-rus-2024.10.10.src.tar.xz" urllib.request.urlretrieve(url, freedict_path) print("Parsing FreeDict TEI XML...") with tarfile.open(freedict_path, "r:xz") as tar: f = tar.extractfile("eng-rus/eng-rus.tei") for event, elem in ET.iterparse(f, events=("end",)): tag = elem.tag.split("}")[-1] if tag == "entry": orth = elem.find(".//{*}orth") if orth is not None and orth.text: term = orth.text.strip() t_lower = term.lower() pron = elem.find(".//{*}pron") phonetic = pron.text.strip() if pron is not None and pron.text else "" pos_elem = elem.find(".//{*}pos") raw_pos = pos_elem.text.strip() if pos_elem is not None and pos_elem.text else "" pos = POS_MAP.get(raw_pos.lower(), raw_pos) quotes = [q.text.strip() for q in elem.findall(".//{*}quote") if q.text] tr = clean_translation(", ".join(quotes)) if tr: if t_lower not in db_dict: db_dict[t_lower] = { "term": term, "translation": tr, "phonetic": phonetic, "pos": pos, } else: cur = db_dict[t_lower] if not cur["phonetic"] and phonetic: cur["phonetic"] = phonetic if not cur["pos"] and pos: cur["pos"] = pos elem.clear() print(f"FreeDict loaded: {len(db_dict)} entries") # ------------------------------------------------------------------------- # 2. Wiktionary DSL # ------------------------------------------------------------------------- dsl_path = SCRATCH_DIR / "en-ru.dsl.dz" if not dsl_path.exists(): print("Downloading Wiktionary DSL dataset...") url = "https://raw.githubusercontent.com/open-dsl-dict/wiktionary-dict/master/dsl/oneway/en-ru-enwiktionary.dsl.dz" urllib.request.urlretrieve(url, dsl_path) print("Parsing Wiktionary DSL...") with gzip.open(dsl_path, "rt", encoding="utf-16-le") as f: current_term = None current_ipa = "" current_pos = "" current_translations = [] for line in f: line = line.rstrip("\r\n") if not line or line.startswith("#"): continue if not line.startswith("\t"): if current_term and current_translations: t_lower = current_term.lower() tr = clean_translation(", ".join(current_translations)) if tr: if t_lower not in db_dict: db_dict[t_lower] = { "term": current_term, "translation": tr, "phonetic": current_ipa, "pos": current_pos, } else: cur = db_dict[t_lower] if not cur["phonetic"] and current_ipa: cur["phonetic"] = current_ipa if not cur["pos"] and current_pos: cur["pos"] = current_pos # Merge translations if short cur_parts = [p.strip() for p in cur["translation"].split(",") if p.strip()] for p in tr.split(","): p = p.strip() if p and p.lower() not in [x.lower() for x in cur_parts]: cur_parts.append(p) cur["translation"] = ", ".join(cur_parts[:4]) current_term = line.strip() current_ipa = "" current_pos = "" current_translations = [] else: ipa_m = re.search(r"\[c gray\](/[^/]+/)\[/c\]", line) if ipa_m and not current_ipa: current_ipa = ipa_m.group(1) pos_m = re.search(r"\[p\]<([^>]+)>\[/p\]", line) if pos_m and not current_pos: current_pos = POS_MAP.get(pos_m.group(1).lower(), pos_m.group(1)) if "[m1]" in line: tr_part = re.sub(r"\[m\d\]", "", line) tr_part = re.sub(r"\[/m\]", "", tr_part) current_translations.append(tr_part) print(f"Combined base lemmas: {len(db_dict)} entries") # ------------------------------------------------------------------------- # 3. Comprehensive Verb Forms # ------------------------------------------------------------------------- verbs_path = SCRATCH_DIR / "verbs.json" if not verbs_path.exists(): url = "https://raw.githubusercontent.com/monolithpl/verb.forms.dictionary/master/json/verbs-all.json" urllib.request.urlretrieve(url, verbs_path) with open(verbs_path, "r", encoding="utf-8", errors="replace") as f: verb_data = json.load(f) added_verbs = 0 for row in verb_data: if not row or not isinstance(row, list): continue lemma = row[0].strip() base_info = db_dict.get(lemma.lower()) if not base_info: continue base_trans = base_info["translation"] forms = [ (row[1], f"{base_trans} (3-е л. ед. ч. от {lemma})"), (row[2], f"{base_trans} (прош. вр. от {lemma})"), (row[3], f"{base_trans} (причастие прош. вр. от {lemma})"), (row[4], f"{base_trans} (причастие наст. вр. от {lemma})") if len(row) > 4 else ("", "") ] for form, desc in forms: form = form.strip() if not form or form == lemma: continue f_lower = form.lower() if f_lower not in db_dict: db_dict[f_lower] = { "term": form, "translation": desc, "phonetic": "", "pos": "Verb", } added_verbs += 1 print(f"Added {added_verbs} irregular/standard verb forms (total: {len(db_dict)})") # ------------------------------------------------------------------------- # 4. Irregular Nouns & General Plurals # ------------------------------------------------------------------------- irregular_nouns = { 'man': 'men', 'woman': 'women', 'child': 'children', 'tooth': 'teeth', 'foot': 'feet', 'mouse': 'mice', 'goose': 'geese', 'person': 'people', 'ox': 'oxen', 'leaf': 'leaves', 'half': 'halves', 'knife': 'knives', 'wife': 'wives', 'life': 'lives', 'wolf': 'wolves', 'thief': 'thieves', 'shelf': 'shelves', 'scarf': 'scarves', 'calf': 'calves', 'elf': 'elves', 'loaf': 'loaves', 'sheaf': 'sheaves', 'crisis': 'crises', 'thesis': 'theses', 'basis': 'bases', 'axis': 'axes', 'analysis': 'analyses', 'hypothesis': 'hypotheses', 'phenomenon': 'phenomena', 'criterion': 'criteria', 'datum': 'data', 'medium': 'media', 'index': 'indices', 'matrix': 'matrices', 'appendix': 'appendices', 'cactus': 'cacti', 'focus': 'foci', 'nucleus': 'nuclei', 'syllabus': 'syllabi' } added_plurals = 0 for sing, pl in irregular_nouns.items(): sing_info = db_dict.get(sing) if sing_info and pl not in db_dict: db_dict[pl] = { 'term': pl, 'translation': f"{sing_info['translation']} (мн. ч. от {sing})", 'phonetic': '', 'pos': 'Noun' } added_plurals += 1 # Regular noun plurals (-s, -es, -ies) for t_lower, info in list(db_dict.items()): if info.get('pos') == 'Noun' and len(t_lower) > 2 and ' ' not in t_lower and t_lower.isalpha(): if t_lower.endswith(('s', 'x', 'z', 'ch', 'sh')): pl = t_lower + 'es' elif t_lower.endswith('y') and t_lower[-2] not in 'aeiou': pl = t_lower[:-1] + 'ies' elif t_lower.endswith('o') and t_lower[-2] not in 'aeiou': pl = t_lower + 'es' else: pl = t_lower + 's' if pl not in db_dict: db_dict[pl] = { 'term': pl, 'translation': f"{info['translation']} (мн. ч. от {info['term']})", 'phonetic': '', 'pos': 'Noun' } added_plurals += 1 print(f"Added {added_plurals} noun plurals (total: {len(db_dict)})") # ------------------------------------------------------------------------- # 5. Regular Verb Inflections (-ed, -ing, -s) for remaining verbs # ------------------------------------------------------------------------- added_reg_verbs = 0 for t_lower, info in list(db_dict.items()): if info.get('pos') == 'Verb' and len(t_lower) > 2 and ' ' not in t_lower and t_lower.isalpha(): lemma = info['term'] # -s / -es if t_lower.endswith(('s', 'x', 'z', 'ch', 'sh')): v_3rd = t_lower + 'es' elif t_lower.endswith('y') and t_lower[-2] not in 'aeiou': v_3rd = t_lower[:-1] + 'ies' else: v_3rd = t_lower + 's' if v_3rd not in db_dict: db_dict[v_3rd] = {'term': v_3rd, 'translation': f"{info['translation']} (3-е л. ед. ч. от {lemma})", 'phonetic': '', 'pos': 'Verb'} added_reg_verbs += 1 # -ed if t_lower.endswith('e'): v_ed = t_lower + 'd' elif t_lower.endswith('y') and t_lower[-2] not in 'aeiou': v_ed = t_lower[:-1] + 'ied' else: v_ed = t_lower + 'ed' if v_ed not in db_dict: db_dict[v_ed] = {'term': v_ed, 'translation': f"{info['translation']} (прош. вр. от {lemma})", 'phonetic': '', 'pos': 'Verb'} added_reg_verbs += 1 # -ing if t_lower.endswith('ie'): v_ing = t_lower[:-2] + 'ying' elif t_lower.endswith('e') and not t_lower.endswith(('ee', 'oe', 'ye')): v_ing = t_lower[:-1] + 'ing' else: v_ing = t_lower + 'ing' if v_ing not in db_dict: db_dict[v_ing] = {'term': v_ing, 'translation': f"{info['translation']} (причастие от {lemma})", 'phonetic': '', 'pos': 'Verb'} added_reg_verbs += 1 print(f"Added {added_reg_verbs} regular verb inflections (total: {len(db_dict)})") # ------------------------------------------------------------------------- # 6. IT / Tech / Networking Terminology # ------------------------------------------------------------------------- tech_glossary = { 'packet': ('пакет, блок данных', '/ˈpæk.ɪt/', 'Noun'), 'packets': ('пакеты данных', '', 'Noun'), 'router': ('маршрутизатор, роутер', '/ˈɹuː.tə/', 'Noun'), 'routers': ('маршрутизаторы, роутеры', '', 'Noun'), 'switch': ('коммутатор, свитч, переключатель', '/swɪtʃ/', 'Noun'), 'switches': ('коммутаторы, свитчи', '', 'Noun'), 'switching': ('коммутация (пакетов)', '', 'Noun'), 'subnet': ('подсеть', '/ˈsʌb.nɛt/', 'Noun'), 'subnets': ('подсети', '', 'Noun'), 'subnetting': ('разбиение на подсети, субнеттинг', '', 'Noun'), 'subnetted': ('разделенный на подсети', '', 'Adjective'), 'bandwidth': ('пропускная способность, ширина полосы пропускания', '/ˈbænd.wɪdθ/', 'Noun'), 'latency': ('задержка, время ожидания, латентность', '/ˈleɪ.tən.si/', 'Noun'), 'throughput': ('пропускная способность, производительность передачи', '/ˈθɹuː.pʊt/', 'Noun'), 'payload': ('полезная нагрузка (пакета данных)', '/ˈpeɪ.loʊd/', 'Noun'), 'handshake': ('рукопожатие, согласование связи (TCP/TLS)', '/ˈhænd.ʃeɪk/', 'Noun'), 'encapsulation': ('инкапсуляция данных', '/ɪnˌkæp.sjʊˈleɪ.ʃən/', 'Noun'), 'decapsulation': ('декапсуляция данных', '', 'Noun'), 'gateway': ('сетевой шлюз', '/ˈɡeɪt.weɪ/', 'Noun'), 'gateways': ('сетевые шлюзы', '', 'Noun'), 'firewall': ('межсетевой экран, файрвол, брандмауэр', '/ˈfaɪə.wɔːl/', 'Noun'), 'firewalls': ('межсетевые экраны, файрволы', '', 'Noun'), 'broadcast': ('широковещательный, широковещание', '/ˈbɹɔːd.kɑːst/', 'Noun'), 'multicast': ('многоадресная рассылка, мультикаст', '/ˈmʌl.ti.kæst/', 'Noun'), 'unicast': ('одноадресная рассылка, уникаст', '/ˈjuː.nɪ.kæst/', 'Noun'), 'loopback': ('петлевой интерфейс (loopback)', '/ˈluːp.bæk/', 'Noun'), 'topology': ('топология сети', '/təˈpɒl.ə.dʒi/', 'Noun'), 'topologies': ('сетевые топологии', '', 'Noun'), 'interface': ('интерфейс, сетевой порт', '/ˈɪn.tə.feɪs/', 'Noun'), 'interfaces': ('интерфейсы, сетевые порты', '', 'Noun'), 'vlan': ('виртуальная локальная сеть (VLAN)', '/viː.læn/', 'Noun'), 'vlans': ('виртуальные локальные сети (VLAN)', '', 'Noun'), 'trunk': ('магистральный канал, транк', '/tɹʌŋk/', 'Noun'), 'trunking': ('транкинг (объединение каналов/VLAN)', '', 'Noun'), 'dhcp': ('протокол динамической настройки узла (DHCP)', '', 'Noun'), 'dns': ('система доменных имен (DNS)', '', 'Noun'), 'arp': ('протокол разрешения адресов (ARP)', '', 'Noun'), 'icmp': ('протокол управляющих сообщений сети (ICMP)', '', 'Noun'), 'tcp': ('протокол управления передачей (TCP)', '', 'Noun'), 'udp': ('протокол пользовательских датаграмм (UDP)', '', 'Noun'), 'ipv4': ('протокол IPv4 (32-битная адресация)', '', 'Noun'), 'ipv6': ('протокол IPv6 (128-битная адресация)', '', 'Noun'), 'socket': ('сокет, программный интерфейс связи', '/ˈsɒk.ɪt/', 'Noun'), 'sockets': ('сокеты', '', 'Noun'), 'endpoint': ('конечная точка, эндпоинт', '/ˈɛndˌpɔɪnt/', 'Noun'), 'endpoints': ('конечные точки, эндпоинты', '', 'Noun'), 'daemon': ('фоновый процесс, демон', '/ˈdiː.mən/', 'Noun'), 'daemons': ('фоновые процессы, демоны', '', 'Noun'), 'buffer': ('буфер, промежуточная память', '/ˈbʌf.ə/', 'Noun'), 'buffering': ('буферизация', '', 'Noun'), 'buffers': ('буферы', '', 'Noun'), 'overflow': ('переполнение (буфера)', '/ˌoʊ.vɚˈfloʊ/', 'Noun'), 'underflow': ('исчерпание буфера', '', 'Noun'), 'checksum': ('контрольная сумма', '/ˈtʃɛk.sʌm/', 'Noun'), 'checksums': ('контрольные суммы', '', 'Noun'), 'hop': ('хоп, промежуточный узел маршрута', '/hɒp/', 'Noun'), 'hops': ('хопы, узлы маршрута', '', 'Noun'), 'duplex': ('дуплексный режим связи', '/ˈdjuː.plɛks/', 'Noun'), 'half-duplex': ('полудуплексный режим', '', 'Noun'), 'full-duplex': ('полнодуплексный режим', '', 'Noun'), 'attenuation': ('затухание сигнала', '/əˌtɛn.juˈeɪ.ʃən/', 'Noun'), 'crosstalk': ('перекрестные помехи', '/ˈkɹɔːs.tɔːk/', 'Noun'), 'jitter': ('джиттер, фазовое дрожание задержки', '/ˈdʒɪt.ə/', 'Noun'), 'collision': ('коллизия, столкновение пакетов', '/kəˈlɪʒ.ən/', 'Noun'), 'collisions': ('коллизии пакетов', '', 'Noun'), 'forwarding': ('пересылка, продвижение пакетов', '', 'Noun'), 'default-gateway': ('основной шлюз', '', 'Noun'), 'keepalive': ('поддержание соединения, keep-alive', '', 'Noun'), 'keep-alive': ('сигнал проверки активности соединения', '', 'Noun'), 'wildcard': ('маска подсети, групповой символ', '/ˈwaɪldˌkɑːd/', 'Noun'), 'acknowledgment': ('подтверждение приема (ACK)', '/əkˈnɒl.ɪdʒ.mənt/', 'Noun'), 'acknowledgements': ('подтверждения приема (ACK)', '', 'Noun'), 'fragmentation': ('фрагментация пакетов', '/ˌfɹæɡ.mənˈteɪ.ʃən/', 'Noun'), 'reassembly': ('сборка пакетов из фрагментов', '', 'Noun'), } added_tech = 0 for term, (tr, phon, pos) in tech_glossary.items(): t_lower = term.lower() if t_lower not in db_dict or not db_dict[t_lower]["phonetic"]: db_dict[t_lower] = { "term": term, "translation": tr, "phonetic": phon or (db_dict.get(t_lower, {}).get("phonetic", "")), "pos": pos, } added_tech += 1 print(f"Added {added_tech} IT/tech terms. Final database size: {len(db_dict)} entries") # ------------------------------------------------------------------------- # 7. Write out compressed TSV # ------------------------------------------------------------------------- print(f"Writing compressed dataset to {OUT_FILE}...") with gzip.open(OUT_FILE, "wt", encoding="utf-8") as f: for info in db_dict.values(): term = info["term"].replace("\t", " ").replace("\n", " ").strip() tr = info["translation"].replace("\t", " ").replace("\n", " ").strip() phon = info.get("phonetic", "").replace("\t", " ").replace("\n", " ").strip() pos = info.get("pos", "").replace("\t", " ").replace("\n", " ").strip() if term and tr: f.write(f"{term}\t{tr}\t{phon}\t{pos}\n") size_mb = os.path.getsize(OUT_FILE) / (1024 * 1024) t1 = time.time() print(f"✅ Generated {len(db_dict)} articles in {OUT_FILE} ({size_mb:.2f} MB) in {t1-t0:.2f}s") if __name__ == "__main__": build()