import sqlite3 import contextlib from pathlib import Path from typing import Generator from app import config def get_db_connection() -> sqlite3.Connection: conn = sqlite3.connect(config.DB_PATH, timeout=15.0) conn.row_factory = sqlite3.Row conn.execute("PRAGMA synchronous = NORMAL;") conn.execute("PRAGMA mmap_size = 268435456;") # 256MB memory-mapped I/O conn.execute("PRAGMA cache_size = -32000;") # 32MB RAM page cache conn.execute("PRAGMA temp_store = MEMORY;") # In-memory temporary tables & sorts conn.execute("PRAGMA foreign_keys = ON;") return conn @contextlib.contextmanager def get_db() -> Generator[sqlite3.Connection, None, None]: conn = get_db_connection() try: yield conn conn.commit() except Exception: conn.rollback() raise finally: conn.close() def _migrate(conn: sqlite3.Connection): """Safe schema migrations: add new columns/tables/indexes without touching existing data.""" conn.execute("PRAGMA journal_mode=WAL;") # --- vocabulary_cards new columns --- existing_cols = { row[1] for row in conn.execute("PRAGMA table_info(vocabulary_cards)").fetchall() } card_migrations = [ ("collection", "TEXT NOT NULL DEFAULT 'General'"), ("srs_reps", "INTEGER DEFAULT 0"), ("srs_interval", "INTEGER DEFAULT 0"), ("srs_ease", "REAL DEFAULT 2.5"), ("srs_next_review", "TIMESTAMP DEFAULT CURRENT_TIMESTAMP"), ("srs_last_review", "TIMESTAMP"), ("srs_state", "TEXT DEFAULT 'new'"), ("error_count", "INTEGER DEFAULT 0"), ] for col, definition in card_migrations: if col not in existing_cols: conn.execute(f"ALTER TABLE vocabulary_cards ADD COLUMN {col} {definition}") # Ensure srs_logs table exists conn.execute(""" CREATE TABLE IF NOT EXISTS srs_logs ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL, card_id INTEGER NOT NULL, rating INTEGER NOT NULL, interval INTEGER NOT NULL, ease REAL NOT NULL, reviewed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, FOREIGN KEY (card_id) REFERENCES vocabulary_cards(id) ON DELETE CASCADE ); """) # Ensure books table exists conn.execute(""" CREATE TABLE IF NOT EXISTS books ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL, title TEXT NOT NULL, author TEXT DEFAULT '', source_type TEXT NOT NULL, chapters_json TEXT NOT NULL, current_chapter INTEGER DEFAULT 0, current_position REAL DEFAULT 0, total_words INTEGER DEFAULT 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ); """) # --- Deduplicate vocabulary_cards (case-insensitive) and enforce UNIQUE index --- dup_rows = conn.execute(""" SELECT user_id, lower(trim(term)) as clean_term, COUNT(*) as cnt FROM vocabulary_cards GROUP BY user_id, lower(trim(term)) HAVING cnt > 1 """).fetchall() for d in dup_rows: u_id = d["user_id"] c_term = d["clean_term"] cards = conn.execute( "SELECT * FROM vocabulary_cards WHERE user_id = ? AND lower(trim(term)) = ? ORDER BY id ASC", (u_id, c_term), ).fetchall() if not cards: continue primary = cards[0] primary_id = primary["id"] merged_contexts = [primary["context_sentence"]] if primary["context_sentence"] else [] merged_tags = set([t.strip() for t in (primary["tags"] or "").split(",") if t.strip()]) max_reps = primary["srs_reps"] or 0 max_interval = primary["srs_interval"] or 0 max_ease = primary["srs_ease"] or 2.5 latest_review = primary["srs_last_review"] srs_state = primary["srs_state"] for dup in cards[1:]: dup_id = dup["id"] if dup["context_sentence"] and dup["context_sentence"] not in merged_contexts: merged_contexts.append(dup["context_sentence"]) for t in (dup["tags"] or "").split(","): if t.strip(): merged_tags.add(t.strip()) if (dup["srs_reps"] or 0) > max_reps: max_reps = dup["srs_reps"] max_interval = dup["srs_interval"] max_ease = dup["srs_ease"] latest_review = dup["srs_last_review"] srs_state = dup["srs_state"] # Reassign SRS logs to primary card conn.execute("UPDATE srs_logs SET card_id = ? WHERE card_id = ?", (primary_id, dup_id)) # Delete duplicate row conn.execute("DELETE FROM vocabulary_cards WHERE id = ?", (dup_id,)) # Update primary card with merged data conn.execute( """ UPDATE vocabulary_cards SET context_sentence = ?, tags = ?, srs_reps = ?, srs_interval = ?, srs_ease = ?, srs_last_review = ?, srs_state = ? WHERE id = ? """, ( "\n\n".join(merged_contexts), ", ".join(sorted(merged_tags)), max_reps, max_interval, max_ease, latest_review, srs_state, primary_id, ), ) # Composite and single column indexes for high-speed queries conn.execute( "CREATE UNIQUE INDEX IF NOT EXISTS idx_vocab_user_term_nocase ON vocabulary_cards(user_id, term COLLATE NOCASE);" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_vocab_user_created ON vocabulary_cards(user_id, created_at DESC);" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_state ON vocabulary_cards(user_id, srs_state, srs_next_review);" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_created ON vocabulary_cards(user_id, srs_state, created_at DESC);" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_vocab_collection ON vocabulary_cards(user_id, collection);" ) conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_user ON srs_logs(user_id);") conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_reviewed ON srs_logs(user_id, reviewed_at);") conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_card ON srs_logs(card_id);") conn.execute("CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id);") conn.execute("CREATE INDEX IF NOT EXISTS idx_books_user_updated ON books(user_id, updated_at DESC);") conn.execute("CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id);") # Ensure offline_dictionary table exists conn.execute(""" CREATE TABLE IF NOT EXISTS offline_dictionary ( term TEXT PRIMARY KEY COLLATE NOCASE, translation TEXT NOT NULL, phonetic TEXT DEFAULT '', pos TEXT DEFAULT '' ); """) conn.execute("CREATE INDEX IF NOT EXISTS idx_offline_dict_term ON offline_dictionary(term COLLATE NOCASE);") # Auto-seed offline dictionary if empty _seed_offline_dictionary(conn) def _seed_offline_dictionary(conn: sqlite3.Connection): """Seeds the offline dictionary from data/offline_dict.tsv.gz if the table is empty.""" try: count = conn.execute("SELECT COUNT(*) FROM offline_dictionary").fetchone()[0] if count > 0: return except Exception: return archive_candidates = [ config.BASE_DIR / "data" / "offline_dict.tsv.gz", config.BASE_DIR / "app" / "data" / "offline_dict.tsv.gz", Path("/app/data/offline_dict.tsv.gz"), ] archive_path = next((p for p in archive_candidates if p.is_file()), None) if not archive_path: return import gzip batch = [] with gzip.open(archive_path, "rt", encoding="utf-8") as f: for line in f: parts = line.rstrip("\r\n").split("\t") if len(parts) == 4: batch.append(parts) if len(batch) >= 10000: conn.executemany( "INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos) VALUES (?, ?, ?, ?)", batch, ) batch.clear() if batch: conn.executemany( "INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos) VALUES (?, ?, ?, ?)", batch, ) def init_db(): config.DB_PATH.parent.mkdir(parents=True, exist_ok=True) with get_db() as conn: conn.execute("PRAGMA journal_mode=WAL;") conn.executescript(""" CREATE TABLE IF NOT EXISTS users ( id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT UNIQUE NOT NULL COLLATE NOCASE, password_hash TEXT NOT NULL, salt TEXT NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); CREATE TABLE IF NOT EXISTS sessions ( token TEXT PRIMARY KEY, user_id INTEGER NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, expires_at TIMESTAMP NOT NULL, user_agent TEXT, ip TEXT, FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id); CREATE TABLE IF NOT EXISTS vocabulary_cards ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL, term TEXT NOT NULL, translation TEXT NOT NULL, context_sentence TEXT DEFAULT '', phonetic TEXT DEFAULT '', audio_url TEXT DEFAULT '', part_of_speech TEXT DEFAULT '', definition_en TEXT DEFAULT '', tags TEXT DEFAULT 'general', notes TEXT DEFAULT '', collection TEXT NOT NULL DEFAULT 'General', srs_reps INTEGER DEFAULT 0, srs_interval INTEGER DEFAULT 0, srs_ease REAL DEFAULT 2.5, srs_next_review TIMESTAMP DEFAULT CURRENT_TIMESTAMP, srs_last_review TIMESTAMP, srs_state TEXT DEFAULT 'new', error_count INTEGER DEFAULT 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_vocab_user ON vocabulary_cards(user_id); CREATE INDEX IF NOT EXISTS idx_vocab_term ON vocabulary_cards(term); CREATE INDEX IF NOT EXISTS idx_vocab_user_created ON vocabulary_cards(user_id, created_at DESC); CREATE INDEX IF NOT EXISTS idx_vocab_collection ON vocabulary_cards(user_id, collection); CREATE INDEX IF NOT EXISTS idx_vocab_srs_queue ON vocabulary_cards(user_id, srs_next_review, srs_state); CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_state ON vocabulary_cards(user_id, srs_state, srs_next_review); CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_created ON vocabulary_cards(user_id, srs_state, created_at DESC); CREATE UNIQUE INDEX IF NOT EXISTS idx_vocab_user_term_nocase ON vocabulary_cards(user_id, term COLLATE NOCASE); CREATE TABLE IF NOT EXISTS srs_logs ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL, card_id INTEGER NOT NULL, rating INTEGER NOT NULL, interval INTEGER NOT NULL, ease REAL NOT NULL, reviewed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, FOREIGN KEY (card_id) REFERENCES vocabulary_cards(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_srs_user ON srs_logs(user_id); CREATE INDEX IF NOT EXISTS idx_srs_reviewed ON srs_logs(user_id, reviewed_at); CREATE INDEX IF NOT EXISTS idx_srs_card ON srs_logs(card_id); CREATE TABLE IF NOT EXISTS dictionary_cache ( query TEXT PRIMARY KEY COLLATE NOCASE, response_json TEXT NOT NULL, cached_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); CREATE TABLE IF NOT EXISTS books ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL, title TEXT NOT NULL, author TEXT DEFAULT '', source_type TEXT NOT NULL, -- 'epub', 'txt', 'md', 'text', 'pdf' chapters_json TEXT NOT NULL, -- JSON array: [{"title": "Chapter 1", "content": "..."}] current_chapter INTEGER DEFAULT 0, current_position REAL DEFAULT 0, total_words INTEGER DEFAULT 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id); CREATE INDEX IF NOT EXISTS idx_books_user_updated ON books(user_id, updated_at DESC); CREATE TABLE IF NOT EXISTS offline_dictionary ( term TEXT PRIMARY KEY COLLATE NOCASE, translation TEXT NOT NULL, phonetic TEXT DEFAULT '', pos TEXT DEFAULT '' ); CREATE INDEX IF NOT EXISTS idx_offline_dict_term ON offline_dictionary(term COLLATE NOCASE); """) # Apply migrations for pre-existing databases _migrate(conn)