459 lines
18 KiB
Python
459 lines
18 KiB
Python
import sqlite3
|
|
import contextlib
|
|
import hashlib
|
|
from pathlib import Path
|
|
from typing import Generator
|
|
from app import config
|
|
|
|
|
|
def get_db_connection() -> sqlite3.Connection:
|
|
conn = sqlite3.connect(config.DB_PATH, timeout=15.0)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA synchronous = NORMAL;")
|
|
conn.execute("PRAGMA mmap_size = 268435456;") # 256MB memory-mapped I/O
|
|
conn.execute("PRAGMA cache_size = -32000;") # 32MB RAM page cache
|
|
conn.execute("PRAGMA temp_store = MEMORY;") # In-memory temporary tables & sorts
|
|
conn.execute("PRAGMA foreign_keys = ON;")
|
|
return conn
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def get_db() -> Generator[sqlite3.Connection, None, None]:
|
|
conn = get_db_connection()
|
|
try:
|
|
yield conn
|
|
conn.commit()
|
|
except Exception:
|
|
conn.rollback()
|
|
raise
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _migrate(conn: sqlite3.Connection):
|
|
"""Safe schema migrations: add new columns/tables/indexes without touching existing data."""
|
|
conn.execute("PRAGMA journal_mode=WAL;")
|
|
|
|
# --- vocabulary_cards new columns ---
|
|
existing_cols = {
|
|
row[1] for row in conn.execute("PRAGMA table_info(vocabulary_cards)").fetchall()
|
|
}
|
|
card_migrations = [
|
|
("collection", "TEXT NOT NULL DEFAULT 'General'"),
|
|
("srs_reps", "INTEGER DEFAULT 0"),
|
|
("srs_interval", "INTEGER DEFAULT 0"),
|
|
("srs_ease", "REAL DEFAULT 2.5"),
|
|
("srs_next_review", "TIMESTAMP DEFAULT CURRENT_TIMESTAMP"),
|
|
("srs_last_review", "TIMESTAMP"),
|
|
("srs_state", "TEXT DEFAULT 'new'"),
|
|
("error_count", "INTEGER DEFAULT 0"),
|
|
]
|
|
for col, definition in card_migrations:
|
|
if col not in existing_cols:
|
|
conn.execute(f"ALTER TABLE vocabulary_cards ADD COLUMN {col} {definition}")
|
|
|
|
# Ensure srs_logs table exists
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS srs_logs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
user_id INTEGER NOT NULL,
|
|
card_id INTEGER NOT NULL,
|
|
rating INTEGER NOT NULL,
|
|
interval INTEGER NOT NULL,
|
|
ease REAL NOT NULL,
|
|
reviewed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
|
|
FOREIGN KEY (card_id) REFERENCES vocabulary_cards(id) ON DELETE CASCADE
|
|
);
|
|
""")
|
|
|
|
# Ensure books table exists
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS books (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
user_id INTEGER NOT NULL,
|
|
title TEXT NOT NULL,
|
|
author TEXT DEFAULT '',
|
|
source_type TEXT NOT NULL,
|
|
chapters_json TEXT NOT NULL,
|
|
current_chapter INTEGER DEFAULT 0,
|
|
current_position REAL DEFAULT 0,
|
|
total_words INTEGER DEFAULT 0,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
|
|
);
|
|
""")
|
|
|
|
# --- Deduplicate vocabulary_cards (case-insensitive) and enforce UNIQUE index ---
|
|
dup_rows = conn.execute("""
|
|
SELECT user_id, lower(trim(term)) as clean_term, COUNT(*) as cnt
|
|
FROM vocabulary_cards
|
|
GROUP BY user_id, lower(trim(term))
|
|
HAVING cnt > 1
|
|
""").fetchall()
|
|
|
|
for d in dup_rows:
|
|
u_id = d["user_id"]
|
|
c_term = d["clean_term"]
|
|
cards = conn.execute(
|
|
"SELECT * FROM vocabulary_cards WHERE user_id = ? AND lower(trim(term)) = ? ORDER BY id ASC",
|
|
(u_id, c_term),
|
|
).fetchall()
|
|
if not cards:
|
|
continue
|
|
primary = cards[0]
|
|
primary_id = primary["id"]
|
|
merged_contexts = [primary["context_sentence"]] if primary["context_sentence"] else []
|
|
merged_tags = set([t.strip() for t in (primary["tags"] or "").split(",") if t.strip()])
|
|
max_reps = primary["srs_reps"] or 0
|
|
max_interval = primary["srs_interval"] or 0
|
|
max_ease = primary["srs_ease"] or 2.5
|
|
latest_review = primary["srs_last_review"]
|
|
srs_state = primary["srs_state"]
|
|
|
|
for dup in cards[1:]:
|
|
dup_id = dup["id"]
|
|
if dup["context_sentence"] and dup["context_sentence"] not in merged_contexts:
|
|
merged_contexts.append(dup["context_sentence"])
|
|
for t in (dup["tags"] or "").split(","):
|
|
if t.strip():
|
|
merged_tags.add(t.strip())
|
|
if (dup["srs_reps"] or 0) > max_reps:
|
|
max_reps = dup["srs_reps"]
|
|
max_interval = dup["srs_interval"]
|
|
max_ease = dup["srs_ease"]
|
|
latest_review = dup["srs_last_review"]
|
|
srs_state = dup["srs_state"]
|
|
# Reassign SRS logs to primary card
|
|
conn.execute("UPDATE srs_logs SET card_id = ? WHERE card_id = ?", (primary_id, dup_id))
|
|
# Delete duplicate row
|
|
conn.execute("DELETE FROM vocabulary_cards WHERE id = ?", (dup_id,))
|
|
|
|
# Update primary card with merged data
|
|
conn.execute(
|
|
"""
|
|
UPDATE vocabulary_cards
|
|
SET context_sentence = ?, tags = ?, srs_reps = ?, srs_interval = ?, srs_ease = ?,
|
|
srs_last_review = ?, srs_state = ?
|
|
WHERE id = ?
|
|
""",
|
|
(
|
|
"\n\n".join(merged_contexts),
|
|
", ".join(sorted(merged_tags)),
|
|
max_reps,
|
|
max_interval,
|
|
max_ease,
|
|
latest_review,
|
|
srs_state,
|
|
primary_id,
|
|
),
|
|
)
|
|
|
|
# Composite and single column indexes for high-speed queries
|
|
conn.execute(
|
|
"CREATE UNIQUE INDEX IF NOT EXISTS idx_vocab_user_term_nocase ON vocabulary_cards(user_id, term COLLATE NOCASE);"
|
|
)
|
|
conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_vocab_user_created ON vocabulary_cards(user_id, created_at DESC);"
|
|
)
|
|
conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_state ON vocabulary_cards(user_id, srs_state, srs_next_review);"
|
|
)
|
|
conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_created ON vocabulary_cards(user_id, srs_state, created_at DESC);"
|
|
)
|
|
conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_vocab_collection ON vocabulary_cards(user_id, collection);"
|
|
)
|
|
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_user ON srs_logs(user_id);")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_reviewed ON srs_logs(user_id, reviewed_at);")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_card ON srs_logs(card_id);")
|
|
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id);")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_books_user_updated ON books(user_id, updated_at DESC);")
|
|
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id);")
|
|
|
|
# Ensure offline_dictionary table exists
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS offline_dictionary (
|
|
term TEXT PRIMARY KEY COLLATE NOCASE,
|
|
translation TEXT NOT NULL,
|
|
phonetic TEXT DEFAULT '',
|
|
pos TEXT DEFAULT ''
|
|
);
|
|
""")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_offline_dict_term ON offline_dictionary(term COLLATE NOCASE);")
|
|
|
|
# Ensure sentence_translations table exists
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS sentence_translations (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
sentence_hash TEXT NOT NULL,
|
|
original_text TEXT NOT NULL,
|
|
translated_text TEXT NOT NULL,
|
|
target_word TEXT DEFAULT '',
|
|
word_meaning TEXT DEFAULT '',
|
|
source TEXT DEFAULT 'local_llm',
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
""")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_sentence_trans_hash ON sentence_translations(sentence_hash);")
|
|
|
|
# Ensure cefr_questions and cefr_progress tables exist
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS cefr_questions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
level TEXT NOT NULL,
|
|
term TEXT NOT NULL COLLATE NOCASE,
|
|
translation TEXT NOT NULL,
|
|
pos TEXT DEFAULT '',
|
|
phonetic TEXT DEFAULT '',
|
|
sentence TEXT NOT NULL,
|
|
options_json TEXT NOT NULL,
|
|
explanation TEXT DEFAULT '',
|
|
frequency_rank INTEGER DEFAULT 0,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
UNIQUE(level, term COLLATE NOCASE)
|
|
);
|
|
""")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_level ON cefr_questions(level);")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_term ON cefr_questions(term COLLATE NOCASE);")
|
|
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS cefr_progress (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
user_id INTEGER NOT NULL,
|
|
question_id INTEGER NOT NULL,
|
|
level TEXT NOT NULL,
|
|
is_mastered INTEGER DEFAULT 0,
|
|
attempts INTEGER DEFAULT 0,
|
|
mistakes INTEGER DEFAULT 0,
|
|
last_tested_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
|
|
FOREIGN KEY (question_id) REFERENCES cefr_questions(id) ON DELETE CASCADE,
|
|
UNIQUE(user_id, question_id)
|
|
);
|
|
""")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_prog_user_lvl ON cefr_progress(user_id, level);")
|
|
|
|
# Auto-seed offline dictionary if empty
|
|
_seed_offline_dictionary(conn)
|
|
|
|
# Seed curated phrasal verbs
|
|
try:
|
|
from app.phrasal import seed_curated_phrasal_verbs
|
|
seed_curated_phrasal_verbs(conn)
|
|
except Exception:
|
|
pass
|
|
|
|
# Seed curated CEFR questions
|
|
try:
|
|
from app.cefr import seed_cefr_questions
|
|
seed_cefr_questions(conn)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _seed_offline_dictionary(conn: sqlite3.Connection):
|
|
"""Seeds the offline dictionary from data/offline_dict.tsv.gz if the table is empty."""
|
|
try:
|
|
count = conn.execute("SELECT COUNT(*) FROM offline_dictionary").fetchone()[0]
|
|
if count > 0:
|
|
return
|
|
except Exception:
|
|
return
|
|
|
|
archive_candidates = [
|
|
config.BASE_DIR / "data" / "offline_dict.tsv.gz",
|
|
config.BASE_DIR / "app" / "data" / "offline_dict.tsv.gz",
|
|
Path("/app/data/offline_dict.tsv.gz"),
|
|
]
|
|
archive_path = next((p for p in archive_candidates if p.is_file()), None)
|
|
if not archive_path:
|
|
return
|
|
|
|
import gzip
|
|
|
|
batch = []
|
|
with gzip.open(archive_path, "rt", encoding="utf-8") as f:
|
|
for line in f:
|
|
parts = line.rstrip("\r\n").split("\t")
|
|
if len(parts) == 4:
|
|
batch.append(parts)
|
|
if len(batch) >= 10000:
|
|
conn.executemany(
|
|
"INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos) VALUES (?, ?, ?, ?)",
|
|
batch,
|
|
)
|
|
batch.clear()
|
|
if batch:
|
|
conn.executemany(
|
|
"INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos) VALUES (?, ?, ?, ?)",
|
|
batch,
|
|
)
|
|
|
|
|
|
def init_db():
|
|
config.DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
with get_db() as conn:
|
|
conn.execute("PRAGMA journal_mode=WAL;")
|
|
conn.executescript("""
|
|
CREATE TABLE IF NOT EXISTS users (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
username TEXT UNIQUE NOT NULL COLLATE NOCASE,
|
|
password_hash TEXT NOT NULL,
|
|
salt TEXT NOT NULL,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS sessions (
|
|
token TEXT PRIMARY KEY,
|
|
user_id INTEGER NOT NULL,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
expires_at TIMESTAMP NOT NULL,
|
|
user_agent TEXT,
|
|
ip TEXT,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id);
|
|
|
|
CREATE TABLE IF NOT EXISTS vocabulary_cards (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
user_id INTEGER NOT NULL,
|
|
term TEXT NOT NULL,
|
|
translation TEXT NOT NULL,
|
|
context_sentence TEXT DEFAULT '',
|
|
phonetic TEXT DEFAULT '',
|
|
audio_url TEXT DEFAULT '',
|
|
part_of_speech TEXT DEFAULT '',
|
|
definition_en TEXT DEFAULT '',
|
|
tags TEXT DEFAULT 'general',
|
|
notes TEXT DEFAULT '',
|
|
collection TEXT NOT NULL DEFAULT 'General',
|
|
srs_reps INTEGER DEFAULT 0,
|
|
srs_interval INTEGER DEFAULT 0,
|
|
srs_ease REAL DEFAULT 2.5,
|
|
srs_next_review TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
srs_last_review TIMESTAMP,
|
|
srs_state TEXT DEFAULT 'new',
|
|
error_count INTEGER DEFAULT 0,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_vocab_user ON vocabulary_cards(user_id);
|
|
CREATE INDEX IF NOT EXISTS idx_vocab_term ON vocabulary_cards(term);
|
|
CREATE INDEX IF NOT EXISTS idx_vocab_user_created ON vocabulary_cards(user_id, created_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_vocab_collection ON vocabulary_cards(user_id, collection);
|
|
CREATE INDEX IF NOT EXISTS idx_vocab_srs_queue ON vocabulary_cards(user_id, srs_next_review, srs_state);
|
|
CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_state ON vocabulary_cards(user_id, srs_state, srs_next_review);
|
|
CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_created ON vocabulary_cards(user_id, srs_state, created_at DESC);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_vocab_user_term_nocase ON vocabulary_cards(user_id, term COLLATE NOCASE);
|
|
|
|
CREATE TABLE IF NOT EXISTS srs_logs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
user_id INTEGER NOT NULL,
|
|
card_id INTEGER NOT NULL,
|
|
rating INTEGER NOT NULL,
|
|
interval INTEGER NOT NULL,
|
|
ease REAL NOT NULL,
|
|
reviewed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
|
|
FOREIGN KEY (card_id) REFERENCES vocabulary_cards(id) ON DELETE CASCADE
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_srs_user ON srs_logs(user_id);
|
|
CREATE INDEX IF NOT EXISTS idx_srs_reviewed ON srs_logs(user_id, reviewed_at);
|
|
CREATE INDEX IF NOT EXISTS idx_srs_card ON srs_logs(card_id);
|
|
|
|
CREATE TABLE IF NOT EXISTS dictionary_cache (
|
|
query TEXT PRIMARY KEY COLLATE NOCASE,
|
|
response_json TEXT NOT NULL,
|
|
cached_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS books (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
user_id INTEGER NOT NULL,
|
|
title TEXT NOT NULL,
|
|
author TEXT DEFAULT '',
|
|
source_type TEXT NOT NULL, -- 'epub', 'txt', 'md', 'text', 'pdf'
|
|
chapters_json TEXT NOT NULL, -- JSON array: [{"title": "Chapter 1", "content": "..."}]
|
|
current_chapter INTEGER DEFAULT 0,
|
|
current_position REAL DEFAULT 0,
|
|
total_words INTEGER DEFAULT 0,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id);
|
|
CREATE INDEX IF NOT EXISTS idx_books_user_updated ON books(user_id, updated_at DESC);
|
|
|
|
CREATE TABLE IF NOT EXISTS offline_dictionary (
|
|
term TEXT PRIMARY KEY COLLATE NOCASE,
|
|
translation TEXT NOT NULL,
|
|
phonetic TEXT DEFAULT '',
|
|
pos TEXT DEFAULT ''
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_offline_dict_term ON offline_dictionary(term COLLATE NOCASE);
|
|
|
|
CREATE TABLE IF NOT EXISTS sentence_translations (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
sentence_hash TEXT NOT NULL,
|
|
original_text TEXT NOT NULL,
|
|
translated_text TEXT NOT NULL,
|
|
target_word TEXT DEFAULT '',
|
|
word_meaning TEXT DEFAULT '',
|
|
source TEXT DEFAULT 'local_llm',
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_sentence_trans_hash ON sentence_translations(sentence_hash);
|
|
""")
|
|
# Apply migrations for pre-existing databases
|
|
_migrate(conn)
|
|
|
|
|
|
def compute_sentence_hash(sentence: str, target_word: str = "") -> str:
|
|
norm = f"{sentence.strip().lower()}||{target_word.strip().lower()}"
|
|
return hashlib.sha256(norm.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def get_cached_sentence_translation(sentence: str, target_word: str = "") -> dict | None:
|
|
s_hash = compute_sentence_hash(sentence, target_word)
|
|
with get_db() as conn:
|
|
row = conn.execute(
|
|
"""
|
|
SELECT original_text, translated_text, target_word, word_meaning, source, created_at
|
|
FROM sentence_translations
|
|
WHERE sentence_hash = ?
|
|
ORDER BY id DESC LIMIT 1
|
|
""",
|
|
(s_hash,),
|
|
).fetchone()
|
|
if row:
|
|
return dict(row)
|
|
return None
|
|
|
|
|
|
def save_sentence_translation(
|
|
sentence: str,
|
|
translated_text: str,
|
|
target_word: str = "",
|
|
word_meaning: str = "",
|
|
source: str = "local_llm",
|
|
) -> None:
|
|
s_hash = compute_sentence_hash(sentence, target_word)
|
|
with get_db() as conn:
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO sentence_translations (sentence_hash, original_text, translated_text, target_word, word_meaning, source)
|
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(s_hash, sentence.strip(), translated_text.strip(), target_word.strip(), word_meaning.strip(), source),
|
|
)
|
|
|