feat: background tasks manager and book sentence mining for CEFR tests

app/database.py

@@ -216,12 +216,71 @@ def _migrate(conn: sqlite3.Connection):
         explanation TEXT DEFAULT '',
         frequency_rank INTEGER DEFAULT 0,
         created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-        UNIQUE(level, term COLLATE NOCASE)
+        source_book_id INTEGER DEFAULT NULL,
+        source_book_title TEXT DEFAULT '',
+        source_chapter_title TEXT DEFAULT ''
     );
     """)
     conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_level ON cefr_questions(level);")
     conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_term ON cefr_questions(term COLLATE NOCASE);")
 
+    # Safe migration: if old table had UNIQUE(level, term), migrate to allow rich book-mined sentences
+    cefr_ddl_row = conn.execute(
+        "SELECT sql FROM sqlite_master WHERE type='table' AND name='cefr_questions'"
+    ).fetchone()
+    if cefr_ddl_row and "UNIQUE(level, term" in cefr_ddl_row[0]:
+        conn.execute("PRAGMA foreign_keys = OFF;")
+        conn.execute("""
+        CREATE TABLE cefr_questions_new (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            level TEXT NOT NULL,
+            term TEXT NOT NULL COLLATE NOCASE,
+            translation TEXT NOT NULL,
+            pos TEXT DEFAULT '',
+            phonetic TEXT DEFAULT '',
+            sentence TEXT NOT NULL,
+            options_json TEXT NOT NULL,
+            explanation TEXT DEFAULT '',
+            frequency_rank INTEGER DEFAULT 0,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            source_book_id INTEGER DEFAULT NULL,
+            source_book_title TEXT DEFAULT '',
+            source_chapter_title TEXT DEFAULT ''
+        );
+        """)
+        conn.execute("""
+        INSERT INTO cefr_questions_new (
+            id, level, term, translation, pos, phonetic, sentence, options_json,
+            explanation, frequency_rank, created_at, source_book_id, source_book_title, source_chapter_title
+        )
+        SELECT id, level, term, translation, pos, phonetic, sentence, options_json,
+               explanation, frequency_rank, created_at,
+               COALESCE(source_book_id, NULL),
+               COALESCE(source_book_title, ''),
+               COALESCE(source_chapter_title, '')
+        FROM cefr_questions;
+        """)
+        conn.execute("DROP TABLE cefr_questions;")
+        conn.execute("ALTER TABLE cefr_questions_new RENAME TO cefr_questions;")
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_level ON cefr_questions(level);")
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_term ON cefr_questions(term COLLATE NOCASE);")
+        conn.execute("PRAGMA foreign_keys = ON;")
+
+    # Safe migration: add book source metadata columns if not present
+    existing_cefr_cols = {
+        row[1] for row in conn.execute("PRAGMA table_info(cefr_questions)").fetchall()
+    }
+    cefr_migrations = [
+        ("source_book_id", "INTEGER DEFAULT NULL"),
+        ("source_book_title", "TEXT DEFAULT ''"),
+        ("source_chapter_title", "TEXT DEFAULT ''"),
+    ]
+    for col, definition in cefr_migrations:
+        if col not in existing_cefr_cols:
+            conn.execute(f"ALTER TABLE cefr_questions ADD COLUMN {col} {definition}")
+
+    conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_cefr_q_level_sentence ON cefr_questions(level, sentence);")
+
     conn.execute("""
     CREATE TABLE IF NOT EXISTS cefr_progress (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -299,6 +358,27 @@ def _migrate(conn: sqlite3.Connection):
     except Exception:
         pass
 
+    # Ensure background_tasks table exists (Step 1: Background Tasks Engine)
+    conn.execute("""
+    CREATE TABLE IF NOT EXISTS background_tasks (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        user_id INTEGER NOT NULL,
+        task_type TEXT NOT NULL,
+        status TEXT NOT NULL DEFAULT 'pending',
+        progress_current INTEGER DEFAULT 0,
+        progress_total INTEGER DEFAULT 0,
+        message TEXT DEFAULT '',
+        params_json TEXT DEFAULT '{}',
+        result_json TEXT DEFAULT '{}',
+        error_message TEXT DEFAULT '',
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    );
+    """)
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_bg_tasks_user_status ON background_tasks(user_id, status);")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_bg_tasks_status ON background_tasks(status);")
+
 
 
 def _seed_offline_dictionary(conn: sqlite3.Connection):