fix(cefr): resolve authentic context sentence per term and repair legacy SRS cards

- Add resolve_context_sentence_for_term to replace placeholder blanks and lookup authentic sentences from cefr_questions or books
- Fix dual mistake ingestion in process_cefr_answer to assign distinct context to target and chosen distractor
- Provide authentic context_sentence in options_breakdown and pass it via data-context in cefr.js
- Add repair_mismatched_cefr_contexts in database.py to automatically repair cards with placeholder blanks
- Bump Service Worker cache to notes-v5.9
- Add test_cefr_context_resolution_and_repair covering resolution, dual ingestion, manual add, and DB repair

app/cefr.py

@@ -9,6 +9,7 @@ Features:
 - Per-level progress tracking (mastered, learning, percentage).
 """
 
+import re
 import json
 import sqlite3
 import random
@@ -2039,6 +2040,64 @@ def get_cefr_quiz_batch(
     return quiz_items
 
 
+def resolve_context_sentence_for_term(
+    conn: sqlite3.Connection,
+    term: str,
+    candidate_sentence: str = "",
+) -> str:
+    """
+    Resolves the best authentic context sentence for a given term:
+    1. If candidate_sentence contains the term, use it (with ___ replaced).
+    2. Look up in cefr_questions specifically for this term.
+    3. Look up in books table for an authentic sentence containing this term.
+    4. Fallback to clean natural sentence.
+    """
+    clean_term = term.strip()
+    if not clean_term:
+        return ""
+
+    # 1. If candidate_sentence already contains this term (case-insensitive word boundary)
+    if candidate_sentence:
+        cand_clean = candidate_sentence.strip()
+        if re.search(rf"\b{re.escape(clean_term)}\b", cand_clean, re.IGNORECASE):
+            return re.sub(r"\s+", " ", cand_clean.replace("___", clean_term)).strip()
+
+    # 2. Look up in cefr_questions specifically for this term
+    try:
+        q_row = conn.execute(
+            "SELECT sentence FROM cefr_questions WHERE lower(trim(term)) = lower(trim(?)) ORDER BY id ASC LIMIT 1",
+            (clean_term,),
+        ).fetchone()
+        if q_row and q_row["sentence"]:
+            return re.sub(r"\s+", " ", q_row["sentence"].replace("___", clean_term)).strip()
+    except Exception:
+        pass
+
+    # 3. Look up in books table
+    try:
+        books = conn.execute("SELECT chapters_json FROM books ORDER BY id DESC LIMIT 10").fetchall()
+        for b in books:
+            try:
+                chapters = json.loads(b["chapters_json"])
+            except Exception:
+                continue
+            for ch in chapters:
+                content = ch.get("content", "")
+                if clean_term.lower() in content.lower():
+                    for s in re.split(r"(?<=[.!?])\s+", content):
+                        s_clean = s.strip()
+                        if 20 <= len(s_clean) <= 180 and re.search(rf"\b{re.escape(clean_term)}\b", s_clean, re.IGNORECASE):
+                            return re.sub(r"\s+", " ", s_clean).strip()
+    except Exception:
+        pass
+
+    # 4. If candidate_sentence had '___' and came from this term
+    if candidate_sentence and "___" in candidate_sentence:
+        return re.sub(r"\s+", " ", candidate_sentence.replace("___", clean_term)).strip()
+
+    return f"Using '{clean_term}' in meaningful sentences reinforces active recall and long-term fluency."
+
+
 def process_cefr_answer(
     conn: sqlite3.Connection,
     user_id: int,
@@ -2105,9 +2164,10 @@ def process_cefr_answer(
     if not is_correct:
         # =====================================================================
         # DUAL AUTOMATED GAP INGESTION (Crucial Pedagogical Requirement)
-        # 1. Correct target word -> goes 100% to vocabulary
-        # 2. Chosen wrong word -> also goes to vocabulary!
+        # 1. Correct target word -> goes 100% to vocabulary with its sentence
+        # 2. Chosen wrong word -> also goes to vocabulary with its OWN context!
         # =====================================================================
+        target_ctx = resolve_context_sentence_for_term(conn, target_term, sentence_with_term)
         target_card_info = _ingest_card_for_user(
             conn,
             user_id=user_id,
@@ -2115,7 +2175,7 @@ def process_cefr_answer(
             translation=translation,
             phonetic=phonetic,
             pos=pos,
-            context_sentence=sentence_with_term,
+            context_sentence=target_ctx,
             explanation=explanation,
             level=level,
             tag_suffix="test-mistake",
@@ -2130,6 +2190,7 @@ def process_cefr_answer(
         sel_trans = sel_row["translation"] if sel_row else selected_clean
         sel_phon = sel_row["phonetic"] if sel_row else ""
         sel_pos = sel_row["pos"] if sel_row else ""
+        sel_ctx = resolve_context_sentence_for_term(conn, selected_clean)
 
         selected_card_info = _ingest_card_for_user(
             conn,
@@ -2138,7 +2199,7 @@ def process_cefr_answer(
             translation=sel_trans,
             phonetic=sel_phon,
             pos=sel_pos,
-            context_sentence=sentence_with_term,
+            context_sentence=sel_ctx,
             explanation=f"Выбрано по ошибке в вопросе про '{target_term}'",
             level=level,
             tag_suffix="test-mistake-chosen",
@@ -2193,11 +2254,14 @@ def process_cefr_answer(
         if not is_correct and (is_target or was_selected):
             is_auto_added = True
 
+        opt_ctx = resolve_context_sentence_for_term(conn, opt_clean, sentence_with_term if is_target else "")
+
         options_breakdown.append({
             "term": opt_clean,
             "translation": opt_trans,
             "phonetic": opt_phon,
             "pos": opt_pos,
+            "context_sentence": opt_ctx,
             "is_correct": is_target,
             "was_selected": was_selected,
             "in_library": bool(user_card) or is_auto_added,
@@ -2287,8 +2351,11 @@ def _ingest_card_for_user(
         existing_tags.add(tag_suffix)
         updated_tags = ", ".join(sorted(existing_tags))
 
-        updated_context = existing_card["context_sentence"] or ""
-        if context_sentence and context_sentence not in updated_context:
+        updated_context = (existing_card["context_sentence"] or "").strip()
+        # If existing context has "___" or lacks the term, overwrite with new authentic context!
+        if ("___" in updated_context or not re.search(rf"\b{re.escape(term_clean)}\b", updated_context, re.IGNORECASE)) and context_sentence:
+            updated_context = context_sentence
+        elif context_sentence and context_sentence not in updated_context:
             updated_context = f"{updated_context}\n\n{context_sentence}".strip()
 
         conn.execute(
@@ -2337,6 +2404,8 @@ def ingest_cefr_word(
     clean_term = term.strip()
     clean_level = level.upper().strip()
 
+    resolved_ctx = resolve_context_sentence_for_term(conn, clean_term, context_sentence)
+
     row = conn.execute(
         "SELECT term, translation, phonetic, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE",
         (clean_term,),
@@ -2352,7 +2421,7 @@ def ingest_cefr_word(
         translation=trans,
         phonetic=phon,
         pos=pos,
-        context_sentence=context_sentence,
+        context_sentence=resolved_ctx,
         explanation=f"Добавлено вручную из вариантов вопроса теста CEFR ({clean_level})",
         level=clean_level,
         tag_suffix="manual-test-add",
@@ -2364,6 +2433,7 @@ def ingest_cefr_word(
         "term": clean_term,
         "translation": trans,
         "phonetic": phon,
+        "context_sentence": resolved_ctx,
     }
 
 

app/database.py

@@ -379,6 +379,43 @@ def _migrate(conn: sqlite3.Connection):
     conn.execute("CREATE INDEX IF NOT EXISTS idx_bg_tasks_user_status ON background_tasks(user_id, status);")
     conn.execute("CREATE INDEX IF NOT EXISTS idx_bg_tasks_status ON background_tasks(status);")
 
+    # Repair legacy cards with placeholder blanks or mismatched context
+    repair_mismatched_cefr_contexts(conn)
+
+
+def repair_mismatched_cefr_contexts(conn: sqlite3.Connection):
+    """
+    Repairs legacy vocabulary cards that were imported with placeholder blanks ('___')
+    or with mismatched contexts from CEFR test questions.
+    """
+    try:
+        from app.cefr import resolve_context_sentence_for_term
+
+        # Check if table exists
+        tbl = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='vocabulary_cards'").fetchone()
+        if not tbl:
+            return
+
+        rows = conn.execute(
+            """
+            SELECT id, term, context_sentence
+            FROM vocabulary_cards
+            WHERE instr(context_sentence, '___') > 0
+            """
+        ).fetchall()
+
+        for r in rows:
+            card_id = r["id"]
+            term = r["term"]
+            old_ctx = r["context_sentence"] or ""
+            new_ctx = resolve_context_sentence_for_term(conn, term, old_ctx)
+            if new_ctx and new_ctx != old_ctx:
+                conn.execute(
+                    "UPDATE vocabulary_cards SET context_sentence = ? WHERE id = ?",
+                    (new_ctx, card_id),
+                )
+    except Exception:
+        pass
 
 
 def _seed_offline_dictionary(conn: sqlite3.Connection):

app/static/js/modules/cefr.js

@@ -676,7 +676,7 @@ function renderOptionsBreakdown(breakdown) {
                     } else if (inLib) {
                         actionHtml = `<span class="wb-badge badge-in-lib"><i class="ph-check"></i> В библиотеке</span>`;
                     } else {
-                        actionHtml = `<button type="button" class="btn btn-xs btn-outline btn-add-option-word" data-term="${escapeHtml(b.term)}">+ В словарь</button>`;
+                        actionHtml = `<button type="button" class="btn btn-xs btn-outline btn-add-option-word" data-term="${escapeHtml(b.term)}" data-context="${escapeHtml(b.context_sentence || '')}">+ В словарь</button>`;
                     }
 
                     return `
@@ -708,7 +708,8 @@ function attachBreakdownListeners(container, contextSentence) {
             btn.innerHTML = '...';
 
             try {
-                await api.cefr.addWord(currentLevel, term, contextSentence || '');
+                const termContext = btn.dataset.context || contextSentence || '';
+                await api.cefr.addWord(currentLevel, term, termContext);
                 sessionStats.ingested++;
                 btn.outerHTML = '<span class="wb-badge badge-just-added"><i class="ph-check-circle-fill"></i> Добавлено!</span>';
                 showToast(`Слово «${term}» добавлено в утренний тренажёр SRS!`, 'success');

app/static/sw.js

@@ -1,7 +1,7 @@
 // Service Worker for Notes English PWA (Stage 2)
 // Strategy: Cache-first for static assets, Network-first for API calls.
 
-const CACHE_NAME = 'notes-v5.8';
+const CACHE_NAME = 'notes-v5.9';
 const OFFLINE_URL = '/';
 
 const PRECACHE_URLS = [