fix(cefr): resolve authentic context sentence per term and repair legacy SRS cards

- Add resolve_context_sentence_for_term to replace placeholder blanks and lookup authentic sentences from cefr_questions or books
- Fix dual mistake ingestion in process_cefr_answer to assign distinct context to target and chosen distractor
- Provide authentic context_sentence in options_breakdown and pass it via data-context in cefr.js
- Add repair_mismatched_cefr_contexts in database.py to automatically repair cards with placeholder blanks
- Bump Service Worker cache to notes-v5.9
- Add test_cefr_context_resolution_and_repair covering resolution, dual ingestion, manual add, and DB repair

app/cefr.py

@@ -9,6 +9,7 @@ Features:
 - Per-level progress tracking (mastered, learning, percentage).
 """
 
+import re
 import json
 import sqlite3
 import random
@@ -2039,6 +2040,64 @@ def get_cefr_quiz_batch(
     return quiz_items
 
 
+def resolve_context_sentence_for_term(
+    conn: sqlite3.Connection,
+    term: str,
+    candidate_sentence: str = "",
+) -> str:
+    """
+    Resolves the best authentic context sentence for a given term:
+    1. If candidate_sentence contains the term, use it (with ___ replaced).
+    2. Look up in cefr_questions specifically for this term.
+    3. Look up in books table for an authentic sentence containing this term.
+    4. Fallback to clean natural sentence.
+    """
+    clean_term = term.strip()
+    if not clean_term:
+        return ""
+
+    # 1. If candidate_sentence already contains this term (case-insensitive word boundary)
+    if candidate_sentence:
+        cand_clean = candidate_sentence.strip()
+        if re.search(rf"\b{re.escape(clean_term)}\b", cand_clean, re.IGNORECASE):
+            return re.sub(r"\s+", " ", cand_clean.replace("___", clean_term)).strip()
+
+    # 2. Look up in cefr_questions specifically for this term
+    try:
+        q_row = conn.execute(
+            "SELECT sentence FROM cefr_questions WHERE lower(trim(term)) = lower(trim(?)) ORDER BY id ASC LIMIT 1",
+            (clean_term,),
+        ).fetchone()
+        if q_row and q_row["sentence"]:
+            return re.sub(r"\s+", " ", q_row["sentence"].replace("___", clean_term)).strip()
+    except Exception:
+        pass
+
+    # 3. Look up in books table
+    try:
+        books = conn.execute("SELECT chapters_json FROM books ORDER BY id DESC LIMIT 10").fetchall()
+        for b in books:
+            try:
+                chapters = json.loads(b["chapters_json"])
+            except Exception:
+                continue
+            for ch in chapters:
+                content = ch.get("content", "")
+                if clean_term.lower() in content.lower():
+                    for s in re.split(r"(?<=[.!?])\s+", content):
+                        s_clean = s.strip()
+                        if 20 <= len(s_clean) <= 180 and re.search(rf"\b{re.escape(clean_term)}\b", s_clean, re.IGNORECASE):
+                            return re.sub(r"\s+", " ", s_clean).strip()
+    except Exception:
+        pass
+
+    # 4. If candidate_sentence had '___' and came from this term
+    if candidate_sentence and "___" in candidate_sentence:
+        return re.sub(r"\s+", " ", candidate_sentence.replace("___", clean_term)).strip()
+
+    return f"Using '{clean_term}' in meaningful sentences reinforces active recall and long-term fluency."
+
+
 def process_cefr_answer(
     conn: sqlite3.Connection,
     user_id: int,
@@ -2105,9 +2164,10 @@ def process_cefr_answer(
     if not is_correct:
         # =====================================================================
         # DUAL AUTOMATED GAP INGESTION (Crucial Pedagogical Requirement)
-        # 1. Correct target word -> goes 100% to vocabulary
+        # 1. Correct target word -> goes 100% to vocabulary with its sentence
-        # 2. Chosen wrong word -> also goes to vocabulary!
+        # 2. Chosen wrong word -> also goes to vocabulary with its OWN context!
         # =====================================================================
+        target_ctx = resolve_context_sentence_for_term(conn, target_term, sentence_with_term)
         target_card_info = _ingest_card_for_user(
             conn,
             user_id=user_id,
@@ -2115,7 +2175,7 @@ def process_cefr_answer(
             translation=translation,
             phonetic=phonetic,
             pos=pos,
-            context_sentence=sentence_with_term,
+            context_sentence=target_ctx,
             explanation=explanation,
             level=level,
             tag_suffix="test-mistake",
@@ -2130,6 +2190,7 @@ def process_cefr_answer(
         sel_trans = sel_row["translation"] if sel_row else selected_clean
         sel_phon = sel_row["phonetic"] if sel_row else ""
         sel_pos = sel_row["pos"] if sel_row else ""
+        sel_ctx = resolve_context_sentence_for_term(conn, selected_clean)
 
         selected_card_info = _ingest_card_for_user(
             conn,
@@ -2138,7 +2199,7 @@ def process_cefr_answer(
             translation=sel_trans,
             phonetic=sel_phon,
             pos=sel_pos,
-            context_sentence=sentence_with_term,
+            context_sentence=sel_ctx,
             explanation=f"Выбрано по ошибке в вопросе про '{target_term}'",
             level=level,
             tag_suffix="test-mistake-chosen",
@@ -2193,11 +2254,14 @@ def process_cefr_answer(
         if not is_correct and (is_target or was_selected):
             is_auto_added = True
 
+        opt_ctx = resolve_context_sentence_for_term(conn, opt_clean, sentence_with_term if is_target else "")
+
         options_breakdown.append({
             "term": opt_clean,
             "translation": opt_trans,
             "phonetic": opt_phon,
             "pos": opt_pos,
+            "context_sentence": opt_ctx,
             "is_correct": is_target,
             "was_selected": was_selected,
             "in_library": bool(user_card) or is_auto_added,
@@ -2287,8 +2351,11 @@ def _ingest_card_for_user(
         existing_tags.add(tag_suffix)
         updated_tags = ", ".join(sorted(existing_tags))
 
-        updated_context = existing_card["context_sentence"] or ""
+        updated_context = (existing_card["context_sentence"] or "").strip()
-        if context_sentence and context_sentence not in updated_context:
+        # If existing context has "___" or lacks the term, overwrite with new authentic context!
+        if ("___" in updated_context or not re.search(rf"\b{re.escape(term_clean)}\b", updated_context, re.IGNORECASE)) and context_sentence:
+            updated_context = context_sentence
+        elif context_sentence and context_sentence not in updated_context:
             updated_context = f"{updated_context}\n\n{context_sentence}".strip()
 
         conn.execute(
@@ -2337,6 +2404,8 @@ def ingest_cefr_word(
     clean_term = term.strip()
     clean_level = level.upper().strip()
 
+    resolved_ctx = resolve_context_sentence_for_term(conn, clean_term, context_sentence)
+
     row = conn.execute(
         "SELECT term, translation, phonetic, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE",
         (clean_term,),
@@ -2352,7 +2421,7 @@ def ingest_cefr_word(
         translation=trans,
         phonetic=phon,
         pos=pos,
-        context_sentence=context_sentence,
+        context_sentence=resolved_ctx,
         explanation=f"Добавлено вручную из вариантов вопроса теста CEFR ({clean_level})",
         level=clean_level,
         tag_suffix="manual-test-add",
@@ -2364,6 +2433,7 @@ def ingest_cefr_word(
         "term": clean_term,
         "translation": trans,
         "phonetic": phon,
+        "context_sentence": resolved_ctx,
     }
 
 

app/database.py

@@ -379,6 +379,43 @@ def _migrate(conn: sqlite3.Connection):
     conn.execute("CREATE INDEX IF NOT EXISTS idx_bg_tasks_user_status ON background_tasks(user_id, status);")
     conn.execute("CREATE INDEX IF NOT EXISTS idx_bg_tasks_status ON background_tasks(status);")
 
+    # Repair legacy cards with placeholder blanks or mismatched context
+    repair_mismatched_cefr_contexts(conn)
+
+
+def repair_mismatched_cefr_contexts(conn: sqlite3.Connection):
+    """
+    Repairs legacy vocabulary cards that were imported with placeholder blanks ('___')
+    or with mismatched contexts from CEFR test questions.
+    """
+    try:
+        from app.cefr import resolve_context_sentence_for_term
+
+        # Check if table exists
+        tbl = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='vocabulary_cards'").fetchone()
+        if not tbl:
+            return
+
+        rows = conn.execute(
+            """
+            SELECT id, term, context_sentence
+            FROM vocabulary_cards
+            WHERE instr(context_sentence, '___') > 0
+            """
+        ).fetchall()
+
+        for r in rows:
+            card_id = r["id"]
+            term = r["term"]
+            old_ctx = r["context_sentence"] or ""
+            new_ctx = resolve_context_sentence_for_term(conn, term, old_ctx)
+            if new_ctx and new_ctx != old_ctx:
+                conn.execute(
+                    "UPDATE vocabulary_cards SET context_sentence = ? WHERE id = ?",
+                    (new_ctx, card_id),
+                )
+    except Exception:
+        pass
 
 
 def _seed_offline_dictionary(conn: sqlite3.Connection):

app/static/js/modules/cefr.js

@@ -676,7 +676,7 @@ function renderOptionsBreakdown(breakdown) {
                     } else if (inLib) {
                         actionHtml = `<span class="wb-badge badge-in-lib"><i class="ph-check"></i> В библиотеке</span>`;
                     } else {
-                        actionHtml = `<button type="button" class="btn btn-xs btn-outline btn-add-option-word" data-term="${escapeHtml(b.term)}">+ В словарь</button>`;
+                        actionHtml = `<button type="button" class="btn btn-xs btn-outline btn-add-option-word" data-term="${escapeHtml(b.term)}" data-context="${escapeHtml(b.context_sentence || '')}">+ В словарь</button>`;
                     }
 
                     return `
@@ -708,7 +708,8 @@ function attachBreakdownListeners(container, contextSentence) {
             btn.innerHTML = '...';
 
             try {
-                await api.cefr.addWord(currentLevel, term, contextSentence || '');
+                const termContext = btn.dataset.context || contextSentence || '';
+                await api.cefr.addWord(currentLevel, term, termContext);
                 sessionStats.ingested++;
                 btn.outerHTML = '<span class="wb-badge badge-just-added"><i class="ph-check-circle-fill"></i> Добавлено!</span>';
                 showToast(`Слово «${term}» добавлено в утренний тренажёр SRS!`, 'success');

app/static/sw.js

@@ -1,7 +1,7 @@
 // Service Worker for Notes English PWA (Stage 2)
 // Strategy: Cache-first for static assets, Network-first for API calls.
 
-const CACHE_NAME = 'notes-v5.8';
+const CACHE_NAME = 'notes-v5.9';
 const OFFLINE_URL = '/';
 
 const PRECACHE_URLS = [

tests/test_api.py

@@ -1756,6 +1756,118 @@ Authors' Addresses
     assert rfc_books[0]["id"] == book_id
 
 
+def test_cefr_context_resolution_and_repair():
+    from app.cefr import resolve_context_sentence_for_term, process_cefr_answer, ingest_cefr_word
+    from app.database import repair_mismatched_cefr_contexts
+
+    with get_db() as conn:
+        # 1. Test resolve_context_sentence_for_term
+        # Case A: candidate has the term and placeholder '___'
+        res_a = resolve_context_sentence_for_term(
+            conn,
+            "decision",
+            "Choosing a career path is a major life ___."
+        )
+        assert res_a == "Choosing a career path is a major life decision."
+        assert "___" not in res_a
+
+        # Case B: candidate does NOT match the term, term exists in cefr_questions
+        res_b = resolve_context_sentence_for_term(
+            conn,
+            "opportunity",
+            "Choosing a career path is a major life ___."
+        )
+        assert "opportunity" in res_b.lower()
+        assert "career path" not in res_b
+        assert "___" not in res_b
+
+        # 2. Test process_cefr_answer mistake ingestion
+        # Seed test user
+        user_row = conn.execute("SELECT id FROM users WHERE username = 'testadmin'").fetchone()
+        if not user_row:
+            create_user("testadmin", "testpass12345")
+            user_row = conn.execute("SELECT id FROM users WHERE username = 'testadmin'").fetchone()
+        assert user_row is not None
+        user_id = user_row["id"]
+
+        # Find or create a question for 'decision'
+        q = conn.execute(
+            "SELECT id, level, term, sentence, options_json FROM cefr_questions WHERE term = 'decision' LIMIT 1"
+        ).fetchone()
+        if not q:
+            conn.execute(
+                """
+                INSERT INTO cefr_questions (level, term, translation, sentence, options_json, explanation)
+                VALUES ('B1', 'decision', 'решение', 'Choosing a career path is a major life ___.', '["decision", "opportunity", "behavior", "relationship"]', 'Major decision.')
+                """
+            )
+            q = conn.execute("SELECT id, level, term, sentence, options_json FROM cefr_questions WHERE term = 'decision' LIMIT 1").fetchone()
+
+        result = process_cefr_answer(conn, user_id, q["id"], "opportunity")
+        assert result["correct"] is False
+        assert result["correct_option"] == "decision"
+        assert result["selected_option"] == "opportunity"
+
+        # Check options_breakdown has distinct authentic context sentences
+        breakdown = result["options_breakdown"]
+        assert len(breakdown) == 4
+        for item in breakdown:
+            assert "context_sentence" in item
+            assert "___" not in item["context_sentence"]
+            assert item["term"].lower() in item["context_sentence"].lower()
+
+        # Check vocabulary_cards in DB:
+        target_card = conn.execute(
+            "SELECT * FROM vocabulary_cards WHERE user_id = ? AND lower(term) = 'decision'",
+            (user_id,)
+        ).fetchone()
+        assert target_card is not None
+        assert "decision" in target_card["context_sentence"].lower()
+        assert "___" not in target_card["context_sentence"]
+
+        chosen_card = conn.execute(
+            "SELECT * FROM vocabulary_cards WHERE user_id = ? AND lower(term) = 'opportunity'",
+            (user_id,)
+        ).fetchone()
+        assert chosen_card is not None
+        assert "opportunity" in chosen_card["context_sentence"].lower()
+        assert "career path" not in chosen_card["context_sentence"]
+        assert "___" not in chosen_card["context_sentence"]
+
+        # 3. Test ingest_cefr_word
+        added_word = ingest_cefr_word(
+            conn,
+            user_id=user_id,
+            level="B1",
+            term="behavior",
+            context_sentence="Choosing a career path is a major life ___."
+        )
+        assert added_word["term"] == "behavior"
+        assert "behavior" in added_word["context_sentence"].lower()
+        assert "career path" not in added_word["context_sentence"]
+        assert "___" not in added_word["context_sentence"]
+
+        # 4. Test repair_mismatched_cefr_contexts
+        # Insert a card with corrupted blank
+        conn.execute("DELETE FROM vocabulary_cards WHERE user_id = ? AND lower(term) = 'negotiate'", (user_id,))
+        conn.execute(
+            """
+            INSERT INTO vocabulary_cards (user_id, term, translation, context_sentence, tags)
+            VALUES (?, 'negotiate', 'вести переговоры', 'With persistent practice, you can ___ fluency in English.', 'cefr-test')
+            """,
+            (user_id,)
+        )
+        corrupted = conn.execute("SELECT context_sentence FROM vocabulary_cards WHERE user_id = ? AND term = 'negotiate'", (user_id,)).fetchone()
+        assert "___" in corrupted["context_sentence"]
+
+        repair_mismatched_cefr_contexts(conn)
+
+        repaired = conn.execute("SELECT context_sentence FROM vocabulary_cards WHERE user_id = ? AND term = 'negotiate'", (user_id,)).fetchone()
+        assert "___" not in repaired["context_sentence"]
+        assert "negotiate" in repaired["context_sentence"].lower()
+
+
+