From 3098d3016d3d8be5cbcd74fe60689abd51c3e900 Mon Sep 17 00:00:00 2001 From: Antigravity Date: Tue, 6 Oct 2026 22:17:51 +0300 Subject: [PATCH] fix(cefr): resolve authentic context sentence per term and repair legacy SRS cards - Add resolve_context_sentence_for_term to replace placeholder blanks and lookup authentic sentences from cefr_questions or books - Fix dual mistake ingestion in process_cefr_answer to assign distinct context to target and chosen distractor - Provide authentic context_sentence in options_breakdown and pass it via data-context in cefr.js - Add repair_mismatched_cefr_contexts in database.py to automatically repair cards with placeholder blanks - Bump Service Worker cache to notes-v5.9 - Add test_cefr_context_resolution_and_repair covering resolution, dual ingestion, manual add, and DB repair --- app/cefr.py | 84 ++++++++++++++++++++++--- app/database.py | 37 +++++++++++ app/static/js/modules/cefr.js | 5 +- app/static/sw.js | 2 +- tests/test_api.py | 112 ++++++++++++++++++++++++++++++++++ 5 files changed, 230 insertions(+), 10 deletions(-) diff --git a/app/cefr.py b/app/cefr.py index 6bdb085..e6f0c9a 100644 --- a/app/cefr.py +++ b/app/cefr.py @@ -9,6 +9,7 @@ Features: - Per-level progress tracking (mastered, learning, percentage). """ +import re import json import sqlite3 import random @@ -2039,6 +2040,64 @@ def get_cefr_quiz_batch( return quiz_items +def resolve_context_sentence_for_term( + conn: sqlite3.Connection, + term: str, + candidate_sentence: str = "", +) -> str: + """ + Resolves the best authentic context sentence for a given term: + 1. If candidate_sentence contains the term, use it (with ___ replaced). + 2. Look up in cefr_questions specifically for this term. + 3. Look up in books table for an authentic sentence containing this term. + 4. Fallback to clean natural sentence. + """ + clean_term = term.strip() + if not clean_term: + return "" + + # 1. If candidate_sentence already contains this term (case-insensitive word boundary) + if candidate_sentence: + cand_clean = candidate_sentence.strip() + if re.search(rf"\b{re.escape(clean_term)}\b", cand_clean, re.IGNORECASE): + return re.sub(r"\s+", " ", cand_clean.replace("___", clean_term)).strip() + + # 2. Look up in cefr_questions specifically for this term + try: + q_row = conn.execute( + "SELECT sentence FROM cefr_questions WHERE lower(trim(term)) = lower(trim(?)) ORDER BY id ASC LIMIT 1", + (clean_term,), + ).fetchone() + if q_row and q_row["sentence"]: + return re.sub(r"\s+", " ", q_row["sentence"].replace("___", clean_term)).strip() + except Exception: + pass + + # 3. Look up in books table + try: + books = conn.execute("SELECT chapters_json FROM books ORDER BY id DESC LIMIT 10").fetchall() + for b in books: + try: + chapters = json.loads(b["chapters_json"]) + except Exception: + continue + for ch in chapters: + content = ch.get("content", "") + if clean_term.lower() in content.lower(): + for s in re.split(r"(?<=[.!?])\s+", content): + s_clean = s.strip() + if 20 <= len(s_clean) <= 180 and re.search(rf"\b{re.escape(clean_term)}\b", s_clean, re.IGNORECASE): + return re.sub(r"\s+", " ", s_clean).strip() + except Exception: + pass + + # 4. If candidate_sentence had '___' and came from this term + if candidate_sentence and "___" in candidate_sentence: + return re.sub(r"\s+", " ", candidate_sentence.replace("___", clean_term)).strip() + + return f"Using '{clean_term}' in meaningful sentences reinforces active recall and long-term fluency." + + def process_cefr_answer( conn: sqlite3.Connection, user_id: int, @@ -2105,9 +2164,10 @@ def process_cefr_answer( if not is_correct: # ===================================================================== # DUAL AUTOMATED GAP INGESTION (Crucial Pedagogical Requirement) - # 1. Correct target word -> goes 100% to vocabulary - # 2. Chosen wrong word -> also goes to vocabulary! + # 1. Correct target word -> goes 100% to vocabulary with its sentence + # 2. Chosen wrong word -> also goes to vocabulary with its OWN context! # ===================================================================== + target_ctx = resolve_context_sentence_for_term(conn, target_term, sentence_with_term) target_card_info = _ingest_card_for_user( conn, user_id=user_id, @@ -2115,7 +2175,7 @@ def process_cefr_answer( translation=translation, phonetic=phonetic, pos=pos, - context_sentence=sentence_with_term, + context_sentence=target_ctx, explanation=explanation, level=level, tag_suffix="test-mistake", @@ -2130,6 +2190,7 @@ def process_cefr_answer( sel_trans = sel_row["translation"] if sel_row else selected_clean sel_phon = sel_row["phonetic"] if sel_row else "" sel_pos = sel_row["pos"] if sel_row else "" + sel_ctx = resolve_context_sentence_for_term(conn, selected_clean) selected_card_info = _ingest_card_for_user( conn, @@ -2138,7 +2199,7 @@ def process_cefr_answer( translation=sel_trans, phonetic=sel_phon, pos=sel_pos, - context_sentence=sentence_with_term, + context_sentence=sel_ctx, explanation=f"Выбрано по ошибке в вопросе про '{target_term}'", level=level, tag_suffix="test-mistake-chosen", @@ -2193,11 +2254,14 @@ def process_cefr_answer( if not is_correct and (is_target or was_selected): is_auto_added = True + opt_ctx = resolve_context_sentence_for_term(conn, opt_clean, sentence_with_term if is_target else "") + options_breakdown.append({ "term": opt_clean, "translation": opt_trans, "phonetic": opt_phon, "pos": opt_pos, + "context_sentence": opt_ctx, "is_correct": is_target, "was_selected": was_selected, "in_library": bool(user_card) or is_auto_added, @@ -2287,8 +2351,11 @@ def _ingest_card_for_user( existing_tags.add(tag_suffix) updated_tags = ", ".join(sorted(existing_tags)) - updated_context = existing_card["context_sentence"] or "" - if context_sentence and context_sentence not in updated_context: + updated_context = (existing_card["context_sentence"] or "").strip() + # If existing context has "___" or lacks the term, overwrite with new authentic context! + if ("___" in updated_context or not re.search(rf"\b{re.escape(term_clean)}\b", updated_context, re.IGNORECASE)) and context_sentence: + updated_context = context_sentence + elif context_sentence and context_sentence not in updated_context: updated_context = f"{updated_context}\n\n{context_sentence}".strip() conn.execute( @@ -2337,6 +2404,8 @@ def ingest_cefr_word( clean_term = term.strip() clean_level = level.upper().strip() + resolved_ctx = resolve_context_sentence_for_term(conn, clean_term, context_sentence) + row = conn.execute( "SELECT term, translation, phonetic, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE", (clean_term,), @@ -2352,7 +2421,7 @@ def ingest_cefr_word( translation=trans, phonetic=phon, pos=pos, - context_sentence=context_sentence, + context_sentence=resolved_ctx, explanation=f"Добавлено вручную из вариантов вопроса теста CEFR ({clean_level})", level=clean_level, tag_suffix="manual-test-add", @@ -2364,6 +2433,7 @@ def ingest_cefr_word( "term": clean_term, "translation": trans, "phonetic": phon, + "context_sentence": resolved_ctx, } diff --git a/app/database.py b/app/database.py index 0f3a744..ee3d094 100644 --- a/app/database.py +++ b/app/database.py @@ -379,6 +379,43 @@ def _migrate(conn: sqlite3.Connection): conn.execute("CREATE INDEX IF NOT EXISTS idx_bg_tasks_user_status ON background_tasks(user_id, status);") conn.execute("CREATE INDEX IF NOT EXISTS idx_bg_tasks_status ON background_tasks(status);") + # Repair legacy cards with placeholder blanks or mismatched context + repair_mismatched_cefr_contexts(conn) + + +def repair_mismatched_cefr_contexts(conn: sqlite3.Connection): + """ + Repairs legacy vocabulary cards that were imported with placeholder blanks ('___') + or with mismatched contexts from CEFR test questions. + """ + try: + from app.cefr import resolve_context_sentence_for_term + + # Check if table exists + tbl = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='vocabulary_cards'").fetchone() + if not tbl: + return + + rows = conn.execute( + """ + SELECT id, term, context_sentence + FROM vocabulary_cards + WHERE instr(context_sentence, '___') > 0 + """ + ).fetchall() + + for r in rows: + card_id = r["id"] + term = r["term"] + old_ctx = r["context_sentence"] or "" + new_ctx = resolve_context_sentence_for_term(conn, term, old_ctx) + if new_ctx and new_ctx != old_ctx: + conn.execute( + "UPDATE vocabulary_cards SET context_sentence = ? WHERE id = ?", + (new_ctx, card_id), + ) + except Exception: + pass def _seed_offline_dictionary(conn: sqlite3.Connection): diff --git a/app/static/js/modules/cefr.js b/app/static/js/modules/cefr.js index 5778092..f5ccd9e 100644 --- a/app/static/js/modules/cefr.js +++ b/app/static/js/modules/cefr.js @@ -676,7 +676,7 @@ function renderOptionsBreakdown(breakdown) { } else if (inLib) { actionHtml = ` В библиотеке`; } else { - actionHtml = ``; + actionHtml = ``; } return ` @@ -708,7 +708,8 @@ function attachBreakdownListeners(container, contextSentence) { btn.innerHTML = '...'; try { - await api.cefr.addWord(currentLevel, term, contextSentence || ''); + const termContext = btn.dataset.context || contextSentence || ''; + await api.cefr.addWord(currentLevel, term, termContext); sessionStats.ingested++; btn.outerHTML = ' Добавлено!'; showToast(`Слово «${term}» добавлено в утренний тренажёр SRS!`, 'success'); diff --git a/app/static/sw.js b/app/static/sw.js index 4ac9e4c..da07bde 100644 --- a/app/static/sw.js +++ b/app/static/sw.js @@ -1,7 +1,7 @@ // Service Worker for Notes English PWA (Stage 2) // Strategy: Cache-first for static assets, Network-first for API calls. -const CACHE_NAME = 'notes-v5.8'; +const CACHE_NAME = 'notes-v5.9'; const OFFLINE_URL = '/'; const PRECACHE_URLS = [ diff --git a/tests/test_api.py b/tests/test_api.py index 1047352..38ef067 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1756,6 +1756,118 @@ Authors' Addresses assert rfc_books[0]["id"] == book_id +def test_cefr_context_resolution_and_repair(): + from app.cefr import resolve_context_sentence_for_term, process_cefr_answer, ingest_cefr_word + from app.database import repair_mismatched_cefr_contexts + + with get_db() as conn: + # 1. Test resolve_context_sentence_for_term + # Case A: candidate has the term and placeholder '___' + res_a = resolve_context_sentence_for_term( + conn, + "decision", + "Choosing a career path is a major life ___." + ) + assert res_a == "Choosing a career path is a major life decision." + assert "___" not in res_a + + # Case B: candidate does NOT match the term, term exists in cefr_questions + res_b = resolve_context_sentence_for_term( + conn, + "opportunity", + "Choosing a career path is a major life ___." + ) + assert "opportunity" in res_b.lower() + assert "career path" not in res_b + assert "___" not in res_b + + # 2. Test process_cefr_answer mistake ingestion + # Seed test user + user_row = conn.execute("SELECT id FROM users WHERE username = 'testadmin'").fetchone() + if not user_row: + create_user("testadmin", "testpass12345") + user_row = conn.execute("SELECT id FROM users WHERE username = 'testadmin'").fetchone() + assert user_row is not None + user_id = user_row["id"] + + # Find or create a question for 'decision' + q = conn.execute( + "SELECT id, level, term, sentence, options_json FROM cefr_questions WHERE term = 'decision' LIMIT 1" + ).fetchone() + if not q: + conn.execute( + """ + INSERT INTO cefr_questions (level, term, translation, sentence, options_json, explanation) + VALUES ('B1', 'decision', 'решение', 'Choosing a career path is a major life ___.', '["decision", "opportunity", "behavior", "relationship"]', 'Major decision.') + """ + ) + q = conn.execute("SELECT id, level, term, sentence, options_json FROM cefr_questions WHERE term = 'decision' LIMIT 1").fetchone() + + result = process_cefr_answer(conn, user_id, q["id"], "opportunity") + assert result["correct"] is False + assert result["correct_option"] == "decision" + assert result["selected_option"] == "opportunity" + + # Check options_breakdown has distinct authentic context sentences + breakdown = result["options_breakdown"] + assert len(breakdown) == 4 + for item in breakdown: + assert "context_sentence" in item + assert "___" not in item["context_sentence"] + assert item["term"].lower() in item["context_sentence"].lower() + + # Check vocabulary_cards in DB: + target_card = conn.execute( + "SELECT * FROM vocabulary_cards WHERE user_id = ? AND lower(term) = 'decision'", + (user_id,) + ).fetchone() + assert target_card is not None + assert "decision" in target_card["context_sentence"].lower() + assert "___" not in target_card["context_sentence"] + + chosen_card = conn.execute( + "SELECT * FROM vocabulary_cards WHERE user_id = ? AND lower(term) = 'opportunity'", + (user_id,) + ).fetchone() + assert chosen_card is not None + assert "opportunity" in chosen_card["context_sentence"].lower() + assert "career path" not in chosen_card["context_sentence"] + assert "___" not in chosen_card["context_sentence"] + + # 3. Test ingest_cefr_word + added_word = ingest_cefr_word( + conn, + user_id=user_id, + level="B1", + term="behavior", + context_sentence="Choosing a career path is a major life ___." + ) + assert added_word["term"] == "behavior" + assert "behavior" in added_word["context_sentence"].lower() + assert "career path" not in added_word["context_sentence"] + assert "___" not in added_word["context_sentence"] + + # 4. Test repair_mismatched_cefr_contexts + # Insert a card with corrupted blank + conn.execute("DELETE FROM vocabulary_cards WHERE user_id = ? AND lower(term) = 'negotiate'", (user_id,)) + conn.execute( + """ + INSERT INTO vocabulary_cards (user_id, term, translation, context_sentence, tags) + VALUES (?, 'negotiate', 'вести переговоры', 'With persistent practice, you can ___ fluency in English.', 'cefr-test') + """, + (user_id,) + ) + corrupted = conn.execute("SELECT context_sentence FROM vocabulary_cards WHERE user_id = ? AND term = 'negotiate'", (user_id,)).fetchone() + assert "___" in corrupted["context_sentence"] + + repair_mismatched_cefr_contexts(conn) + + repaired = conn.execute("SELECT context_sentence FROM vocabulary_cards WHERE user_id = ? AND term = 'negotiate'", (user_id,)).fetchone() + assert "___" not in repaired["context_sentence"] + assert "negotiate" in repaired["context_sentence"].lower() + + +