266 lines
11 KiB
Python
266 lines
11 KiB
Python
"""
|
|
Phrasal Verb & Multi-Word Collocation Detector.
|
|
Analyzes context sentence tokens around a clicked word to identify
|
|
phrasal verbs (e.g. 'take off', 'look forward to', 'turn it off') and idioms.
|
|
100% offline, zero-dependency, pure Python stdlib + SQLite.
|
|
"""
|
|
|
|
import re
|
|
import sqlite3
|
|
from typing import Optional, Dict, Any, List
|
|
from app.database import get_db
|
|
|
|
IRREGULARS = {
|
|
"took": "take", "taken": "take", "takes": "take", "taking": "take",
|
|
"gave": "give", "given": "give", "gives": "give", "giving": "give",
|
|
"got": "get", "gotten": "get", "gets": "get", "getting": "get",
|
|
"went": "go", "gone": "go", "goes": "go", "going": "go",
|
|
"came": "come", "comes": "come", "coming": "come",
|
|
"made": "make", "makes": "make", "making": "make",
|
|
"put": "put", "puts": "put", "putting": "put",
|
|
"ran": "run", "runs": "run", "running": "run",
|
|
"held": "hold", "holds": "hold", "holding": "hold",
|
|
"brought": "bring", "brings": "bring", "bringing": "bring",
|
|
"broke": "break", "broken": "break", "breaks": "break", "breaking": "break",
|
|
"fell": "fall", "fallen": "fall", "falls": "fall", "falling": "fall",
|
|
"kept": "keep", "keeps": "keep", "keeping": "keep",
|
|
"stood": "stand", "stands": "stand", "standing": "stand",
|
|
"drew": "draw", "drawn": "draw", "draws": "draw", "drawing": "draw",
|
|
"found": "find", "finds": "find", "finding": "find",
|
|
"blew": "blow", "blown": "blow", "blows": "blow", "blowing": "blow",
|
|
"hung": "hang", "hangs": "hang", "hanging": "hang",
|
|
"caught": "catch", "catches": "catch", "catching": "catch",
|
|
"threw": "throw", "thrown": "throw", "throws": "throw", "throwing": "throw",
|
|
"told": "tell", "tells": "tell", "telling": "tell",
|
|
"left": "leave", "leaves": "leave", "leaving": "leave",
|
|
"grew": "grow", "grown": "grow", "grows": "grow", "growing": "grow",
|
|
"paid": "pay", "pays": "pay", "paying": "pay",
|
|
"wound": "wind", "winds": "wind", "winding": "wind",
|
|
"built": "build", "builds": "build", "building": "build",
|
|
"cut": "cut", "cuts": "cut", "cutting": "cut",
|
|
"shut": "shut", "shuts": "shut", "shutting": "shut",
|
|
"set": "set", "sets": "set", "setting": "set",
|
|
"split": "split", "splits": "split", "splitting": "split",
|
|
"spread": "spread", "spreads": "spread", "spreading": "spread",
|
|
}
|
|
|
|
PRONOUNS_AND_OBJECTS = {
|
|
"it", "him", "her", "them", "us", "me", "you", "this", "that", "these", "those",
|
|
"things", "something", "everything", "anything", "all"
|
|
}
|
|
|
|
PHRASAL_PARTICLES = {
|
|
"about", "across", "ahead", "along", "apart", "around", "round", "aside",
|
|
"at", "away", "back", "by", "down", "for", "forth", "forward", "in",
|
|
"into", "off", "on", "onto", "out", "over", "past", "round", "through",
|
|
"to", "together", "under", "up", "upon", "with"
|
|
}
|
|
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from app import config
|
|
|
|
_CURATED_PHRASAL_VERBS_CACHE: Optional[Dict[str, str]] = None
|
|
|
|
def get_curated_phrasal_verbs() -> Dict[str, str]:
|
|
global _CURATED_PHRASAL_VERBS_CACHE
|
|
if _CURATED_PHRASAL_VERBS_CACHE is None:
|
|
seeds_file = config.BASE_DIR / "data" / "seeds" / "phrasal_verbs.json"
|
|
if seeds_file.is_file():
|
|
try:
|
|
with open(seeds_file, "r", encoding="utf-8") as f:
|
|
_CURATED_PHRASAL_VERBS_CACHE = json.load(f)
|
|
except Exception:
|
|
_CURATED_PHRASAL_VERBS_CACHE = {}
|
|
else:
|
|
_CURATED_PHRASAL_VERBS_CACHE = {}
|
|
return _CURATED_PHRASAL_VERBS_CACHE
|
|
|
|
CURATED_PHRASAL_VERBS: Dict[str, str] = get_curated_phrasal_verbs()
|
|
|
|
|
|
|
|
def lemmatize_word(w: str) -> List[str]:
|
|
"""Generates candidate base forms for English verbs and nouns."""
|
|
w = w.lower()
|
|
res = [w]
|
|
if w in IRREGULARS:
|
|
res.append(IRREGULARS[w])
|
|
|
|
if w.endswith("ing") and len(w) > 4:
|
|
stem = w[:-3]
|
|
res.extend([stem, stem + "e"])
|
|
if len(stem) > 2 and stem[-1] == stem[-2]:
|
|
res.append(stem[:-1])
|
|
elif w.endswith("ed") and len(w) > 3:
|
|
stem = w[:-2]
|
|
res.extend([stem, stem + "e"])
|
|
if stem.endswith("i"):
|
|
res.append(stem[:-1] + "y")
|
|
if len(stem) > 2 and stem[-1] == stem[-2]:
|
|
res.append(stem[:-1])
|
|
elif w.endswith("s") and len(w) > 3:
|
|
if w.endswith("es"):
|
|
res.extend([w[:-2], w[:-1]])
|
|
if w.endswith("ies"):
|
|
res.append(w[:-3] + "y")
|
|
else:
|
|
res.append(w[:-1])
|
|
|
|
# De-duplicate preserving order
|
|
return list(dict.fromkeys(res))
|
|
|
|
|
|
def detect_phrasal_verb(
|
|
sentence: str,
|
|
target_word: str,
|
|
user_id: Optional[int] = None,
|
|
conn: Optional[sqlite3.Connection] = None,
|
|
) -> Optional[Dict[str, Any]]:
|
|
"""
|
|
Detects if target_word in sentence is part of a phrasal verb or collocation.
|
|
Checks window sizes 4, 3, 2 and separable phrasal verbs.
|
|
Returns matched term, raw snippet in text, translation, separable flag, and library status.
|
|
"""
|
|
if not sentence or not target_word:
|
|
return None
|
|
|
|
tokens = re.findall(r"\b[a-zA-Z\'-]+\b", sentence)
|
|
lower_tokens = [t.lower() for t in tokens]
|
|
target_lower = target_word.strip().lower()
|
|
|
|
target_indices = [i for i, t in enumerate(lower_tokens) if t == target_lower]
|
|
if not target_indices:
|
|
return None
|
|
|
|
def execute_query(query: str, params: tuple):
|
|
if conn is not None:
|
|
return conn.execute(query, params).fetchone()
|
|
with get_db() as c:
|
|
return c.execute(query, params).fetchone()
|
|
|
|
match_result = None
|
|
|
|
for idx in target_indices:
|
|
# 1. Contiguous spans: 4, 3, 2 words containing target_word
|
|
for span_len in (4, 3, 2):
|
|
for start in range(max(0, idx - span_len + 1), min(len(tokens) - span_len + 1, idx + 1)):
|
|
span_tokens = lower_tokens[start : start + span_len]
|
|
raw_match = " ".join(tokens[start : start + span_len])
|
|
v_lemmas = lemmatize_word(span_tokens[0])
|
|
|
|
for v in v_lemmas:
|
|
cand = " ".join([v] + span_tokens[1:])
|
|
if cand in CURATED_PHRASAL_VERBS:
|
|
match_result = {
|
|
"term": cand,
|
|
"raw": raw_match,
|
|
"translation": CURATED_PHRASAL_VERBS[cand],
|
|
"separable": False,
|
|
"matched_words": tokens[start : start + span_len],
|
|
}
|
|
break
|
|
|
|
# Check offline dictionary only for genuine phrasal verbs:
|
|
# 1. Candidate must end with a known particle (e.g. 'turn on', 'come up with')
|
|
# 2. pos in offline_dictionary must explicitly be a verb / verb phrase
|
|
cand_words = cand.split()
|
|
if len(cand_words) >= 2 and cand_words[-1] in PHRASAL_PARTICLES:
|
|
row = execute_query(
|
|
"SELECT term, translation, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE",
|
|
(cand,),
|
|
)
|
|
if row:
|
|
pos_val = (row[2] or "").lower()
|
|
if "verb" in pos_val:
|
|
match_result = {
|
|
"term": row[0],
|
|
"raw": raw_match,
|
|
"translation": row[1],
|
|
"separable": False,
|
|
"matched_words": tokens[start : start + span_len],
|
|
}
|
|
break
|
|
if match_result:
|
|
break
|
|
if match_result:
|
|
break
|
|
if match_result:
|
|
break
|
|
|
|
# 2. Separable phrasal verbs: verb + pronoun/short object + particle (e.g. turned it off -> turn off)
|
|
for v_idx in range(max(0, idx - 2), min(len(tokens) - 2, idx + 1)):
|
|
if lower_tokens[v_idx + 1] in PRONOUNS_AND_OBJECTS:
|
|
p_idx = v_idx + 2
|
|
if idx in (v_idx, p_idx):
|
|
particle = lower_tokens[p_idx]
|
|
if particle not in PHRASAL_PARTICLES:
|
|
continue
|
|
v_lemmas = lemmatize_word(lower_tokens[v_idx])
|
|
raw_match = " ".join(tokens[v_idx : p_idx + 1])
|
|
for v in v_lemmas:
|
|
cand = f"{v} {particle}"
|
|
if cand in CURATED_PHRASAL_VERBS:
|
|
match_result = {
|
|
"term": cand,
|
|
"raw": raw_match,
|
|
"translation": CURATED_PHRASAL_VERBS[cand],
|
|
"separable": True,
|
|
"matched_words": [tokens[v_idx], tokens[p_idx]],
|
|
}
|
|
break
|
|
row = execute_query(
|
|
"SELECT term, translation, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE",
|
|
(cand,),
|
|
)
|
|
if row:
|
|
pos_val = (row[2] or "").lower()
|
|
if "verb" in pos_val:
|
|
match_result = {
|
|
"term": row[0],
|
|
"raw": raw_match,
|
|
"translation": row[1],
|
|
"separable": True,
|
|
"matched_words": [tokens[v_idx], tokens[p_idx]],
|
|
}
|
|
break
|
|
if match_result:
|
|
break
|
|
if match_result:
|
|
break
|
|
|
|
if not match_result:
|
|
return None
|
|
|
|
# Check if this phrasal verb already exists in user's vocabulary
|
|
match_result["in_library"] = False
|
|
match_result["library_card"] = None
|
|
|
|
if user_id is not None:
|
|
row = execute_query(
|
|
"SELECT id, term, translation, context_sentence, tags, collection, srs_state, srs_reps "
|
|
"FROM vocabulary_cards WHERE user_id = ? AND lower(trim(term)) = lower(trim(?))",
|
|
(user_id, match_result["term"]),
|
|
)
|
|
if row:
|
|
match_result["in_library"] = True
|
|
match_result["library_card"] = dict(row)
|
|
|
|
return match_result
|
|
|
|
|
|
def seed_curated_phrasal_verbs(conn: sqlite3.Connection):
|
|
"""Ensures all curated phrasal verbs are present in offline_dictionary table."""
|
|
try:
|
|
for term, trans in CURATED_PHRASAL_VERBS.items():
|
|
conn.execute(
|
|
"""
|
|
INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos)
|
|
VALUES (?, ?, '', 'verb phrase')
|
|
""",
|
|
(term, trans),
|
|
)
|
|
except Exception:
|
|
pass
|