fix(ai,reader): robust sentence segmentation without abbreviation cutoffs & context-aware literary translation
- Implement robustSentenceSplit across reader.js and backend (miner.py, cefr.py) to prevent false cutoffs on honorifics (Mr., Mrs., Dr.), single-letter initials (John F. Kennedy), decimals, acronyms (U.S., U.K.), and abbreviations (No. 4, etc.) - Upgrade translation prompt to professional literary Russian, eliminating literal word-for-word translation and preserving titles/honorifics - Inject surrounding paragraph context into translation API to avoid out-of-context translations - Inject offline dictionary hints into LLM prompt for accurate contextual word meaning - Prefer qwen2.5:3b for higher translation fluency and fidelity - Purge truncated cache entries from sentence_translations and restore Card 30 (briefcase) with authentic full sentence - Bump Service Worker cache to notes-v6.0 - Add unit tests for robust sentence splitting, prompt formatting, and translation API
This commit is contained in:
1 parent
3098d3016d
commit
532e2ec3b2
11 files changed
+313
-89
No files matched your search
@@ -1867,6 +1867,94 @@ def test_cefr_context_resolution_and_repair():
|
||||
assert "negotiate" in repaired["context_sentence"].lower()
|
||||
|
||||
|
||||
def test_robust_sentence_split_and_ai_translation(monkeypatch):
|
||||
from app.miner import robust_sentence_split
|
||||
from app.ai import build_translation_prompt
|
||||
|
||||
# 1. Test robust sentence segmentation
|
||||
# Case A: Initials in names (John F. Kennedy)
|
||||
s_jfk = "Perhaps the most famous is the John F. Kennedy International Airport in New York, but there is also the John F. Kennedy Space Center."
|
||||
splits_jfk = robust_sentence_split(s_jfk)
|
||||
assert len(splits_jfk) == 1
|
||||
assert "John F. Kennedy International Airport" in splits_jfk[0]
|
||||
|
||||
# Case B: Honorifics with dots (Mr., Mrs., Dr., etc.)
|
||||
s_dursley = "When Mr. and Mrs. Dursley woke up on the dull, gray Tuesday our story starts, there was nothing about the cloudy sky outside to suggest that strange things would happen."
|
||||
splits_dursley = robust_sentence_split(s_dursley)
|
||||
assert len(splits_dursley) == 1
|
||||
assert "Mr. and Mrs. Dursley" in splits_dursley[0]
|
||||
|
||||
# Case C: Abbreviations (No. 4, U.S. Navy)
|
||||
s_no4 = "They lived at No. 4 Privet Drive, which was a very quiet street. Next sentence starts here."
|
||||
splits_no4 = robust_sentence_split(s_no4)
|
||||
assert len(splits_no4) == 2
|
||||
assert splits_no4[0] == "They lived at No. 4 Privet Drive, which was a very quiet street."
|
||||
assert splits_no4[1] == "Next sentence starts here."
|
||||
|
||||
s_us = "The men thought that the U.S. Navy would send a plane to look for them. They waited patiently."
|
||||
splits_us = robust_sentence_split(s_us)
|
||||
assert len(splits_us) == 2
|
||||
assert "U.S. Navy" in splits_us[0]
|
||||
|
||||
# Case D: Complex paragraph with Mr., Mrs., dialogue
|
||||
s_hp = (
|
||||
"At half past eight, Mr. Dursley picked up his briefcase, pecked Mrs. Dursley on the cheek, "
|
||||
"and tried to kiss Dudley good-bye but missed, because Dudley was now having a tantrum and "
|
||||
"throwing his cereal at the walls. “Little tyke,” chortled Mr. Dursley as he left the house."
|
||||
)
|
||||
splits_hp = robust_sentence_split(s_hp)
|
||||
assert len(splits_hp) == 2
|
||||
assert "pecked Mrs. Dursley on the cheek" in splits_hp[0]
|
||||
assert splits_hp[0].startswith("At half past eight, Mr. Dursley")
|
||||
assert "Little tyke" in splits_hp[1]
|
||||
|
||||
# 2. Test build_translation_prompt
|
||||
prompt = build_translation_prompt(
|
||||
sentence="At half past eight, Mr. Dursley picked up his briefcase.",
|
||||
target_word="briefcase",
|
||||
context="Full context paragraph here.",
|
||||
dict_hint="портфель, кейс, дипломат",
|
||||
)
|
||||
assert "Контекст абзаца:" in prompt
|
||||
assert "Full context paragraph here." in prompt
|
||||
assert "briefcase" in prompt
|
||||
assert "портфель, кейс, дипломат" in prompt
|
||||
assert "Mr. -> мистер" in prompt
|
||||
assert "НЕ дословным, без топорного калькирования" in prompt
|
||||
|
||||
# 3. Test API endpoint /api/translate-sentence
|
||||
# Authenticate client
|
||||
login_resp = client.post("/api/login", json={"username": "testadmin", "password": "testpass12345"})
|
||||
assert login_resp.status_code == 200
|
||||
|
||||
# Mock translate_sentence_ai
|
||||
import app.main as main_mod
|
||||
|
||||
async def mock_translate_sentence_ai(sentence, target_word="", context=""):
|
||||
return {
|
||||
"original_sentence": sentence,
|
||||
"sentence_ru": "В половине девятого мистер Дурсли взял свой портфель.",
|
||||
"target_word": target_word,
|
||||
"word_meaning": "портфель",
|
||||
"source": "mock_ollama",
|
||||
"cached": False,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(main_mod, "translate_sentence_ai", mock_translate_sentence_ai)
|
||||
|
||||
tr_resp = client.post("/api/translate-sentence", json={
|
||||
"sentence": "At half past eight, Mr. Dursley picked up his briefcase.",
|
||||
"target_word": "briefcase",
|
||||
"context": "Context paragraph",
|
||||
})
|
||||
assert tr_resp.status_code == 200
|
||||
tr_data = tr_resp.json()
|
||||
assert tr_data["sentence_ru"] == "В половине девятого мистер Дурсли взял свой портфель."
|
||||
assert tr_data["word_meaning"] == "портфель"
|
||||
assert tr_data["target_word"] == "briefcase"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user