feat(reader): autonomous Wikipedia import and reading with media, tables and settings
This commit is contained in:
Antigravity committed
1 parent
52db766b23
commit
dee9b8d10b
9 files changed
+1659
-24
No files matched your search
@@ -79,12 +79,18 @@ def _migrate(conn: sqlite3.Connection):
|
||||
current_chapter INTEGER DEFAULT 0,
|
||||
current_position REAL DEFAULT 0,
|
||||
total_words INTEGER DEFAULT 0,
|
||||
cover_url TEXT DEFAULT '',
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
|
||||
);
|
||||
""")
|
||||
|
||||
# Safe migration: ensure cover_url column exists in books
|
||||
books_cols = {row[1] for row in conn.execute("PRAGMA table_info(books)").fetchall()}
|
||||
if "cover_url" not in books_cols:
|
||||
conn.execute("ALTER TABLE books ADD COLUMN cover_url TEXT DEFAULT ''")
|
||||
|
||||
# --- Deduplicate vocabulary_cards (case-insensitive) and enforce UNIQUE index ---
|
||||
dup_rows = conn.execute("""
|
||||
SELECT user_id, lower(trim(term)) as clean_term, COUNT(*) as cnt
|
||||
|
||||
+126
-7
@@ -30,7 +30,11 @@ from app.auth import (
|
||||
from app.dictionary import lookup_term, USER_AGENT, close_http_client
|
||||
from app.export import export_csv, export_json, export_anki_tsv, export_markdown
|
||||
from app.srs import calculate_sm2, generate_quiz_options, build_activity_heatmap, calculate_cefr_level
|
||||
from app.reader import parse_epub, parse_text, parse_pdf, count_words, extract_rfc_number, fetch_and_parse_rfc, parse_rfc_text
|
||||
from app.reader import (
|
||||
parse_epub, parse_text, parse_pdf, count_words, extract_rfc_number,
|
||||
fetch_and_parse_rfc, parse_rfc_text, fetch_and_parse_wikipedia,
|
||||
extract_wikipedia_title_and_lang
|
||||
)
|
||||
from app.ai import translate_sentence_ai, get_ollama_status
|
||||
from app.phrasal import detect_phrasal_verb
|
||||
from app.cefr import (
|
||||
@@ -187,6 +191,14 @@ class RfcFetchRequest(BaseModel):
|
||||
force: Optional[bool] = False
|
||||
|
||||
|
||||
class WikipediaFetchRequest(BaseModel):
|
||||
query: Optional[str] = ""
|
||||
lang: Optional[str] = "simple"
|
||||
time_budget: Optional[str] = "10-15"
|
||||
topic: Optional[str] = "random"
|
||||
force: Optional[bool] = False
|
||||
|
||||
|
||||
class SentenceTranslateRequest(BaseModel):
|
||||
sentence: str = Field(..., min_length=1)
|
||||
target_word: Optional[str] = ""
|
||||
@@ -1194,7 +1206,7 @@ async def api_get_books(user: dict = Depends(require_user)):
|
||||
"""
|
||||
SELECT id, user_id, title, author, source_type,
|
||||
json_array_length(chapters_json) AS chapters_count,
|
||||
total_words, current_chapter, current_position,
|
||||
total_words, current_chapter, current_position, cover_url,
|
||||
created_at, updated_at
|
||||
FROM books
|
||||
WHERE user_id = ?
|
||||
@@ -1309,7 +1321,7 @@ async def api_create_book(
|
||||
"""
|
||||
SELECT id, user_id, title, author, source_type, chapters_json,
|
||||
json_array_length(chapters_json) AS chapters_count,
|
||||
current_chapter, current_position, total_words,
|
||||
current_chapter, current_position, total_words, cover_url,
|
||||
created_at, updated_at
|
||||
FROM books WHERE id = ?
|
||||
""",
|
||||
@@ -1347,7 +1359,7 @@ async def api_fetch_rfc(
|
||||
"""
|
||||
SELECT id, user_id, title, author, source_type, chapters_json,
|
||||
json_array_length(chapters_json) AS chapters_count,
|
||||
current_chapter, current_position, total_words,
|
||||
current_chapter, current_position, total_words, cover_url,
|
||||
created_at, updated_at
|
||||
FROM books
|
||||
WHERE user_id = ? AND source_type = 'rfc' AND (title LIKE ? OR title LIKE ?)
|
||||
@@ -1388,7 +1400,114 @@ async def api_fetch_rfc(
|
||||
"""
|
||||
SELECT id, user_id, title, author, source_type, chapters_json,
|
||||
json_array_length(chapters_json) AS chapters_count,
|
||||
current_chapter, current_position, total_words,
|
||||
current_chapter, current_position, total_words, cover_url,
|
||||
created_at, updated_at
|
||||
FROM books WHERE id = ?
|
||||
""",
|
||||
(new_id,),
|
||||
).fetchone()
|
||||
|
||||
res = dict(row)
|
||||
res["chapters"] = chapters
|
||||
res["already_existed"] = False
|
||||
return res
|
||||
|
||||
|
||||
@app.post("/api/books/fetch-wikipedia")
|
||||
async def api_fetch_wikipedia(
|
||||
req: WikipediaFetchRequest,
|
||||
user: dict = Depends(require_user),
|
||||
):
|
||||
"""
|
||||
Fetches a Wikipedia article (by title/URL or random topic & duration),
|
||||
parses sections into readable chapters with media and tables,
|
||||
and stores autonomously in the user's SQLite bookshelf (source_type = 'wikipedia').
|
||||
"""
|
||||
raw_query = (req.query or "").strip()
|
||||
target_lang = (req.lang or "simple").strip()
|
||||
time_budget = (req.time_budget or "10-15").strip()
|
||||
topic = (req.topic or "random").strip()
|
||||
|
||||
# If specific title or URL provided and force is False, check for existing book
|
||||
if raw_query and not req.force:
|
||||
title_cand, lang_cand = extract_wikipedia_title_and_lang(raw_query, default_lang=target_lang)
|
||||
search_title = title_cand.replace("_", " ")
|
||||
with get_db() as conn:
|
||||
existing = conn.execute(
|
||||
"""
|
||||
SELECT id, user_id, title, author, source_type, chapters_json,
|
||||
json_array_length(chapters_json) AS chapters_count,
|
||||
current_chapter, current_position, total_words, cover_url,
|
||||
created_at, updated_at
|
||||
FROM books
|
||||
WHERE user_id = ? AND source_type = 'wikipedia' AND lower(title) = lower(?)
|
||||
ORDER BY id DESC LIMIT 1
|
||||
""",
|
||||
(user["id"], search_title),
|
||||
).fetchone()
|
||||
if existing:
|
||||
res = dict(existing)
|
||||
try:
|
||||
res["chapters"] = json.loads(res["chapters_json"])
|
||||
except Exception:
|
||||
res["chapters"] = []
|
||||
res["already_existed"] = True
|
||||
return res
|
||||
|
||||
try:
|
||||
title, author, chapters, cover_url, total_words, wiki_url = fetch_and_parse_wikipedia(
|
||||
title_or_url=raw_query,
|
||||
lang=target_lang,
|
||||
time_budget=time_budget,
|
||||
topic=topic,
|
||||
)
|
||||
except ValueError as ve:
|
||||
raise HTTPException(status_code=400, detail=str(ve))
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Ошибка загрузки статьи Wikipedia: {e}")
|
||||
|
||||
# Check again by resolved title if not forcing
|
||||
if not req.force:
|
||||
with get_db() as conn:
|
||||
existing = conn.execute(
|
||||
"""
|
||||
SELECT id, user_id, title, author, source_type, chapters_json,
|
||||
json_array_length(chapters_json) AS chapters_count,
|
||||
current_chapter, current_position, total_words, cover_url,
|
||||
created_at, updated_at
|
||||
FROM books
|
||||
WHERE user_id = ? AND source_type = 'wikipedia' AND lower(title) = lower(?)
|
||||
ORDER BY id DESC LIMIT 1
|
||||
""",
|
||||
(user["id"], title),
|
||||
).fetchone()
|
||||
if existing:
|
||||
res = dict(existing)
|
||||
try:
|
||||
res["chapters"] = json.loads(res["chapters_json"])
|
||||
except Exception:
|
||||
res["chapters"] = []
|
||||
res["already_existed"] = True
|
||||
return res
|
||||
|
||||
chapters_json = json.dumps(chapters, ensure_ascii=False)
|
||||
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
INSERT INTO books (
|
||||
user_id, title, author, source_type, chapters_json,
|
||||
current_chapter, current_position, total_words, cover_url
|
||||
) VALUES (?, ?, ?, 'wikipedia', ?, 0, 0.0, ?, ?)
|
||||
""",
|
||||
(user["id"], title, author, chapters_json, total_words, cover_url),
|
||||
)
|
||||
new_id = cursor.lastrowid
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id, user_id, title, author, source_type, chapters_json,
|
||||
json_array_length(chapters_json) AS chapters_count,
|
||||
current_chapter, current_position, total_words, cover_url,
|
||||
created_at, updated_at
|
||||
FROM books WHERE id = ?
|
||||
""",
|
||||
@@ -1411,7 +1530,7 @@ async def api_get_book(book_id: int, user: dict = Depends(require_user)):
|
||||
"""
|
||||
SELECT id, user_id, title, author, source_type, chapters_json,
|
||||
json_array_length(chapters_json) AS chapters_count,
|
||||
current_chapter, current_position, total_words,
|
||||
current_chapter, current_position, total_words, cover_url,
|
||||
created_at, updated_at
|
||||
FROM books
|
||||
WHERE id = ? AND user_id = ?
|
||||
@@ -1488,7 +1607,7 @@ async def api_update_book_progress(
|
||||
"""
|
||||
SELECT id, user_id, title, author, source_type, chapters_json,
|
||||
json_array_length(chapters_json) AS chapters_count,
|
||||
current_chapter, current_position, total_words,
|
||||
current_chapter, current_position, total_words, cover_url,
|
||||
created_at, updated_at
|
||||
FROM books
|
||||
WHERE id = ?
|
||||
|
||||
+381
-1
@@ -11,7 +11,7 @@ import urllib.error
|
||||
from pathlib import Path
|
||||
import xml.etree.ElementTree as ET
|
||||
from html.parser import HTMLParser
|
||||
from typing import Tuple, List, Dict
|
||||
from typing import Tuple, List, Dict, Optional
|
||||
|
||||
|
||||
def count_words(text: str) -> int:
|
||||
@@ -1087,3 +1087,383 @@ def fetch_and_parse_rfc(query: str) -> Tuple[str, str, List[Dict[str, str]], int
|
||||
return clean_title, clean_author, chapters, rfc_num, total_words
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wikipedia Autonomous Import & Reader Module
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
WIKIPEDIA_USER_AGENT = "StudyReaderApp/1.0 (https://study.qpid.pro; admin@study.qpid.pro) edu-reader"
|
||||
|
||||
WIKIPEDIA_TOPIC_POOLS: Dict[str, Dict[str, List[str]]] = {
|
||||
"science": {
|
||||
"simple": [
|
||||
"Solar_System", "Sun", "Moon", "Mars", "Black_hole", "Milky_Way", "Speed_of_light",
|
||||
"Gravity", "Atom", "Dinosaur", "Evolution", "Photosynthesis", "Asteroid", "Comet",
|
||||
"Volcano", "Plate_tectonics", "Hubble_Space_Telescope", "James_Webb_Space_Telescope",
|
||||
"Planet", "Universe", "Earthquake", "Atmosphere"
|
||||
],
|
||||
"en": [
|
||||
"Solar_System", "Sun", "Moon", "Mars", "Black_hole", "Milky_Way", "General_relativity",
|
||||
"Quantum_mechanics", "Speed_of_light", "Plate_tectonics", "Supernova", "Exoplanet",
|
||||
"Dark_matter", "Hubble_Space_Telescope", "James_Webb_Space_Telescope", "Pendulum",
|
||||
"Thermodynamics", "Stellar_evolution", "Gravitational_wave"
|
||||
]
|
||||
},
|
||||
"tech": {
|
||||
"simple": [
|
||||
"Computer", "Internet", "World_Wide_Web", "Artificial_intelligence", "Smartphone",
|
||||
"Robot", "Video_game", "Software", "Linux", "Algorithm", "Satellite", "Electricity",
|
||||
"Machine_learning", "Computer_network", "Fiber_optic", "Microprocessor"
|
||||
],
|
||||
"en": [
|
||||
"Artificial_intelligence", "Computer", "Internet", "World_Wide_Web", "Quantum_computing",
|
||||
"Machine_learning", "Operating_system", "Cryptography", "Blockchain", "Transistor",
|
||||
"Linux", "Neural_network", "Optical_fiber", "Semiconductor"
|
||||
]
|
||||
},
|
||||
"history": {
|
||||
"simple": [
|
||||
"Ancient_Egypt", "Roman_Empire", "Ancient_Greece", "Middle_Ages", "Industrial_Revolution",
|
||||
"Silk_Road", "Renaissance", "Aztec", "World_War_I", "French_Revolution", "Great_Wall_of_China",
|
||||
"Mesopotamia", "Byzantine_Empire", "Inca_Empire", "Voyages_of_Christopher_Columbus"
|
||||
],
|
||||
"en": [
|
||||
"Ancient_Egypt", "Roman_Empire", "Renaissance", "Industrial_Revolution", "Silk_Road",
|
||||
"Byzantine_Empire", "Age_of_Discovery", "Classical_Athens", "Mongol_Empire", "French_Revolution",
|
||||
"Machu_Picchu", "Hanseatic_League", "Printing_press", "Maya_civilization"
|
||||
]
|
||||
},
|
||||
"nature": {
|
||||
"simple": [
|
||||
"Blue_whale", "Tiger", "African_elephant", "Honey_bee", "Amazon_rainforest", "Coral_reef",
|
||||
"Penguin", "Dolphin", "Wolf", "Sahara", "Giant_panda", "Cheetah", "Kangaroo", "Giraffe",
|
||||
"Ocean", "Forest", "Rainforest", "Migration"
|
||||
],
|
||||
"en": [
|
||||
"Blue_whale", "Tiger", "African_elephant", "Amazon_rainforest", "Great_Barrier_Reef",
|
||||
"Deep_sea", "Polar_bear", "Galapagos_Islands", "Bioluminescence", "Coast_redwood",
|
||||
"Honey_bee", "Monarch_butterfly", "Humpback_whale", "Ecosystem"
|
||||
]
|
||||
},
|
||||
"psychology": {
|
||||
"simple": [
|
||||
"Memory", "Emotion", "Consciousness", "Sleep", "Brain", "Intelligence", "Dream",
|
||||
"Perception", "Learning", "Happiness", "Stress_(biology)", "Mind", "Behavior"
|
||||
],
|
||||
"en": [
|
||||
"Human_memory", "Emotion", "Consciousness", "Sleep", "Cognitive_bias", "Neuroplasticity",
|
||||
"Empathy", "Perception", "Motivation", "Decision-making", "Working_memory",
|
||||
"Attention", "Lateralization_of_brain_function"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def extract_wikipedia_title_and_lang(query: str, default_lang: str = "simple") -> Tuple[str, str]:
|
||||
"""
|
||||
Parses a Wikipedia URL or plain title into (article_title, lang).
|
||||
Examples:
|
||||
https://simple.wikipedia.org/wiki/Solar_System -> ('Solar_System', 'simple')
|
||||
https://en.wikipedia.org/wiki/Quantum_mechanics -> ('Quantum_mechanics', 'en')
|
||||
'Quantum mechanics' -> ('Quantum_mechanics', default_lang)
|
||||
"""
|
||||
raw = (query or "").strip()
|
||||
if not raw:
|
||||
return "", default_lang
|
||||
|
||||
# Check URL format
|
||||
m_url = re.search(r"https?://([a-z]+)\.(?:m\.)?wikipedia\.org/wiki/([^?#]+)", raw, re.IGNORECASE)
|
||||
if m_url:
|
||||
lang = m_url.group(1).lower()
|
||||
title_part = urllib.parse.unquote(m_url.group(2)).strip()
|
||||
# Convert spaces
|
||||
clean_title = title_part.replace(" ", "_")
|
||||
return clean_title, lang
|
||||
|
||||
# Plain title or search query
|
||||
clean_title = raw.replace(" ", "_")
|
||||
return clean_title, default_lang
|
||||
|
||||
|
||||
def clean_wiki_html(raw_html: str, article_title: str = "") -> Tuple[List[Dict[str, str]], str, int]:
|
||||
"""
|
||||
Parses MediaWiki Parsoid HTML into clean, structured chapters for the interactive reader.
|
||||
Extracts cover image/thumbnail, preserves figures, animated GIFs/SVGs, and clean data tables,
|
||||
and strips wiki clutter ([1], [citation needed], infobox rows, navboxes).
|
||||
"""
|
||||
cover_image = ""
|
||||
|
||||
# 1. Lead image: check infobox or first figure
|
||||
lead_img_m = re.search(
|
||||
r'<table\b[^>]*class=["\x27][^"\x27]*infobox[^"\x27]*["\x27][^>]*>.*?<img\b([^>]*)>',
|
||||
raw_html,
|
||||
re.DOTALL | re.IGNORECASE
|
||||
)
|
||||
if not lead_img_m:
|
||||
lead_img_m = re.search(r'<figure\b[^>]*>.*?<img\b([^>]*)>', raw_html, re.DOTALL | re.IGNORECASE)
|
||||
|
||||
if lead_img_m:
|
||||
src_m = re.search(r'src=["\x27]([^\s"\x27]+)["\x27]', lead_img_m.group(1), re.IGNORECASE)
|
||||
if src_m:
|
||||
src = src_m.group(1)
|
||||
if src.startswith("//"):
|
||||
src = "https:" + src
|
||||
elif not src.startswith("http"):
|
||||
src = "https://" + src.lstrip("/")
|
||||
cover_image = src
|
||||
|
||||
# 2. Extract body
|
||||
body_m = re.search(r"<body[^>]*>(.*?)</body>", raw_html, re.DOTALL | re.IGNORECASE)
|
||||
body = body_m.group(1) if body_m else raw_html
|
||||
|
||||
# Strip style, script, noscript tags
|
||||
body = re.sub(r'<style\b[^>]*>.*?</style>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<script\b[^>]*>.*?</script>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<noscript\b[^>]*>.*?</noscript>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# 3. Strip unwanted wiki elements
|
||||
# Remove references: <sup class="reference">...</sup> and bracketed reference markers
|
||||
body = re.sub(r'<sup\b[^>]*class=["\x27][^"\x27]*reference[^"\x27]*["\x27][^>]*>.*?</sup>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<sup\b[^>]*>\s*\[\s*[\d\w\s,-]+\s*\]\s*</sup>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<span\b[^>]*class=["\x27][^"\x27]*mw-editsection[^"\x27]*["\x27][^>]*>.*?</span>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<div\b[^>]*class=["\x27][^"\x27]*(?:navbox|ambox|metadata|hatnote|sistersitebox|portal)[^"\x27]*["\x27][^>]*>.*?</div>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<table\b[^>]*class=["\x27][^"\x27]*(?:navbox|ambox|metadata|nowraplinks)[^"\x27]*["\x27][^>]*>.*?</table>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# Remove infobox entirely from text flow (lead image was already captured)
|
||||
body = re.sub(r'<table\b[^>]*class=["\x27][^"\x27]*infobox[^"\x27]*["\x27][^>]*>.*?</table>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<div\b[^>]*class=["\x27][^"\x27]*infobox[^"\x27]*["\x27][^>]*>.*?</div>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# 4. Standardize figures and images (preserving animated GIFs/SVGs/PNGs)
|
||||
def clean_figure(m: re.Match) -> str:
|
||||
f_html = m.group(0)
|
||||
img_m = re.search(r'<img\b([^>]*)>', f_html, re.IGNORECASE)
|
||||
if not img_m:
|
||||
return ""
|
||||
attrs = img_m.group(1)
|
||||
src_m = re.search(r'src=["\x27]([^\s"\x27]+)["\x27]', attrs, re.IGNORECASE)
|
||||
if not src_m:
|
||||
return ""
|
||||
src = src_m.group(1)
|
||||
if src.startswith("//"):
|
||||
src = "https:" + src
|
||||
elif not src.startswith("http"):
|
||||
src = "https://" + src.lstrip("/")
|
||||
|
||||
alt_m = re.search(r'alt=["\x27]([^\"]*)["\x27]', attrs, re.IGNORECASE)
|
||||
alt = alt_m.group(1) if alt_m else ""
|
||||
|
||||
cap_m = re.search(r'<figcaption\b[^>]*>(.*?)</figcaption>', f_html, re.DOTALL | re.IGNORECASE)
|
||||
cap = ""
|
||||
if cap_m:
|
||||
cap_clean = re.sub(r'<[^>]+>', ' ', cap_m.group(1))
|
||||
cap = " ".join(cap_clean.split()).strip()
|
||||
|
||||
return f'\n\n<figure class="reader-figure"><img src="{src}" alt="{html.escape(alt)}" loading="lazy" /><figcaption>{html.escape(cap)}</figcaption></figure>\n\n'
|
||||
|
||||
body = re.sub(r'<figure\b[^>]*>.*?</figure>', clean_figure, body, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# 5. Clean data tables (tables that are not navboxes/infoboxes)
|
||||
def clean_data_table(m: re.Match) -> str:
|
||||
tbl_html = m.group(0)
|
||||
if re.search(r'class=["\x27][^"\x27]*(?:navbox|ambox|metadata|nowraplinks|infobox)[^"\x27]*["\x27]', tbl_html, re.IGNORECASE):
|
||||
return ""
|
||||
cleaned_tbl = re.sub(
|
||||
r'\s+(?:style|bgcolor|align|valign|width|height|border|cellspacing|cellpadding|id)=["\x27][^"\x27]*["\x27]',
|
||||
'',
|
||||
tbl_html,
|
||||
flags=re.IGNORECASE
|
||||
)
|
||||
cleaned_tbl = re.sub(r'<table[^>]*>', '<div class="reader-table-wrap"><table class="reader-table">', cleaned_tbl, count=1, flags=re.IGNORECASE)
|
||||
cleaned_tbl = re.sub(r'</table>', '</table></div>', cleaned_tbl, count=1, flags=re.IGNORECASE)
|
||||
return f'\n\n{cleaned_tbl}\n\n'
|
||||
|
||||
body = re.sub(r'<table\b[^>]*>.*?</table>', clean_data_table, body, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# 6. Split into sections by <h2>
|
||||
ignore_sections = {
|
||||
"references", "reference", "see also", "external links", "further reading",
|
||||
"notes", "footnotes", "sources", "bibliography", "other websites",
|
||||
"related pages", "citations", "works cited", "navigation menu"
|
||||
}
|
||||
|
||||
parts = re.split(r'(<h2\b[^>]*>.*?</h2>)', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
raw_chapters: List[Tuple[str, str]] = []
|
||||
lead_content = parts[0].strip()
|
||||
if lead_content:
|
||||
raw_chapters.append(("Overview", lead_content))
|
||||
|
||||
idx = 1
|
||||
while idx < len(parts):
|
||||
h2_tag = parts[idx]
|
||||
h_title = re.sub(r'<[^>]+>', '', h2_tag).strip()
|
||||
h_title_clean = " ".join(h_title.split())
|
||||
sec_content = parts[idx + 1].strip() if idx + 1 < len(parts) else ""
|
||||
idx += 2
|
||||
|
||||
if h_title_clean.lower() in ignore_sections:
|
||||
continue
|
||||
|
||||
raw_chapters.append((h_title_clean, sec_content))
|
||||
|
||||
# 7. Convert HTML within each chapter into clean reader content
|
||||
chapters: List[Dict[str, str]] = []
|
||||
|
||||
def format_section_body(sec_html: str) -> str:
|
||||
s = re.sub(r'<h3\b[^>]*>(.*?)</h3>', r'\n\n### \1\n\n', sec_html, flags=re.DOTALL | re.IGNORECASE)
|
||||
s = re.sub(r'<h4\b[^>]*>(.*?)</h4>', r'\n\n#### \1\n\n', s, flags=re.DOTALL | re.IGNORECASE)
|
||||
s = re.sub(r'<li\b[^>]*>(.*?)</li>', r'\n- \1', s, flags=re.DOTALL | re.IGNORECASE)
|
||||
s = re.sub(r'</?[uo]l[^>]*>', '\n', s, flags=re.IGNORECASE)
|
||||
s = re.sub(r'<p\b[^>]*>(.*?)</p>', r'\n\n\1\n\n', s, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# Protect <figure...> and <div class="reader-table-wrap">
|
||||
blocks_protected = []
|
||||
def stash_block(match: re.Match) -> str:
|
||||
token = f"§§BLOCK_{len(blocks_protected)}§§"
|
||||
blocks_protected.append(match.group(0))
|
||||
return f"\n\n{token}\n\n"
|
||||
|
||||
s = re.sub(r'<figure\b[^>]*>.*?</figure>', stash_block, s, flags=re.DOTALL | re.IGNORECASE)
|
||||
s = re.sub(r'<div class="reader-table-wrap">.*?</div>', stash_block, s, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
s = re.sub(r'<[^>]+>', ' ', s)
|
||||
s = html.unescape(s)
|
||||
|
||||
lines = []
|
||||
for line in s.splitlines():
|
||||
trimmed = " ".join(line.split())
|
||||
if trimmed:
|
||||
lines.append(trimmed)
|
||||
elif lines and lines[-1] != "":
|
||||
lines.append("")
|
||||
|
||||
res_text = "\n".join(lines).strip()
|
||||
|
||||
for b_idx, block_html in enumerate(blocks_protected):
|
||||
res_text = res_text.replace(f"§§BLOCK_{b_idx}§§", f"\n\n{block_html}\n\n")
|
||||
|
||||
res_text = re.sub(r'\n{3,}', '\n\n', res_text).strip()
|
||||
return res_text
|
||||
|
||||
for title, content_html in raw_chapters:
|
||||
cleaned_body = format_section_body(content_html)
|
||||
w_count = count_words(cleaned_body)
|
||||
if w_count >= 15 or "<figure" in cleaned_body or "<table" in cleaned_body:
|
||||
chapters.append({
|
||||
"title": title,
|
||||
"content": cleaned_body
|
||||
})
|
||||
|
||||
if not chapters and raw_chapters:
|
||||
for title, content_html in raw_chapters:
|
||||
cleaned_body = format_section_body(content_html)
|
||||
if cleaned_body.strip():
|
||||
chapters.append({"title": title, "content": cleaned_body.strip()})
|
||||
|
||||
total_words = sum(count_words(c["content"]) for c in chapters)
|
||||
return chapters, cover_image, total_words
|
||||
|
||||
|
||||
def fetch_and_parse_wikipedia(
|
||||
title_or_url: Optional[str] = None,
|
||||
lang: str = "simple",
|
||||
time_budget: str = "10-15",
|
||||
topic: str = "random",
|
||||
) -> Tuple[str, str, List[Dict[str, str]], str, int, str]:
|
||||
"""
|
||||
Fetches a Wikipedia article (by title/URL or by random topic & time budget),
|
||||
parses into chapters with preserved media and tables, and returns:
|
||||
(clean_title, author, chapters, cover_url, total_words, wiki_url)
|
||||
"""
|
||||
import random
|
||||
from datetime import datetime
|
||||
|
||||
clean_lang = "en" if (lang or "").lower() in {"en", "standard", "standard english"} else "simple"
|
||||
target_title = ""
|
||||
|
||||
# Case 1: Specific title or URL provided
|
||||
if title_or_url and title_or_url.strip():
|
||||
extracted_title, detected_lang = extract_wikipedia_title_and_lang(title_or_url, default_lang=clean_lang)
|
||||
target_title = extracted_title
|
||||
if detected_lang:
|
||||
clean_lang = detected_lang
|
||||
|
||||
# Case 2: Pick random article by topic and time budget
|
||||
if not target_title:
|
||||
candidates = []
|
||||
clean_topic = (topic or "random").lower().strip()
|
||||
|
||||
# Handle 'featured' article of the day
|
||||
if clean_topic == "featured":
|
||||
try:
|
||||
now = datetime.utcnow()
|
||||
feat_url = f"https://{clean_lang}.wikipedia.org/api/rest_v1/feed/featured/{now.year}/{now.month:02d}/{now.day:02d}"
|
||||
req = urllib.request.Request(feat_url, headers={"User-Agent": WIKIPEDIA_USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=5) as r:
|
||||
feed_data = json.loads(r.read().decode("utf-8"))
|
||||
if "tfa" in feed_data and feed_data["tfa"].get("title"):
|
||||
target_title = feed_data["tfa"]["title"]
|
||||
elif "mostread" in feed_data and feed_data["mostread"].get("articles"):
|
||||
art_list = [a["title"] for a in feed_data["mostread"]["articles"] if a.get("title")]
|
||||
if art_list:
|
||||
target_title = random.choice(art_list[:5])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not target_title:
|
||||
if clean_topic in WIKIPEDIA_TOPIC_POOLS:
|
||||
candidates = list(WIKIPEDIA_TOPIC_POOLS[clean_topic].get(clean_lang, []))
|
||||
else:
|
||||
for pool_dict in WIKIPEDIA_TOPIC_POOLS.values():
|
||||
candidates.extend(pool_dict.get(clean_lang, []))
|
||||
|
||||
random.shuffle(candidates)
|
||||
target_title = candidates[0] if candidates else "Solar_System"
|
||||
|
||||
# Fetch article summary to get clean title and thumbnail
|
||||
clean_title = target_title.replace("_", " ")
|
||||
cover_url = ""
|
||||
wiki_url = f"https://{clean_lang}.wikipedia.org/wiki/{urllib.parse.quote(target_title)}"
|
||||
|
||||
try:
|
||||
sum_url = f"https://{clean_lang}.wikipedia.org/api/rest_v1/page/summary/{urllib.parse.quote(target_title)}"
|
||||
req = urllib.request.Request(sum_url, headers={"User-Agent": WIKIPEDIA_USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=8) as r:
|
||||
sum_data = json.loads(r.read().decode("utf-8"))
|
||||
if sum_data.get("title"):
|
||||
clean_title = sum_data["title"]
|
||||
if sum_data.get("thumbnail") and sum_data["thumbnail"].get("source"):
|
||||
cover_url = sum_data["thumbnail"]["source"]
|
||||
except urllib.error.HTTPError as he:
|
||||
if he.code == 404:
|
||||
raise ValueError(f"Статья «{clean_title}» не найдена в Википедии ({clean_lang}.wikipedia.org)")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fetch full Parsoid HTML
|
||||
raw_html = ""
|
||||
try:
|
||||
html_url = f"https://{clean_lang}.wikipedia.org/api/rest_v1/page/html/{urllib.parse.quote(target_title)}"
|
||||
req = urllib.request.Request(html_url, headers={"User-Agent": WIKIPEDIA_USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=12) as r:
|
||||
raw_html = r.read().decode("utf-8", errors="replace")
|
||||
except urllib.error.HTTPError as he:
|
||||
if he.code == 404:
|
||||
raise ValueError(f"Статья «{clean_title}» не найдена в Википедии ({clean_lang}.wikipedia.org)")
|
||||
raise ValueError(f"Ошибка загрузки статьи из Wikipedia: HTTP {he.code}")
|
||||
except Exception as e:
|
||||
raise ValueError(f"Не удалось соединиться с сервером Википедии: {e}")
|
||||
|
||||
chapters, html_cover, total_words = clean_wiki_html(raw_html, article_title=clean_title)
|
||||
if not cover_url and html_cover:
|
||||
cover_url = html_cover
|
||||
|
||||
if not chapters:
|
||||
raise ValueError(f"В статье «{clean_title}» не найдено текстового содержимого для чтения")
|
||||
|
||||
edition_name = "Simple English" if clean_lang == "simple" else "Standard English"
|
||||
author = f"Wikipedia ({edition_name})"
|
||||
|
||||
return clean_title, author, chapters, cover_url, total_words, wiki_url
|
||||
|
||||
|
||||
|
||||
@@ -3603,6 +3603,408 @@ body.drawer-open {
|
||||
}
|
||||
}
|
||||
|
||||
/* ==========================================================================
|
||||
Wikipedia Articles Section & Settings Modal
|
||||
========================================================================== */
|
||||
|
||||
.wiki-section {
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.wiki-header {
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.wiki-title-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.wiki-icon-badge {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 44px;
|
||||
height: 44px;
|
||||
font-size: 24px;
|
||||
background: rgba(168, 85, 247, 0.12);
|
||||
border: 1px solid rgba(168, 85, 247, 0.28);
|
||||
border-radius: var(--radius-md);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.wiki-heading {
|
||||
font-size: 20px;
|
||||
font-weight: 700;
|
||||
color: var(--text-main);
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.wiki-action-box {
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: var(--radius-lg);
|
||||
padding: 20px 24px;
|
||||
margin-bottom: 24px;
|
||||
box-shadow: var(--shadow-sm);
|
||||
}
|
||||
|
||||
.wiki-primary-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
flex-wrap: wrap;
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
|
||||
.wiki-read-btn {
|
||||
padding: 10px 24px;
|
||||
font-weight: 700;
|
||||
font-size: 15px;
|
||||
background: linear-gradient(135deg, #6366f1, #8b5cf6);
|
||||
border: none;
|
||||
box-shadow: 0 4px 14px rgba(99, 102, 241, 0.35);
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.wiki-read-btn:hover {
|
||||
background: linear-gradient(135deg, #4f46e5, #7c3aed);
|
||||
transform: translateY(-1px);
|
||||
box-shadow: 0 6px 18px rgba(99, 102, 241, 0.45);
|
||||
}
|
||||
|
||||
.wiki-settings-btn {
|
||||
padding: 10px 18px;
|
||||
font-weight: 600;
|
||||
border-color: rgba(168, 85, 247, 0.35);
|
||||
color: var(--text-main);
|
||||
}
|
||||
|
||||
.wiki-settings-btn:hover {
|
||||
background: rgba(168, 85, 247, 0.12);
|
||||
border-color: #a855f7;
|
||||
color: #c084fc;
|
||||
}
|
||||
|
||||
.wiki-badges-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
flex-wrap: wrap;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.wiki-badges-label {
|
||||
color: var(--text-dim);
|
||||
font-size: 12px;
|
||||
text-transform: uppercase;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.05em;
|
||||
margin-right: 4px;
|
||||
}
|
||||
|
||||
.wiki-badge-chip {
|
||||
background: rgba(168, 85, 247, 0.12);
|
||||
color: #c084fc;
|
||||
border: 1px solid rgba(168, 85, 247, 0.28);
|
||||
border-radius: var(--radius-full);
|
||||
padding: 4px 12px;
|
||||
font-weight: 500;
|
||||
font-size: 12px;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.wiki-fetch-status {
|
||||
margin-top: 14px;
|
||||
padding: 10px 16px;
|
||||
font-size: 13.5px;
|
||||
border-radius: var(--radius-md);
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.wiki-status-loading {
|
||||
background: rgba(99, 102, 241, 0.12);
|
||||
color: #818cf8;
|
||||
border: 1px solid rgba(99, 102, 241, 0.3);
|
||||
}
|
||||
|
||||
.wiki-status-success {
|
||||
background: rgba(34, 197, 94, 0.12);
|
||||
color: #4ade80;
|
||||
border: 1px solid rgba(34, 197, 94, 0.25);
|
||||
}
|
||||
|
||||
.wiki-status-error {
|
||||
background: rgba(239, 68, 68, 0.12);
|
||||
color: #f87171;
|
||||
border: 1px solid rgba(239, 68, 68, 0.25);
|
||||
}
|
||||
|
||||
.wiki-empty-state {
|
||||
padding: 36px 20px;
|
||||
background: var(--bg-surface);
|
||||
border: 1px dashed var(--border-color);
|
||||
border-radius: var(--radius-lg);
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
|
||||
/* Card styling for Wikipedia and thumbnails */
|
||||
.wiki-card {
|
||||
border-color: rgba(168, 85, 247, 0.22);
|
||||
}
|
||||
|
||||
.wiki-card:hover {
|
||||
border-color: rgba(168, 85, 247, 0.55);
|
||||
box-shadow: 0 8px 24px rgba(168, 85, 247, 0.15);
|
||||
}
|
||||
|
||||
.wiki-cover-icon {
|
||||
background: rgba(168, 85, 247, 0.12);
|
||||
border-color: rgba(168, 85, 247, 0.3);
|
||||
color: #c084fc;
|
||||
}
|
||||
|
||||
.badge-wiki {
|
||||
background: rgba(168, 85, 247, 0.18);
|
||||
color: #c084fc;
|
||||
border: 1px solid rgba(168, 85, 247, 0.35);
|
||||
}
|
||||
|
||||
.wiki-progress-fill {
|
||||
background: linear-gradient(90deg, #7c3aed, #a855f7);
|
||||
}
|
||||
|
||||
.book-cover-img-wrap {
|
||||
width: 52px;
|
||||
height: 64px;
|
||||
border-radius: var(--radius-sm);
|
||||
overflow: hidden;
|
||||
flex-shrink: 0;
|
||||
background: rgba(0, 0, 0, 0.2);
|
||||
border: 1px solid var(--border-color);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.book-cover-img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.badge-time {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 3px;
|
||||
background: rgba(255, 255, 255, 0.06);
|
||||
color: var(--text-dim);
|
||||
font-size: 11px;
|
||||
font-weight: 500;
|
||||
padding: 2px 7px;
|
||||
border-radius: var(--radius-sm);
|
||||
border: 1px solid rgba(255, 255, 255, 0.08);
|
||||
}
|
||||
|
||||
/* Wikipedia Settings Modal */
|
||||
.wiki-modal-window {
|
||||
max-width: 620px;
|
||||
width: 95%;
|
||||
}
|
||||
|
||||
.wiki-modal-body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 22px;
|
||||
}
|
||||
|
||||
.wiki-setting-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.wiki-group-label {
|
||||
font-size: 13.5px;
|
||||
font-weight: 700;
|
||||
color: var(--text-main);
|
||||
letter-spacing: 0.02em;
|
||||
}
|
||||
|
||||
.wiki-radio-cards {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.wiki-radio-card {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 12px;
|
||||
padding: 14px 16px;
|
||||
background: var(--bg-input);
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: var(--radius-md);
|
||||
cursor: pointer;
|
||||
transition: all 0.2s ease;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
.wiki-radio-card:hover {
|
||||
border-color: rgba(168, 85, 247, 0.4);
|
||||
background: rgba(168, 85, 247, 0.04);
|
||||
}
|
||||
|
||||
.wiki-radio-card:has(input:checked) {
|
||||
border-color: #a855f7;
|
||||
background: rgba(168, 85, 247, 0.1);
|
||||
box-shadow: 0 0 0 1px #a855f7;
|
||||
}
|
||||
|
||||
.wiki-radio-card input[type="radio"] {
|
||||
margin-top: 3px;
|
||||
accent-color: #a855f7;
|
||||
}
|
||||
|
||||
.wiki-radio-card-content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 5px;
|
||||
}
|
||||
|
||||
.wiki-card-title-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.wiki-card-desc {
|
||||
font-size: 12px;
|
||||
color: var(--text-dim);
|
||||
line-height: 1.45;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.badge-level-pill {
|
||||
font-size: 10.5px;
|
||||
font-weight: 700;
|
||||
padding: 1px 7px;
|
||||
border-radius: var(--radius-full);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
|
||||
.badge-level-a2 {
|
||||
background: rgba(34, 197, 94, 0.2);
|
||||
color: #4ade80;
|
||||
border: 1px solid rgba(34, 197, 94, 0.35);
|
||||
}
|
||||
|
||||
.badge-level-b2 {
|
||||
background: rgba(59, 130, 246, 0.2);
|
||||
color: #60a5fa;
|
||||
border: 1px solid rgba(59, 130, 246, 0.35);
|
||||
}
|
||||
|
||||
.wiki-time-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(130px, 1fr));
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.wiki-time-pill {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
position: relative;
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
.wiki-time-pill input[type="radio"] {
|
||||
position: absolute;
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.wiki-pill-text {
|
||||
width: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
text-align: center;
|
||||
padding: 10px 8px;
|
||||
background: var(--bg-input);
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: var(--radius-md);
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: var(--text-main);
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.wiki-pill-text small {
|
||||
font-size: 11px;
|
||||
font-weight: normal;
|
||||
margin-top: 2px;
|
||||
}
|
||||
|
||||
.wiki-time-pill:hover .wiki-pill-text {
|
||||
border-color: rgba(168, 85, 247, 0.4);
|
||||
}
|
||||
|
||||
.wiki-time-pill input[type="radio"]:checked + .wiki-pill-text {
|
||||
border-color: #a855f7;
|
||||
background: rgba(168, 85, 247, 0.12);
|
||||
color: #c084fc;
|
||||
box-shadow: 0 0 0 1px #a855f7;
|
||||
}
|
||||
|
||||
.wiki-select {
|
||||
width: 100%;
|
||||
padding: 10px 14px;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.wiki-direct-import-group {
|
||||
border-top: 1px dashed var(--border-color);
|
||||
padding-top: 18px;
|
||||
}
|
||||
|
||||
.wiki-custom-input-wrap {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.wiki-custom-input {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.wiki-custom-fetch-btn {
|
||||
white-space: nowrap;
|
||||
padding: 0 18px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
@media (max-width: 640px) {
|
||||
.wiki-radio-cards {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
.wiki-custom-input-wrap {
|
||||
flex-direction: column;
|
||||
}
|
||||
.wiki-custom-fetch-btn {
|
||||
width: 100%;
|
||||
padding: 10px 16px;
|
||||
}
|
||||
}
|
||||
|
||||
/* Reader Interface */
|
||||
.reader-interface {
|
||||
background-color: var(--bg-surface);
|
||||
@@ -4273,6 +4675,48 @@ body.drawer-open {
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
/* Figures and Media (Images, Diagrams, Animated GIFs, SVGs) */
|
||||
.reader-figure {
|
||||
margin: 28px auto;
|
||||
padding: 14px;
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: var(--radius-lg);
|
||||
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.15);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
max-width: 95%;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
.reader-figure-img {
|
||||
max-width: 100%;
|
||||
max-height: 480px;
|
||||
width: auto;
|
||||
height: auto;
|
||||
border-radius: var(--radius-md);
|
||||
object-fit: contain;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.reader-figcaption {
|
||||
margin-top: 12px;
|
||||
padding-top: 10px;
|
||||
border-top: 1px solid rgba(255, 255, 255, 0.08);
|
||||
font-size: 13.5px;
|
||||
line-height: 1.55;
|
||||
color: var(--text-dim);
|
||||
text-align: center;
|
||||
max-width: 90%;
|
||||
font-style: italic;
|
||||
user-select: text;
|
||||
}
|
||||
|
||||
.reader-figcaption .reader-word {
|
||||
font-style: normal;
|
||||
}
|
||||
|
||||
.reader-content-body p[style*="text-align: center"],
|
||||
.reader-content-body p.text-center {
|
||||
text-indent: 0 !important;
|
||||
|
||||
+137
-1
@@ -546,7 +546,52 @@
|
||||
</div>
|
||||
<div id="bookshelf-grid" class="bookshelf-grid"></div>
|
||||
|
||||
<!-- Divider between General Books & RFC Specifications -->
|
||||
<!-- Divider between General Books & Wikipedia Section -->
|
||||
<hr class="bookshelf-divider">
|
||||
|
||||
<!-- Wikipedia Articles Section -->
|
||||
<div id="wiki-section" class="wiki-section">
|
||||
<div class="wiki-header">
|
||||
<div class="wiki-title-wrap">
|
||||
<div class="wiki-title-row">
|
||||
<span class="wiki-icon-badge">🌐</span>
|
||||
<div>
|
||||
<h3 class="wiki-heading">Статьи Википедии (Wikipedia Reader)</h3>
|
||||
<p class="subtitle">Автономное чтение статей Simple и Standard Wikipedia с иллюстрациями, таблицами и клик-переводом</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Wikipedia Action Bar & Current Settings Badges -->
|
||||
<div class="wiki-action-box">
|
||||
<div class="wiki-primary-actions">
|
||||
<button type="button" id="wiki-read-btn" class="btn btn-primary wiki-read-btn">
|
||||
<span>📖 Читать статью</span>
|
||||
</button>
|
||||
<button type="button" id="wiki-settings-btn" class="btn btn-outline wiki-settings-btn" title="Настроить сложность, объем и тематику статьи">
|
||||
<span>⚙️ Настройки</span>
|
||||
</button>
|
||||
</div>
|
||||
<div class="wiki-badges-row">
|
||||
<span class="wiki-badges-label">Текущие фильтры:</span>
|
||||
<span id="wiki-active-lang-badge" class="badge-chip wiki-badge-chip">Simple English (A2–B1)</span>
|
||||
<span id="wiki-active-time-badge" class="badge-chip wiki-badge-chip">⏱️ ~10-15 мин</span>
|
||||
<span id="wiki-active-topic-badge" class="badge-chip wiki-badge-chip">🎲 Любая тема</span>
|
||||
</div>
|
||||
<div id="wiki-fetch-status" class="wiki-fetch-status hidden"></div>
|
||||
</div>
|
||||
|
||||
<!-- Wikipedia Articles Grid / Empty State -->
|
||||
<div id="wiki-empty-state" class="empty-state wiki-empty-state hidden">
|
||||
<div class="empty-icon">🌐</div>
|
||||
<h4>Нет сохраненных статей из Википедии</h4>
|
||||
<p>Нажмите «Читать статью» выше, чтобы подобрать случайную познавательную статью по вашим настройкам, или настройте тему и объем через шестерёнку ⚙️.</p>
|
||||
</div>
|
||||
<div id="wiki-grid" class="bookshelf-grid wiki-grid"></div>
|
||||
</div>
|
||||
|
||||
<!-- Divider between Wikipedia & RFC Specifications -->
|
||||
<hr class="bookshelf-divider">
|
||||
|
||||
<!-- RFC Specifications Section -->
|
||||
@@ -1253,6 +1298,97 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Wikipedia Settings Modal -->
|
||||
<div id="wiki-settings-modal" class="modal-backdrop hidden">
|
||||
<div class="modal-window wiki-modal-window">
|
||||
<div class="modal-header">
|
||||
<h3>⚙️ Настройки статей Википедии</h3>
|
||||
<button id="wiki-settings-close-btn" class="icon-btn" title="Закрыть">✕</button>
|
||||
</div>
|
||||
<div class="modal-body wiki-modal-body">
|
||||
<!-- Setting 1: Language / Complexity -->
|
||||
<div class="wiki-setting-group">
|
||||
<label class="wiki-group-label">Сложность языка (Wikipedia Edition):</label>
|
||||
<div class="wiki-radio-cards">
|
||||
<label class="wiki-radio-card">
|
||||
<input type="radio" name="wiki-lang" value="simple" checked>
|
||||
<div class="wiki-radio-card-content">
|
||||
<div class="wiki-card-title-row">
|
||||
<strong>Simple English</strong>
|
||||
<span class="badge-level-pill badge-level-a2">A2–B1</span>
|
||||
</div>
|
||||
<p class="wiki-card-desc">Простая грамматика и базовая лексика (simple.wikipedia.org). Рекомендуется для уверенного чтения без перегрузки редкими терминами.</p>
|
||||
</div>
|
||||
</label>
|
||||
<label class="wiki-radio-card">
|
||||
<input type="radio" name="wiki-lang" value="en">
|
||||
<div class="wiki-radio-card-content">
|
||||
<div class="wiki-card-title-row">
|
||||
<strong>Standard English</strong>
|
||||
<span class="badge-level-pill badge-level-b2">B2–C1</span>
|
||||
</div>
|
||||
<p class="wiki-card-desc">Полный оригинальный текст (en.wikipedia.org). Академическая лексика, развернутые описания и глубокая научная терминология.</p>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Setting 2: Reading Volume / Duration -->
|
||||
<div class="wiki-setting-group">
|
||||
<label class="wiki-group-label">Объем статьи / Время чтения:</label>
|
||||
<div class="wiki-time-grid">
|
||||
<label class="wiki-time-pill">
|
||||
<input type="radio" name="wiki-time" value="5">
|
||||
<span class="wiki-pill-text">⚡ ~5 мин <small class="text-dim">(800–1200 слов)</small></span>
|
||||
</label>
|
||||
<label class="wiki-time-pill">
|
||||
<input type="radio" name="wiki-time" value="10-15" checked>
|
||||
<span class="wiki-pill-text">⏱️ ~10–15 мин <small class="text-dim">(1500–2500 слов)</small></span>
|
||||
</label>
|
||||
<label class="wiki-time-pill">
|
||||
<input type="radio" name="wiki-time" value="20+">
|
||||
<span class="wiki-pill-text">📚 20+ мин <small class="text-dim">(лонгрид от 3000 слов)</small></span>
|
||||
</label>
|
||||
<label class="wiki-time-pill">
|
||||
<input type="radio" name="wiki-time" value="any">
|
||||
<span class="wiki-pill-text">🎲 Любой объем</span>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Setting 3: Topic Presets -->
|
||||
<div class="wiki-setting-group">
|
||||
<label class="wiki-group-label">Тематика статьи:</label>
|
||||
<select id="wiki-topic-select" class="input-styled wiki-select">
|
||||
<option value="random">🎲 Любая / Случайная тема</option>
|
||||
<option value="science">🚀 Наука, космос и астрономия</option>
|
||||
<option value="tech">💻 Технологии, IT и интернет</option>
|
||||
<option value="history">🏛️ История и великие цивилизации</option>
|
||||
<option value="nature">🌿 Природа, животные и экология</option>
|
||||
<option value="psychology">🧠 Психология, мозг и память</option>
|
||||
<option value="featured">⭐ Статья дня (Featured Article)</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<!-- Setting 4: Direct Article URL or Title -->
|
||||
<div class="wiki-setting-group wiki-direct-import-group">
|
||||
<label class="wiki-group-label">Или загрузить конкретную статью по ссылке/названию:</label>
|
||||
<form id="wiki-custom-form" class="wiki-custom-form">
|
||||
<div class="wiki-custom-input-wrap">
|
||||
<input type="text" id="wiki-custom-input" class="input-styled wiki-custom-input" placeholder="Название (напр. Black hole, Solar System) или URL статьи" autocomplete="off" spellcheck="false">
|
||||
<button type="submit" id="wiki-custom-fetch-btn" class="btn btn-primary wiki-custom-fetch-btn">
|
||||
<span>📥 Загрузить</span>
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" id="wiki-settings-save-btn" class="btn btn-primary">Применить настройки</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Move Collection Modal -->
|
||||
<div id="move-col-modal" class="modal-backdrop hidden">
|
||||
<div class="modal-window modal-sm">
|
||||
|
||||
@@ -230,6 +230,26 @@ export const api = {
|
||||
detail = errData.detail || detail;
|
||||
} catch (_) {}
|
||||
throw new Error(detail);
|
||||
},
|
||||
async fetchWikipedia({ query = '', lang = 'simple', time_budget = '10-15', topic = 'random', force = false } = {}) {
|
||||
const resp = await fetch('/api/books/fetch-wikipedia', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
query: query,
|
||||
lang: lang,
|
||||
time_budget: time_budget,
|
||||
topic: topic,
|
||||
force: force
|
||||
})
|
||||
});
|
||||
if (resp.ok) return await resp.json();
|
||||
let detail = 'Не удалось загрузить статью из Wikipedia';
|
||||
try {
|
||||
const errData = await resp.json();
|
||||
detail = errData.detail || detail;
|
||||
} catch (_) {}
|
||||
throw new Error(detail);
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
+359
-14
@@ -42,7 +42,10 @@ export function getCurrentChapterIndex() {
|
||||
function createBookCard(b) {
|
||||
const card = document.createElement('div');
|
||||
const isRfc = (b.source_type === 'rfc');
|
||||
card.className = isRfc ? 'book-card rfc-card' : 'book-card';
|
||||
const isWiki = (b.source_type === 'wikipedia');
|
||||
if (isWiki) card.className = 'book-card wiki-card';
|
||||
else if (isRfc) card.className = 'book-card rfc-card';
|
||||
else card.className = 'book-card';
|
||||
|
||||
const totalCh = Math.max(1, b.chapters_count || 1);
|
||||
const chIndex = Math.max(0, Math.min(totalCh - 1, b.current_chapter || 0));
|
||||
@@ -56,22 +59,42 @@ function createBookCard(b) {
|
||||
? `<span>1 глава • ${totalPct}%</span><span>${totalPct}%</span>`
|
||||
: `<span>Глава ${chIndex + 1} из ${totalCh} (${chPct}%)</span><span>${totalPct}%</span>`;
|
||||
|
||||
let coverIcon = '📄';
|
||||
if (b.source_type === 'epub') coverIcon = '📕';
|
||||
else if (b.source_type === 'pdf') coverIcon = '📑';
|
||||
else if (b.source_type === 'rfc') coverIcon = '🌐';
|
||||
let coverHtml = '';
|
||||
if (b.cover_url) {
|
||||
coverHtml = `<div class="book-cover-img-wrap"><img src="${escapeHtml(b.cover_url)}" class="book-cover-img" alt="Cover" loading="lazy" onerror="this.parentElement.innerHTML='<div class=\\\'book-cover-icon ${isWiki ? 'wiki-cover-icon' : (isRfc ? 'rfc-cover-icon' : '')}\\\'>${isWiki ? '🌐' : (isRfc ? '📜' : '📄')}</div>';"/></div>`;
|
||||
} else {
|
||||
let coverIcon = '📄';
|
||||
if (b.source_type === 'epub') coverIcon = '📕';
|
||||
else if (b.source_type === 'pdf') coverIcon = '📑';
|
||||
else if (isRfc) coverIcon = '📜';
|
||||
else if (isWiki) coverIcon = '🌐';
|
||||
const coverIconClass = isWiki ? 'wiki-cover-icon' : (isRfc ? 'rfc-cover-icon' : '');
|
||||
coverHtml = `<div class="book-cover-icon ${coverIconClass}">${coverIcon}</div>`;
|
||||
}
|
||||
|
||||
const badgeClass = isRfc ? 'badge-format badge-rfc' : 'badge-format';
|
||||
let badgeClass = 'badge-format';
|
||||
if (isWiki) badgeClass = 'badge-format badge-wiki';
|
||||
else if (isRfc) badgeClass = 'badge-format badge-rfc';
|
||||
|
||||
const readMinutes = Math.max(1, Math.round((b.total_words || 800) / 180));
|
||||
const timeBadge = (b.total_words && b.total_words > 0)
|
||||
? `<span class="badge-time" title="Примерное время чтения: ~${readMinutes} минут">⏱️ ~${readMinutes} мин</span>`
|
||||
: '';
|
||||
const progressFillClass = isWiki ? 'wiki-progress-fill' : (isRfc ? 'rfc-progress-fill' : '');
|
||||
|
||||
const authorDisplay = b.author || (isWiki ? 'Wikipedia' : (isRfc ? 'IETF' : 'Неизвестный автор'));
|
||||
const sourceLabel = isWiki ? 'WIKIPEDIA' : (isRfc ? 'RFC' : (b.source_type || 'text').toUpperCase());
|
||||
|
||||
card.innerHTML = `
|
||||
<div>
|
||||
<div class="book-card-top">
|
||||
<div class="book-cover-icon ${isRfc ? 'rfc-cover-icon' : ''}">${coverIcon}</div>
|
||||
${coverHtml}
|
||||
<div class="book-card-info">
|
||||
<h4 class="book-card-title" title="${escapeHtml(b.title)}">${escapeHtml(b.title)}</h4>
|
||||
<p class="book-card-author">${escapeHtml(b.author || (isRfc ? 'IETF' : 'Неизвестный автор'))}</p>
|
||||
<p class="book-card-author">${escapeHtml(authorDisplay)}</p>
|
||||
<div class="book-card-badges">
|
||||
<span class="${badgeClass}">${escapeHtml((b.source_type || 'text').toUpperCase())}</span>
|
||||
<span class="${badgeClass}">${escapeHtml(sourceLabel)}</span>
|
||||
${timeBadge}
|
||||
<span class="badge-words">${b.total_words ? b.total_words.toLocaleString() + ' слов' : ''}</span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -81,14 +104,14 @@ function createBookCard(b) {
|
||||
${progressLabels}
|
||||
</div>
|
||||
<div class="book-progress-bar-bg">
|
||||
<div class="book-progress-bar-fill ${isRfc ? 'rfc-progress-fill' : ''}" style="width: ${totalPct}%"></div>
|
||||
<div class="book-progress-bar-fill ${progressFillClass}" style="width: ${totalPct}%"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="book-card-actions">
|
||||
<button class="btn btn-primary btn-sm read-book-btn" data-id="${b.id}">📖 Читать</button>
|
||||
<button class="btn btn-outline btn-xs train-book-btn" data-title="${escapeHtml(b.title)}" title="Тренировать слова из этой книги">🎓 Слова книги</button>
|
||||
<button class="icon-btn delete-book-btn" data-id="${b.id}" title="Удалить книгу">🗑️</button>
|
||||
<button class="icon-btn delete-book-btn" data-id="${b.id}" title="Удалить из библиотеки">🗑️</button>
|
||||
</div>
|
||||
`;
|
||||
|
||||
@@ -100,7 +123,7 @@ function createBookCard(b) {
|
||||
window.location.hash = `#srs/${encodeURIComponent(bookTag)}`;
|
||||
});
|
||||
card.querySelector('.delete-book-btn').addEventListener('click', async () => {
|
||||
const typeName = isRfc ? 'спецификацию' : 'книгу';
|
||||
const typeName = isWiki ? 'статью Википедии' : (isRfc ? 'спецификацию' : 'книгу');
|
||||
if (confirm(`Удалить ${typeName} "${b.title}" из библиотеки?`)) {
|
||||
await api.books.delete(b.id);
|
||||
loadBookshelf();
|
||||
@@ -117,6 +140,8 @@ export async function loadBookshelf() {
|
||||
const bookshelfEmpty = document.getElementById('bookshelf-empty');
|
||||
const rfcGrid = document.getElementById('rfc-grid');
|
||||
const rfcEmptyState = document.getElementById('rfc-empty-state');
|
||||
const wikiGrid = document.getElementById('wiki-grid');
|
||||
const wikiEmptyState = document.getElementById('wiki-empty-state');
|
||||
|
||||
if (!bookshelfContainer) return;
|
||||
if (!window.location.hash.includes('/book/')) {
|
||||
@@ -125,8 +150,10 @@ export async function loadBookshelf() {
|
||||
}
|
||||
if (bookshelfGrid) bookshelfGrid.innerHTML = '';
|
||||
if (rfcGrid) rfcGrid.innerHTML = '';
|
||||
if (wikiGrid) wikiGrid.innerHTML = '';
|
||||
if (bookshelfEmpty) bookshelfEmpty.classList.add('hidden');
|
||||
if (rfcEmptyState) rfcEmptyState.classList.add('hidden');
|
||||
if (wikiEmptyState) wikiEmptyState.classList.add('hidden');
|
||||
|
||||
try {
|
||||
const books = await api.books.getAll();
|
||||
@@ -141,12 +168,16 @@ export function renderBookshelf(books) {
|
||||
const bookshelfEmpty = document.getElementById('bookshelf-empty');
|
||||
const rfcGrid = document.getElementById('rfc-grid');
|
||||
const rfcEmptyState = document.getElementById('rfc-empty-state');
|
||||
const wikiGrid = document.getElementById('wiki-grid');
|
||||
const wikiEmptyState = document.getElementById('wiki-empty-state');
|
||||
|
||||
if (bookshelfGrid) bookshelfGrid.innerHTML = '';
|
||||
if (rfcGrid) rfcGrid.innerHTML = '';
|
||||
if (wikiGrid) wikiGrid.innerHTML = '';
|
||||
|
||||
const generalBooks = (books || []).filter(b => b.source_type !== 'rfc');
|
||||
const generalBooks = (books || []).filter(b => b.source_type !== 'rfc' && b.source_type !== 'wikipedia');
|
||||
const rfcBooks = (books || []).filter(b => b.source_type === 'rfc');
|
||||
const wikiBooks = (books || []).filter(b => b.source_type === 'wikipedia');
|
||||
|
||||
if (bookshelfEmpty) {
|
||||
bookshelfEmpty.classList.toggle('hidden', generalBooks.length > 0);
|
||||
@@ -157,6 +188,15 @@ export function renderBookshelf(books) {
|
||||
});
|
||||
}
|
||||
|
||||
if (wikiEmptyState) {
|
||||
wikiEmptyState.classList.toggle('hidden', wikiBooks.length > 0);
|
||||
}
|
||||
if (wikiGrid) {
|
||||
wikiBooks.forEach(b => {
|
||||
wikiGrid.appendChild(createBookCard(b));
|
||||
});
|
||||
}
|
||||
|
||||
if (rfcEmptyState) {
|
||||
rfcEmptyState.classList.toggle('hidden', rfcBooks.length > 0);
|
||||
}
|
||||
@@ -375,6 +415,24 @@ export function buildChapterDOM(chapterIdx, knownWordsMap, isContinuous = false)
|
||||
|
||||
blockEl.appendChild(header);
|
||||
blockEl.appendChild(pre);
|
||||
} else if (block.type === 'figure') {
|
||||
blockEl = document.createElement('figure');
|
||||
blockEl.className = 'reader-figure';
|
||||
|
||||
const img = document.createElement('img');
|
||||
img.className = 'reader-figure-img';
|
||||
img.src = block.src;
|
||||
img.alt = block.alt || '';
|
||||
img.loading = 'lazy';
|
||||
img.onerror = () => { blockEl.style.display = 'none'; };
|
||||
blockEl.appendChild(img);
|
||||
|
||||
if (block.caption) {
|
||||
const figcaption = document.createElement('figcaption');
|
||||
figcaption.className = 'reader-figcaption';
|
||||
renderInteractiveSpans(figcaption, block.caption, block.caption, knownWordsMap);
|
||||
blockEl.appendChild(figcaption);
|
||||
}
|
||||
} else if (block.type === 'table') {
|
||||
blockEl = document.createElement('div');
|
||||
blockEl.className = 'reader-table-wrap';
|
||||
@@ -804,6 +862,86 @@ export function parseContentBlocks(content) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 2a. HTML Figure Block: <figure ...> ... </figure>
|
||||
if (/^<figure\b/i.test(trimmed)) {
|
||||
const figLines = [];
|
||||
while (i < lines.length) {
|
||||
figLines.push(lines[i]);
|
||||
if (/<\/figure>/i.test(lines[i])) {
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
const figHtml = figLines.join('\n');
|
||||
const imgMatch = figHtml.match(/<img\b([^>]*)>/i);
|
||||
let src = '';
|
||||
let alt = '';
|
||||
if (imgMatch) {
|
||||
const srcM = imgMatch[1].match(/src=["']([^"']+)["']/i);
|
||||
const altM = imgMatch[1].match(/alt=["']([^"']*)["']/i);
|
||||
if (srcM) src = srcM[1];
|
||||
if (altM) alt = altM[1];
|
||||
}
|
||||
let caption = '';
|
||||
const capMatch = figHtml.match(/<figcaption\b[^>]*>([\s\S]*?)<\/figcaption>/i);
|
||||
if (capMatch) {
|
||||
caption = capMatch[1].replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
if (src) {
|
||||
blocks.push({
|
||||
type: 'figure',
|
||||
src: src,
|
||||
alt: alt,
|
||||
caption: caption
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// 2b. HTML Data Table: <div class="reader-table-wrap"><table ...> or <table ...>
|
||||
if (/^(?:<div[^>]*class=["'][^"']*reader-table-wrap[^"']*["'][^>]*>\s*)?<table\b/i.test(trimmed)) {
|
||||
const tableLines = [];
|
||||
while (i < lines.length) {
|
||||
tableLines.push(lines[i]);
|
||||
if (/<\/table>(?:\s*<\/div>)?/i.test(lines[i])) {
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
const fullTableHtml = tableLines.join('\n');
|
||||
const tpl = document.createElement('template');
|
||||
tpl.innerHTML = fullTableHtml;
|
||||
const tableEl = tpl.content.querySelector('table');
|
||||
if (tableEl) {
|
||||
const headers = [];
|
||||
const ths = tableEl.querySelectorAll('thead th, tr:first-child th');
|
||||
ths.forEach(th => headers.push(th.textContent.trim()));
|
||||
|
||||
const rows = [];
|
||||
const trs = tableEl.querySelectorAll('tbody tr, tr');
|
||||
trs.forEach((tr) => {
|
||||
const tds = tr.querySelectorAll('td');
|
||||
if (tds.length > 0) {
|
||||
const rowCells = [];
|
||||
tds.forEach(td => rowCells.push(td.textContent.trim()));
|
||||
rows.push(rowCells);
|
||||
}
|
||||
});
|
||||
|
||||
if (headers.length > 0 || rows.length > 0) {
|
||||
blocks.push({
|
||||
type: 'table',
|
||||
headers: headers,
|
||||
alignments: headers.map(() => 'left'),
|
||||
rows: rows
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Markdown Table: lines starting with | and ending with |
|
||||
if (trimmed.startsWith('|') && trimmed.endsWith('|')) {
|
||||
const tableLines = [];
|
||||
@@ -950,6 +1088,9 @@ export function parseContentBlocks(content) {
|
||||
/^[-*+]\s+/.test(curTrim) ||
|
||||
/^\d+\.\s+/.test(curTrim) ||
|
||||
/^(\-{3,}|\*{3,}|\_{3,})$/.test(curTrim) ||
|
||||
/^<figure\b/i.test(curTrim) ||
|
||||
/^<table\b/i.test(curTrim) ||
|
||||
curTrim.includes('reader-table-wrap') ||
|
||||
/^<(?:p|div)[^>]*align=["']?center/i.test(curTrim)) {
|
||||
break;
|
||||
}
|
||||
@@ -1899,7 +2040,7 @@ export function initReader() {
|
||||
if (wordSpan) {
|
||||
e.stopPropagation();
|
||||
const word = wordSpan.dataset.word;
|
||||
const context = wordSpan.dataset.context || wordSpan.closest('p, div, li, td, th, blockquote')?.textContent || word;
|
||||
const context = wordSpan.dataset.context || wordSpan.closest('p, div, li, td, th, blockquote, figcaption')?.textContent || word;
|
||||
handleWordClick(word, context, wordSpan);
|
||||
}
|
||||
});
|
||||
@@ -2448,4 +2589,208 @@ export function initReader() {
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Wikipedia Autonomous Import & Settings
|
||||
// -----------------------------------------------------------------------
|
||||
const WIKI_SETTINGS_KEY = 'notes_wiki_settings';
|
||||
const wikiReadBtn = document.getElementById('wiki-read-btn');
|
||||
const wikiSettingsBtn = document.getElementById('wiki-settings-btn');
|
||||
const wikiSettingsModal = document.getElementById('wiki-settings-modal');
|
||||
const wikiSettingsCloseBtn = document.getElementById('wiki-settings-close-btn');
|
||||
const wikiSettingsSaveBtn = document.getElementById('wiki-settings-save-btn');
|
||||
const wikiFetchStatus = document.getElementById('wiki-fetch-status');
|
||||
const wikiCustomForm = document.getElementById('wiki-custom-form');
|
||||
const wikiCustomInput = document.getElementById('wiki-custom-input');
|
||||
const wikiCustomFetchBtn = document.getElementById('wiki-custom-fetch-btn');
|
||||
|
||||
const wikiActiveLangBadge = document.getElementById('wiki-active-lang-badge');
|
||||
const wikiActiveTimeBadge = document.getElementById('wiki-active-time-badge');
|
||||
const wikiActiveTopicBadge = document.getElementById('wiki-active-topic-badge');
|
||||
|
||||
const TOPIC_LABELS = {
|
||||
'random': '🎲 Любая тема',
|
||||
'science': '🚀 Наука и космос',
|
||||
'tech': '💻 Технологии и IT',
|
||||
'history': '🏛️ История',
|
||||
'nature': '🌿 Природа',
|
||||
'psychology': '🧠 Психология',
|
||||
'featured': '⭐ Статья дня'
|
||||
};
|
||||
|
||||
const TIME_LABELS = {
|
||||
'5': '⏱️ ~5 мин',
|
||||
'10-15': '⏱️ ~10-15 мин',
|
||||
'20+': '⏱️ 20+ мин',
|
||||
'any': '⏱️ Любой объем'
|
||||
};
|
||||
|
||||
function getStoredWikiSettings() {
|
||||
try {
|
||||
const raw = localStorage.getItem(WIKI_SETTINGS_KEY);
|
||||
if (raw) {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (parsed && typeof parsed === 'object') {
|
||||
return {
|
||||
lang: parsed.lang || 'simple',
|
||||
time_budget: parsed.time_budget || '10-15',
|
||||
topic: parsed.topic || 'random'
|
||||
};
|
||||
}
|
||||
}
|
||||
} catch (_) {}
|
||||
return { lang: 'simple', time_budget: '10-15', topic: 'random' };
|
||||
}
|
||||
|
||||
function saveStoredWikiSettings(settings) {
|
||||
try {
|
||||
localStorage.setItem(WIKI_SETTINGS_KEY, JSON.stringify(settings));
|
||||
} catch (_) {}
|
||||
updateWikiUIBadges(settings);
|
||||
}
|
||||
|
||||
function updateWikiUIBadges(settings) {
|
||||
if (wikiActiveLangBadge) {
|
||||
wikiActiveLangBadge.textContent = settings.lang === 'en' ? 'Standard English (B2–C1)' : 'Simple English (A2–B1)';
|
||||
}
|
||||
if (wikiActiveTimeBadge) {
|
||||
wikiActiveTimeBadge.textContent = TIME_LABELS[settings.time_budget] || '⏱️ ~10-15 мин';
|
||||
}
|
||||
if (wikiActiveTopicBadge) {
|
||||
wikiActiveTopicBadge.textContent = TOPIC_LABELS[settings.topic] || '🎲 Любая тема';
|
||||
}
|
||||
}
|
||||
|
||||
function syncWikiModalInputs(settings) {
|
||||
const langRadios = document.querySelectorAll('input[name="wiki-lang"]');
|
||||
langRadios.forEach(r => {
|
||||
r.checked = (r.value === settings.lang);
|
||||
});
|
||||
const timeRadios = document.querySelectorAll('input[name="wiki-time"]');
|
||||
timeRadios.forEach(r => {
|
||||
r.checked = (r.value === settings.time_budget);
|
||||
});
|
||||
const topicSelect = document.getElementById('wiki-topic-select');
|
||||
if (topicSelect) {
|
||||
topicSelect.value = settings.topic || 'random';
|
||||
}
|
||||
}
|
||||
|
||||
function readWikiModalInputs() {
|
||||
const langRadio = document.querySelector('input[name="wiki-lang"]:checked');
|
||||
const timeRadio = document.querySelector('input[name="wiki-time"]:checked');
|
||||
const topicSelect = document.getElementById('wiki-topic-select');
|
||||
|
||||
return {
|
||||
lang: langRadio ? langRadio.value : 'simple',
|
||||
time_budget: timeRadio ? timeRadio.value : '10-15',
|
||||
topic: topicSelect ? topicSelect.value : 'random'
|
||||
};
|
||||
}
|
||||
|
||||
const currentWikiSettings = getStoredWikiSettings();
|
||||
updateWikiUIBadges(currentWikiSettings);
|
||||
|
||||
if (wikiSettingsBtn && wikiSettingsModal) {
|
||||
wikiSettingsBtn.addEventListener('click', () => {
|
||||
syncWikiModalInputs(getStoredWikiSettings());
|
||||
wikiSettingsModal.classList.remove('hidden');
|
||||
});
|
||||
}
|
||||
|
||||
if (wikiSettingsCloseBtn && wikiSettingsModal) {
|
||||
wikiSettingsCloseBtn.addEventListener('click', () => {
|
||||
wikiSettingsModal.classList.add('hidden');
|
||||
});
|
||||
}
|
||||
|
||||
if (wikiSettingsModal) {
|
||||
wikiSettingsModal.addEventListener('click', (e) => {
|
||||
if (e.target === wikiSettingsModal) {
|
||||
wikiSettingsModal.classList.add('hidden');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (wikiSettingsSaveBtn && wikiSettingsModal) {
|
||||
wikiSettingsSaveBtn.addEventListener('click', () => {
|
||||
const newSettings = readWikiModalInputs();
|
||||
saveStoredWikiSettings(newSettings);
|
||||
wikiSettingsModal.classList.add('hidden');
|
||||
showToast('Настройки статей Википедии сохранены!');
|
||||
});
|
||||
}
|
||||
|
||||
async function handleWikiFetch(params) {
|
||||
const isCustom = Boolean(params.query);
|
||||
const activeBtn = isCustom ? wikiCustomFetchBtn : wikiReadBtn;
|
||||
|
||||
if (activeBtn) activeBtn.disabled = true;
|
||||
if (wikiReadBtn) wikiReadBtn.disabled = true;
|
||||
if (wikiCustomInput) wikiCustomInput.disabled = true;
|
||||
|
||||
if (wikiFetchStatus) {
|
||||
wikiFetchStatus.className = 'wiki-fetch-status wiki-status-loading';
|
||||
wikiFetchStatus.innerHTML = `<span>⏳ ${isCustom ? 'Загрузка статьи по запросу...' : 'Подбираем и загружаем статью из Википедии...'}</span>`;
|
||||
wikiFetchStatus.classList.remove('hidden');
|
||||
}
|
||||
|
||||
try {
|
||||
const book = await api.books.fetchWikipedia(params);
|
||||
if (wikiFetchStatus) {
|
||||
wikiFetchStatus.className = 'wiki-fetch-status wiki-status-success';
|
||||
const estMin = Math.max(1, Math.round((book.total_words || 800) / 180));
|
||||
wikiFetchStatus.innerHTML = `<span>✅ Статья «${escapeHtml(book.title)}» сохранена (~${estMin} мин, ${book.total_words ? book.total_words.toLocaleString() : ''} слов)!</span>`;
|
||||
}
|
||||
showToast(book.already_existed ? `Открываем ${book.title}` : `Статья Википедии сохранена офлайн!`);
|
||||
if (wikiCustomInput) wikiCustomInput.value = '';
|
||||
await loadBookshelf();
|
||||
setTimeout(() => {
|
||||
if (wikiFetchStatus) wikiFetchStatus.classList.add('hidden');
|
||||
if (wikiSettingsModal) wikiSettingsModal.classList.add('hidden');
|
||||
window.location.hash = `#reader/book/${book.id}`;
|
||||
}, 500);
|
||||
} catch (err) {
|
||||
console.error('Wikipedia fetch error:', err);
|
||||
if (wikiFetchStatus) {
|
||||
wikiFetchStatus.className = 'wiki-fetch-status wiki-status-error';
|
||||
wikiFetchStatus.innerHTML = `<span>❌ Ошибка: ${escapeHtml(err.message || 'Не удалось загрузить статью')}</span>`;
|
||||
wikiFetchStatus.classList.remove('hidden');
|
||||
}
|
||||
showToast(err.message || 'Ошибка загрузки статьи Wikipedia', 4000);
|
||||
} finally {
|
||||
if (activeBtn) activeBtn.disabled = false;
|
||||
if (wikiReadBtn) wikiReadBtn.disabled = false;
|
||||
if (wikiCustomInput) wikiCustomInput.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (wikiReadBtn) {
|
||||
wikiReadBtn.addEventListener('click', () => {
|
||||
const currentSettings = getStoredWikiSettings();
|
||||
handleWikiFetch({
|
||||
query: '',
|
||||
lang: currentSettings.lang,
|
||||
time_budget: currentSettings.time_budget,
|
||||
topic: currentSettings.topic,
|
||||
force: false
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
if (wikiCustomForm) {
|
||||
wikiCustomForm.addEventListener('submit', (e) => {
|
||||
e.preventDefault();
|
||||
const val = wikiCustomInput ? wikiCustomInput.value.trim() : '';
|
||||
if (!val) return;
|
||||
const currentSettings = getStoredWikiSettings();
|
||||
handleWikiFetch({
|
||||
query: val,
|
||||
lang: currentSettings.lang,
|
||||
time_budget: currentSettings.time_budget,
|
||||
topic: currentSettings.topic,
|
||||
force: false
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
// Service Worker for Notes English PWA (Stage 2)
|
||||
// Strategy: Cache-first for static assets, Network-first for API calls.
|
||||
|
||||
const CACHE_NAME = 'notes-v6.2';
|
||||
const CACHE_NAME = 'notes-v6.3';
|
||||
const OFFLINE_URL = '/';
|
||||
|
||||
const PRECACHE_URLS = [
|
||||
|
||||
@@ -2046,6 +2046,191 @@ def test_clean_russian_translation_removes_romanization():
|
||||
assert clean_like == "подобный, похожий, как, типа"
|
||||
|
||||
|
||||
def test_extract_wikipedia_title_and_lang():
|
||||
from app.reader import extract_wikipedia_title_and_lang
|
||||
|
||||
t1, l1 = extract_wikipedia_title_and_lang("https://simple.wikipedia.org/wiki/Solar_System")
|
||||
assert t1 == "Solar_System"
|
||||
assert l1 == "simple"
|
||||
|
||||
t2, l2 = extract_wikipedia_title_and_lang("https://en.wikipedia.org/wiki/Quantum_mechanics")
|
||||
assert t2 == "Quantum_mechanics"
|
||||
assert l2 == "en"
|
||||
|
||||
t3, l3 = extract_wikipedia_title_and_lang("Artificial Intelligence", default_lang="simple")
|
||||
assert t3 == "Artificial_Intelligence"
|
||||
assert l3 == "simple"
|
||||
|
||||
|
||||
def test_clean_wiki_html_unit():
|
||||
from app.reader import clean_wiki_html
|
||||
|
||||
sample_parsoid_html = """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<style>.some-css { color: red; }</style>
|
||||
<script>console.log("bad");</script>
|
||||
<div class="hatnote">This article is about astronomy.</div>
|
||||
<table class="infobox">
|
||||
<tr><th>Lead</th></tr>
|
||||
<tr><td><img src="//thumb.wikimedia.org/wikipedia/commons/thumb/a/a1/Sun.jpg/300px-Sun.jpg" alt="The Sun"></td></tr>
|
||||
<tr><td>Orbital info</td></tr>
|
||||
</table>
|
||||
<p>The <b>Sun</b> is the star at the center of the Solar System<sup class="reference">[1]</sup>.</p>
|
||||
<figure typeof="mw:File/Thumb">
|
||||
<img src="//thumb.wikimedia.org/wikipedia/commons/thumb/b/b2/Solar_flare.jpg/400px-Solar_flare.jpg" alt="Solar flare">
|
||||
<figcaption>A powerful <a href="./Solar_flare">solar flare</a> erupting from the surface</figcaption>
|
||||
</figure>
|
||||
<p>It is a nearly perfect ball of hot plasma<sup class="reference">[citation needed]</sup>.</p>
|
||||
|
||||
<h2><span class="mw-headline">Characteristics</span><span class="mw-editsection">[edit]</span></h2>
|
||||
<p>The Sun is a G-type main-sequence star.</p>
|
||||
<table class="wikitable">
|
||||
<thead><tr><th>Parameter</th><th>Value</th></tr></thead>
|
||||
<tbody><tr><td>Radius</td><td>696,340 km</td></tr></tbody>
|
||||
</table>
|
||||
|
||||
<h2>References</h2>
|
||||
<p>1. Solar Physics Journal 2024.</p>
|
||||
|
||||
<h2>Other websites</h2>
|
||||
<ul><li>NASA Sun info</li></ul>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
chapters, cover_url, total_words = clean_wiki_html(sample_parsoid_html, article_title="Sun")
|
||||
|
||||
assert cover_url.startswith("https://thumb.wikimedia.org/")
|
||||
assert len(chapters) == 2 # Overview and Characteristics (References and Other websites ignored)
|
||||
|
||||
ch1 = chapters[0]
|
||||
assert ch1["title"] == "Overview"
|
||||
assert "star at the center of the Solar System" in ch1["content"]
|
||||
assert "[1]" not in ch1["content"]
|
||||
assert "[citation needed]" not in ch1["content"]
|
||||
assert ".some-css" not in ch1["content"]
|
||||
assert "bad" not in ch1["content"]
|
||||
assert "<figure class=\"reader-figure\">" in ch1["content"]
|
||||
assert "https://thumb.wikimedia.org/wikipedia/commons/thumb/b/b2/Solar_flare.jpg/400px-Solar_flare.jpg" in ch1["content"]
|
||||
assert "A powerful solar flare erupting from the surface" in ch1["content"]
|
||||
|
||||
ch2 = chapters[1]
|
||||
assert ch2["title"] == "Characteristics"
|
||||
assert "[edit]" not in ch2["title"]
|
||||
assert "G-type main-sequence star" in ch2["content"]
|
||||
assert "reader-table" in ch2["content"]
|
||||
assert "696,340 km" in ch2["content"]
|
||||
assert total_words > 20
|
||||
|
||||
|
||||
def test_wikipedia_api_endpoints_workflow(monkeypatch):
|
||||
import urllib.request
|
||||
import io
|
||||
from app.reader import clean_wiki_html
|
||||
|
||||
sample_html = """
|
||||
<html><body>
|
||||
<table class="infobox">
|
||||
<tr><td><img src="//thumb.wikimedia.org/test/cover.jpg" alt="Planet"></td></tr>
|
||||
</table>
|
||||
<p>Earth is the third planet from the Sun and the only astronomical object known to harbor life.</p>
|
||||
<h2>Atmosphere</h2>
|
||||
<p>The atmosphere of Earth is composed mostly of nitrogen and oxygen.</p>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
sample_summary = {
|
||||
"title": "Earth",
|
||||
"thumbnail": {"source": "https://thumb.wikimedia.org/test/cover.jpg"}
|
||||
}
|
||||
|
||||
class MockResponse:
|
||||
def __init__(self, data_bytes, status=200):
|
||||
self.data_bytes = data_bytes
|
||||
self.status = status
|
||||
|
||||
def read(self, *args, **kwargs):
|
||||
return self.data_bytes
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
return self.data_bytes.decode(*args, **kwargs)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
def mock_urlopen(req, timeout=None):
|
||||
url = req.full_url if hasattr(req, "full_url") else str(req)
|
||||
if "summary" in url:
|
||||
return MockResponse(json.dumps(sample_summary).encode("utf-8"))
|
||||
elif "html" in url:
|
||||
return MockResponse(sample_html.encode("utf-8"))
|
||||
return MockResponse(b"{}")
|
||||
|
||||
monkeypatch.setattr(urllib.request, "urlopen", mock_urlopen)
|
||||
|
||||
# 1. Login user
|
||||
login_resp = client.post("/api/login", json={"username": "testadmin", "password": "testpass12345"})
|
||||
assert login_resp.status_code == 200
|
||||
token = login_resp.cookies.get(config.SESSION_COOKIE_NAME)
|
||||
auth_headers = {"Cookie": f"{config.SESSION_COOKIE_NAME}={token}"}
|
||||
|
||||
# 2. Fetch Wikipedia article
|
||||
resp = client.post(
|
||||
"/api/books/fetch-wikipedia",
|
||||
json={"query": "Earth", "lang": "simple", "topic": "science"},
|
||||
headers=auth_headers
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
book_data = resp.json()
|
||||
assert book_data["title"] == "Earth"
|
||||
assert book_data["source_type"] == "wikipedia"
|
||||
assert "cover.jpg" in book_data["cover_url"]
|
||||
assert len(book_data["chapters"]) >= 1
|
||||
assert book_data["already_existed"] is False
|
||||
book_id = book_data["id"]
|
||||
|
||||
# 3. Fetch again without force - should return already_existed=True
|
||||
resp2 = client.post(
|
||||
"/api/books/fetch-wikipedia",
|
||||
json={"query": "Earth", "lang": "simple", "force": False},
|
||||
headers=auth_headers
|
||||
)
|
||||
assert resp2.status_code == 200
|
||||
assert resp2.json()["already_existed"] is True
|
||||
assert resp2.json()["id"] == book_id
|
||||
|
||||
# 4. Check book list includes cover_url and source_type
|
||||
list_resp = client.get("/api/books", headers=auth_headers)
|
||||
assert list_resp.status_code == 200
|
||||
books = list_resp.json()
|
||||
wiki_book = next((b for b in books if b["id"] == book_id), None)
|
||||
assert wiki_book is not None
|
||||
assert wiki_book["source_type"] == "wikipedia"
|
||||
assert "cover.jpg" in wiki_book["cover_url"]
|
||||
|
||||
# 5. Update reading progress
|
||||
prog_resp = client.put(
|
||||
f"/api/books/{book_id}/progress",
|
||||
json={"current_chapter": 1, "current_position": 0.5},
|
||||
headers=auth_headers
|
||||
)
|
||||
assert prog_resp.status_code == 200
|
||||
assert prog_resp.json()["current_chapter"] == 1
|
||||
assert prog_resp.json()["current_position"] == 0.5
|
||||
assert prog_resp.json()["cover_url"] != ""
|
||||
|
||||
# 6. Delete book
|
||||
del_resp = client.delete(f"/api/books/{book_id}", headers=auth_headers)
|
||||
assert del_resp.status_code == 200
|
||||
assert del_resp.json()["deleted_id"] == book_id
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user