diff --git a/app/main.py b/app/main.py index cd445b7..72075a9 100644 --- a/app/main.py +++ b/app/main.py @@ -30,7 +30,7 @@ from app.auth import ( from app.dictionary import lookup_term, USER_AGENT, close_http_client from app.export import export_csv, export_json, export_anki_tsv, export_markdown from app.srs import calculate_sm2, generate_quiz_options, build_activity_heatmap, calculate_cefr_level -from app.reader import parse_epub, parse_text, parse_pdf, count_words +from app.reader import parse_epub, parse_text, parse_pdf, count_words, extract_rfc_number, fetch_and_parse_rfc, parse_rfc_text from app.ai import translate_sentence_ai, get_ollama_status from app.phrasal import detect_phrasal_verb from app.cefr import ( @@ -182,6 +182,11 @@ class BookProgressRequest(BaseModel): position: Optional[float] = None +class RfcFetchRequest(BaseModel): + rfc: str + force: Optional[bool] = False + + class SentenceTranslateRequest(BaseModel): sentence: str = Field(..., min_length=1) target_word: Optional[str] = "" @@ -1315,6 +1320,86 @@ async def api_create_book( return res +@app.post("/api/books/fetch-rfc") +async def api_fetch_rfc( + req: RfcFetchRequest, + user: dict = Depends(require_user), +): + """ + Fetches an RFC specification by number or URL from RFC Editor & Datatracker, + parses sections into readable chapters, and saves to the user's bookshelf. + If the RFC already exists on user's shelf and force=False, returns the existing book. + """ + raw_query = (req.rfc or "").strip() + if not raw_query: + raise HTTPException(status_code=400, detail="Укажите номер или URL спецификации RFC") + + try: + rfc_num = extract_rfc_number(raw_query) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + + # Check if user already loaded this RFC + if not req.force: + with get_db() as conn: + existing = conn.execute( + """ + SELECT id, user_id, title, author, source_type, chapters_json, + json_array_length(chapters_json) AS chapters_count, + current_chapter, current_position, total_words, + created_at, updated_at + FROM books + WHERE user_id = ? AND source_type = 'rfc' AND (title LIKE ? OR title LIKE ?) + ORDER BY id DESC LIMIT 1 + """, + (user["id"], f"RFC {rfc_num}:%", f"RFC {rfc_num}"), + ).fetchone() + if existing: + res = dict(existing) + try: + res["chapters"] = json.loads(res["chapters_json"]) + except Exception: + res["chapters"] = [] + res["already_existed"] = True + return res + + try: + title, author, chapters, rfc_num, total_words = fetch_and_parse_rfc(raw_query) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as e: + raise HTTPException(status_code=500, detail=f"Ошибка загрузки RFC: {e}") + + chapters_json = json.dumps(chapters, ensure_ascii=False) + + with get_db() as conn: + cursor = conn.execute( + """ + INSERT INTO books ( + user_id, title, author, source_type, chapters_json, + current_chapter, current_position, total_words + ) VALUES (?, ?, ?, 'rfc', ?, 0, 0.0, ?) + """, + (user["id"], title, author, chapters_json, total_words), + ) + new_id = cursor.lastrowid + row = conn.execute( + """ + SELECT id, user_id, title, author, source_type, chapters_json, + json_array_length(chapters_json) AS chapters_count, + current_chapter, current_position, total_words, + created_at, updated_at + FROM books WHERE id = ? + """, + (new_id,), + ).fetchone() + + res = dict(row) + res["chapters"] = chapters + res["already_existed"] = False + return res + + @app.get("/api/books/{book_id}") async def api_get_book(book_id: int, user: dict = Depends(require_user)): """ diff --git a/app/reader.py b/app/reader.py index 3f8efaa..afc9366 100644 --- a/app/reader.py +++ b/app/reader.py @@ -2,9 +2,12 @@ import io import re import sys import html +import json import zipfile import posixpath import urllib.parse +import urllib.request +import urllib.error from pathlib import Path import xml.etree.ElementTree as ET from html.parser import HTMLParser @@ -846,3 +849,241 @@ def parse_pdf(file_bytes: bytes, filename: str = "") -> Tuple[str, str, List[Dic return title, author, chapters + +# --------------------------------------------------------------------------- +# RFC Specifications Parser & Fetcher (IETF & RFC Editor) +# --------------------------------------------------------------------------- + +def extract_rfc_number(query: str) -> int: + """ + Extracts RFC number from a string, supporting inputs like: + - "9110", "rfc 9110", "RFC-9110", "rfc9110" + - "https://www.rfc-editor.org/rfc/rfc9110.txt" + - "https://datatracker.ietf.org/doc/rfc9110/" + """ + query = (query or "").strip() + if re.search(r"-\s*\d+", query) and not re.search(r"rfc-\d+", query, re.IGNORECASE): + raise ValueError(f"Недопустимый отрицательный номер RFC: '{query}'") + + m = re.search(r"(?:rfc[-_ ]?|doc/(?:html/)?rfc|/rfc)?(\d+)", query, re.IGNORECASE) + if not m: + raise ValueError(f"Не удалось определить номер RFC из строки: '{query}'") + num = int(m.group(1)) + if num <= 0 or num > 100000: + raise ValueError(f"Недопустимый номер RFC: {num}") + return num + + +def parse_rfc_text( + raw_text: str, + rfc_number: int, + title: str = "", + author: str = "" +) -> Tuple[str, str, List[Dict[str, str]], int]: + """ + Cleans raw RFC text, strips pagination artifacts and form feeds, + and splits the specification into structured readable chapters. + Returns: (clean_title, clean_author, chapters, total_words) + """ + raw_text = raw_text.replace("\r\n", "\n").replace("\r", "\n").lstrip("\ufeff") + + # 1. Clean pagination artifacts (running footers, form-feed \x0c, running headers) + clean = re.sub( + r"\n[^\n]*\[Page [0-9ivxlc]+\]\s*[\n\x0c]+\s*(?:RFC[^\n]*\n)?", + "\n\n", + raw_text, + flags=re.IGNORECASE + ) + clean = clean.replace("\x0c", "\n\n") + + lines = clean.splitlines() + + # 2. If title wasn't provided, extract from the document header + if not title: + for l in lines[0:65]: + s = l.strip() + if not s: + continue + if re.match( + r"^(Internet Engineering Task Force|Request for Comments|STD:|Updates:|Obsoletes:|Category:|ISSN:|Network Working Group|BCP:|FYI:|RFC\s*\d+)", + s, + re.IGNORECASE + ): + continue + if re.match(r"^Table of Contents\b", s, re.IGNORECASE): + break + if re.match(r"^Abstract\b", s, re.IGNORECASE): + break + if 4 <= len(s) <= 80 and not s.endswith((":", ";", ".")): + title = s + break + + clean_title = f"RFC {rfc_number}: {title}" if title and not title.lower().startswith("rfc") else (title or f"RFC {rfc_number}") + clean_author = author or "IETF" + + # 3. Detect chapters + in_toc = False + past_toc = False + headings = [] + bad_sentence_starters = { + "if", "the", "when", "where", "this", "that", "these", "those", + "allow", "rather", "a", "an", "in", "for", "on", "to", "there", + "clients", "optionally", "both", "all", "each", "such", "see", "note" + } + + for i, line in enumerate(lines): + trimmed = line.strip() + if not trimmed: + continue + + if re.search(r"\.{4,}", line): + in_toc = True + continue + + if not past_toc: + if re.match(r"^Table of Contents\b", trimmed, re.IGNORECASE): + in_toc = True + continue + if in_toc: + if (line.startswith("1.") or line.startswith("1 ") or re.match(r"^\s{10,}1\.\s+[A-Z]", line)) and not re.search(r"\.{3,}", line) and not re.search(r"\s+\d+$", trimmed): + in_toc = False + past_toc = True + else: + continue + + prev_blank = (i == 0) or (not lines[i - 1].strip()) + if not prev_blank: + continue + + m = re.match( + r"^(?:([0-9]{1,2}\.?)\s+([A-Za-z][^\n]{1,65})|(Appendix\s+[A-Z]\.?\s*[^\n]{0,65})|(Authors?'? Addresses\b.*)|(Full Copyright Statement\b.*))$", + trimmed, + re.IGNORECASE + ) + if not m and re.match(r"^[0-9]{1,2}\.\s+[A-Z\s]{3,50}$", trimmed): + m = True + + if m: + if trimmed.endswith((".", ",", ":", ";", "-")): + continue + if len(trimmed) > 70: + continue + words = re.findall(r"\b[A-Za-z]+\b", trimmed) + if len(words) >= 2: + sec_word = words[0].lower() if not words[0].isdigit() else (words[1].lower() if len(words) > 1 else "") + if sec_word in bad_sentence_starters: + continue + headings.append((i, trimmed)) + + # Deduplicate headings that are right next to each other + deduped_headings = [] + for h in headings: + if not deduped_headings or (h[0] - deduped_headings[-1][0] >= 3): + deduped_headings.append(h) + + chapters: List[Dict[str, str]] = [] + if deduped_headings: + first_line_idx = deduped_headings[0][0] + intro_content = "\n".join(lines[:first_line_idx]).strip() + if count_words(intro_content) >= 30: + chapters.append({ + "title": "Overview & Abstract", + "content": intro_content + }) + + for idx, (line_idx, h_title) in enumerate(deduped_headings): + next_line_idx = deduped_headings[idx + 1][0] if idx + 1 < len(deduped_headings) else len(lines) + body = "\n".join(lines[line_idx:next_line_idx]).strip() + if body: + chapters.append({ + "title": h_title, + "content": body + }) + + if not chapters: + paras = [p.strip() for p in re.split(r"\n{2,}", clean) if p.strip()] + chunks = [] + curr = [] + curr_w = 0 + for p in paras: + w = count_words(p) + if curr_w + w > 1500 and curr: + chunks.append("\n\n".join(curr)) + curr = [p] + curr_w = w + else: + curr.append(p) + curr_w += w + if curr: + chunks.append("\n\n".join(curr)) + + for idx, chunk in enumerate(chunks, 1): + chapters.append({ + "title": f"Section {idx}", + "content": chunk + }) + + total_words = sum(count_words(ch["content"]) for ch in chapters) + return clean_title, clean_author, chapters, total_words + + +def fetch_and_parse_rfc(query: str) -> Tuple[str, str, List[Dict[str, str]], int, int]: + """ + Fetches an RFC by number or URL from RFC Editor & IETF Datatracker, + cleans formatting, parses chapters, and returns: + (clean_title, author, chapters, rfc_number, total_words) + """ + rfc_num = extract_rfc_number(query) + + # 1. Fetch metadata from IETF Datatracker (title, authors) + dt_title = "" + dt_authors = [] + try: + dt_url = f"https://datatracker.ietf.org/doc/rfc{rfc_num}/doc.json" + req = urllib.request.Request(dt_url, headers={"User-Agent": "study-qpid-pro/1.0 (educational-reader)"}) + with urllib.request.urlopen(req, timeout=5) as resp: + data = json.loads(resp.read().decode("utf-8", errors="replace")) + dt_title = (data.get("title") or "").strip() + for a in data.get("authors", []): + name = a.get("name") or a.get("person", {}).get("plain_name") + if name and name not in dt_authors: + dt_authors.append(name.strip()) + except Exception: + pass + + # 2. Fetch raw text from RFC Editor (primary) or Datatracker (fallback) + raw_text = None + urls_to_try = [ + f"https://www.rfc-editor.org/rfc/rfc{rfc_num}.txt", + f"https://datatracker.ietf.org/doc/rfc{rfc_num}.txt", + ] + fetch_err = None + for u in urls_to_try: + try: + req = urllib.request.Request(u, headers={"User-Agent": "study-qpid-pro/1.0 (educational-reader)"}) + with urllib.request.urlopen(req, timeout=8) as resp: + raw_bytes = resp.read() + raw_text = raw_bytes.decode("utf-8", errors="replace") + break + except urllib.error.HTTPError as he: + fetch_err = he + if he.code == 404: + continue + except Exception as e: + fetch_err = e + continue + + if not raw_text: + if isinstance(fetch_err, urllib.error.HTTPError) and fetch_err.code == 404: + raise ValueError(f"RFC {rfc_num} не найден в базе RFC Editor (404 Not Found)") + err_msg = str(fetch_err) if fetch_err else "документ недоступен" + raise ValueError(f"Не удалось загрузить RFC {rfc_num}: {err_msg}") + + author = ("IETF (" + ", ".join(dt_authors) + ")") if dt_authors else "IETF" + clean_title, clean_author, chapters, total_words = parse_rfc_text( + raw_text, rfc_number=rfc_num, title=dt_title, author=author + ) + + return clean_title, clean_author, chapters, rfc_num, total_words + + diff --git a/app/static/css/style.css b/app/static/css/style.css index e6bb59d..9f79281 100644 --- a/app/static/css/style.css +++ b/app/static/css/style.css @@ -3399,6 +3399,210 @@ body.drawer-open { border-top: 1px solid var(--border-color); } +/* Bookshelf Divider between General Books & RFC Specifications */ +.bookshelf-divider { + margin: 48px 0 36px; + border: 0; + height: 1px; + background: linear-gradient(90deg, transparent, var(--border-color) 15%, var(--border-color) 85%, transparent); +} + +/* RFC Specifications Section */ +.rfc-section { + margin-top: 8px; +} + +.rfc-header { + margin-bottom: 20px; +} + +.rfc-title-row { + display: flex; + align-items: center; + gap: 14px; +} + +.rfc-icon-badge { + display: inline-flex; + align-items: center; + justify-content: center; + width: 44px; + height: 44px; + font-size: 24px; + background: rgba(14, 165, 233, 0.12); + border: 1px solid rgba(14, 165, 233, 0.28); + border-radius: var(--radius-md); + flex-shrink: 0; +} + +.rfc-heading { + font-size: 20px; + font-weight: 700; + color: var(--text-main); + margin: 0; +} + +.rfc-fetch-box { + background: var(--bg-surface); + border: 1px solid var(--border-color); + border-radius: var(--radius-lg); + padding: 20px 24px; + margin-bottom: 24px; + box-shadow: var(--shadow-sm); +} + +.rfc-fetch-form { + margin-bottom: 16px; +} + +.rfc-input-wrap { + display: flex; + align-items: stretch; + gap: 10px; +} + +.rfc-input-prefix { + display: inline-flex; + align-items: center; + padding: 0 14px; + font-weight: 700; + font-size: 13px; + letter-spacing: 0.05em; + color: #38bdf8; + background: rgba(56, 189, 248, 0.12); + border: 1px solid rgba(56, 189, 248, 0.25); + border-radius: var(--radius-md); + user-select: none; +} + +.rfc-input { + flex: 1; + min-width: 0; +} + +.rfc-fetch-btn { + white-space: nowrap; + padding: 0 20px; + display: inline-flex; + align-items: center; + justify-content: center; + font-weight: 600; +} + +.rfc-fetch-status { + margin-top: 10px; + padding: 8px 14px; + font-size: 13px; + border-radius: var(--radius-md); + transition: all 0.2s ease; +} + +.rfc-status-loading { + background: rgba(14, 165, 233, 0.1); + color: #38bdf8; + border: 1px solid rgba(14, 165, 233, 0.25); +} + +.rfc-status-success { + background: rgba(34, 197, 94, 0.12); + color: #4ade80; + border: 1px solid rgba(34, 197, 94, 0.25); +} + +.rfc-status-error { + background: rgba(239, 68, 68, 0.12); + color: #f87171; + border: 1px solid rgba(239, 68, 68, 0.25); +} + +.rfc-presets-wrap { + display: flex; + flex-direction: column; + gap: 10px; +} + +.rfc-presets-label { + font-size: 12px; + text-transform: uppercase; + font-weight: 600; + letter-spacing: 0.05em; + color: var(--text-dim); +} + +.rfc-presets-list { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.rfc-preset-chip { + background: var(--bg-input); + color: var(--text-main); + border: 1px solid var(--border-color); + border-radius: var(--radius-full); + padding: 6px 14px; + font-size: 12.5px; + font-weight: 500; + cursor: pointer; + transition: var(--transition); + user-select: none; + display: inline-flex; + align-items: center; + gap: 5px; +} + +.rfc-preset-chip:hover { + background: rgba(14, 165, 233, 0.15); + border-color: #38bdf8; + color: #38bdf8; + transform: translateY(-1px); +} + +.rfc-empty-state { + padding: 36px 20px; + background: var(--bg-surface); + border: 1px dashed var(--border-color); + border-radius: var(--radius-lg); + margin-bottom: 24px; +} + +.rfc-card { + border-color: rgba(56, 189, 248, 0.2); +} + +.rfc-card:hover { + border-color: rgba(56, 189, 248, 0.5); + box-shadow: 0 8px 24px rgba(14, 165, 233, 0.12); +} + +.rfc-cover-icon { + background: rgba(14, 165, 233, 0.1); + border-color: rgba(14, 165, 233, 0.25); +} + +.badge-rfc { + background: rgba(14, 165, 233, 0.2); + color: #38bdf8; + border: 1px solid rgba(14, 165, 233, 0.35); +} + +.rfc-progress-fill { + background: linear-gradient(90deg, #0284c7, #38bdf8); +} + +@media (max-width: 640px) { + .rfc-input-wrap { + flex-direction: column; + } + .rfc-input-prefix { + display: none; + } + .rfc-fetch-btn { + width: 100%; + padding: 10px 16px; + } +} + /* Reader Interface */ .reader-interface { background-color: var(--bg-surface); diff --git a/app/static/index.html b/app/static/index.html index 5b56d6d..734245b 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -545,6 +545,60 @@
Загрузите книгу в формате EPUB или PDF, текстовый файл TXT/MD или вставьте текст статьи, чтобы начать интерактивное чтение.
+ + +Интерактивное чтение официальных стандартов интернета на английском с клик-переводом и авто-словарем
+