feat(reader): добавить парсер RFC из RFC Editor и отдельную группу спецификаций на книжной полке
- Реализован парсер RFC в app/reader.py с авто-загрузкой с rfc-editor.org и datatracker.ietf.org - Очистка пагинации ([Page X], \x0c) и разбиение спецификаций на структурированные главы (TOC, разделы) - Эндпоинт POST /api/books/fetch-rfc с защитой от дублирования и поддержкой номеров/URL - Визуальное разделение на книжной полке через hr-разделитель и блок спецификаций IETF & RFC - Быстрые пресеты стандартов (HTTP/1.1, HTTP/2, TLS 1.3, OAuth 2.0, WebSocket, QUIC, TCP, DNS) - Интеграция в интерактивную читалку (перевод по клику, авто-захват контекста, SRS карточки) - Unit-тесты парсера и API (31/31 passed) и обновление Service Worker кэша
This commit is contained in:
Antigravity committed
1 parent
e73d3470b3
commit
82b7a654ca
8 files changed
+852
-41
No files matched your search
+86
-1
@@ -30,7 +30,7 @@ from app.auth import (
|
||||
from app.dictionary import lookup_term, USER_AGENT, close_http_client
|
||||
from app.export import export_csv, export_json, export_anki_tsv, export_markdown
|
||||
from app.srs import calculate_sm2, generate_quiz_options, build_activity_heatmap, calculate_cefr_level
|
||||
from app.reader import parse_epub, parse_text, parse_pdf, count_words
|
||||
from app.reader import parse_epub, parse_text, parse_pdf, count_words, extract_rfc_number, fetch_and_parse_rfc, parse_rfc_text
|
||||
from app.ai import translate_sentence_ai, get_ollama_status
|
||||
from app.phrasal import detect_phrasal_verb
|
||||
from app.cefr import (
|
||||
@@ -182,6 +182,11 @@ class BookProgressRequest(BaseModel):
|
||||
position: Optional[float] = None
|
||||
|
||||
|
||||
class RfcFetchRequest(BaseModel):
|
||||
rfc: str
|
||||
force: Optional[bool] = False
|
||||
|
||||
|
||||
class SentenceTranslateRequest(BaseModel):
|
||||
sentence: str = Field(..., min_length=1)
|
||||
target_word: Optional[str] = ""
|
||||
@@ -1315,6 +1320,86 @@ async def api_create_book(
|
||||
return res
|
||||
|
||||
|
||||
@app.post("/api/books/fetch-rfc")
|
||||
async def api_fetch_rfc(
|
||||
req: RfcFetchRequest,
|
||||
user: dict = Depends(require_user),
|
||||
):
|
||||
"""
|
||||
Fetches an RFC specification by number or URL from RFC Editor & Datatracker,
|
||||
parses sections into readable chapters, and saves to the user's bookshelf.
|
||||
If the RFC already exists on user's shelf and force=False, returns the existing book.
|
||||
"""
|
||||
raw_query = (req.rfc or "").strip()
|
||||
if not raw_query:
|
||||
raise HTTPException(status_code=400, detail="Укажите номер или URL спецификации RFC")
|
||||
|
||||
try:
|
||||
rfc_num = extract_rfc_number(raw_query)
|
||||
except ValueError as ve:
|
||||
raise HTTPException(status_code=400, detail=str(ve))
|
||||
|
||||
# Check if user already loaded this RFC
|
||||
if not req.force:
|
||||
with get_db() as conn:
|
||||
existing = conn.execute(
|
||||
"""
|
||||
SELECT id, user_id, title, author, source_type, chapters_json,
|
||||
json_array_length(chapters_json) AS chapters_count,
|
||||
current_chapter, current_position, total_words,
|
||||
created_at, updated_at
|
||||
FROM books
|
||||
WHERE user_id = ? AND source_type = 'rfc' AND (title LIKE ? OR title LIKE ?)
|
||||
ORDER BY id DESC LIMIT 1
|
||||
""",
|
||||
(user["id"], f"RFC {rfc_num}:%", f"RFC {rfc_num}"),
|
||||
).fetchone()
|
||||
if existing:
|
||||
res = dict(existing)
|
||||
try:
|
||||
res["chapters"] = json.loads(res["chapters_json"])
|
||||
except Exception:
|
||||
res["chapters"] = []
|
||||
res["already_existed"] = True
|
||||
return res
|
||||
|
||||
try:
|
||||
title, author, chapters, rfc_num, total_words = fetch_and_parse_rfc(raw_query)
|
||||
except ValueError as ve:
|
||||
raise HTTPException(status_code=400, detail=str(ve))
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Ошибка загрузки RFC: {e}")
|
||||
|
||||
chapters_json = json.dumps(chapters, ensure_ascii=False)
|
||||
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
INSERT INTO books (
|
||||
user_id, title, author, source_type, chapters_json,
|
||||
current_chapter, current_position, total_words
|
||||
) VALUES (?, ?, ?, 'rfc', ?, 0, 0.0, ?)
|
||||
""",
|
||||
(user["id"], title, author, chapters_json, total_words),
|
||||
)
|
||||
new_id = cursor.lastrowid
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id, user_id, title, author, source_type, chapters_json,
|
||||
json_array_length(chapters_json) AS chapters_count,
|
||||
current_chapter, current_position, total_words,
|
||||
created_at, updated_at
|
||||
FROM books WHERE id = ?
|
||||
""",
|
||||
(new_id,),
|
||||
).fetchone()
|
||||
|
||||
res = dict(row)
|
||||
res["chapters"] = chapters
|
||||
res["already_existed"] = False
|
||||
return res
|
||||
|
||||
|
||||
@app.get("/api/books/{book_id}")
|
||||
async def api_get_book(book_id: int, user: dict = Depends(require_user)):
|
||||
"""
|
||||
|
||||
+241
@@ -2,9 +2,12 @@ import io
|
||||
import re
|
||||
import sys
|
||||
import html
|
||||
import json
|
||||
import zipfile
|
||||
import posixpath
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
import xml.etree.ElementTree as ET
|
||||
from html.parser import HTMLParser
|
||||
@@ -846,3 +849,241 @@ def parse_pdf(file_bytes: bytes, filename: str = "") -> Tuple[str, str, List[Dic
|
||||
|
||||
return title, author, chapters
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# RFC Specifications Parser & Fetcher (IETF & RFC Editor)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def extract_rfc_number(query: str) -> int:
|
||||
"""
|
||||
Extracts RFC number from a string, supporting inputs like:
|
||||
- "9110", "rfc 9110", "RFC-9110", "rfc9110"
|
||||
- "https://www.rfc-editor.org/rfc/rfc9110.txt"
|
||||
- "https://datatracker.ietf.org/doc/rfc9110/"
|
||||
"""
|
||||
query = (query or "").strip()
|
||||
if re.search(r"-\s*\d+", query) and not re.search(r"rfc-\d+", query, re.IGNORECASE):
|
||||
raise ValueError(f"Недопустимый отрицательный номер RFC: '{query}'")
|
||||
|
||||
m = re.search(r"(?:rfc[-_ ]?|doc/(?:html/)?rfc|/rfc)?(\d+)", query, re.IGNORECASE)
|
||||
if not m:
|
||||
raise ValueError(f"Не удалось определить номер RFC из строки: '{query}'")
|
||||
num = int(m.group(1))
|
||||
if num <= 0 or num > 100000:
|
||||
raise ValueError(f"Недопустимый номер RFC: {num}")
|
||||
return num
|
||||
|
||||
|
||||
def parse_rfc_text(
|
||||
raw_text: str,
|
||||
rfc_number: int,
|
||||
title: str = "",
|
||||
author: str = ""
|
||||
) -> Tuple[str, str, List[Dict[str, str]], int]:
|
||||
"""
|
||||
Cleans raw RFC text, strips pagination artifacts and form feeds,
|
||||
and splits the specification into structured readable chapters.
|
||||
Returns: (clean_title, clean_author, chapters, total_words)
|
||||
"""
|
||||
raw_text = raw_text.replace("\r\n", "\n").replace("\r", "\n").lstrip("\ufeff")
|
||||
|
||||
# 1. Clean pagination artifacts (running footers, form-feed \x0c, running headers)
|
||||
clean = re.sub(
|
||||
r"\n[^\n]*\[Page [0-9ivxlc]+\]\s*[\n\x0c]+\s*(?:RFC[^\n]*\n)?",
|
||||
"\n\n",
|
||||
raw_text,
|
||||
flags=re.IGNORECASE
|
||||
)
|
||||
clean = clean.replace("\x0c", "\n\n")
|
||||
|
||||
lines = clean.splitlines()
|
||||
|
||||
# 2. If title wasn't provided, extract from the document header
|
||||
if not title:
|
||||
for l in lines[0:65]:
|
||||
s = l.strip()
|
||||
if not s:
|
||||
continue
|
||||
if re.match(
|
||||
r"^(Internet Engineering Task Force|Request for Comments|STD:|Updates:|Obsoletes:|Category:|ISSN:|Network Working Group|BCP:|FYI:|RFC\s*\d+)",
|
||||
s,
|
||||
re.IGNORECASE
|
||||
):
|
||||
continue
|
||||
if re.match(r"^Table of Contents\b", s, re.IGNORECASE):
|
||||
break
|
||||
if re.match(r"^Abstract\b", s, re.IGNORECASE):
|
||||
break
|
||||
if 4 <= len(s) <= 80 and not s.endswith((":", ";", ".")):
|
||||
title = s
|
||||
break
|
||||
|
||||
clean_title = f"RFC {rfc_number}: {title}" if title and not title.lower().startswith("rfc") else (title or f"RFC {rfc_number}")
|
||||
clean_author = author or "IETF"
|
||||
|
||||
# 3. Detect chapters
|
||||
in_toc = False
|
||||
past_toc = False
|
||||
headings = []
|
||||
bad_sentence_starters = {
|
||||
"if", "the", "when", "where", "this", "that", "these", "those",
|
||||
"allow", "rather", "a", "an", "in", "for", "on", "to", "there",
|
||||
"clients", "optionally", "both", "all", "each", "such", "see", "note"
|
||||
}
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
trimmed = line.strip()
|
||||
if not trimmed:
|
||||
continue
|
||||
|
||||
if re.search(r"\.{4,}", line):
|
||||
in_toc = True
|
||||
continue
|
||||
|
||||
if not past_toc:
|
||||
if re.match(r"^Table of Contents\b", trimmed, re.IGNORECASE):
|
||||
in_toc = True
|
||||
continue
|
||||
if in_toc:
|
||||
if (line.startswith("1.") or line.startswith("1 ") or re.match(r"^\s{10,}1\.\s+[A-Z]", line)) and not re.search(r"\.{3,}", line) and not re.search(r"\s+\d+$", trimmed):
|
||||
in_toc = False
|
||||
past_toc = True
|
||||
else:
|
||||
continue
|
||||
|
||||
prev_blank = (i == 0) or (not lines[i - 1].strip())
|
||||
if not prev_blank:
|
||||
continue
|
||||
|
||||
m = re.match(
|
||||
r"^(?:([0-9]{1,2}\.?)\s+([A-Za-z][^\n]{1,65})|(Appendix\s+[A-Z]\.?\s*[^\n]{0,65})|(Authors?'? Addresses\b.*)|(Full Copyright Statement\b.*))$",
|
||||
trimmed,
|
||||
re.IGNORECASE
|
||||
)
|
||||
if not m and re.match(r"^[0-9]{1,2}\.\s+[A-Z\s]{3,50}$", trimmed):
|
||||
m = True
|
||||
|
||||
if m:
|
||||
if trimmed.endswith((".", ",", ":", ";", "-")):
|
||||
continue
|
||||
if len(trimmed) > 70:
|
||||
continue
|
||||
words = re.findall(r"\b[A-Za-z]+\b", trimmed)
|
||||
if len(words) >= 2:
|
||||
sec_word = words[0].lower() if not words[0].isdigit() else (words[1].lower() if len(words) > 1 else "")
|
||||
if sec_word in bad_sentence_starters:
|
||||
continue
|
||||
headings.append((i, trimmed))
|
||||
|
||||
# Deduplicate headings that are right next to each other
|
||||
deduped_headings = []
|
||||
for h in headings:
|
||||
if not deduped_headings or (h[0] - deduped_headings[-1][0] >= 3):
|
||||
deduped_headings.append(h)
|
||||
|
||||
chapters: List[Dict[str, str]] = []
|
||||
if deduped_headings:
|
||||
first_line_idx = deduped_headings[0][0]
|
||||
intro_content = "\n".join(lines[:first_line_idx]).strip()
|
||||
if count_words(intro_content) >= 30:
|
||||
chapters.append({
|
||||
"title": "Overview & Abstract",
|
||||
"content": intro_content
|
||||
})
|
||||
|
||||
for idx, (line_idx, h_title) in enumerate(deduped_headings):
|
||||
next_line_idx = deduped_headings[idx + 1][0] if idx + 1 < len(deduped_headings) else len(lines)
|
||||
body = "\n".join(lines[line_idx:next_line_idx]).strip()
|
||||
if body:
|
||||
chapters.append({
|
||||
"title": h_title,
|
||||
"content": body
|
||||
})
|
||||
|
||||
if not chapters:
|
||||
paras = [p.strip() for p in re.split(r"\n{2,}", clean) if p.strip()]
|
||||
chunks = []
|
||||
curr = []
|
||||
curr_w = 0
|
||||
for p in paras:
|
||||
w = count_words(p)
|
||||
if curr_w + w > 1500 and curr:
|
||||
chunks.append("\n\n".join(curr))
|
||||
curr = [p]
|
||||
curr_w = w
|
||||
else:
|
||||
curr.append(p)
|
||||
curr_w += w
|
||||
if curr:
|
||||
chunks.append("\n\n".join(curr))
|
||||
|
||||
for idx, chunk in enumerate(chunks, 1):
|
||||
chapters.append({
|
||||
"title": f"Section {idx}",
|
||||
"content": chunk
|
||||
})
|
||||
|
||||
total_words = sum(count_words(ch["content"]) for ch in chapters)
|
||||
return clean_title, clean_author, chapters, total_words
|
||||
|
||||
|
||||
def fetch_and_parse_rfc(query: str) -> Tuple[str, str, List[Dict[str, str]], int, int]:
|
||||
"""
|
||||
Fetches an RFC by number or URL from RFC Editor & IETF Datatracker,
|
||||
cleans formatting, parses chapters, and returns:
|
||||
(clean_title, author, chapters, rfc_number, total_words)
|
||||
"""
|
||||
rfc_num = extract_rfc_number(query)
|
||||
|
||||
# 1. Fetch metadata from IETF Datatracker (title, authors)
|
||||
dt_title = ""
|
||||
dt_authors = []
|
||||
try:
|
||||
dt_url = f"https://datatracker.ietf.org/doc/rfc{rfc_num}/doc.json"
|
||||
req = urllib.request.Request(dt_url, headers={"User-Agent": "study-qpid-pro/1.0 (educational-reader)"})
|
||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8", errors="replace"))
|
||||
dt_title = (data.get("title") or "").strip()
|
||||
for a in data.get("authors", []):
|
||||
name = a.get("name") or a.get("person", {}).get("plain_name")
|
||||
if name and name not in dt_authors:
|
||||
dt_authors.append(name.strip())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2. Fetch raw text from RFC Editor (primary) or Datatracker (fallback)
|
||||
raw_text = None
|
||||
urls_to_try = [
|
||||
f"https://www.rfc-editor.org/rfc/rfc{rfc_num}.txt",
|
||||
f"https://datatracker.ietf.org/doc/rfc{rfc_num}.txt",
|
||||
]
|
||||
fetch_err = None
|
||||
for u in urls_to_try:
|
||||
try:
|
||||
req = urllib.request.Request(u, headers={"User-Agent": "study-qpid-pro/1.0 (educational-reader)"})
|
||||
with urllib.request.urlopen(req, timeout=8) as resp:
|
||||
raw_bytes = resp.read()
|
||||
raw_text = raw_bytes.decode("utf-8", errors="replace")
|
||||
break
|
||||
except urllib.error.HTTPError as he:
|
||||
fetch_err = he
|
||||
if he.code == 404:
|
||||
continue
|
||||
except Exception as e:
|
||||
fetch_err = e
|
||||
continue
|
||||
|
||||
if not raw_text:
|
||||
if isinstance(fetch_err, urllib.error.HTTPError) and fetch_err.code == 404:
|
||||
raise ValueError(f"RFC {rfc_num} не найден в базе RFC Editor (404 Not Found)")
|
||||
err_msg = str(fetch_err) if fetch_err else "документ недоступен"
|
||||
raise ValueError(f"Не удалось загрузить RFC {rfc_num}: {err_msg}")
|
||||
|
||||
author = ("IETF (" + ", ".join(dt_authors) + ")") if dt_authors else "IETF"
|
||||
clean_title, clean_author, chapters, total_words = parse_rfc_text(
|
||||
raw_text, rfc_number=rfc_num, title=dt_title, author=author
|
||||
)
|
||||
|
||||
return clean_title, clean_author, chapters, rfc_num, total_words
|
||||
|
||||
|
||||
@@ -3399,6 +3399,210 @@ body.drawer-open {
|
||||
border-top: 1px solid var(--border-color);
|
||||
}
|
||||
|
||||
/* Bookshelf Divider between General Books & RFC Specifications */
|
||||
.bookshelf-divider {
|
||||
margin: 48px 0 36px;
|
||||
border: 0;
|
||||
height: 1px;
|
||||
background: linear-gradient(90deg, transparent, var(--border-color) 15%, var(--border-color) 85%, transparent);
|
||||
}
|
||||
|
||||
/* RFC Specifications Section */
|
||||
.rfc-section {
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.rfc-header {
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.rfc-title-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.rfc-icon-badge {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 44px;
|
||||
height: 44px;
|
||||
font-size: 24px;
|
||||
background: rgba(14, 165, 233, 0.12);
|
||||
border: 1px solid rgba(14, 165, 233, 0.28);
|
||||
border-radius: var(--radius-md);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.rfc-heading {
|
||||
font-size: 20px;
|
||||
font-weight: 700;
|
||||
color: var(--text-main);
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.rfc-fetch-box {
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: var(--radius-lg);
|
||||
padding: 20px 24px;
|
||||
margin-bottom: 24px;
|
||||
box-shadow: var(--shadow-sm);
|
||||
}
|
||||
|
||||
.rfc-fetch-form {
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.rfc-input-wrap {
|
||||
display: flex;
|
||||
align-items: stretch;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.rfc-input-prefix {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
padding: 0 14px;
|
||||
font-weight: 700;
|
||||
font-size: 13px;
|
||||
letter-spacing: 0.05em;
|
||||
color: #38bdf8;
|
||||
background: rgba(56, 189, 248, 0.12);
|
||||
border: 1px solid rgba(56, 189, 248, 0.25);
|
||||
border-radius: var(--radius-md);
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
.rfc-input {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.rfc-fetch-btn {
|
||||
white-space: nowrap;
|
||||
padding: 0 20px;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.rfc-fetch-status {
|
||||
margin-top: 10px;
|
||||
padding: 8px 14px;
|
||||
font-size: 13px;
|
||||
border-radius: var(--radius-md);
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.rfc-status-loading {
|
||||
background: rgba(14, 165, 233, 0.1);
|
||||
color: #38bdf8;
|
||||
border: 1px solid rgba(14, 165, 233, 0.25);
|
||||
}
|
||||
|
||||
.rfc-status-success {
|
||||
background: rgba(34, 197, 94, 0.12);
|
||||
color: #4ade80;
|
||||
border: 1px solid rgba(34, 197, 94, 0.25);
|
||||
}
|
||||
|
||||
.rfc-status-error {
|
||||
background: rgba(239, 68, 68, 0.12);
|
||||
color: #f87171;
|
||||
border: 1px solid rgba(239, 68, 68, 0.25);
|
||||
}
|
||||
|
||||
.rfc-presets-wrap {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.rfc-presets-label {
|
||||
font-size: 12px;
|
||||
text-transform: uppercase;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.05em;
|
||||
color: var(--text-dim);
|
||||
}
|
||||
|
||||
.rfc-presets-list {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.rfc-preset-chip {
|
||||
background: var(--bg-input);
|
||||
color: var(--text-main);
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: var(--radius-full);
|
||||
padding: 6px 14px;
|
||||
font-size: 12.5px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
transition: var(--transition);
|
||||
user-select: none;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 5px;
|
||||
}
|
||||
|
||||
.rfc-preset-chip:hover {
|
||||
background: rgba(14, 165, 233, 0.15);
|
||||
border-color: #38bdf8;
|
||||
color: #38bdf8;
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.rfc-empty-state {
|
||||
padding: 36px 20px;
|
||||
background: var(--bg-surface);
|
||||
border: 1px dashed var(--border-color);
|
||||
border-radius: var(--radius-lg);
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
|
||||
.rfc-card {
|
||||
border-color: rgba(56, 189, 248, 0.2);
|
||||
}
|
||||
|
||||
.rfc-card:hover {
|
||||
border-color: rgba(56, 189, 248, 0.5);
|
||||
box-shadow: 0 8px 24px rgba(14, 165, 233, 0.12);
|
||||
}
|
||||
|
||||
.rfc-cover-icon {
|
||||
background: rgba(14, 165, 233, 0.1);
|
||||
border-color: rgba(14, 165, 233, 0.25);
|
||||
}
|
||||
|
||||
.badge-rfc {
|
||||
background: rgba(14, 165, 233, 0.2);
|
||||
color: #38bdf8;
|
||||
border: 1px solid rgba(14, 165, 233, 0.35);
|
||||
}
|
||||
|
||||
.rfc-progress-fill {
|
||||
background: linear-gradient(90deg, #0284c7, #38bdf8);
|
||||
}
|
||||
|
||||
@media (max-width: 640px) {
|
||||
.rfc-input-wrap {
|
||||
flex-direction: column;
|
||||
}
|
||||
.rfc-input-prefix {
|
||||
display: none;
|
||||
}
|
||||
.rfc-fetch-btn {
|
||||
width: 100%;
|
||||
padding: 10px 16px;
|
||||
}
|
||||
}
|
||||
|
||||
/* Reader Interface */
|
||||
.reader-interface {
|
||||
background-color: var(--bg-surface);
|
||||
|
||||
@@ -545,6 +545,60 @@
|
||||
<p>Загрузите книгу в формате EPUB или PDF, текстовый файл TXT/MD или вставьте текст статьи, чтобы начать интерактивное чтение.</p>
|
||||
</div>
|
||||
<div id="bookshelf-grid" class="bookshelf-grid"></div>
|
||||
|
||||
<!-- Divider between General Books & RFC Specifications -->
|
||||
<hr class="bookshelf-divider">
|
||||
|
||||
<!-- RFC Specifications Section -->
|
||||
<div id="rfc-section" class="rfc-section">
|
||||
<div class="rfc-header">
|
||||
<div class="rfc-title-wrap">
|
||||
<div class="rfc-title-row">
|
||||
<span class="rfc-icon-badge">🌐</span>
|
||||
<div>
|
||||
<h3 class="rfc-heading">Спецификации IETF & RFC Editor</h3>
|
||||
<p class="subtitle">Интерактивное чтение официальных стандартов интернета на английском с клик-переводом и авто-словарем</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- RFC Fetch Form & Preset Chips -->
|
||||
<div class="rfc-fetch-box">
|
||||
<form id="rfc-fetch-form" class="rfc-fetch-form">
|
||||
<div class="rfc-input-wrap">
|
||||
<span class="rfc-input-prefix">RFC</span>
|
||||
<input type="text" id="rfc-input" class="input-styled rfc-input" placeholder="Номер (напр. 9110, 7540, 8446, 6749) или URL rfc-editor.org" autocomplete="off" spellcheck="false">
|
||||
<button type="submit" id="rfc-fetch-btn" class="btn btn-primary rfc-fetch-btn">
|
||||
<span>⚡ Загрузить RFC</span>
|
||||
</button>
|
||||
</div>
|
||||
<div id="rfc-fetch-status" class="rfc-fetch-status hidden"></div>
|
||||
</form>
|
||||
|
||||
<div class="rfc-presets-wrap">
|
||||
<span class="rfc-presets-label">Популярные стандарты:</span>
|
||||
<div class="rfc-presets-list">
|
||||
<button type="button" class="rfc-preset-chip" data-rfc="9110" title="HTTP Semantics (RFC 9110)">🌐 HTTP/1.1 (9110)</button>
|
||||
<button type="button" class="rfc-preset-chip" data-rfc="7540" title="HTTP/2 (RFC 7540)">⚡ HTTP/2 (7540)</button>
|
||||
<button type="button" class="rfc-preset-chip" data-rfc="9000" title="QUIC Transport Protocol (RFC 9000)">🚀 QUIC (9000)</button>
|
||||
<button type="button" class="rfc-preset-chip" data-rfc="8446" title="TLS Protocol 1.3 (RFC 8446)">🔒 TLS 1.3 (8446)</button>
|
||||
<button type="button" class="rfc-preset-chip" data-rfc="6749" title="OAuth 2.0 Authorization Framework (RFC 6749)">🔑 OAuth 2.0 (6749)</button>
|
||||
<button type="button" class="rfc-preset-chip" data-rfc="6455" title="The WebSocket Protocol (RFC 6455)">🔌 WebSocket (6455)</button>
|
||||
<button type="button" class="rfc-preset-chip" data-rfc="793" title="Transmission Control Protocol (RFC 793)">📡 TCP (793)</button>
|
||||
<button type="button" class="rfc-preset-chip" data-rfc="1035" title="Domain Names Implementation (RFC 1035)">🧭 DNS (1035)</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- RFC Books Grid / Empty State -->
|
||||
<div id="rfc-empty-state" class="empty-state rfc-empty-state hidden">
|
||||
<div class="empty-icon">📜</div>
|
||||
<h4>Нет сохраненных RFC спецификаций</h4>
|
||||
<p>Введите номер стандарта выше или нажмите на одну из быстрых кнопок, чтобы разобрать спецификацию на главы и читать в интерактивном режиме.</p>
|
||||
</div>
|
||||
<div id="rfc-grid" class="bookshelf-grid rfc-grid"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Reading Interface (Active when a book is open) -->
|
||||
|
||||
@@ -215,6 +215,20 @@ export const api = {
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
},
|
||||
async fetchRfc(query, force = false) {
|
||||
const resp = await fetch('/api/books/fetch-rfc', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ rfc: query, force: force })
|
||||
});
|
||||
if (resp.ok) return await resp.json();
|
||||
let detail = 'Не удалось загрузить RFC';
|
||||
try {
|
||||
const errData = await resp.json();
|
||||
detail = errData.detail || detail;
|
||||
} catch (_) {}
|
||||
throw new Error(detail);
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
+140
-39
@@ -39,40 +39,10 @@ export function getCurrentChapterIndex() {
|
||||
// -----------------------------------------------------------------------
|
||||
// Bookshelf Actions & Rendering
|
||||
// -----------------------------------------------------------------------
|
||||
export async function loadBookshelf() {
|
||||
const bookshelfContainer = document.getElementById('bookshelf-container');
|
||||
const readerInterface = document.getElementById('reader-interface');
|
||||
const bookshelfGrid = document.getElementById('bookshelf-grid');
|
||||
const bookshelfEmpty = document.getElementById('bookshelf-empty');
|
||||
|
||||
if (!bookshelfContainer) return;
|
||||
if (!window.location.hash.includes('/book/')) {
|
||||
bookshelfContainer.classList.remove('hidden');
|
||||
if (readerInterface) readerInterface.classList.add('hidden');
|
||||
}
|
||||
if (bookshelfGrid) bookshelfGrid.innerHTML = '';
|
||||
if (bookshelfEmpty) bookshelfEmpty.classList.add('hidden');
|
||||
|
||||
try {
|
||||
const books = await api.books.getAll();
|
||||
if (!books || books.length === 0) {
|
||||
if (bookshelfEmpty) bookshelfEmpty.classList.remove('hidden');
|
||||
} else {
|
||||
renderBookshelf(books);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error loading bookshelf:', e);
|
||||
}
|
||||
}
|
||||
|
||||
export function renderBookshelf(books) {
|
||||
const bookshelfGrid = document.getElementById('bookshelf-grid');
|
||||
if (!bookshelfGrid) return;
|
||||
bookshelfGrid.innerHTML = '';
|
||||
|
||||
books.forEach(b => {
|
||||
function createBookCard(b) {
|
||||
const card = document.createElement('div');
|
||||
card.className = 'book-card';
|
||||
const isRfc = (b.source_type === 'rfc');
|
||||
card.className = isRfc ? 'book-card rfc-card' : 'book-card';
|
||||
|
||||
const totalCh = Math.max(1, b.chapters_count || 1);
|
||||
const chIndex = Math.max(0, Math.min(totalCh - 1, b.current_chapter || 0));
|
||||
@@ -86,15 +56,22 @@ export function renderBookshelf(books) {
|
||||
? `<span>1 глава • ${totalPct}%</span><span>${totalPct}%</span>`
|
||||
: `<span>Глава ${chIndex + 1} из ${totalCh} (${chPct}%)</span><span>${totalPct}%</span>`;
|
||||
|
||||
let coverIcon = '📄';
|
||||
if (b.source_type === 'epub') coverIcon = '📕';
|
||||
else if (b.source_type === 'pdf') coverIcon = '📑';
|
||||
else if (b.source_type === 'rfc') coverIcon = '🌐';
|
||||
|
||||
const badgeClass = isRfc ? 'badge-format badge-rfc' : 'badge-format';
|
||||
|
||||
card.innerHTML = `
|
||||
<div>
|
||||
<div class="book-card-top">
|
||||
<div class="book-cover-icon">${b.source_type === 'epub' ? '📕' : (b.source_type === 'pdf' ? '📑' : '📄')}</div>
|
||||
<div class="book-cover-icon ${isRfc ? 'rfc-cover-icon' : ''}">${coverIcon}</div>
|
||||
<div class="book-card-info">
|
||||
<h4 class="book-card-title" title="${escapeHtml(b.title)}">${escapeHtml(b.title)}</h4>
|
||||
<p class="book-card-author">${escapeHtml(b.author || 'Неизвестный автор')}</p>
|
||||
<p class="book-card-author">${escapeHtml(b.author || (isRfc ? 'IETF' : 'Неизвестный автор'))}</p>
|
||||
<div class="book-card-badges">
|
||||
<span class="badge-format">${escapeHtml((b.source_type || 'text').toUpperCase())}</span>
|
||||
<span class="${badgeClass}">${escapeHtml((b.source_type || 'text').toUpperCase())}</span>
|
||||
<span class="badge-words">${b.total_words ? b.total_words.toLocaleString() + ' слов' : ''}</span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -104,7 +81,7 @@ export function renderBookshelf(books) {
|
||||
${progressLabels}
|
||||
</div>
|
||||
<div class="book-progress-bar-bg">
|
||||
<div class="book-progress-bar-fill" style="width: ${totalPct}%"></div>
|
||||
<div class="book-progress-bar-fill ${isRfc ? 'rfc-progress-fill' : ''}" style="width: ${totalPct}%"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -123,14 +100,71 @@ export function renderBookshelf(books) {
|
||||
window.location.hash = `#srs/${encodeURIComponent(bookTag)}`;
|
||||
});
|
||||
card.querySelector('.delete-book-btn').addEventListener('click', async () => {
|
||||
if (confirm(`Удалить книгу "${b.title}" из библиотеки?`)) {
|
||||
const typeName = isRfc ? 'спецификацию' : 'книгу';
|
||||
if (confirm(`Удалить ${typeName} "${b.title}" из библиотеки?`)) {
|
||||
await api.books.delete(b.id);
|
||||
loadBookshelf();
|
||||
}
|
||||
});
|
||||
|
||||
bookshelfGrid.appendChild(card);
|
||||
return card;
|
||||
}
|
||||
|
||||
export async function loadBookshelf() {
|
||||
const bookshelfContainer = document.getElementById('bookshelf-container');
|
||||
const readerInterface = document.getElementById('reader-interface');
|
||||
const bookshelfGrid = document.getElementById('bookshelf-grid');
|
||||
const bookshelfEmpty = document.getElementById('bookshelf-empty');
|
||||
const rfcGrid = document.getElementById('rfc-grid');
|
||||
const rfcEmptyState = document.getElementById('rfc-empty-state');
|
||||
|
||||
if (!bookshelfContainer) return;
|
||||
if (!window.location.hash.includes('/book/')) {
|
||||
bookshelfContainer.classList.remove('hidden');
|
||||
if (readerInterface) readerInterface.classList.add('hidden');
|
||||
}
|
||||
if (bookshelfGrid) bookshelfGrid.innerHTML = '';
|
||||
if (rfcGrid) rfcGrid.innerHTML = '';
|
||||
if (bookshelfEmpty) bookshelfEmpty.classList.add('hidden');
|
||||
if (rfcEmptyState) rfcEmptyState.classList.add('hidden');
|
||||
|
||||
try {
|
||||
const books = await api.books.getAll();
|
||||
renderBookshelf(books || []);
|
||||
} catch (e) {
|
||||
console.error('Error loading bookshelf:', e);
|
||||
}
|
||||
}
|
||||
|
||||
export function renderBookshelf(books) {
|
||||
const bookshelfGrid = document.getElementById('bookshelf-grid');
|
||||
const bookshelfEmpty = document.getElementById('bookshelf-empty');
|
||||
const rfcGrid = document.getElementById('rfc-grid');
|
||||
const rfcEmptyState = document.getElementById('rfc-empty-state');
|
||||
|
||||
if (bookshelfGrid) bookshelfGrid.innerHTML = '';
|
||||
if (rfcGrid) rfcGrid.innerHTML = '';
|
||||
|
||||
const generalBooks = (books || []).filter(b => b.source_type !== 'rfc');
|
||||
const rfcBooks = (books || []).filter(b => b.source_type === 'rfc');
|
||||
|
||||
if (bookshelfEmpty) {
|
||||
bookshelfEmpty.classList.toggle('hidden', generalBooks.length > 0);
|
||||
}
|
||||
if (bookshelfGrid) {
|
||||
generalBooks.forEach(b => {
|
||||
bookshelfGrid.appendChild(createBookCard(b));
|
||||
});
|
||||
}
|
||||
|
||||
if (rfcEmptyState) {
|
||||
rfcEmptyState.classList.toggle('hidden', rfcBooks.length > 0);
|
||||
}
|
||||
if (rfcGrid) {
|
||||
rfcBooks.forEach(b => {
|
||||
rfcGrid.appendChild(createBookCard(b));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -2335,4 +2369,71 @@ export function initReader() {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// RFC Fetching & Presets
|
||||
const rfcFetchForm = document.getElementById('rfc-fetch-form');
|
||||
const rfcInput = document.getElementById('rfc-input');
|
||||
const rfcFetchBtn = document.getElementById('rfc-fetch-btn');
|
||||
const rfcFetchStatus = document.getElementById('rfc-fetch-status');
|
||||
const rfcPresetChips = document.querySelectorAll('.rfc-preset-chip');
|
||||
|
||||
async function handleRfcFetch(query) {
|
||||
if (!query) return;
|
||||
query = query.trim();
|
||||
if (!query) return;
|
||||
|
||||
if (rfcFetchBtn) rfcFetchBtn.disabled = true;
|
||||
if (rfcInput) rfcInput.disabled = true;
|
||||
if (rfcFetchStatus) {
|
||||
rfcFetchStatus.className = 'rfc-fetch-status rfc-status-loading';
|
||||
rfcFetchStatus.innerHTML = `<span>⏳ Загрузка и разбор спецификации RFC...</span>`;
|
||||
rfcFetchStatus.classList.remove('hidden');
|
||||
}
|
||||
|
||||
try {
|
||||
const book = await api.books.fetchRfc(query);
|
||||
if (rfcFetchStatus) {
|
||||
rfcFetchStatus.className = 'rfc-fetch-status rfc-status-success';
|
||||
rfcFetchStatus.innerHTML = `<span>✅ ${escapeHtml(book.title)} успешно добавлен!</span>`;
|
||||
}
|
||||
showToast(book.already_existed ? `Открываем ${book.title}` : `RFC спецификация загружена!`);
|
||||
if (rfcInput) rfcInput.value = '';
|
||||
setTimeout(() => {
|
||||
if (rfcFetchStatus) rfcFetchStatus.classList.add('hidden');
|
||||
window.location.hash = `#reader/book/${book.id}`;
|
||||
}, 400);
|
||||
} catch (err) {
|
||||
console.error('RFC fetch error:', err);
|
||||
if (rfcFetchStatus) {
|
||||
rfcFetchStatus.className = 'rfc-fetch-status rfc-status-error';
|
||||
rfcFetchStatus.innerHTML = `<span>❌ Ошибка: ${escapeHtml(err.message || 'Не удалось загрузить RFC')}</span>`;
|
||||
rfcFetchStatus.classList.remove('hidden');
|
||||
}
|
||||
showToast(err.message || 'Ошибка загрузки RFC', 4000);
|
||||
} finally {
|
||||
if (rfcFetchBtn) rfcFetchBtn.disabled = false;
|
||||
if (rfcInput) {
|
||||
rfcInput.disabled = false;
|
||||
rfcInput.focus();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (rfcFetchForm) {
|
||||
rfcFetchForm.addEventListener('submit', (e) => {
|
||||
e.preventDefault();
|
||||
const val = rfcInput ? rfcInput.value : '';
|
||||
handleRfcFetch(val);
|
||||
});
|
||||
}
|
||||
|
||||
if (rfcPresetChips && rfcPresetChips.length > 0) {
|
||||
rfcPresetChips.forEach(chip => {
|
||||
chip.addEventListener('click', () => {
|
||||
const rfcVal = chip.dataset.rfc;
|
||||
if (rfcInput) rfcInput.value = rfcVal;
|
||||
handleRfcFetch(rfcVal);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
// Service Worker for Notes English PWA (Stage 2)
|
||||
// Strategy: Cache-first for static assets, Network-first for API calls.
|
||||
|
||||
const CACHE_NAME = 'notes-v5.7';
|
||||
const CACHE_NAME = 'notes-v5.8';
|
||||
const OFFLINE_URL = '/';
|
||||
|
||||
const PRECACHE_URLS = [
|
||||
|
||||
@@ -1645,6 +1645,118 @@ def test_background_tasks_and_book_mining():
|
||||
assert cancel_resp.json()["cancelled"] is True
|
||||
|
||||
|
||||
def test_rfc_parser_and_api(monkeypatch):
|
||||
from app.reader import extract_rfc_number, parse_rfc_text
|
||||
|
||||
# 1. Number extraction
|
||||
assert extract_rfc_number("9110") == 9110
|
||||
assert extract_rfc_number("RFC 8446") == 8446
|
||||
assert extract_rfc_number("rfc-6749") == 6749
|
||||
assert extract_rfc_number("https://www.rfc-editor.org/rfc/rfc7540.txt") == 7540
|
||||
assert extract_rfc_number("https://datatracker.ietf.org/doc/rfc9000/") == 9000
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
extract_rfc_number("invalid-rfc-string-without-number")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
extract_rfc_number("rfc -5")
|
||||
|
||||
# 2. Parsing raw text with pagination artifacts
|
||||
sample_rfc = """Internet Engineering Task Force (IETF)
|
||||
Request for Comments: 9999
|
||||
Category: Standards Track
|
||||
|
||||
Test Protocol Specification
|
||||
|
||||
Abstract
|
||||
|
||||
This is a test abstract for RFC 9999 demonstrating cleaning.
|
||||
|
||||
Table of Contents
|
||||
|
||||
1. Introduction .................................................... 1
|
||||
2. Protocol Framing ................................................ 2
|
||||
Authors' Addresses ................................................. 3
|
||||
|
||||
Standards Track [Page 1]
|
||||
\x0c
|
||||
RFC 9999 Test Protocol Specification June 2026
|
||||
|
||||
1. Introduction
|
||||
|
||||
The Test Protocol is designed for testing RFC parsers.
|
||||
It provides ultra-clean chapter segmentation.
|
||||
|
||||
2. Protocol Framing
|
||||
|
||||
Framing uses standard binary encodings.
|
||||
|
||||
Authors' Addresses
|
||||
|
||||
Jane Doe
|
||||
Example Corp
|
||||
"""
|
||||
clean_title, author, chapters, total_words = parse_rfc_text(sample_rfc, rfc_number=9999)
|
||||
assert clean_title == "RFC 9999: Test Protocol Specification"
|
||||
assert author == "IETF"
|
||||
assert len(chapters) >= 3
|
||||
# Check that pagination footer/header was stripped
|
||||
for ch in chapters:
|
||||
assert "[Page 1]" not in ch["content"]
|
||||
assert "\x0c" not in ch["content"]
|
||||
assert any("Introduction" in ch["title"] for ch in chapters)
|
||||
assert any("Framing" in ch["title"] for ch in chapters)
|
||||
assert any("Addresses" in ch["title"] for ch in chapters)
|
||||
assert total_words > 20
|
||||
|
||||
# 3. Test API endpoint unauthenticated
|
||||
anon_client = TestClient(app)
|
||||
resp = anon_client.post("/api/books/fetch-rfc", json={"rfc": "9999"})
|
||||
assert resp.status_code == 401
|
||||
|
||||
# 4. Test API endpoint authenticated with mock fetch
|
||||
import app.main as main_mod
|
||||
|
||||
def mock_fetch_and_parse_rfc(query):
|
||||
return ("RFC 9999: Test Protocol Specification", "IETF (Jane Doe)", chapters, 9999, total_words)
|
||||
|
||||
monkeypatch.setattr(main_mod, "fetch_and_parse_rfc", mock_fetch_and_parse_rfc)
|
||||
|
||||
# Login demo user
|
||||
login_resp = client.post("/api/login", json={"username": "demo_user", "password": "demo_pass_999"})
|
||||
assert login_resp.status_code == 200
|
||||
|
||||
# Invalid input
|
||||
bad_resp = client.post("/api/books/fetch-rfc", json={"rfc": "no_number_here"})
|
||||
assert bad_resp.status_code == 400
|
||||
|
||||
# Fetch valid RFC
|
||||
fetch_resp = client.post("/api/books/fetch-rfc", json={"rfc": "9999"})
|
||||
assert fetch_resp.status_code == 200
|
||||
book_data = fetch_resp.json()
|
||||
assert book_data["source_type"] == "rfc"
|
||||
assert book_data["title"] == "RFC 9999: Test Protocol Specification"
|
||||
assert book_data["already_existed"] is False
|
||||
assert len(book_data["chapters"]) >= 3
|
||||
book_id = book_data["id"]
|
||||
|
||||
# Fetch again without force -> should return existing book
|
||||
fetch_again = client.post("/api/books/fetch-rfc", json={"rfc": "9999"})
|
||||
assert fetch_again.status_code == 200
|
||||
again_data = fetch_again.json()
|
||||
assert again_data["id"] == book_id
|
||||
assert again_data["already_existed"] is True
|
||||
|
||||
# Check bookshelf retrieval includes RFC book
|
||||
shelf_resp = client.get("/api/books")
|
||||
assert shelf_resp.status_code == 200
|
||||
shelf_books = shelf_resp.json()
|
||||
rfc_books = [b for b in shelf_books if b["source_type"] == "rfc"]
|
||||
assert len(rfc_books) > 0
|
||||
assert rfc_books[0]["id"] == book_id
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user