feat(reader): добавить парсер RFC из RFC Editor и отдельную группу спецификаций на книжной полке
- Реализован парсер RFC в app/reader.py с авто-загрузкой с rfc-editor.org и datatracker.ietf.org - Очистка пагинации ([Page X], \x0c) и разбиение спецификаций на структурированные главы (TOC, разделы) - Эндпоинт POST /api/books/fetch-rfc с защитой от дублирования и поддержкой номеров/URL - Визуальное разделение на книжной полке через hr-разделитель и блок спецификаций IETF & RFC - Быстрые пресеты стандартов (HTTP/1.1, HTTP/2, TLS 1.3, OAuth 2.0, WebSocket, QUIC, TCP, DNS) - Интеграция в интерактивную читалку (перевод по клику, авто-захват контекста, SRS карточки) - Unit-тесты парсера и API (31/31 passed) и обновление Service Worker кэша
This commit is contained in:
Antigravity committed
1 parent
e73d3470b3
commit
82b7a654ca
8 files changed
+877
-66
No files matched your search
+241
@@ -2,9 +2,12 @@ import io
|
||||
import re
|
||||
import sys
|
||||
import html
|
||||
import json
|
||||
import zipfile
|
||||
import posixpath
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
import xml.etree.ElementTree as ET
|
||||
from html.parser import HTMLParser
|
||||
@@ -846,3 +849,241 @@ def parse_pdf(file_bytes: bytes, filename: str = "") -> Tuple[str, str, List[Dic
|
||||
|
||||
return title, author, chapters
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# RFC Specifications Parser & Fetcher (IETF & RFC Editor)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def extract_rfc_number(query: str) -> int:
|
||||
"""
|
||||
Extracts RFC number from a string, supporting inputs like:
|
||||
- "9110", "rfc 9110", "RFC-9110", "rfc9110"
|
||||
- "https://www.rfc-editor.org/rfc/rfc9110.txt"
|
||||
- "https://datatracker.ietf.org/doc/rfc9110/"
|
||||
"""
|
||||
query = (query or "").strip()
|
||||
if re.search(r"-\s*\d+", query) and not re.search(r"rfc-\d+", query, re.IGNORECASE):
|
||||
raise ValueError(f"Недопустимый отрицательный номер RFC: '{query}'")
|
||||
|
||||
m = re.search(r"(?:rfc[-_ ]?|doc/(?:html/)?rfc|/rfc)?(\d+)", query, re.IGNORECASE)
|
||||
if not m:
|
||||
raise ValueError(f"Не удалось определить номер RFC из строки: '{query}'")
|
||||
num = int(m.group(1))
|
||||
if num <= 0 or num > 100000:
|
||||
raise ValueError(f"Недопустимый номер RFC: {num}")
|
||||
return num
|
||||
|
||||
|
||||
def parse_rfc_text(
|
||||
raw_text: str,
|
||||
rfc_number: int,
|
||||
title: str = "",
|
||||
author: str = ""
|
||||
) -> Tuple[str, str, List[Dict[str, str]], int]:
|
||||
"""
|
||||
Cleans raw RFC text, strips pagination artifacts and form feeds,
|
||||
and splits the specification into structured readable chapters.
|
||||
Returns: (clean_title, clean_author, chapters, total_words)
|
||||
"""
|
||||
raw_text = raw_text.replace("\r\n", "\n").replace("\r", "\n").lstrip("\ufeff")
|
||||
|
||||
# 1. Clean pagination artifacts (running footers, form-feed \x0c, running headers)
|
||||
clean = re.sub(
|
||||
r"\n[^\n]*\[Page [0-9ivxlc]+\]\s*[\n\x0c]+\s*(?:RFC[^\n]*\n)?",
|
||||
"\n\n",
|
||||
raw_text,
|
||||
flags=re.IGNORECASE
|
||||
)
|
||||
clean = clean.replace("\x0c", "\n\n")
|
||||
|
||||
lines = clean.splitlines()
|
||||
|
||||
# 2. If title wasn't provided, extract from the document header
|
||||
if not title:
|
||||
for l in lines[0:65]:
|
||||
s = l.strip()
|
||||
if not s:
|
||||
continue
|
||||
if re.match(
|
||||
r"^(Internet Engineering Task Force|Request for Comments|STD:|Updates:|Obsoletes:|Category:|ISSN:|Network Working Group|BCP:|FYI:|RFC\s*\d+)",
|
||||
s,
|
||||
re.IGNORECASE
|
||||
):
|
||||
continue
|
||||
if re.match(r"^Table of Contents\b", s, re.IGNORECASE):
|
||||
break
|
||||
if re.match(r"^Abstract\b", s, re.IGNORECASE):
|
||||
break
|
||||
if 4 <= len(s) <= 80 and not s.endswith((":", ";", ".")):
|
||||
title = s
|
||||
break
|
||||
|
||||
clean_title = f"RFC {rfc_number}: {title}" if title and not title.lower().startswith("rfc") else (title or f"RFC {rfc_number}")
|
||||
clean_author = author or "IETF"
|
||||
|
||||
# 3. Detect chapters
|
||||
in_toc = False
|
||||
past_toc = False
|
||||
headings = []
|
||||
bad_sentence_starters = {
|
||||
"if", "the", "when", "where", "this", "that", "these", "those",
|
||||
"allow", "rather", "a", "an", "in", "for", "on", "to", "there",
|
||||
"clients", "optionally", "both", "all", "each", "such", "see", "note"
|
||||
}
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
trimmed = line.strip()
|
||||
if not trimmed:
|
||||
continue
|
||||
|
||||
if re.search(r"\.{4,}", line):
|
||||
in_toc = True
|
||||
continue
|
||||
|
||||
if not past_toc:
|
||||
if re.match(r"^Table of Contents\b", trimmed, re.IGNORECASE):
|
||||
in_toc = True
|
||||
continue
|
||||
if in_toc:
|
||||
if (line.startswith("1.") or line.startswith("1 ") or re.match(r"^\s{10,}1\.\s+[A-Z]", line)) and not re.search(r"\.{3,}", line) and not re.search(r"\s+\d+$", trimmed):
|
||||
in_toc = False
|
||||
past_toc = True
|
||||
else:
|
||||
continue
|
||||
|
||||
prev_blank = (i == 0) or (not lines[i - 1].strip())
|
||||
if not prev_blank:
|
||||
continue
|
||||
|
||||
m = re.match(
|
||||
r"^(?:([0-9]{1,2}\.?)\s+([A-Za-z][^\n]{1,65})|(Appendix\s+[A-Z]\.?\s*[^\n]{0,65})|(Authors?'? Addresses\b.*)|(Full Copyright Statement\b.*))$",
|
||||
trimmed,
|
||||
re.IGNORECASE
|
||||
)
|
||||
if not m and re.match(r"^[0-9]{1,2}\.\s+[A-Z\s]{3,50}$", trimmed):
|
||||
m = True
|
||||
|
||||
if m:
|
||||
if trimmed.endswith((".", ",", ":", ";", "-")):
|
||||
continue
|
||||
if len(trimmed) > 70:
|
||||
continue
|
||||
words = re.findall(r"\b[A-Za-z]+\b", trimmed)
|
||||
if len(words) >= 2:
|
||||
sec_word = words[0].lower() if not words[0].isdigit() else (words[1].lower() if len(words) > 1 else "")
|
||||
if sec_word in bad_sentence_starters:
|
||||
continue
|
||||
headings.append((i, trimmed))
|
||||
|
||||
# Deduplicate headings that are right next to each other
|
||||
deduped_headings = []
|
||||
for h in headings:
|
||||
if not deduped_headings or (h[0] - deduped_headings[-1][0] >= 3):
|
||||
deduped_headings.append(h)
|
||||
|
||||
chapters: List[Dict[str, str]] = []
|
||||
if deduped_headings:
|
||||
first_line_idx = deduped_headings[0][0]
|
||||
intro_content = "\n".join(lines[:first_line_idx]).strip()
|
||||
if count_words(intro_content) >= 30:
|
||||
chapters.append({
|
||||
"title": "Overview & Abstract",
|
||||
"content": intro_content
|
||||
})
|
||||
|
||||
for idx, (line_idx, h_title) in enumerate(deduped_headings):
|
||||
next_line_idx = deduped_headings[idx + 1][0] if idx + 1 < len(deduped_headings) else len(lines)
|
||||
body = "\n".join(lines[line_idx:next_line_idx]).strip()
|
||||
if body:
|
||||
chapters.append({
|
||||
"title": h_title,
|
||||
"content": body
|
||||
})
|
||||
|
||||
if not chapters:
|
||||
paras = [p.strip() for p in re.split(r"\n{2,}", clean) if p.strip()]
|
||||
chunks = []
|
||||
curr = []
|
||||
curr_w = 0
|
||||
for p in paras:
|
||||
w = count_words(p)
|
||||
if curr_w + w > 1500 and curr:
|
||||
chunks.append("\n\n".join(curr))
|
||||
curr = [p]
|
||||
curr_w = w
|
||||
else:
|
||||
curr.append(p)
|
||||
curr_w += w
|
||||
if curr:
|
||||
chunks.append("\n\n".join(curr))
|
||||
|
||||
for idx, chunk in enumerate(chunks, 1):
|
||||
chapters.append({
|
||||
"title": f"Section {idx}",
|
||||
"content": chunk
|
||||
})
|
||||
|
||||
total_words = sum(count_words(ch["content"]) for ch in chapters)
|
||||
return clean_title, clean_author, chapters, total_words
|
||||
|
||||
|
||||
def fetch_and_parse_rfc(query: str) -> Tuple[str, str, List[Dict[str, str]], int, int]:
|
||||
"""
|
||||
Fetches an RFC by number or URL from RFC Editor & IETF Datatracker,
|
||||
cleans formatting, parses chapters, and returns:
|
||||
(clean_title, author, chapters, rfc_number, total_words)
|
||||
"""
|
||||
rfc_num = extract_rfc_number(query)
|
||||
|
||||
# 1. Fetch metadata from IETF Datatracker (title, authors)
|
||||
dt_title = ""
|
||||
dt_authors = []
|
||||
try:
|
||||
dt_url = f"https://datatracker.ietf.org/doc/rfc{rfc_num}/doc.json"
|
||||
req = urllib.request.Request(dt_url, headers={"User-Agent": "study-qpid-pro/1.0 (educational-reader)"})
|
||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8", errors="replace"))
|
||||
dt_title = (data.get("title") or "").strip()
|
||||
for a in data.get("authors", []):
|
||||
name = a.get("name") or a.get("person", {}).get("plain_name")
|
||||
if name and name not in dt_authors:
|
||||
dt_authors.append(name.strip())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2. Fetch raw text from RFC Editor (primary) or Datatracker (fallback)
|
||||
raw_text = None
|
||||
urls_to_try = [
|
||||
f"https://www.rfc-editor.org/rfc/rfc{rfc_num}.txt",
|
||||
f"https://datatracker.ietf.org/doc/rfc{rfc_num}.txt",
|
||||
]
|
||||
fetch_err = None
|
||||
for u in urls_to_try:
|
||||
try:
|
||||
req = urllib.request.Request(u, headers={"User-Agent": "study-qpid-pro/1.0 (educational-reader)"})
|
||||
with urllib.request.urlopen(req, timeout=8) as resp:
|
||||
raw_bytes = resp.read()
|
||||
raw_text = raw_bytes.decode("utf-8", errors="replace")
|
||||
break
|
||||
except urllib.error.HTTPError as he:
|
||||
fetch_err = he
|
||||
if he.code == 404:
|
||||
continue
|
||||
except Exception as e:
|
||||
fetch_err = e
|
||||
continue
|
||||
|
||||
if not raw_text:
|
||||
if isinstance(fetch_err, urllib.error.HTTPError) and fetch_err.code == 404:
|
||||
raise ValueError(f"RFC {rfc_num} не найден в базе RFC Editor (404 Not Found)")
|
||||
err_msg = str(fetch_err) if fetch_err else "документ недоступен"
|
||||
raise ValueError(f"Не удалось загрузить RFC {rfc_num}: {err_msg}")
|
||||
|
||||
author = ("IETF (" + ", ".join(dt_authors) + ")") if dt_authors else "IETF"
|
||||
clean_title, clean_author, chapters, total_words = parse_rfc_text(
|
||||
raw_text, rfc_number=rfc_num, title=dt_title, author=author
|
||||
)
|
||||
|
||||
return clean_title, clean_author, chapters, rfc_num, total_words
|
||||
|
||||
|
||||
Reference in new issue
Block a user