diff --git a/TODO.md b/TODO.md index fb49cee..f874aa5 100644 --- a/TODO.md +++ b/TODO.md @@ -155,22 +155,22 @@ --- -### 7. 📑 Поддержка чтения PDF (Cisco CCNA, O'Reilly & Tech Books с оглавлением и текстовым слоем) -- [ ] **Легковесный Pure-Python парсер (`pypdf`, Zero C-Dependency)**: +### 7. 📑 Поддержка чтения PDF (Cisco CCNA, O'Reilly & Tech Books с оглавлением и текстовым слоем) (Completed) +- [x] **Легковесный Pure-Python парсер (`pypdf`, Zero C-Dependency)**: - Подключение `pypdf` (< 400 КБ, чистый Python, без раздувания Docker-образа и без системных библиотек `poppler-utils` / `mupdf`). -- [ ] **Извлечение оглавления и закладок PDF (`reader.outline`)**: +- [x] **Извлечение оглавления и закладок PDF (`reader.outline`)**: - Извлечение авторского оглавления (ToC / Bookmarks), стандартного для книг Cisco Press (CCNA Official Cert Guide), O'Reilly и Wiley. - Разбивка книги на логические главы и разделы (`Part I`, `Chapter 1`, `Chapter 2`) точно по оглавлению автора. -- [ ] **Fallback-разбивка при отсутствии оглавления**: +- [x] **Fallback-разбивка при отсутствии оглавления**: - При отсутствии PDF-закладок — автоопределение заголовков через regex (`Chapter \d+`, `Глава \d+`) или группировка страниц порциями по ~12–15 страниц. -- [ ] **Очистка шума и колонтитулов**: +- [x] **Очистка шума и колонтитулов**: - Фильтрация повторяющихся верхних/нижних колонтитулов («CCNA 200-301 Official Cert Guide», «Page 45 of 900»). -- [ ] **Сохранение CLI/Console блоков**: +- [x] **Сохранение CLI/Console блоков**: - Распознавание консольного вывода и команд сетевого оборудования (`Router# show ip interface brief`, `Switch(config)#`) и оборачивание в блоки кода Markdown для читалки с кнопкой «Копировать». -- [ ] **Защита от сканов без OCR (Scanned PDF Guard)**: +- [x] **Защита от сканов без OCR (Scanned PDF Guard)**: - Валидация наличия текстового слоя: если суммарный объём извлечённого текста < 100 слов, возврат информативной ошибки с предложением использовать текстовый PDF или OCR. -- [ ] **Интеграция в интерфейс читалки**: - - Разрешение загрузки `.pdf` в ``. +- [x] **Интеграция в интерфейс читалки**: + - Разрешение загрузки `.pdf` в ``. - Обновление кнопки в библиотеке: `📁 Загрузить EPUB / PDF / TXT`. --- diff --git a/app/main.py b/app/main.py index f271a2b..edee002 100644 --- a/app/main.py +++ b/app/main.py @@ -28,7 +28,7 @@ from app.auth import ( from app.dictionary import lookup_term, USER_AGENT from app.export import export_csv, export_json, export_anki_tsv, export_markdown from app.srs import calculate_sm2, generate_quiz_options, build_activity_heatmap, calculate_cefr_level -from app.reader import parse_epub, parse_text, count_words +from app.reader import parse_epub, parse_text, parse_pdf, count_words @asynccontextmanager @@ -1088,29 +1088,37 @@ async def api_create_book( stem = Path(filename).stem lower_name = filename.lower() - if lower_name.endswith(".epub"): - source_type = "epub" - parsed_title, parsed_author, chapters = parse_epub(file_bytes) - title = form_title or (parsed_title if parsed_title != "Untitled" else "") or stem - author = form_author or parsed_author - elif lower_name.endswith(".md"): - source_type = "md" - text_str = file_bytes.decode("utf-8", errors="replace") - parsed_title, parsed_author, chapters = parse_text(text_str, title=form_title or "Untitled", author=form_author) - title = form_title or (parsed_title if parsed_title != "Untitled" else "") or stem - author = form_author or parsed_author - elif lower_name.endswith(".txt"): - source_type = "txt" - text_str = file_bytes.decode("utf-8", errors="replace") - parsed_title, parsed_author, chapters = parse_text(text_str, title=form_title or "Untitled", author=form_author) - title = form_title or (parsed_title if parsed_title != "Untitled" else "") or stem - author = form_author or parsed_author - else: - source_type = "text" - text_str = file_bytes.decode("utf-8", errors="replace") - parsed_title, parsed_author, chapters = parse_text(text_str, title=form_title or "Untitled", author=form_author) - title = form_title or (parsed_title if parsed_title != "Untitled" else "") or stem - author = form_author or parsed_author + try: + if lower_name.endswith(".epub"): + source_type = "epub" + parsed_title, parsed_author, chapters = parse_epub(file_bytes) + title = form_title or (parsed_title if parsed_title != "Untitled" else "") or stem + author = form_author or parsed_author + elif lower_name.endswith(".pdf"): + source_type = "pdf" + parsed_title, parsed_author, chapters = parse_pdf(file_bytes, filename=filename) + title = form_title or (parsed_title if parsed_title != "Untitled" else "") or stem + author = form_author or parsed_author + elif lower_name.endswith(".md"): + source_type = "md" + text_str = file_bytes.decode("utf-8", errors="replace") + parsed_title, parsed_author, chapters = parse_text(text_str, title=form_title or "Untitled", author=form_author) + title = form_title or (parsed_title if parsed_title != "Untitled" else "") or stem + author = form_author or parsed_author + elif lower_name.endswith(".txt"): + source_type = "txt" + text_str = file_bytes.decode("utf-8", errors="replace") + parsed_title, parsed_author, chapters = parse_text(text_str, title=form_title or "Untitled", author=form_author) + title = form_title or (parsed_title if parsed_title != "Untitled" else "") or stem + author = form_author or parsed_author + else: + source_type = "text" + text_str = file_bytes.decode("utf-8", errors="replace") + parsed_title, parsed_author, chapters = parse_text(text_str, title=form_title or "Untitled", author=form_author) + title = form_title or (parsed_title if parsed_title != "Untitled" else "") or stem + author = form_author or parsed_author + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) else: try: diff --git a/app/reader.py b/app/reader.py index 7f26db8..2c51b8a 100644 --- a/app/reader.py +++ b/app/reader.py @@ -1,9 +1,11 @@ import io import re +import sys import html import zipfile import posixpath import urllib.parse +from pathlib import Path import xml.etree.ElementTree as ET from html.parser import HTMLParser from typing import Tuple, List, Dict @@ -496,3 +498,318 @@ def parse_text(content: str, title: str = "Untitled", author: str = "") -> Tuple return title, author, chapters + +# --------------------------------------------------------------------------- +# PDF Parsing & Chapter Extraction (pypdf, Bookmarks/Outline, Cisco CLI Blocks) +# --------------------------------------------------------------------------- + +CLI_PROMPT_RE = re.compile( + r"^(?:[A-Za-z0-9_.\-]+(?:\([a-zA-Z0-9_\-\.\s]+\))?[>#]|" + r"[A-Za-z0-9_.\-]+@[A-Za-z0-9_.\-]+:[~/\w\.\-]*[$#]|" + r"(?:[A-Z]):\\[^>\n\r]*>|" + r"PS\s+[A-Z]:\\[^>\n\r]*>)" +) + + +def extract_outline_entries(outline, reader) -> List[Tuple[str, int]]: + """ + Recursively extracts bookmarks / outline entries from PDF. + Returns a list of (title, 0-indexed page_number). + """ + entries: List[Tuple[str, int]] = [] + + def _traverse(items): + for item in items: + if isinstance(item, list): + _traverse(item) + else: + try: + title = getattr(item, "title", None) or (item.get("/Title") if hasattr(item, "get") else None) + page_num = reader.get_destination_page_number(item) + if title and page_num is not None: + clean_title = str(title).strip() + if clean_title: + entries.append((clean_title, int(page_num))) + except Exception: + continue + + if outline: + _traverse(outline) + return entries + + +def clean_page_lines(raw_pages_text: List[str]) -> List[str]: + """ + Filters running headers, footers, isolated page numbers, and copyright noise. + """ + if not raw_pages_text: + return [] + + num_pages = len(raw_pages_text) + top_candidates: Dict[str, int] = {} + bottom_candidates: Dict[str, int] = {} + + for pt in raw_pages_text: + lines = [l.strip() for l in pt.split("\n") if l.strip()] + if not lines: + continue + if len(lines) >= 1: + top_candidates[lines[0]] = top_candidates.get(lines[0], 0) + 1 + bottom_candidates[lines[-1]] = bottom_candidates.get(lines[-1], 0) + 1 + + # Lines repeated on >= 20% of pages (min 3 pages) are running headers/footers + threshold = max(3, int(num_pages * 0.20)) + common_headers = {l for l, c in top_candidates.items() if c >= threshold} + common_footers = {l for l, c in bottom_candidates.items() if c >= threshold} + + page_num_re = re.compile(r"^(?:page\s+)?\d+(?:\s*(?:of|/)\s*\d+)?$", re.IGNORECASE) + url_re = re.compile(r"^(?:https?://|www\.)\S+$", re.IGNORECASE) + copyright_re = re.compile(r"^(?:©|\(c\)|copyright)\s*\d{4}.*$", re.IGNORECASE) + + cleaned_pages: List[str] = [] + for pt in raw_pages_text: + lines = pt.split("\n") + filtered: List[str] = [] + n_lines = len(lines) + for idx, l in enumerate(lines): + stripped = l.strip() + if not stripped: + filtered.append("") + continue + is_top = (idx <= 2) + is_bottom = (idx >= n_lines - 3) + + if (is_top or is_bottom) and page_num_re.match(stripped): + continue + if is_top and stripped in common_headers: + continue + if is_bottom and (stripped in common_footers or url_re.match(stripped) or copyright_re.match(stripped)): + continue + filtered.append(l) + cleaned_pages.append("\n".join(filtered)) + + return cleaned_pages + + +def format_cli_blocks(text: str) -> str: + """ + Detects Cisco IOS / CLI terminal sessions in text and wraps them + in Markdown fenced code blocks (```text ... ```) for syntax highlighting and copying. + """ + if not text: + return "" + + lines = text.split("\n") + out: List[str] = [] + in_cli = False + cli_lines: List[str] = [] + + def is_likely_cli_line(line: str) -> bool: + s = line.strip() + if not s: + return True + if CLI_PROMPT_RE.match(s): + return True + if s.startswith(("%", "!", "--", ">>", "Enter configuration", "Building configuration", "Current configuration")): + return True + if len(re.findall(r"\S\s{2,}\S", line)) >= 2: + return True + if re.match(r"^(?:show|ip|no|interface|switchport|vlan|router|network|exit|end|enable|disable|configure|ping|traceroute|debug)\s+", s, re.IGNORECASE): + return True + return False + + for line in lines: + s = line.strip() + if not in_cli: + if CLI_PROMPT_RE.match(s): + in_cli = True + cli_lines = [line] + else: + out.append(line) + else: + if is_likely_cli_line(line): + cli_lines.append(line) + else: + while cli_lines and not cli_lines[-1].strip(): + cli_lines.pop() + if cli_lines: + out.append("```text") + out.extend(cli_lines) + out.append("```") + in_cli = False + cli_lines = [] + out.append(line) + + if in_cli and cli_lines: + while cli_lines and not cli_lines[-1].strip(): + cli_lines.pop() + if cli_lines: + out.append("```text") + out.extend(cli_lines) + out.append("```") + + return "\n".join(out) + + +def parse_pdf(file_bytes: bytes, filename: str = "") -> Tuple[str, str, List[Dict[str, str]]]: + """ + Parses a PDF file into readable chapters: + - Extracts title and author from metadata or filename. + - Enforces Scanned PDF Guard (raises ValueError if total_words < 100). + - Cleans repetitive headers, footers, and page numbers. + - Wraps Cisco IOS / CLI terminal blocks in Markdown code blocks. + - Splits chapters by PDF outline / bookmarks (Cisco Press, O'Reilly). + - Fallback to heading regex or ~10-15 page chunks if no bookmarks exist. + """ + try: + import pypdf + except ImportError: + vendor_dir = Path(__file__).resolve().parent / "vendor" + if str(vendor_dir) not in sys.path: + sys.path.insert(0, str(vendor_dir)) + import pypdf + + try: + reader = pypdf.PdfReader(io.BytesIO(file_bytes)) + except Exception as e: + raise ValueError(f"Не удалось открыть PDF документ: {e}") + + if reader.is_encrypted: + try: + reader.decrypt("") + except Exception: + raise ValueError("Этот PDF документ защищен паролем. Пожалуйста, снимите пароль для чтения.") + + num_pages = len(reader.pages) + if num_pages == 0: + raise ValueError("PDF документ не содержит страниц") + + # Metadata extraction + meta = reader.metadata or {} + meta_title = getattr(meta, "title", None) or (meta.get("/Title") if hasattr(meta, "get") else None) or "" + meta_author = getattr(meta, "author", None) or (meta.get("/Author") if hasattr(meta, "get") else None) or "" + + stem = Path(filename).stem if filename else "Untitled Document" + clean_title = str(meta_title).strip() + if not clean_title or clean_title.lower() in {"untitled", "untitled document", "default", "temp", "output"}: + title = stem + else: + title = clean_title + + author = str(meta_author).strip() + + # Page-by-page text extraction + raw_pages_text: List[str] = [] + for p in reader.pages: + try: + pt = p.extract_text() or "" + except Exception: + pt = "" + raw_pages_text.append(pt) + + # Scanned PDF Guard (< 100 words) + total_words = sum(count_words(t) for t in raw_pages_text) + if total_words < 100: + raise ValueError( + "Этот PDF-документ содержит сканы или изображения без текстового слоя (OCR). " + "Пожалуйста, используйте документ с распознанным текстом." + ) + + # Clean noise lines (headers, footers, page numbers) + cleaned_pages = clean_page_lines(raw_pages_text) + + # Apply CLI formatting to each page + formatted_pages = [format_cli_blocks(p) for p in cleaned_pages] + + # Chapter splitting: + # 1. Try outline / bookmarks + outline_entries = extract_outline_entries(getattr(reader, "outline", None), reader) + chapters: List[Dict[str, str]] = [] + + if outline_entries: + valid_entries = [(t, p) for t, p in outline_entries if 0 <= p < num_pages] + valid_entries.sort(key=lambda x: x[1]) + + # Filter duplicates pointing to the same page + deduped_entries: List[Tuple[str, int]] = [] + for t, p in valid_entries: + if not deduped_entries or deduped_entries[-1][1] != p: + deduped_entries.append((t, p)) + + if deduped_entries: + first_p = deduped_entries[0][1] + if first_p > 0: + intro_content = "\n\n".join(formatted_pages[p].strip() for p in range(0, first_p) if formatted_pages[p].strip()) + if count_words(intro_content) >= 20: + is_ru = any(ord(c) > 127 for c in intro_content[:300]) + chapters.append({ + "title": "Введение" if is_ru else "Introduction", + "content": intro_content + }) + + for i, (entry_title, start_p) in enumerate(deduped_entries): + end_p = deduped_entries[i + 1][1] if i + 1 < len(deduped_entries) else num_pages + ch_content = "\n\n".join(formatted_pages[p].strip() for p in range(start_p, end_p) if formatted_pages[p].strip()) + if ch_content: + chapters.append({ + "title": entry_title, + "content": ch_content + }) + + # 2. Fallback: No outline found, check for page-level chapter headings + if not chapters: + chapter_heading_re = re.compile( + r"^\s*(?:chapter|глава|part|часть|section|раздел|module|модуль)\s+([0-9ivxlcdm]+)[\s:.\-–—\n]+(.*)$", + re.IGNORECASE + ) + detected_pages: List[Tuple[str, int]] = [] + for p_idx, p_text in enumerate(formatted_pages): + first_lines = [l.strip() for l in p_text.split("\n") if l.strip()][:3] + for l in first_lines: + m = chapter_heading_re.match(l) + if m: + h_title = l.strip("#").strip() + detected_pages.append((h_title, p_idx)) + break + + if detected_pages: + if detected_pages[0][1] > 0: + intro_content = "\n\n".join(formatted_pages[p].strip() for p in range(0, detected_pages[0][1]) if formatted_pages[p].strip()) + if count_words(intro_content) >= 20: + is_ru = any(ord(c) > 127 for c in intro_content[:300]) + chapters.append({ + "title": "Введение" if is_ru else "Introduction", + "content": intro_content + }) + + for i, (h_title, start_p) in enumerate(detected_pages): + end_p = detected_pages[i + 1][1] if i + 1 < len(detected_pages) else num_pages + ch_content = "\n\n".join(formatted_pages[p].strip() for p in range(start_p, end_p) if formatted_pages[p].strip()) + if ch_content: + chapters.append({ + "title": h_title, + "content": ch_content + }) + + # 3. Fallback: Group pages in chunks of 12-15 pages + if not chapters: + chunk_size = 12 + sample_text = "".join(formatted_pages[:3]) + is_ru = any(ord(c) > 127 for c in sample_text[:300]) + prefix = "Страницы" if is_ru else "Pages" + + for start_p in range(0, num_pages, chunk_size): + end_p = min(start_p + chunk_size, num_pages) + ch_content = "\n\n".join(formatted_pages[p].strip() for p in range(start_p, end_p) if formatted_pages[p].strip()) + if ch_content: + chapters.append({ + "title": f"{prefix} {start_p + 1}–{end_p}", + "content": ch_content + }) + + if not chapters: + full_content = "\n\n".join(p.strip() for p in formatted_pages if p.strip()) + chapters = [{"title": title, "content": full_content}] + + return title, author, chapters + diff --git a/app/static/index.html b/app/static/index.html index de7f02f..26abce6 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -5,7 +5,7 @@