feat(reader): add PDF reading support with outline chapters, CLI blocks, and noise filtering (task 7)

app/reader.py

@@ -1,9 +1,11 @@
 import io
 import re
+import sys
 import html
 import zipfile
 import posixpath
 import urllib.parse
+from pathlib import Path
 import xml.etree.ElementTree as ET
 from html.parser import HTMLParser
 from typing import Tuple, List, Dict
@@ -496,3 +498,318 @@ def parse_text(content: str, title: str = "Untitled", author: str = "") -> Tuple
 
     return title, author, chapters
 
+
+# ---------------------------------------------------------------------------
+# PDF Parsing & Chapter Extraction (pypdf, Bookmarks/Outline, Cisco CLI Blocks)
+# ---------------------------------------------------------------------------
+
+CLI_PROMPT_RE = re.compile(
+    r"^(?:[A-Za-z0-9_.\-]+(?:\([a-zA-Z0-9_\-\.\s]+\))?[>#]|"
+    r"[A-Za-z0-9_.\-]+@[A-Za-z0-9_.\-]+:[~/\w\.\-]*[$#]|"
+    r"(?:[A-Z]):\\[^>\n\r]*>|"
+    r"PS\s+[A-Z]:\\[^>\n\r]*>)"
+)
+
+
+def extract_outline_entries(outline, reader) -> List[Tuple[str, int]]:
+    """
+    Recursively extracts bookmarks / outline entries from PDF.
+    Returns a list of (title, 0-indexed page_number).
+    """
+    entries: List[Tuple[str, int]] = []
+
+    def _traverse(items):
+        for item in items:
+            if isinstance(item, list):
+                _traverse(item)
+            else:
+                try:
+                    title = getattr(item, "title", None) or (item.get("/Title") if hasattr(item, "get") else None)
+                    page_num = reader.get_destination_page_number(item)
+                    if title and page_num is not None:
+                        clean_title = str(title).strip()
+                        if clean_title:
+                            entries.append((clean_title, int(page_num)))
+                except Exception:
+                    continue
+
+    if outline:
+        _traverse(outline)
+    return entries
+
+
+def clean_page_lines(raw_pages_text: List[str]) -> List[str]:
+    """
+    Filters running headers, footers, isolated page numbers, and copyright noise.
+    """
+    if not raw_pages_text:
+        return []
+
+    num_pages = len(raw_pages_text)
+    top_candidates: Dict[str, int] = {}
+    bottom_candidates: Dict[str, int] = {}
+
+    for pt in raw_pages_text:
+        lines = [l.strip() for l in pt.split("\n") if l.strip()]
+        if not lines:
+            continue
+        if len(lines) >= 1:
+            top_candidates[lines[0]] = top_candidates.get(lines[0], 0) + 1
+            bottom_candidates[lines[-1]] = bottom_candidates.get(lines[-1], 0) + 1
+
+    # Lines repeated on >= 20% of pages (min 3 pages) are running headers/footers
+    threshold = max(3, int(num_pages * 0.20))
+    common_headers = {l for l, c in top_candidates.items() if c >= threshold}
+    common_footers = {l for l, c in bottom_candidates.items() if c >= threshold}
+
+    page_num_re = re.compile(r"^(?:page\s+)?\d+(?:\s*(?:of|/)\s*\d+)?$", re.IGNORECASE)
+    url_re = re.compile(r"^(?:https?://|www\.)\S+$", re.IGNORECASE)
+    copyright_re = re.compile(r"^(?:©|\(c\)|copyright)\s*\d{4}.*$", re.IGNORECASE)
+
+    cleaned_pages: List[str] = []
+    for pt in raw_pages_text:
+        lines = pt.split("\n")
+        filtered: List[str] = []
+        n_lines = len(lines)
+        for idx, l in enumerate(lines):
+            stripped = l.strip()
+            if not stripped:
+                filtered.append("")
+                continue
+            is_top = (idx <= 2)
+            is_bottom = (idx >= n_lines - 3)
+
+            if (is_top or is_bottom) and page_num_re.match(stripped):
+                continue
+            if is_top and stripped in common_headers:
+                continue
+            if is_bottom and (stripped in common_footers or url_re.match(stripped) or copyright_re.match(stripped)):
+                continue
+            filtered.append(l)
+        cleaned_pages.append("\n".join(filtered))
+
+    return cleaned_pages
+
+
+def format_cli_blocks(text: str) -> str:
+    """
+    Detects Cisco IOS / CLI terminal sessions in text and wraps them
+    in Markdown fenced code blocks (```text ... ```) for syntax highlighting and copying.
+    """
+    if not text:
+        return ""
+
+    lines = text.split("\n")
+    out: List[str] = []
+    in_cli = False
+    cli_lines: List[str] = []
+
+    def is_likely_cli_line(line: str) -> bool:
+        s = line.strip()
+        if not s:
+            return True
+        if CLI_PROMPT_RE.match(s):
+            return True
+        if s.startswith(("%", "!", "--", ">>", "Enter configuration", "Building configuration", "Current configuration")):
+            return True
+        if len(re.findall(r"\S\s{2,}\S", line)) >= 2:
+            return True
+        if re.match(r"^(?:show|ip|no|interface|switchport|vlan|router|network|exit|end|enable|disable|configure|ping|traceroute|debug)\s+", s, re.IGNORECASE):
+            return True
+        return False
+
+    for line in lines:
+        s = line.strip()
+        if not in_cli:
+            if CLI_PROMPT_RE.match(s):
+                in_cli = True
+                cli_lines = [line]
+            else:
+                out.append(line)
+        else:
+            if is_likely_cli_line(line):
+                cli_lines.append(line)
+            else:
+                while cli_lines and not cli_lines[-1].strip():
+                    cli_lines.pop()
+                if cli_lines:
+                    out.append("```text")
+                    out.extend(cli_lines)
+                    out.append("```")
+                in_cli = False
+                cli_lines = []
+                out.append(line)
+
+    if in_cli and cli_lines:
+        while cli_lines and not cli_lines[-1].strip():
+            cli_lines.pop()
+        if cli_lines:
+            out.append("```text")
+            out.extend(cli_lines)
+            out.append("```")
+
+    return "\n".join(out)
+
+
+def parse_pdf(file_bytes: bytes, filename: str = "") -> Tuple[str, str, List[Dict[str, str]]]:
+    """
+    Parses a PDF file into readable chapters:
+      - Extracts title and author from metadata or filename.
+      - Enforces Scanned PDF Guard (raises ValueError if total_words < 100).
+      - Cleans repetitive headers, footers, and page numbers.
+      - Wraps Cisco IOS / CLI terminal blocks in Markdown code blocks.
+      - Splits chapters by PDF outline / bookmarks (Cisco Press, O'Reilly).
+      - Fallback to heading regex or ~10-15 page chunks if no bookmarks exist.
+    """
+    try:
+        import pypdf
+    except ImportError:
+        vendor_dir = Path(__file__).resolve().parent / "vendor"
+        if str(vendor_dir) not in sys.path:
+            sys.path.insert(0, str(vendor_dir))
+        import pypdf
+
+    try:
+        reader = pypdf.PdfReader(io.BytesIO(file_bytes))
+    except Exception as e:
+        raise ValueError(f"Не удалось открыть PDF документ: {e}")
+
+    if reader.is_encrypted:
+        try:
+            reader.decrypt("")
+        except Exception:
+            raise ValueError("Этот PDF документ защищен паролем. Пожалуйста, снимите пароль для чтения.")
+
+    num_pages = len(reader.pages)
+    if num_pages == 0:
+        raise ValueError("PDF документ не содержит страниц")
+
+    # Metadata extraction
+    meta = reader.metadata or {}
+    meta_title = getattr(meta, "title", None) or (meta.get("/Title") if hasattr(meta, "get") else None) or ""
+    meta_author = getattr(meta, "author", None) or (meta.get("/Author") if hasattr(meta, "get") else None) or ""
+
+    stem = Path(filename).stem if filename else "Untitled Document"
+    clean_title = str(meta_title).strip()
+    if not clean_title or clean_title.lower() in {"untitled", "untitled document", "default", "temp", "output"}:
+        title = stem
+    else:
+        title = clean_title
+
+    author = str(meta_author).strip()
+
+    # Page-by-page text extraction
+    raw_pages_text: List[str] = []
+    for p in reader.pages:
+        try:
+            pt = p.extract_text() or ""
+        except Exception:
+            pt = ""
+        raw_pages_text.append(pt)
+
+    # Scanned PDF Guard (< 100 words)
+    total_words = sum(count_words(t) for t in raw_pages_text)
+    if total_words < 100:
+        raise ValueError(
+            "Этот PDF-документ содержит сканы или изображения без текстового слоя (OCR). "
+            "Пожалуйста, используйте документ с распознанным текстом."
+        )
+
+    # Clean noise lines (headers, footers, page numbers)
+    cleaned_pages = clean_page_lines(raw_pages_text)
+
+    # Apply CLI formatting to each page
+    formatted_pages = [format_cli_blocks(p) for p in cleaned_pages]
+
+    # Chapter splitting:
+    # 1. Try outline / bookmarks
+    outline_entries = extract_outline_entries(getattr(reader, "outline", None), reader)
+    chapters: List[Dict[str, str]] = []
+
+    if outline_entries:
+        valid_entries = [(t, p) for t, p in outline_entries if 0 <= p < num_pages]
+        valid_entries.sort(key=lambda x: x[1])
+
+        # Filter duplicates pointing to the same page
+        deduped_entries: List[Tuple[str, int]] = []
+        for t, p in valid_entries:
+            if not deduped_entries or deduped_entries[-1][1] != p:
+                deduped_entries.append((t, p))
+
+        if deduped_entries:
+            first_p = deduped_entries[0][1]
+            if first_p > 0:
+                intro_content = "\n\n".join(formatted_pages[p].strip() for p in range(0, first_p) if formatted_pages[p].strip())
+                if count_words(intro_content) >= 20:
+                    is_ru = any(ord(c) > 127 for c in intro_content[:300])
+                    chapters.append({
+                        "title": "Введение" if is_ru else "Introduction",
+                        "content": intro_content
+                    })
+
+            for i, (entry_title, start_p) in enumerate(deduped_entries):
+                end_p = deduped_entries[i + 1][1] if i + 1 < len(deduped_entries) else num_pages
+                ch_content = "\n\n".join(formatted_pages[p].strip() for p in range(start_p, end_p) if formatted_pages[p].strip())
+                if ch_content:
+                    chapters.append({
+                        "title": entry_title,
+                        "content": ch_content
+                    })
+
+    # 2. Fallback: No outline found, check for page-level chapter headings
+    if not chapters:
+        chapter_heading_re = re.compile(
+            r"^\s*(?:chapter|глава|part|часть|section|раздел|module|модуль)\s+([0-9ivxlcdm]+)[\s:.\-–—\n]+(.*)$",
+            re.IGNORECASE
+        )
+        detected_pages: List[Tuple[str, int]] = []
+        for p_idx, p_text in enumerate(formatted_pages):
+            first_lines = [l.strip() for l in p_text.split("\n") if l.strip()][:3]
+            for l in first_lines:
+                m = chapter_heading_re.match(l)
+                if m:
+                    h_title = l.strip("#").strip()
+                    detected_pages.append((h_title, p_idx))
+                    break
+
+        if detected_pages:
+            if detected_pages[0][1] > 0:
+                intro_content = "\n\n".join(formatted_pages[p].strip() for p in range(0, detected_pages[0][1]) if formatted_pages[p].strip())
+                if count_words(intro_content) >= 20:
+                    is_ru = any(ord(c) > 127 for c in intro_content[:300])
+                    chapters.append({
+                        "title": "Введение" if is_ru else "Introduction",
+                        "content": intro_content
+                    })
+
+            for i, (h_title, start_p) in enumerate(detected_pages):
+                end_p = detected_pages[i + 1][1] if i + 1 < len(detected_pages) else num_pages
+                ch_content = "\n\n".join(formatted_pages[p].strip() for p in range(start_p, end_p) if formatted_pages[p].strip())
+                if ch_content:
+                    chapters.append({
+                        "title": h_title,
+                        "content": ch_content
+                    })
+
+    # 3. Fallback: Group pages in chunks of 12-15 pages
+    if not chapters:
+        chunk_size = 12
+        sample_text = "".join(formatted_pages[:3])
+        is_ru = any(ord(c) > 127 for c in sample_text[:300])
+        prefix = "Страницы" if is_ru else "Pages"
+
+        for start_p in range(0, num_pages, chunk_size):
+            end_p = min(start_p + chunk_size, num_pages)
+            ch_content = "\n\n".join(formatted_pages[p].strip() for p in range(start_p, end_p) if formatted_pages[p].strip())
+            if ch_content:
+                chapters.append({
+                    "title": f"{prefix} {start_p + 1}–{end_p}",
+                    "content": ch_content
+                })
+
+    if not chapters:
+        full_content = "\n\n".join(p.strip() for p in formatted_pages if p.strip())
+        chapters = [{"title": title, "content": full_content}]
+
+    return title, author, chapters
+