Antigravity
c4a6edee37
- extract_outline_entries: filter PDF bookmarks by depth and major structural headings, avoiding 550 micro-snippets - renderContinuousBook: load current chapter and seamlessly append next chapters on scroll (virtual continuous) instead of dumping all chapters into DOM synchronously - renderInteractiveSpans: use native HTML template fragment instead of new DOMParser per paragraph - event delegation: handle word clicks on readerContentBody instead of adding hundreds of thousands of individual DOM event listeners - consolidate existing CCNA book in DB into 68 clean chapters
849 lines
32 KiB
Python
849 lines
32 KiB
Python
import io
|
||
import re
|
||
import sys
|
||
import html
|
||
import zipfile
|
||
import posixpath
|
||
import urllib.parse
|
||
from pathlib import Path
|
||
import xml.etree.ElementTree as ET
|
||
from html.parser import HTMLParser
|
||
from typing import Tuple, List, Dict
|
||
|
||
|
||
def count_words(text: str) -> int:
|
||
"""
|
||
Counts words in text across multiple languages, ignoring isolated punctuation.
|
||
"""
|
||
if not text:
|
||
return 0
|
||
return len(re.findall(r"\b[\w]+(?:['’][\w]+)?\b", text))
|
||
|
||
|
||
class HTMLTextExtractor(HTMLParser):
|
||
"""
|
||
Strips HTML tags cleanly while preserving paragraph and heading breaks with newlines.
|
||
Also extracts the first heading and/or <title> tag for chapter naming.
|
||
"""
|
||
def __init__(self):
|
||
super().__init__()
|
||
self.text_parts: List[str] = []
|
||
self.ignore_tags = {"style", "script", "noscript"}
|
||
self.ignore_depth = 0
|
||
self.heading: str | None = None
|
||
self.in_heading = False
|
||
self.heading_parts: List[str] = []
|
||
self.title: str | None = None
|
||
self.in_title = False
|
||
self.title_parts: List[str] = []
|
||
|
||
def handle_starttag(self, tag: str, attrs):
|
||
t = tag.lower()
|
||
if t in self.ignore_tags:
|
||
self.ignore_depth += 1
|
||
elif t == "title":
|
||
self.in_title = True
|
||
self.title_parts = []
|
||
elif t in {"h1", "h2", "h3", "h4", "h5", "h6"}:
|
||
self.text_parts.append("\n\n")
|
||
if not self.heading:
|
||
self.in_heading = True
|
||
self.heading_parts = []
|
||
elif t in {"p", "div", "section", "article", "blockquote", "header", "footer"}:
|
||
self.text_parts.append("\n\n")
|
||
elif t in {"br", "hr", "li", "tr"}:
|
||
self.text_parts.append("\n")
|
||
|
||
def handle_endtag(self, tag: str):
|
||
t = tag.lower()
|
||
if t in self.ignore_tags:
|
||
self.ignore_depth = max(0, self.ignore_depth - 1)
|
||
elif t == "title":
|
||
self.in_title = False
|
||
self.title = "".join(self.title_parts).strip()
|
||
elif t in {"h1", "h2", "h3", "h4", "h5", "h6"}:
|
||
self.text_parts.append("\n\n")
|
||
if self.in_heading:
|
||
self.in_heading = False
|
||
if not self.heading:
|
||
self.heading = "".join(self.heading_parts).strip()
|
||
elif t in {"p", "div", "section", "article", "blockquote", "header", "footer"}:
|
||
self.text_parts.append("\n\n")
|
||
elif t in {"li", "tr"}:
|
||
self.text_parts.append("\n")
|
||
|
||
def handle_data(self, data: str):
|
||
if self.in_title:
|
||
self.title_parts.append(data)
|
||
if self.ignore_depth == 0 and not self.in_title:
|
||
# Replace non-breaking spaces with standard space
|
||
cleaned_data = data.replace("\xa0", " ")
|
||
self.text_parts.append(cleaned_data)
|
||
if self.in_heading:
|
||
self.heading_parts.append(cleaned_data)
|
||
|
||
def get_text(self) -> str:
|
||
raw = "".join(self.text_parts)
|
||
lines = [line.strip() for line in raw.split("\n")]
|
||
clean = "\n".join(lines)
|
||
return re.sub(r"\n{3,}", "\n\n", clean).strip()
|
||
|
||
|
||
def parse_epub(file_bytes: bytes) -> Tuple[str, str, List[Dict[str, str]]]:
|
||
"""
|
||
Parses an EPUB file archive and extracts:
|
||
- title: book title
|
||
- author: creator/author
|
||
- chapters: list of {"title": str, "content": str}
|
||
"""
|
||
try:
|
||
z = zipfile.ZipFile(io.BytesIO(file_bytes))
|
||
except Exception as e:
|
||
raise ValueError(f"Invalid EPUB archive: {e}")
|
||
|
||
# 1. Locate root OPF file from META-INF/container.xml
|
||
opf_path = None
|
||
try:
|
||
container_data = z.read("META-INF/container.xml")
|
||
root = ET.fromstring(container_data)
|
||
rootfile = root.find(".//{*}rootfile")
|
||
if rootfile is not None:
|
||
opf_path = rootfile.attrib.get("full-path")
|
||
except Exception:
|
||
pass
|
||
|
||
if not opf_path:
|
||
for name in z.namelist():
|
||
if name.lower().endswith(".opf"):
|
||
opf_path = name
|
||
break
|
||
|
||
if not opf_path:
|
||
raise ValueError("Invalid EPUB: no OPF file found")
|
||
|
||
opf_path = opf_path.replace("\\", "/").lstrip("/")
|
||
opf_dir = posixpath.dirname(opf_path)
|
||
|
||
try:
|
||
opf_content = z.read(opf_path)
|
||
except KeyError:
|
||
# Try lowercase matching
|
||
name_map = {n.lower(): n for n in z.namelist()}
|
||
if opf_path.lower() in name_map:
|
||
opf_content = z.read(name_map[opf_path.lower()])
|
||
else:
|
||
raise ValueError(f"OPF file not found in archive: {opf_path}")
|
||
|
||
# 2. Extract metadata (title, author)
|
||
title = "Untitled"
|
||
author = ""
|
||
opf_tree = None
|
||
try:
|
||
opf_tree = ET.fromstring(opf_content)
|
||
title_el = opf_tree.find(".//{*}title")
|
||
if title_el is not None and title_el.text:
|
||
title = title_el.text.strip()
|
||
creator_el = opf_tree.find(".//{*}creator")
|
||
if creator_el is not None and creator_el.text:
|
||
author = creator_el.text.strip()
|
||
except Exception:
|
||
opf_str = opf_content.decode("utf-8", errors="ignore")
|
||
tm = re.search(r"<[^:]*:?title[^>]*>(.*?)</[^:]*:?title>", opf_str, re.I | re.S)
|
||
if tm:
|
||
title = re.sub(r"<[^>]+>", "", tm.group(1)).strip()
|
||
cm = re.search(r"<[^:]*:?creator[^>]*>(.*?)</[^:]*:?creator>", opf_str, re.I | re.S)
|
||
if cm:
|
||
author = re.sub(r"<[^>]+>", "", cm.group(1)).strip()
|
||
|
||
# 3. Parse Manifest (id -> href, media-type)
|
||
manifest: Dict[str, Dict[str, str]] = {}
|
||
if opf_tree is not None:
|
||
for item in opf_tree.findall(".//{*}item"):
|
||
iid = item.attrib.get("id")
|
||
href = item.attrib.get("href")
|
||
media_type = item.attrib.get("media-type", "").lower()
|
||
if iid and href:
|
||
manifest[iid] = {"href": href, "media_type": media_type}
|
||
else:
|
||
opf_str = opf_content.decode("utf-8", errors="ignore")
|
||
for m in re.finditer(r'<item\s+[^>]*id=["\']([^"\']+)["\'][^>]*href=["\']([^"\']+)["\']', opf_str, re.I):
|
||
manifest[m.group(1)] = {"href": m.group(2), "media_type": ""}
|
||
|
||
# 4. Spine items in reading order
|
||
spine_ids: List[str] = []
|
||
if opf_tree is not None:
|
||
for itemref in opf_tree.findall(".//{*}itemref"):
|
||
idref = itemref.attrib.get("idref")
|
||
if idref:
|
||
spine_ids.append(idref)
|
||
else:
|
||
opf_str = opf_content.decode("utf-8", errors="ignore")
|
||
for m in re.finditer(r'<itemref\s+[^>]*idref=["\']([^"\']+)["\']', opf_str, re.I):
|
||
spine_ids.append(m.group(1))
|
||
|
||
# Fallback if spine is empty: find all html/xhtml files in manifest
|
||
candidate_paths: List[str] = []
|
||
if spine_ids:
|
||
for idref in spine_ids:
|
||
item = manifest.get(idref)
|
||
if item:
|
||
candidate_paths.append(item["href"])
|
||
else:
|
||
for item in manifest.values():
|
||
href = item["href"]
|
||
if href.lower().endswith((".xhtml", ".html", ".htm")):
|
||
candidate_paths.append(href)
|
||
|
||
namelist_map = {name.lower(): name for name in z.namelist()}
|
||
chapters: List[Dict[str, str]] = []
|
||
chapter_num = 1
|
||
|
||
for href in candidate_paths:
|
||
unquoted = urllib.parse.unquote(href)
|
||
rel_path = posixpath.normpath(posixpath.join(opf_dir, unquoted)) if opf_dir else unquoted
|
||
rel_path = rel_path.replace("\\", "/").lstrip("/")
|
||
|
||
actual_path = None
|
||
if rel_path in z.namelist():
|
||
actual_path = rel_path
|
||
elif rel_path.lower() in namelist_map:
|
||
actual_path = namelist_map[rel_path.lower()]
|
||
|
||
if not actual_path:
|
||
continue
|
||
|
||
try:
|
||
raw_bytes = z.read(actual_path)
|
||
try:
|
||
html_str = raw_bytes.decode("utf-8")
|
||
except UnicodeDecodeError:
|
||
html_str = raw_bytes.decode("latin-1", errors="replace")
|
||
except Exception:
|
||
continue
|
||
|
||
parser = HTMLTextExtractor()
|
||
parser.feed(html_str)
|
||
text = parser.get_text()
|
||
|
||
# Skip chapters with little or no readable text
|
||
if count_words(text) < 1:
|
||
continue
|
||
|
||
ch_title = parser.heading or parser.title or f"Chapter {chapter_num}"
|
||
# If ch_title is identical to book title or very long, format nicely
|
||
if len(ch_title) > 120:
|
||
ch_title = f"Chapter {chapter_num}"
|
||
|
||
chapters.append({"title": ch_title, "content": text})
|
||
chapter_num += 1
|
||
|
||
if not chapters:
|
||
chapters = [{"title": title if title != "Untitled" else "Chapter 1", "content": ""}]
|
||
|
||
return title, author, chapters
|
||
|
||
|
||
def split_long_text(text: str, target_words: int = 3000) -> List[str]:
|
||
"""
|
||
Splits text into chunks of approximately target_words on paragraph boundaries.
|
||
"""
|
||
paragraphs = [p.strip() for p in re.split(r"\n{2,}", text) if p.strip()]
|
||
if not paragraphs:
|
||
return [text.strip()] if text.strip() else []
|
||
|
||
chunks: List[str] = []
|
||
current_paras: List[str] = []
|
||
current_count = 0
|
||
|
||
for p in paragraphs:
|
||
w_count = count_words(p)
|
||
# If single paragraph is massive (> target_words * 1.5), split by words
|
||
if w_count > target_words * 1.5:
|
||
if current_paras:
|
||
chunks.append("\n\n".join(current_paras))
|
||
current_paras = []
|
||
current_count = 0
|
||
words = p.split()
|
||
for i in range(0, len(words), target_words):
|
||
chunk = " ".join(words[i : i + target_words])
|
||
if chunk.strip():
|
||
chunks.append(chunk)
|
||
continue
|
||
|
||
if current_count + w_count > target_words and current_count > 0:
|
||
chunks.append("\n\n".join(current_paras))
|
||
current_paras = [p]
|
||
current_count = w_count
|
||
else:
|
||
current_paras.append(p)
|
||
current_count += w_count
|
||
|
||
if current_paras:
|
||
chunks.append("\n\n".join(current_paras))
|
||
|
||
return chunks
|
||
|
||
|
||
def parse_text(content: str, title: str = "Untitled", author: str = "") -> Tuple[str, str, List[Dict[str, str]]]:
|
||
"""
|
||
Parses plain text or Markdown into readable chapters.
|
||
Splits by explicit chapter markers (e.g. 'Chapter X', '# Chapter') or chunks of ~3000 words.
|
||
"""
|
||
content = content.replace("\r\n", "\n").replace("\r", "\n").strip()
|
||
if not content:
|
||
return title, author, [{"title": title if title != "Untitled" else "Chapter 1", "content": ""}]
|
||
|
||
lines = content.split("\n")
|
||
first_line = lines[0].strip()
|
||
|
||
# Extract title from first line or # Heading / <h1...> if default or generic README
|
||
if title == "Untitled" or title.lower().startswith("readme"):
|
||
if first_line.startswith("# "):
|
||
title = first_line.lstrip("# ").strip()
|
||
if len(lines) > 1:
|
||
content = "\n".join(lines[1:]).strip()
|
||
elif first_line.lower().startswith("title:"):
|
||
title = first_line.split(":", 1)[1].strip()
|
||
if len(lines) > 1:
|
||
content = "\n".join(lines[1:]).strip()
|
||
else:
|
||
h1_match = re.search(r"<h1[^>]*>(.*?)</h1>", content[:1500], re.IGNORECASE)
|
||
if h1_match:
|
||
detected = re.sub(r"<[^>]+>", "", h1_match.group(1)).strip()
|
||
if detected:
|
||
title = detected
|
||
|
||
# Extract author from next line if missing
|
||
if not author:
|
||
lines = content.split("\n")
|
||
if lines and lines[0].lower().startswith(("author:", "by:")):
|
||
author = lines[0].split(":", 1)[1].strip()
|
||
if len(lines) > 1:
|
||
content = "\n".join(lines[1:]).strip()
|
||
|
||
def detect_text_headings(content: str) -> List[Dict]:
|
||
"""
|
||
Detects section and chapter headings with high accuracy across Markdown,
|
||
numbered sections, explicit keywords, bold titles, and Setext styles.
|
||
"""
|
||
lines = content.split("\n")
|
||
line_starts = []
|
||
curr = 0
|
||
for l in lines:
|
||
line_starts.append(curr)
|
||
curr += len(l) + 1
|
||
|
||
kw_regex = re.compile(
|
||
r"^[ \t]*(?:\*\*)?(?:Chapter|CHAPTER|Глава|ГЛАВА|Part|PART|Часть|ЧАСТЬ|Section|SECTION|Раздел|РАЗДЕЛ|Module|MODULE|Модуль|МОДУЛЬ|Unit|UNIT|Lesson|LESSON|Урок|УРОК|Topic|TOPIC|Тема|ТЕМА|Appendix|APPENDIX|Приложение|ПРИЛОЖЕНИЕ|Act|ACT|Scene|SCENE)[ \t]+(?:[0-9IVXLCDMivxlcdm\w\.\-]+)(?:[ \t]*[:\-–—\.][ \t]*[^\n\r*]+|[ \t]+[^\n\r*]+)?(?:\*\*)?[ \t]*$"
|
||
)
|
||
|
||
headings = []
|
||
|
||
for i, line in enumerate(lines):
|
||
trimmed = line.strip()
|
||
if not trimmed:
|
||
continue
|
||
|
||
prev_empty = (i == 0) or (not lines[i - 1].strip())
|
||
next_empty = (i == len(lines) - 1) or (not lines[i + 1].strip())
|
||
|
||
# 1. Markdown # heading (# to #####)
|
||
m_md = re.match(r"^#{1,5}[ \t]+(.+)$", trimmed)
|
||
if m_md:
|
||
clean_title = m_md.group(1).strip("#").strip()
|
||
headings.append({
|
||
"line_idx": i,
|
||
"start": line_starts[i],
|
||
"title": clean_title,
|
||
"type": "markdown",
|
||
})
|
||
continue
|
||
|
||
# 2. Setext heading: Title followed by === or ---
|
||
if i + 1 < len(lines):
|
||
next_line = lines[i + 1].strip()
|
||
if prev_empty and len(trimmed) <= 80 and not trimmed.startswith(("-", "*", "#", "`")):
|
||
if re.match(r"^={3,}$", next_line) or re.match(r"^-{3,}$", next_line):
|
||
headings.append({
|
||
"line_idx": i,
|
||
"start": line_starts[i],
|
||
"title": trimmed,
|
||
"type": "setext",
|
||
})
|
||
continue
|
||
|
||
# 3. Explicit keywords (Chapter 1, Глава 2, Module 3, etc.)
|
||
if kw_regex.match(trimmed):
|
||
clean_title = trimmed.strip("*").strip()
|
||
headings.append({
|
||
"line_idx": i,
|
||
"start": line_starts[i],
|
||
"title": clean_title,
|
||
"type": "keyword",
|
||
})
|
||
continue
|
||
|
||
# 4. Bold heading on standalone line: **Title**
|
||
bold_m = re.match(r"^\*\*([A-ZА-ЯЁ0-9][^\n\r*]{2,70})\*\*$", trimmed)
|
||
if bold_m and prev_empty:
|
||
headings.append({
|
||
"line_idx": i,
|
||
"start": line_starts[i],
|
||
"title": bold_m.group(1).strip(),
|
||
"type": "bold",
|
||
})
|
||
continue
|
||
|
||
# 5. Numbered sections: e.g. 1. Title, 1.1 Title, 1. Title.
|
||
num_m = re.match(r"^([0-9]{1,2}(?:\.[0-9]{1,2})*\.?|[0-9]{1,2}\))[ \t]+([A-ZА-ЯЁ][^\n\r]{2,70})$", trimmed)
|
||
if num_m and prev_empty:
|
||
is_list = False
|
||
if i + 1 < len(lines):
|
||
next_t = lines[i + 1].strip()
|
||
if re.match(r"^[0-9]{1,2}(?:\.[0-9]{1,2})*\.?[ \t]+", next_t):
|
||
is_list = True
|
||
if not is_list and not trimmed.endswith((",", ";")):
|
||
headings.append({
|
||
"line_idx": i,
|
||
"start": line_starts[i],
|
||
"title": trimmed,
|
||
"type": "numbered",
|
||
})
|
||
continue
|
||
|
||
# 6. Standalone ALL CAPS line (e.g. INTRODUCTION TO NETWORKS)
|
||
next_is_allcaps = (i + 1 < len(lines)) and bool(lines[i + 1].strip()) and lines[i + 1].strip().isupper()
|
||
if prev_empty and not next_is_allcaps and 4 <= len(trimmed) <= 60 and not trimmed.endswith((",", ";")):
|
||
if re.match(r"^[A-ZА-ЯЁ0-9\s:\-–—\.]+$", trimmed):
|
||
letters = re.findall(r"[A-ZА-ЯЁ]", trimmed)
|
||
words = trimmed.split()
|
||
if len(letters) >= 4 and (len(words) >= 2 or len(trimmed) >= 6):
|
||
headings.append({
|
||
"line_idx": i,
|
||
"start": line_starts[i],
|
||
"title": trimmed,
|
||
"type": "allcaps",
|
||
})
|
||
continue
|
||
|
||
return headings
|
||
|
||
|
||
def parse_text(content: str, title: str = "Untitled", author: str = "") -> Tuple[str, str, List[Dict[str, str]]]:
|
||
"""
|
||
Parses plain text or Markdown into readable chapters.
|
||
Splits by intelligent heading detection (Markdown, numbered sections, keywords)
|
||
or chunks of ~1000 words.
|
||
"""
|
||
content = content.replace("\r\n", "\n").replace("\r", "\n").strip()
|
||
if not content:
|
||
return title, author, [{"title": title if title != "Untitled" else "Chapter 1", "content": ""}]
|
||
|
||
lines = content.split("\n")
|
||
first_line = lines[0].strip()
|
||
|
||
# Extract title from first line or # Heading / <h1...> if default or generic README
|
||
if title == "Untitled" or title.lower().startswith("readme"):
|
||
if first_line.startswith("# "):
|
||
title = first_line.lstrip("# ").strip()
|
||
if len(lines) > 1:
|
||
content = "\n".join(lines[1:]).strip()
|
||
elif first_line.lower().startswith("title:"):
|
||
title = first_line.split(":", 1)[1].strip()
|
||
if len(lines) > 1:
|
||
content = "\n".join(lines[1:]).strip()
|
||
else:
|
||
h1_match = re.search(r"<h1[^>]*>(.*?)</h1>", content[:1500], re.IGNORECASE)
|
||
if h1_match:
|
||
detected = re.sub(r"<[^>]+>", "", h1_match.group(1)).strip()
|
||
if detected:
|
||
title = detected
|
||
|
||
# Extract author from next line if missing
|
||
if not author:
|
||
lines = content.split("\n")
|
||
if lines and lines[0].lower().startswith(("author:", "by:")):
|
||
author = lines[0].split(":", 1)[1].strip()
|
||
if len(lines) > 1:
|
||
content = "\n".join(lines[1:]).strip()
|
||
|
||
detected_headings = detect_text_headings(content)
|
||
chapters: List[Dict[str, str]] = []
|
||
|
||
if detected_headings:
|
||
# Pre-chapter intro text
|
||
intro = content[: detected_headings[0]["start"]].strip()
|
||
if intro and count_words(intro) > 10:
|
||
is_ru = any(ord(c) > 127 for c in intro[:200])
|
||
chapters.append({"title": "Введение" if is_ru else "Introduction", "content": intro})
|
||
|
||
for i, h in enumerate(detected_headings):
|
||
start = h["start"]
|
||
end = detected_headings[i + 1]["start"] if i + 1 < len(detected_headings) else len(content)
|
||
body = content[start:end].strip()
|
||
if body:
|
||
chapters.append({"title": h["title"], "content": body})
|
||
|
||
if not chapters:
|
||
# No headings detected: split by ~1000 words chunks if long enough
|
||
total_w = count_words(content)
|
||
if total_w > 1200:
|
||
chunks = split_long_text(content, target_words=1000)
|
||
is_ru = any(ord(c) > 127 for c in content[:200])
|
||
prefix = "Часть" if is_ru else "Chapter"
|
||
for idx, chunk in enumerate(chunks, start=1):
|
||
chapters.append({"title": f"{prefix} {idx}", "content": chunk})
|
||
else:
|
||
chapters = [{"title": title if title != "Untitled" else "Chapter 1", "content": content}]
|
||
|
||
return title, author, chapters
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# PDF Parsing & Chapter Extraction (pypdf, Bookmarks/Outline, Cisco CLI Blocks)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
CLI_PROMPT_RE = re.compile(
|
||
r"^(?:[A-Za-z0-9_.\-]+(?:\([a-zA-Z0-9_\-\.\s]+\))?[>#]|"
|
||
r"[A-Za-z0-9_.\-]+@[A-Za-z0-9_.\-]+:[~/\w\.\-]*[$#]|"
|
||
r"(?:[A-Z]):\\[^>\n\r]*>|"
|
||
r"PS\s+[A-Z]:\\[^>\n\r]*>)"
|
||
)
|
||
|
||
|
||
def extract_outline_entries(outline, reader) -> List[Tuple[str, int]]:
|
||
"""
|
||
Recursively extracts bookmarks / outline entries from PDF respecting hierarchy depth.
|
||
For books with deep trees (Parts -> Chapters -> Sections -> Subsections),
|
||
prioritizes major structural headings and prevents fragmenting chapters into
|
||
hundreds of tiny 1-page snippets.
|
||
"""
|
||
raw_entries: List[Tuple[str, int, int]] = []
|
||
|
||
def _traverse(items, depth=0):
|
||
for item in items:
|
||
if isinstance(item, list):
|
||
_traverse(item, depth + 1)
|
||
else:
|
||
try:
|
||
title = getattr(item, "title", None) or (item.get("/Title") if hasattr(item, "get") else None)
|
||
page_num = reader.get_destination_page_number(item)
|
||
if title and page_num is not None:
|
||
clean_title = str(title).strip()
|
||
if clean_title:
|
||
raw_entries.append((clean_title, int(page_num), depth))
|
||
except Exception:
|
||
continue
|
||
|
||
if outline:
|
||
_traverse(outline, 0)
|
||
|
||
if not raw_entries:
|
||
return []
|
||
|
||
# If total entries is reasonable (<= 50), return all
|
||
if len(raw_entries) <= 50:
|
||
return [(t, p) for t, p, _ in raw_entries]
|
||
|
||
# Check depth distribution
|
||
d0_entries = [e for e in raw_entries if e[2] == 0]
|
||
d1_entries = [e for e in raw_entries if e[2] <= 1]
|
||
|
||
# If top-level (depth 0) already has >= 10 chapters, use depth 0
|
||
if len(d0_entries) >= 10:
|
||
return [(t, p) for t, p, _ in d0_entries]
|
||
|
||
# Otherwise, if depth <= 1 gives a good number of chapters (10 to 80), use depth <= 1
|
||
if len(d1_entries) <= 80:
|
||
return [(t, p) for t, p, _ in d1_entries]
|
||
|
||
# If still too many entries (> 80), filter by major chapter keywords
|
||
major_re = re.compile(
|
||
r"^(cover|title\s+page|copyright|contents|introduction|your\s+study\s+plan|"
|
||
r"part\s+[0-9ivxlcdm]+|chapter\s+\d+|appendix\s+[a-z]|glossary|index)",
|
||
re.IGNORECASE
|
||
)
|
||
major_entries = [e for e in d1_entries if major_re.match(e[0])]
|
||
if len(major_entries) >= 10:
|
||
return [(t, p) for t, p, _ in major_entries]
|
||
|
||
return [(t, p) for t, p, _ in d1_entries[:80]]
|
||
|
||
|
||
|
||
def clean_page_lines(raw_pages_text: List[str]) -> List[str]:
|
||
"""
|
||
Filters running headers, footers, isolated page numbers, and copyright noise.
|
||
"""
|
||
if not raw_pages_text:
|
||
return []
|
||
|
||
num_pages = len(raw_pages_text)
|
||
top_candidates: Dict[str, int] = {}
|
||
bottom_candidates: Dict[str, int] = {}
|
||
|
||
for pt in raw_pages_text:
|
||
lines = [l.strip() for l in pt.split("\n") if l.strip()]
|
||
if not lines:
|
||
continue
|
||
if len(lines) >= 1:
|
||
top_candidates[lines[0]] = top_candidates.get(lines[0], 0) + 1
|
||
bottom_candidates[lines[-1]] = bottom_candidates.get(lines[-1], 0) + 1
|
||
|
||
# Lines repeated on >= 20% of pages (min 3 pages) are running headers/footers
|
||
threshold = max(3, int(num_pages * 0.20))
|
||
common_headers = {l for l, c in top_candidates.items() if c >= threshold}
|
||
common_footers = {l for l, c in bottom_candidates.items() if c >= threshold}
|
||
|
||
page_num_re = re.compile(r"^(?:page\s+)?\d+(?:\s*(?:of|/)\s*\d+)?$", re.IGNORECASE)
|
||
url_re = re.compile(r"^(?:https?://|www\.)\S+$", re.IGNORECASE)
|
||
copyright_re = re.compile(r"^(?:©|\(c\)|copyright)\s*\d{4}.*$", re.IGNORECASE)
|
||
|
||
cleaned_pages: List[str] = []
|
||
for pt in raw_pages_text:
|
||
lines = pt.split("\n")
|
||
filtered: List[str] = []
|
||
n_lines = len(lines)
|
||
for idx, l in enumerate(lines):
|
||
stripped = l.strip()
|
||
if not stripped:
|
||
filtered.append("")
|
||
continue
|
||
is_top = (idx <= 2)
|
||
is_bottom = (idx >= n_lines - 3)
|
||
|
||
if (is_top or is_bottom) and page_num_re.match(stripped):
|
||
continue
|
||
if is_top and stripped in common_headers:
|
||
continue
|
||
if is_bottom and (stripped in common_footers or url_re.match(stripped) or copyright_re.match(stripped)):
|
||
continue
|
||
filtered.append(l)
|
||
cleaned_pages.append("\n".join(filtered))
|
||
|
||
return cleaned_pages
|
||
|
||
|
||
def format_cli_blocks(text: str) -> str:
|
||
"""
|
||
Detects Cisco IOS / CLI terminal sessions in text and wraps them
|
||
in Markdown fenced code blocks (```text ... ```) for syntax highlighting and copying.
|
||
"""
|
||
if not text:
|
||
return ""
|
||
|
||
lines = text.split("\n")
|
||
out: List[str] = []
|
||
in_cli = False
|
||
cli_lines: List[str] = []
|
||
|
||
def is_likely_cli_line(line: str) -> bool:
|
||
s = line.strip()
|
||
if not s:
|
||
return True
|
||
if CLI_PROMPT_RE.match(s):
|
||
return True
|
||
if s.startswith(("%", "!", "--", ">>", "Enter configuration", "Building configuration", "Current configuration")):
|
||
return True
|
||
if len(re.findall(r"\S\s{2,}\S", line)) >= 2:
|
||
return True
|
||
if re.match(r"^(?:show|ip|no|interface|switchport|vlan|router|network|exit|end|enable|disable|configure|ping|traceroute|debug)\s+", s, re.IGNORECASE):
|
||
return True
|
||
return False
|
||
|
||
for line in lines:
|
||
s = line.strip()
|
||
if not in_cli:
|
||
if CLI_PROMPT_RE.match(s):
|
||
in_cli = True
|
||
cli_lines = [line]
|
||
else:
|
||
out.append(line)
|
||
else:
|
||
if is_likely_cli_line(line):
|
||
cli_lines.append(line)
|
||
else:
|
||
while cli_lines and not cli_lines[-1].strip():
|
||
cli_lines.pop()
|
||
if cli_lines:
|
||
out.append("```text")
|
||
out.extend(cli_lines)
|
||
out.append("```")
|
||
in_cli = False
|
||
cli_lines = []
|
||
out.append(line)
|
||
|
||
if in_cli and cli_lines:
|
||
while cli_lines and not cli_lines[-1].strip():
|
||
cli_lines.pop()
|
||
if cli_lines:
|
||
out.append("```text")
|
||
out.extend(cli_lines)
|
||
out.append("```")
|
||
|
||
return "\n".join(out)
|
||
|
||
|
||
def parse_pdf(file_bytes: bytes, filename: str = "") -> Tuple[str, str, List[Dict[str, str]]]:
|
||
"""
|
||
Parses a PDF file into readable chapters:
|
||
- Extracts title and author from metadata or filename.
|
||
- Enforces Scanned PDF Guard (raises ValueError if total_words < 100).
|
||
- Cleans repetitive headers, footers, and page numbers.
|
||
- Wraps Cisco IOS / CLI terminal blocks in Markdown code blocks.
|
||
- Splits chapters by PDF outline / bookmarks (Cisco Press, O'Reilly).
|
||
- Fallback to heading regex or ~10-15 page chunks if no bookmarks exist.
|
||
"""
|
||
try:
|
||
import pypdf
|
||
except ImportError:
|
||
vendor_dir = Path(__file__).resolve().parent / "vendor"
|
||
if str(vendor_dir) not in sys.path:
|
||
sys.path.insert(0, str(vendor_dir))
|
||
import pypdf
|
||
|
||
try:
|
||
reader = pypdf.PdfReader(io.BytesIO(file_bytes))
|
||
except Exception as e:
|
||
raise ValueError(f"Не удалось открыть PDF документ: {e}")
|
||
|
||
if reader.is_encrypted:
|
||
try:
|
||
reader.decrypt("")
|
||
except Exception:
|
||
raise ValueError("Этот PDF документ защищен паролем. Пожалуйста, снимите пароль для чтения.")
|
||
|
||
num_pages = len(reader.pages)
|
||
if num_pages == 0:
|
||
raise ValueError("PDF документ не содержит страниц")
|
||
|
||
# Metadata extraction
|
||
meta = reader.metadata or {}
|
||
meta_title = getattr(meta, "title", None) or (meta.get("/Title") if hasattr(meta, "get") else None) or ""
|
||
meta_author = getattr(meta, "author", None) or (meta.get("/Author") if hasattr(meta, "get") else None) or ""
|
||
|
||
stem = Path(filename).stem if filename else "Untitled Document"
|
||
clean_title = str(meta_title).strip()
|
||
if not clean_title or clean_title.lower() in {"untitled", "untitled document", "default", "temp", "output"}:
|
||
title = stem
|
||
else:
|
||
title = clean_title
|
||
|
||
author = str(meta_author).strip()
|
||
|
||
# Page-by-page text extraction
|
||
raw_pages_text: List[str] = []
|
||
for p in reader.pages:
|
||
try:
|
||
pt = p.extract_text() or ""
|
||
except Exception:
|
||
pt = ""
|
||
raw_pages_text.append(pt)
|
||
|
||
# Scanned PDF Guard (< 100 words)
|
||
total_words = sum(count_words(t) for t in raw_pages_text)
|
||
if total_words < 100:
|
||
raise ValueError(
|
||
"Этот PDF-документ содержит сканы или изображения без текстового слоя (OCR). "
|
||
"Пожалуйста, используйте документ с распознанным текстом."
|
||
)
|
||
|
||
# Clean noise lines (headers, footers, page numbers)
|
||
cleaned_pages = clean_page_lines(raw_pages_text)
|
||
|
||
# Apply CLI formatting to each page
|
||
formatted_pages = [format_cli_blocks(p) for p in cleaned_pages]
|
||
|
||
# Chapter splitting:
|
||
# 1. Try outline / bookmarks
|
||
outline_entries = extract_outline_entries(getattr(reader, "outline", None), reader)
|
||
chapters: List[Dict[str, str]] = []
|
||
|
||
if outline_entries:
|
||
valid_entries = [(t, p) for t, p in outline_entries if 0 <= p < num_pages]
|
||
valid_entries.sort(key=lambda x: x[1])
|
||
|
||
# Filter duplicates pointing to the same page
|
||
deduped_entries: List[Tuple[str, int]] = []
|
||
for t, p in valid_entries:
|
||
if not deduped_entries or deduped_entries[-1][1] != p:
|
||
deduped_entries.append((t, p))
|
||
|
||
if deduped_entries:
|
||
first_p = deduped_entries[0][1]
|
||
if first_p > 0:
|
||
intro_content = "\n\n".join(formatted_pages[p].strip() for p in range(0, first_p) if formatted_pages[p].strip())
|
||
if count_words(intro_content) >= 20:
|
||
is_ru = any(ord(c) > 127 for c in intro_content[:300])
|
||
chapters.append({
|
||
"title": "Введение" if is_ru else "Introduction",
|
||
"content": intro_content
|
||
})
|
||
|
||
for i, (entry_title, start_p) in enumerate(deduped_entries):
|
||
end_p = deduped_entries[i + 1][1] if i + 1 < len(deduped_entries) else num_pages
|
||
ch_content = "\n\n".join(formatted_pages[p].strip() for p in range(start_p, end_p) if formatted_pages[p].strip())
|
||
if ch_content:
|
||
chapters.append({
|
||
"title": entry_title,
|
||
"content": ch_content
|
||
})
|
||
|
||
# 2. Fallback: No outline found, check for page-level chapter headings
|
||
if not chapters:
|
||
chapter_heading_re = re.compile(
|
||
r"^\s*(?:chapter|глава|part|часть|section|раздел|module|модуль)\s+([0-9ivxlcdm]+)[\s:.\-–—\n]+(.*)$",
|
||
re.IGNORECASE
|
||
)
|
||
detected_pages: List[Tuple[str, int]] = []
|
||
for p_idx, p_text in enumerate(formatted_pages):
|
||
first_lines = [l.strip() for l in p_text.split("\n") if l.strip()][:3]
|
||
for l in first_lines:
|
||
m = chapter_heading_re.match(l)
|
||
if m:
|
||
h_title = l.strip("#").strip()
|
||
detected_pages.append((h_title, p_idx))
|
||
break
|
||
|
||
if detected_pages:
|
||
if detected_pages[0][1] > 0:
|
||
intro_content = "\n\n".join(formatted_pages[p].strip() for p in range(0, detected_pages[0][1]) if formatted_pages[p].strip())
|
||
if count_words(intro_content) >= 20:
|
||
is_ru = any(ord(c) > 127 for c in intro_content[:300])
|
||
chapters.append({
|
||
"title": "Введение" if is_ru else "Introduction",
|
||
"content": intro_content
|
||
})
|
||
|
||
for i, (h_title, start_p) in enumerate(detected_pages):
|
||
end_p = detected_pages[i + 1][1] if i + 1 < len(detected_pages) else num_pages
|
||
ch_content = "\n\n".join(formatted_pages[p].strip() for p in range(start_p, end_p) if formatted_pages[p].strip())
|
||
if ch_content:
|
||
chapters.append({
|
||
"title": h_title,
|
||
"content": ch_content
|
||
})
|
||
|
||
# 3. Fallback: Group pages in chunks of 12-15 pages
|
||
if not chapters:
|
||
chunk_size = 12
|
||
sample_text = "".join(formatted_pages[:3])
|
||
is_ru = any(ord(c) > 127 for c in sample_text[:300])
|
||
prefix = "Страницы" if is_ru else "Pages"
|
||
|
||
for start_p in range(0, num_pages, chunk_size):
|
||
end_p = min(start_p + chunk_size, num_pages)
|
||
ch_content = "\n\n".join(formatted_pages[p].strip() for p in range(start_p, end_p) if formatted_pages[p].strip())
|
||
if ch_content:
|
||
chapters.append({
|
||
"title": f"{prefix} {start_p + 1}–{end_p}",
|
||
"content": ch_content
|
||
})
|
||
|
||
if not chapters:
|
||
full_content = "\n\n".join(p.strip() for p in formatted_pages if p.strip())
|
||
chapters = [{"title": title, "content": full_content}]
|
||
|
||
return title, author, chapters
|
||
|