499 lines
19 KiB
Python
499 lines
19 KiB
Python
import io
|
||
import re
|
||
import html
|
||
import zipfile
|
||
import posixpath
|
||
import urllib.parse
|
||
import xml.etree.ElementTree as ET
|
||
from html.parser import HTMLParser
|
||
from typing import Tuple, List, Dict
|
||
|
||
|
||
def count_words(text: str) -> int:
|
||
"""
|
||
Counts words in text across multiple languages, ignoring isolated punctuation.
|
||
"""
|
||
if not text:
|
||
return 0
|
||
return len(re.findall(r"\b[\w]+(?:['’][\w]+)?\b", text))
|
||
|
||
|
||
class HTMLTextExtractor(HTMLParser):
|
||
"""
|
||
Strips HTML tags cleanly while preserving paragraph and heading breaks with newlines.
|
||
Also extracts the first heading and/or <title> tag for chapter naming.
|
||
"""
|
||
def __init__(self):
|
||
super().__init__()
|
||
self.text_parts: List[str] = []
|
||
self.ignore_tags = {"style", "script", "noscript"}
|
||
self.ignore_depth = 0
|
||
self.heading: str | None = None
|
||
self.in_heading = False
|
||
self.heading_parts: List[str] = []
|
||
self.title: str | None = None
|
||
self.in_title = False
|
||
self.title_parts: List[str] = []
|
||
|
||
def handle_starttag(self, tag: str, attrs):
|
||
t = tag.lower()
|
||
if t in self.ignore_tags:
|
||
self.ignore_depth += 1
|
||
elif t == "title":
|
||
self.in_title = True
|
||
self.title_parts = []
|
||
elif t in {"h1", "h2", "h3", "h4", "h5", "h6"}:
|
||
self.text_parts.append("\n\n")
|
||
if not self.heading:
|
||
self.in_heading = True
|
||
self.heading_parts = []
|
||
elif t in {"p", "div", "section", "article", "blockquote", "header", "footer"}:
|
||
self.text_parts.append("\n\n")
|
||
elif t in {"br", "hr", "li", "tr"}:
|
||
self.text_parts.append("\n")
|
||
|
||
def handle_endtag(self, tag: str):
|
||
t = tag.lower()
|
||
if t in self.ignore_tags:
|
||
self.ignore_depth = max(0, self.ignore_depth - 1)
|
||
elif t == "title":
|
||
self.in_title = False
|
||
self.title = "".join(self.title_parts).strip()
|
||
elif t in {"h1", "h2", "h3", "h4", "h5", "h6"}:
|
||
self.text_parts.append("\n\n")
|
||
if self.in_heading:
|
||
self.in_heading = False
|
||
if not self.heading:
|
||
self.heading = "".join(self.heading_parts).strip()
|
||
elif t in {"p", "div", "section", "article", "blockquote", "header", "footer"}:
|
||
self.text_parts.append("\n\n")
|
||
elif t in {"li", "tr"}:
|
||
self.text_parts.append("\n")
|
||
|
||
def handle_data(self, data: str):
|
||
if self.in_title:
|
||
self.title_parts.append(data)
|
||
if self.ignore_depth == 0 and not self.in_title:
|
||
# Replace non-breaking spaces with standard space
|
||
cleaned_data = data.replace("\xa0", " ")
|
||
self.text_parts.append(cleaned_data)
|
||
if self.in_heading:
|
||
self.heading_parts.append(cleaned_data)
|
||
|
||
def get_text(self) -> str:
|
||
raw = "".join(self.text_parts)
|
||
lines = [line.strip() for line in raw.split("\n")]
|
||
clean = "\n".join(lines)
|
||
return re.sub(r"\n{3,}", "\n\n", clean).strip()
|
||
|
||
|
||
def parse_epub(file_bytes: bytes) -> Tuple[str, str, List[Dict[str, str]]]:
|
||
"""
|
||
Parses an EPUB file archive and extracts:
|
||
- title: book title
|
||
- author: creator/author
|
||
- chapters: list of {"title": str, "content": str}
|
||
"""
|
||
try:
|
||
z = zipfile.ZipFile(io.BytesIO(file_bytes))
|
||
except Exception as e:
|
||
raise ValueError(f"Invalid EPUB archive: {e}")
|
||
|
||
# 1. Locate root OPF file from META-INF/container.xml
|
||
opf_path = None
|
||
try:
|
||
container_data = z.read("META-INF/container.xml")
|
||
root = ET.fromstring(container_data)
|
||
rootfile = root.find(".//{*}rootfile")
|
||
if rootfile is not None:
|
||
opf_path = rootfile.attrib.get("full-path")
|
||
except Exception:
|
||
pass
|
||
|
||
if not opf_path:
|
||
for name in z.namelist():
|
||
if name.lower().endswith(".opf"):
|
||
opf_path = name
|
||
break
|
||
|
||
if not opf_path:
|
||
raise ValueError("Invalid EPUB: no OPF file found")
|
||
|
||
opf_path = opf_path.replace("\\", "/").lstrip("/")
|
||
opf_dir = posixpath.dirname(opf_path)
|
||
|
||
try:
|
||
opf_content = z.read(opf_path)
|
||
except KeyError:
|
||
# Try lowercase matching
|
||
name_map = {n.lower(): n for n in z.namelist()}
|
||
if opf_path.lower() in name_map:
|
||
opf_content = z.read(name_map[opf_path.lower()])
|
||
else:
|
||
raise ValueError(f"OPF file not found in archive: {opf_path}")
|
||
|
||
# 2. Extract metadata (title, author)
|
||
title = "Untitled"
|
||
author = ""
|
||
opf_tree = None
|
||
try:
|
||
opf_tree = ET.fromstring(opf_content)
|
||
title_el = opf_tree.find(".//{*}title")
|
||
if title_el is not None and title_el.text:
|
||
title = title_el.text.strip()
|
||
creator_el = opf_tree.find(".//{*}creator")
|
||
if creator_el is not None and creator_el.text:
|
||
author = creator_el.text.strip()
|
||
except Exception:
|
||
opf_str = opf_content.decode("utf-8", errors="ignore")
|
||
tm = re.search(r"<[^:]*:?title[^>]*>(.*?)</[^:]*:?title>", opf_str, re.I | re.S)
|
||
if tm:
|
||
title = re.sub(r"<[^>]+>", "", tm.group(1)).strip()
|
||
cm = re.search(r"<[^:]*:?creator[^>]*>(.*?)</[^:]*:?creator>", opf_str, re.I | re.S)
|
||
if cm:
|
||
author = re.sub(r"<[^>]+>", "", cm.group(1)).strip()
|
||
|
||
# 3. Parse Manifest (id -> href, media-type)
|
||
manifest: Dict[str, Dict[str, str]] = {}
|
||
if opf_tree is not None:
|
||
for item in opf_tree.findall(".//{*}item"):
|
||
iid = item.attrib.get("id")
|
||
href = item.attrib.get("href")
|
||
media_type = item.attrib.get("media-type", "").lower()
|
||
if iid and href:
|
||
manifest[iid] = {"href": href, "media_type": media_type}
|
||
else:
|
||
opf_str = opf_content.decode("utf-8", errors="ignore")
|
||
for m in re.finditer(r'<item\s+[^>]*id=["\']([^"\']+)["\'][^>]*href=["\']([^"\']+)["\']', opf_str, re.I):
|
||
manifest[m.group(1)] = {"href": m.group(2), "media_type": ""}
|
||
|
||
# 4. Spine items in reading order
|
||
spine_ids: List[str] = []
|
||
if opf_tree is not None:
|
||
for itemref in opf_tree.findall(".//{*}itemref"):
|
||
idref = itemref.attrib.get("idref")
|
||
if idref:
|
||
spine_ids.append(idref)
|
||
else:
|
||
opf_str = opf_content.decode("utf-8", errors="ignore")
|
||
for m in re.finditer(r'<itemref\s+[^>]*idref=["\']([^"\']+)["\']', opf_str, re.I):
|
||
spine_ids.append(m.group(1))
|
||
|
||
# Fallback if spine is empty: find all html/xhtml files in manifest
|
||
candidate_paths: List[str] = []
|
||
if spine_ids:
|
||
for idref in spine_ids:
|
||
item = manifest.get(idref)
|
||
if item:
|
||
candidate_paths.append(item["href"])
|
||
else:
|
||
for item in manifest.values():
|
||
href = item["href"]
|
||
if href.lower().endswith((".xhtml", ".html", ".htm")):
|
||
candidate_paths.append(href)
|
||
|
||
namelist_map = {name.lower(): name for name in z.namelist()}
|
||
chapters: List[Dict[str, str]] = []
|
||
chapter_num = 1
|
||
|
||
for href in candidate_paths:
|
||
unquoted = urllib.parse.unquote(href)
|
||
rel_path = posixpath.normpath(posixpath.join(opf_dir, unquoted)) if opf_dir else unquoted
|
||
rel_path = rel_path.replace("\\", "/").lstrip("/")
|
||
|
||
actual_path = None
|
||
if rel_path in z.namelist():
|
||
actual_path = rel_path
|
||
elif rel_path.lower() in namelist_map:
|
||
actual_path = namelist_map[rel_path.lower()]
|
||
|
||
if not actual_path:
|
||
continue
|
||
|
||
try:
|
||
raw_bytes = z.read(actual_path)
|
||
try:
|
||
html_str = raw_bytes.decode("utf-8")
|
||
except UnicodeDecodeError:
|
||
html_str = raw_bytes.decode("latin-1", errors="replace")
|
||
except Exception:
|
||
continue
|
||
|
||
parser = HTMLTextExtractor()
|
||
parser.feed(html_str)
|
||
text = parser.get_text()
|
||
|
||
# Skip chapters with little or no readable text
|
||
if count_words(text) < 1:
|
||
continue
|
||
|
||
ch_title = parser.heading or parser.title or f"Chapter {chapter_num}"
|
||
# If ch_title is identical to book title or very long, format nicely
|
||
if len(ch_title) > 120:
|
||
ch_title = f"Chapter {chapter_num}"
|
||
|
||
chapters.append({"title": ch_title, "content": text})
|
||
chapter_num += 1
|
||
|
||
if not chapters:
|
||
chapters = [{"title": title if title != "Untitled" else "Chapter 1", "content": ""}]
|
||
|
||
return title, author, chapters
|
||
|
||
|
||
def split_long_text(text: str, target_words: int = 3000) -> List[str]:
|
||
"""
|
||
Splits text into chunks of approximately target_words on paragraph boundaries.
|
||
"""
|
||
paragraphs = [p.strip() for p in re.split(r"\n{2,}", text) if p.strip()]
|
||
if not paragraphs:
|
||
return [text.strip()] if text.strip() else []
|
||
|
||
chunks: List[str] = []
|
||
current_paras: List[str] = []
|
||
current_count = 0
|
||
|
||
for p in paragraphs:
|
||
w_count = count_words(p)
|
||
# If single paragraph is massive (> target_words * 1.5), split by words
|
||
if w_count > target_words * 1.5:
|
||
if current_paras:
|
||
chunks.append("\n\n".join(current_paras))
|
||
current_paras = []
|
||
current_count = 0
|
||
words = p.split()
|
||
for i in range(0, len(words), target_words):
|
||
chunk = " ".join(words[i : i + target_words])
|
||
if chunk.strip():
|
||
chunks.append(chunk)
|
||
continue
|
||
|
||
if current_count + w_count > target_words and current_count > 0:
|
||
chunks.append("\n\n".join(current_paras))
|
||
current_paras = [p]
|
||
current_count = w_count
|
||
else:
|
||
current_paras.append(p)
|
||
current_count += w_count
|
||
|
||
if current_paras:
|
||
chunks.append("\n\n".join(current_paras))
|
||
|
||
return chunks
|
||
|
||
|
||
def parse_text(content: str, title: str = "Untitled", author: str = "") -> Tuple[str, str, List[Dict[str, str]]]:
|
||
"""
|
||
Parses plain text or Markdown into readable chapters.
|
||
Splits by explicit chapter markers (e.g. 'Chapter X', '# Chapter') or chunks of ~3000 words.
|
||
"""
|
||
content = content.replace("\r\n", "\n").replace("\r", "\n").strip()
|
||
if not content:
|
||
return title, author, [{"title": title if title != "Untitled" else "Chapter 1", "content": ""}]
|
||
|
||
lines = content.split("\n")
|
||
first_line = lines[0].strip()
|
||
|
||
# Extract title from first line or # Heading / <h1...> if default or generic README
|
||
if title == "Untitled" or title.lower().startswith("readme"):
|
||
if first_line.startswith("# "):
|
||
title = first_line.lstrip("# ").strip()
|
||
if len(lines) > 1:
|
||
content = "\n".join(lines[1:]).strip()
|
||
elif first_line.lower().startswith("title:"):
|
||
title = first_line.split(":", 1)[1].strip()
|
||
if len(lines) > 1:
|
||
content = "\n".join(lines[1:]).strip()
|
||
else:
|
||
h1_match = re.search(r"<h1[^>]*>(.*?)</h1>", content[:1500], re.IGNORECASE)
|
||
if h1_match:
|
||
detected = re.sub(r"<[^>]+>", "", h1_match.group(1)).strip()
|
||
if detected:
|
||
title = detected
|
||
|
||
# Extract author from next line if missing
|
||
if not author:
|
||
lines = content.split("\n")
|
||
if lines and lines[0].lower().startswith(("author:", "by:")):
|
||
author = lines[0].split(":", 1)[1].strip()
|
||
if len(lines) > 1:
|
||
content = "\n".join(lines[1:]).strip()
|
||
|
||
def detect_text_headings(content: str) -> List[Dict]:
|
||
"""
|
||
Detects section and chapter headings with high accuracy across Markdown,
|
||
numbered sections, explicit keywords, bold titles, and Setext styles.
|
||
"""
|
||
lines = content.split("\n")
|
||
line_starts = []
|
||
curr = 0
|
||
for l in lines:
|
||
line_starts.append(curr)
|
||
curr += len(l) + 1
|
||
|
||
kw_regex = re.compile(
|
||
r"^[ \t]*(?:\*\*)?(?:Chapter|CHAPTER|Глава|ГЛАВА|Part|PART|Часть|ЧАСТЬ|Section|SECTION|Раздел|РАЗДЕЛ|Module|MODULE|Модуль|МОДУЛЬ|Unit|UNIT|Lesson|LESSON|Урок|УРОК|Topic|TOPIC|Тема|ТЕМА|Appendix|APPENDIX|Приложение|ПРИЛОЖЕНИЕ|Act|ACT|Scene|SCENE)[ \t]+(?:[0-9IVXLCDMivxlcdm\w\.\-]+)(?:[ \t]*[:\-–—\.][ \t]*[^\n\r*]+|[ \t]+[^\n\r*]+)?(?:\*\*)?[ \t]*$"
|
||
)
|
||
|
||
headings = []
|
||
|
||
for i, line in enumerate(lines):
|
||
trimmed = line.strip()
|
||
if not trimmed:
|
||
continue
|
||
|
||
prev_empty = (i == 0) or (not lines[i - 1].strip())
|
||
next_empty = (i == len(lines) - 1) or (not lines[i + 1].strip())
|
||
|
||
# 1. Markdown # heading (# to #####)
|
||
m_md = re.match(r"^#{1,5}[ \t]+(.+)$", trimmed)
|
||
if m_md:
|
||
clean_title = m_md.group(1).strip("#").strip()
|
||
headings.append({
|
||
"line_idx": i,
|
||
"start": line_starts[i],
|
||
"title": clean_title,
|
||
"type": "markdown",
|
||
})
|
||
continue
|
||
|
||
# 2. Setext heading: Title followed by === or ---
|
||
if i + 1 < len(lines):
|
||
next_line = lines[i + 1].strip()
|
||
if prev_empty and len(trimmed) <= 80 and not trimmed.startswith(("-", "*", "#", "`")):
|
||
if re.match(r"^={3,}$", next_line) or re.match(r"^-{3,}$", next_line):
|
||
headings.append({
|
||
"line_idx": i,
|
||
"start": line_starts[i],
|
||
"title": trimmed,
|
||
"type": "setext",
|
||
})
|
||
continue
|
||
|
||
# 3. Explicit keywords (Chapter 1, Глава 2, Module 3, etc.)
|
||
if kw_regex.match(trimmed):
|
||
clean_title = trimmed.strip("*").strip()
|
||
headings.append({
|
||
"line_idx": i,
|
||
"start": line_starts[i],
|
||
"title": clean_title,
|
||
"type": "keyword",
|
||
})
|
||
continue
|
||
|
||
# 4. Bold heading on standalone line: **Title**
|
||
bold_m = re.match(r"^\*\*([A-ZА-ЯЁ0-9][^\n\r*]{2,70})\*\*$", trimmed)
|
||
if bold_m and prev_empty:
|
||
headings.append({
|
||
"line_idx": i,
|
||
"start": line_starts[i],
|
||
"title": bold_m.group(1).strip(),
|
||
"type": "bold",
|
||
})
|
||
continue
|
||
|
||
# 5. Numbered sections: e.g. 1. Title, 1.1 Title, 1. Title.
|
||
num_m = re.match(r"^([0-9]{1,2}(?:\.[0-9]{1,2})*\.?|[0-9]{1,2}\))[ \t]+([A-ZА-ЯЁ][^\n\r]{2,70})$", trimmed)
|
||
if num_m and prev_empty:
|
||
is_list = False
|
||
if i + 1 < len(lines):
|
||
next_t = lines[i + 1].strip()
|
||
if re.match(r"^[0-9]{1,2}(?:\.[0-9]{1,2})*\.?[ \t]+", next_t):
|
||
is_list = True
|
||
if not is_list and not trimmed.endswith((",", ";")):
|
||
headings.append({
|
||
"line_idx": i,
|
||
"start": line_starts[i],
|
||
"title": trimmed,
|
||
"type": "numbered",
|
||
})
|
||
continue
|
||
|
||
# 6. Standalone ALL CAPS line (e.g. INTRODUCTION TO NETWORKS)
|
||
next_is_allcaps = (i + 1 < len(lines)) and bool(lines[i + 1].strip()) and lines[i + 1].strip().isupper()
|
||
if prev_empty and not next_is_allcaps and 4 <= len(trimmed) <= 60 and not trimmed.endswith((",", ";")):
|
||
if re.match(r"^[A-ZА-ЯЁ0-9\s:\-–—\.]+$", trimmed):
|
||
letters = re.findall(r"[A-ZА-ЯЁ]", trimmed)
|
||
words = trimmed.split()
|
||
if len(letters) >= 4 and (len(words) >= 2 or len(trimmed) >= 6):
|
||
headings.append({
|
||
"line_idx": i,
|
||
"start": line_starts[i],
|
||
"title": trimmed,
|
||
"type": "allcaps",
|
||
})
|
||
continue
|
||
|
||
return headings
|
||
|
||
|
||
def parse_text(content: str, title: str = "Untitled", author: str = "") -> Tuple[str, str, List[Dict[str, str]]]:
|
||
"""
|
||
Parses plain text or Markdown into readable chapters.
|
||
Splits by intelligent heading detection (Markdown, numbered sections, keywords)
|
||
or chunks of ~1000 words.
|
||
"""
|
||
content = content.replace("\r\n", "\n").replace("\r", "\n").strip()
|
||
if not content:
|
||
return title, author, [{"title": title if title != "Untitled" else "Chapter 1", "content": ""}]
|
||
|
||
lines = content.split("\n")
|
||
first_line = lines[0].strip()
|
||
|
||
# Extract title from first line or # Heading / <h1...> if default or generic README
|
||
if title == "Untitled" or title.lower().startswith("readme"):
|
||
if first_line.startswith("# "):
|
||
title = first_line.lstrip("# ").strip()
|
||
if len(lines) > 1:
|
||
content = "\n".join(lines[1:]).strip()
|
||
elif first_line.lower().startswith("title:"):
|
||
title = first_line.split(":", 1)[1].strip()
|
||
if len(lines) > 1:
|
||
content = "\n".join(lines[1:]).strip()
|
||
else:
|
||
h1_match = re.search(r"<h1[^>]*>(.*?)</h1>", content[:1500], re.IGNORECASE)
|
||
if h1_match:
|
||
detected = re.sub(r"<[^>]+>", "", h1_match.group(1)).strip()
|
||
if detected:
|
||
title = detected
|
||
|
||
# Extract author from next line if missing
|
||
if not author:
|
||
lines = content.split("\n")
|
||
if lines and lines[0].lower().startswith(("author:", "by:")):
|
||
author = lines[0].split(":", 1)[1].strip()
|
||
if len(lines) > 1:
|
||
content = "\n".join(lines[1:]).strip()
|
||
|
||
detected_headings = detect_text_headings(content)
|
||
chapters: List[Dict[str, str]] = []
|
||
|
||
if detected_headings:
|
||
# Pre-chapter intro text
|
||
intro = content[: detected_headings[0]["start"]].strip()
|
||
if intro and count_words(intro) > 10:
|
||
is_ru = any(ord(c) > 127 for c in intro[:200])
|
||
chapters.append({"title": "Введение" if is_ru else "Introduction", "content": intro})
|
||
|
||
for i, h in enumerate(detected_headings):
|
||
start = h["start"]
|
||
end = detected_headings[i + 1]["start"] if i + 1 < len(detected_headings) else len(content)
|
||
body = content[start:end].strip()
|
||
if body:
|
||
chapters.append({"title": h["title"], "content": body})
|
||
|
||
if not chapters:
|
||
# No headings detected: split by ~1000 words chunks if long enough
|
||
total_w = count_words(content)
|
||
if total_w > 1200:
|
||
chunks = split_long_text(content, target_words=1000)
|
||
is_ru = any(ord(c) > 127 for c in content[:200])
|
||
prefix = "Часть" if is_ru else "Chapter"
|
||
for idx, chunk in enumerate(chunks, start=1):
|
||
chapters.append({"title": f"{prefix} {idx}", "content": chunk})
|
||
else:
|
||
chapters = [{"title": title if title != "Untitled" else "Chapter 1", "content": content}]
|
||
|
||
return title, author, chapters
|
||
|