feat: implement Stage 5 (Interactive Reader, EPUB/TXT parser, Bookshelf, Click-to-Translate) and Stage 4.3 (Strict CEFR level calculation)

app/reader.py

@@ -0,0 +1,350 @@
+import io
+import re
+import html
+import zipfile
+import posixpath
+import urllib.parse
+import xml.etree.ElementTree as ET
+from html.parser import HTMLParser
+from typing import Tuple, List, Dict
+
+
+def count_words(text: str) -> int:
+    """
+    Counts words in text across multiple languages, ignoring isolated punctuation.
+    """
+    if not text:
+        return 0
+    return len(re.findall(r"\b[\w]+(?:['’][\w]+)?\b", text))
+
+
+class HTMLTextExtractor(HTMLParser):
+    """
+    Strips HTML tags cleanly while preserving paragraph and heading breaks with newlines.
+    Also extracts the first heading and/or <title> tag for chapter naming.
+    """
+    def __init__(self):
+        super().__init__()
+        self.text_parts: List[str] = []
+        self.ignore_tags = {"style", "script", "noscript"}
+        self.ignore_depth = 0
+        self.heading: str | None = None
+        self.in_heading = False
+        self.heading_parts: List[str] = []
+        self.title: str | None = None
+        self.in_title = False
+        self.title_parts: List[str] = []
+
+    def handle_starttag(self, tag: str, attrs):
+        t = tag.lower()
+        if t in self.ignore_tags:
+            self.ignore_depth += 1
+        elif t == "title":
+            self.in_title = True
+            self.title_parts = []
+        elif t in {"h1", "h2", "h3", "h4", "h5", "h6"}:
+            self.text_parts.append("\n\n")
+            if not self.heading:
+                self.in_heading = True
+                self.heading_parts = []
+        elif t in {"p", "div", "section", "article", "blockquote", "header", "footer"}:
+            self.text_parts.append("\n\n")
+        elif t in {"br", "hr", "li", "tr"}:
+            self.text_parts.append("\n")
+
+    def handle_endtag(self, tag: str):
+        t = tag.lower()
+        if t in self.ignore_tags:
+            self.ignore_depth = max(0, self.ignore_depth - 1)
+        elif t == "title":
+            self.in_title = False
+            self.title = "".join(self.title_parts).strip()
+        elif t in {"h1", "h2", "h3", "h4", "h5", "h6"}:
+            self.text_parts.append("\n\n")
+            if self.in_heading:
+                self.in_heading = False
+                if not self.heading:
+                    self.heading = "".join(self.heading_parts).strip()
+        elif t in {"p", "div", "section", "article", "blockquote", "header", "footer"}:
+            self.text_parts.append("\n\n")
+        elif t in {"li", "tr"}:
+            self.text_parts.append("\n")
+
+    def handle_data(self, data: str):
+        if self.in_title:
+            self.title_parts.append(data)
+        if self.ignore_depth == 0 and not self.in_title:
+            # Replace non-breaking spaces with standard space
+            cleaned_data = data.replace("\xa0", " ")
+            self.text_parts.append(cleaned_data)
+            if self.in_heading:
+                self.heading_parts.append(cleaned_data)
+
+    def get_text(self) -> str:
+        raw = "".join(self.text_parts)
+        lines = [line.strip() for line in raw.split("\n")]
+        clean = "\n".join(lines)
+        return re.sub(r"\n{3,}", "\n\n", clean).strip()
+
+
+def parse_epub(file_bytes: bytes) -> Tuple[str, str, List[Dict[str, str]]]:
+    """
+    Parses an EPUB file archive and extracts:
+      - title: book title
+      - author: creator/author
+      - chapters: list of {"title": str, "content": str}
+    """
+    try:
+        z = zipfile.ZipFile(io.BytesIO(file_bytes))
+    except Exception as e:
+        raise ValueError(f"Invalid EPUB archive: {e}")
+
+    # 1. Locate root OPF file from META-INF/container.xml
+    opf_path = None
+    try:
+        container_data = z.read("META-INF/container.xml")
+        root = ET.fromstring(container_data)
+        rootfile = root.find(".//{*}rootfile")
+        if rootfile is not None:
+            opf_path = rootfile.attrib.get("full-path")
+    except Exception:
+        pass
+
+    if not opf_path:
+        for name in z.namelist():
+            if name.lower().endswith(".opf"):
+                opf_path = name
+                break
+
+    if not opf_path:
+        raise ValueError("Invalid EPUB: no OPF file found")
+
+    opf_path = opf_path.replace("\\", "/").lstrip("/")
+    opf_dir = posixpath.dirname(opf_path)
+
+    try:
+        opf_content = z.read(opf_path)
+    except KeyError:
+        # Try lowercase matching
+        name_map = {n.lower(): n for n in z.namelist()}
+        if opf_path.lower() in name_map:
+            opf_content = z.read(name_map[opf_path.lower()])
+        else:
+            raise ValueError(f"OPF file not found in archive: {opf_path}")
+
+    # 2. Extract metadata (title, author)
+    title = "Untitled"
+    author = ""
+    opf_tree = None
+    try:
+        opf_tree = ET.fromstring(opf_content)
+        title_el = opf_tree.find(".//{*}title")
+        if title_el is not None and title_el.text:
+            title = title_el.text.strip()
+        creator_el = opf_tree.find(".//{*}creator")
+        if creator_el is not None and creator_el.text:
+            author = creator_el.text.strip()
+    except Exception:
+        opf_str = opf_content.decode("utf-8", errors="ignore")
+        tm = re.search(r"<[^:]*:?title[^>]*>(.*?)</[^:]*:?title>", opf_str, re.I | re.S)
+        if tm:
+            title = re.sub(r"<[^>]+>", "", tm.group(1)).strip()
+        cm = re.search(r"<[^:]*:?creator[^>]*>(.*?)</[^:]*:?creator>", opf_str, re.I | re.S)
+        if cm:
+            author = re.sub(r"<[^>]+>", "", cm.group(1)).strip()
+
+    # 3. Parse Manifest (id -> href, media-type)
+    manifest: Dict[str, Dict[str, str]] = {}
+    if opf_tree is not None:
+        for item in opf_tree.findall(".//{*}item"):
+            iid = item.attrib.get("id")
+            href = item.attrib.get("href")
+            media_type = item.attrib.get("media-type", "").lower()
+            if iid and href:
+                manifest[iid] = {"href": href, "media_type": media_type}
+    else:
+        opf_str = opf_content.decode("utf-8", errors="ignore")
+        for m in re.finditer(r'<item\s+[^>]*id=["\']([^"\']+)["\'][^>]*href=["\']([^"\']+)["\']', opf_str, re.I):
+            manifest[m.group(1)] = {"href": m.group(2), "media_type": ""}
+
+    # 4. Spine items in reading order
+    spine_ids: List[str] = []
+    if opf_tree is not None:
+        for itemref in opf_tree.findall(".//{*}itemref"):
+            idref = itemref.attrib.get("idref")
+            if idref:
+                spine_ids.append(idref)
+    else:
+        opf_str = opf_content.decode("utf-8", errors="ignore")
+        for m in re.finditer(r'<itemref\s+[^>]*idref=["\']([^"\']+)["\']', opf_str, re.I):
+            spine_ids.append(m.group(1))
+
+    # Fallback if spine is empty: find all html/xhtml files in manifest
+    candidate_paths: List[str] = []
+    if spine_ids:
+        for idref in spine_ids:
+            item = manifest.get(idref)
+            if item:
+                candidate_paths.append(item["href"])
+    else:
+        for item in manifest.values():
+            href = item["href"]
+            if href.lower().endswith((".xhtml", ".html", ".htm")):
+                candidate_paths.append(href)
+
+    namelist_map = {name.lower(): name for name in z.namelist()}
+    chapters: List[Dict[str, str]] = []
+    chapter_num = 1
+
+    for href in candidate_paths:
+        unquoted = urllib.parse.unquote(href)
+        rel_path = posixpath.normpath(posixpath.join(opf_dir, unquoted)) if opf_dir else unquoted
+        rel_path = rel_path.replace("\\", "/").lstrip("/")
+
+        actual_path = None
+        if rel_path in z.namelist():
+            actual_path = rel_path
+        elif rel_path.lower() in namelist_map:
+            actual_path = namelist_map[rel_path.lower()]
+
+        if not actual_path:
+            continue
+
+        try:
+            raw_bytes = z.read(actual_path)
+            try:
+                html_str = raw_bytes.decode("utf-8")
+            except UnicodeDecodeError:
+                html_str = raw_bytes.decode("latin-1", errors="replace")
+        except Exception:
+            continue
+
+        parser = HTMLTextExtractor()
+        parser.feed(html_str)
+        text = parser.get_text()
+
+        # Skip chapters with little or no readable text
+        if count_words(text) < 1:
+            continue
+
+        ch_title = parser.heading or parser.title or f"Chapter {chapter_num}"
+        # If ch_title is identical to book title or very long, format nicely
+        if len(ch_title) > 120:
+            ch_title = f"Chapter {chapter_num}"
+
+        chapters.append({"title": ch_title, "content": text})
+        chapter_num += 1
+
+    if not chapters:
+        chapters = [{"title": title if title != "Untitled" else "Chapter 1", "content": ""}]
+
+    return title, author, chapters
+
+
+def split_long_text(text: str, target_words: int = 3000) -> List[str]:
+    """
+    Splits text into chunks of approximately target_words on paragraph boundaries.
+    """
+    paragraphs = [p.strip() for p in re.split(r"\n{2,}", text) if p.strip()]
+    if not paragraphs:
+        return [text.strip()] if text.strip() else []
+
+    chunks: List[str] = []
+    current_paras: List[str] = []
+    current_count = 0
+
+    for p in paragraphs:
+        w_count = count_words(p)
+        # If single paragraph is massive (> target_words * 1.5), split by words
+        if w_count > target_words * 1.5:
+            if current_paras:
+                chunks.append("\n\n".join(current_paras))
+                current_paras = []
+                current_count = 0
+            words = p.split()
+            for i in range(0, len(words), target_words):
+                chunk = " ".join(words[i : i + target_words])
+                if chunk.strip():
+                    chunks.append(chunk)
+            continue
+
+        if current_count + w_count > target_words and current_count > 0:
+            chunks.append("\n\n".join(current_paras))
+            current_paras = [p]
+            current_count = w_count
+        else:
+            current_paras.append(p)
+            current_count += w_count
+
+    if current_paras:
+        chunks.append("\n\n".join(current_paras))
+
+    return chunks
+
+
+def parse_text(content: str, title: str = "Untitled", author: str = "") -> Tuple[str, str, List[Dict[str, str]]]:
+    """
+    Parses plain text or Markdown into readable chapters.
+    Splits by explicit chapter markers (e.g. 'Chapter X', '# Chapter') or chunks of ~3000 words.
+    """
+    content = content.replace("\r\n", "\n").replace("\r", "\n").strip()
+    if not content:
+        return title, author, [{"title": title if title != "Untitled" else "Chapter 1", "content": ""}]
+
+    lines = content.split("\n")
+    first_line = lines[0].strip()
+
+    # Extract title from first line if default
+    if title == "Untitled":
+        if first_line.startswith("# "):
+            title = first_line.lstrip("# ").strip()
+            if len(lines) > 1:
+                content = "\n".join(lines[1:]).strip()
+        elif first_line.lower().startswith("title:"):
+            title = first_line.split(":", 1)[1].strip()
+            if len(lines) > 1:
+                content = "\n".join(lines[1:]).strip()
+
+    # Extract author from next line if missing
+    if not author:
+        lines = content.split("\n")
+        if lines and lines[0].lower().startswith(("author:", "by:")):
+            author = lines[0].split(":", 1)[1].strip()
+            if len(lines) > 1:
+                content = "\n".join(lines[1:]).strip()
+
+    heading_pattern = re.compile(
+        r"^[ \t]*(?:"
+        r"#{1,3}[ \t]+(.+)|"
+        r"((?:Chapter|CHAPTER|Глава|ГЛАВА|Part|PART|Section|SECTION|Act|ACT|Scene|SCENE)[ \t]+[0-9IVXLCDMivxlcdm\w\.\-]+(?:[ \t]*[:\-–—\.][ \t]*.*|[ \t]+[^\n\r]+)?)"
+        r")[ \t]*$",
+        re.MULTILINE,
+    )
+
+    matches = list(heading_pattern.finditer(content))
+    chapters: List[Dict[str, str]] = []
+
+    if len(matches) >= 1:
+        # Pre-chapter intro text
+        intro = content[: matches[0].start()].strip()
+        if intro and count_words(intro) > 10:
+            chapters.append({"title": "Introduction", "content": intro})
+
+        for i, m in enumerate(matches):
+            h_text = (m.group(1) or m.group(2)).strip()
+            start = m.start()
+            end = matches[i + 1].start() if i + 1 < len(matches) else len(content)
+            body = content[start:end].strip()
+            if body:
+                chapters.append({"title": h_text, "content": body})
+
+    if not chapters:
+        # No headings detected: split by ~3000 words chunks
+        chunks = split_long_text(content, target_words=3000)
+        if len(chunks) == 1:
+            chapters = [{"title": title if title != "Untitled" else "Chapter 1", "content": chunks[0]}]
+        else:
+            for idx, chunk in enumerate(chunks, start=1):
+                chapters.append({"title": f"Chapter {idx}", "content": chunk})
+
+    return title, author, chapters