feat(reader): hash router, zero-flicker reload, honest weighted progress, and smart chapter detection
This commit is contained in:
Antigravity committed
1 parent
f642a131ad
commit
4d40b13bce
5 files changed
+510
-130
No files matched your search
+163
-21
@@ -319,38 +319,180 @@ def parse_text(content: str, title: str = "Untitled", author: str = "") -> Tuple
|
||||
if len(lines) > 1:
|
||||
content = "\n".join(lines[1:]).strip()
|
||||
|
||||
heading_pattern = re.compile(
|
||||
r"^[ \t]*(?:"
|
||||
r"#{1,3}[ \t]+(.+)|"
|
||||
r"((?:Chapter|CHAPTER|Глава|ГЛАВА|Part|PART|Section|SECTION|Act|ACT|Scene|SCENE)[ \t]+[0-9IVXLCDMivxlcdm\w\.\-]+(?:[ \t]*[:\-–—\.][ \t]*.*|[ \t]+[^\n\r]+)?)"
|
||||
r")[ \t]*$",
|
||||
re.MULTILINE,
|
||||
def detect_text_headings(content: str) -> List[Dict]:
|
||||
"""
|
||||
Detects section and chapter headings with high accuracy across Markdown,
|
||||
numbered sections, explicit keywords, bold titles, and Setext styles.
|
||||
"""
|
||||
lines = content.split("\n")
|
||||
line_starts = []
|
||||
curr = 0
|
||||
for l in lines:
|
||||
line_starts.append(curr)
|
||||
curr += len(l) + 1
|
||||
|
||||
kw_regex = re.compile(
|
||||
r"^[ \t]*(?:\*\*)?(?:Chapter|CHAPTER|Глава|ГЛАВА|Part|PART|Часть|ЧАСТЬ|Section|SECTION|Раздел|РАЗДЕЛ|Module|MODULE|Модуль|МОДУЛЬ|Unit|UNIT|Lesson|LESSON|Урок|УРОК|Topic|TOPIC|Тема|ТЕМА|Appendix|APPENDIX|Приложение|ПРИЛОЖЕНИЕ|Act|ACT|Scene|SCENE)[ \t]+(?:[0-9IVXLCDMivxlcdm\w\.\-]+)(?:[ \t]*[:\-–—\.][ \t]*[^\n\r*]+|[ \t]+[^\n\r*]+)?(?:\*\*)?[ \t]*$"
|
||||
)
|
||||
|
||||
matches = list(heading_pattern.finditer(content))
|
||||
headings = []
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
trimmed = line.strip()
|
||||
if not trimmed:
|
||||
continue
|
||||
|
||||
prev_empty = (i == 0) or (not lines[i - 1].strip())
|
||||
next_empty = (i == len(lines) - 1) or (not lines[i + 1].strip())
|
||||
|
||||
# 1. Markdown # heading (# to #####)
|
||||
m_md = re.match(r"^#{1,5}[ \t]+(.+)$", trimmed)
|
||||
if m_md:
|
||||
clean_title = m_md.group(1).strip("#").strip()
|
||||
headings.append({
|
||||
"line_idx": i,
|
||||
"start": line_starts[i],
|
||||
"title": clean_title,
|
||||
"type": "markdown",
|
||||
})
|
||||
continue
|
||||
|
||||
# 2. Setext heading: Title followed by === or ---
|
||||
if i + 1 < len(lines):
|
||||
next_line = lines[i + 1].strip()
|
||||
if prev_empty and len(trimmed) <= 80 and not trimmed.startswith(("-", "*", "#", "`")):
|
||||
if re.match(r"^={3,}$", next_line) or re.match(r"^-{3,}$", next_line):
|
||||
headings.append({
|
||||
"line_idx": i,
|
||||
"start": line_starts[i],
|
||||
"title": trimmed,
|
||||
"type": "setext",
|
||||
})
|
||||
continue
|
||||
|
||||
# 3. Explicit keywords (Chapter 1, Глава 2, Module 3, etc.)
|
||||
if kw_regex.match(trimmed):
|
||||
clean_title = trimmed.strip("*").strip()
|
||||
headings.append({
|
||||
"line_idx": i,
|
||||
"start": line_starts[i],
|
||||
"title": clean_title,
|
||||
"type": "keyword",
|
||||
})
|
||||
continue
|
||||
|
||||
# 4. Bold heading on standalone line: **Title**
|
||||
bold_m = re.match(r"^\*\*([A-ZА-ЯЁ0-9][^\n\r*]{2,70})\*\*$", trimmed)
|
||||
if bold_m and prev_empty:
|
||||
headings.append({
|
||||
"line_idx": i,
|
||||
"start": line_starts[i],
|
||||
"title": bold_m.group(1).strip(),
|
||||
"type": "bold",
|
||||
})
|
||||
continue
|
||||
|
||||
# 5. Numbered sections: e.g. 1. Title, 1.1 Title, 1. Title.
|
||||
num_m = re.match(r"^([0-9]{1,2}(?:\.[0-9]{1,2})*\.?|[0-9]{1,2}\))[ \t]+([A-ZА-ЯЁ][^\n\r]{2,70})$", trimmed)
|
||||
if num_m and prev_empty:
|
||||
is_list = False
|
||||
if i + 1 < len(lines):
|
||||
next_t = lines[i + 1].strip()
|
||||
if re.match(r"^[0-9]{1,2}(?:\.[0-9]{1,2})*\.?[ \t]+", next_t):
|
||||
is_list = True
|
||||
if not is_list and not trimmed.endswith((",", ";")):
|
||||
headings.append({
|
||||
"line_idx": i,
|
||||
"start": line_starts[i],
|
||||
"title": trimmed,
|
||||
"type": "numbered",
|
||||
})
|
||||
continue
|
||||
|
||||
# 6. Standalone ALL CAPS line (e.g. INTRODUCTION TO NETWORKS)
|
||||
next_is_allcaps = (i + 1 < len(lines)) and bool(lines[i + 1].strip()) and lines[i + 1].strip().isupper()
|
||||
if prev_empty and not next_is_allcaps and 4 <= len(trimmed) <= 60 and not trimmed.endswith((",", ";")):
|
||||
if re.match(r"^[A-ZА-ЯЁ0-9\s:\-–—\.]+$", trimmed):
|
||||
letters = re.findall(r"[A-ZА-ЯЁ]", trimmed)
|
||||
words = trimmed.split()
|
||||
if len(letters) >= 4 and (len(words) >= 2 or len(trimmed) >= 6):
|
||||
headings.append({
|
||||
"line_idx": i,
|
||||
"start": line_starts[i],
|
||||
"title": trimmed,
|
||||
"type": "allcaps",
|
||||
})
|
||||
continue
|
||||
|
||||
return headings
|
||||
|
||||
|
||||
def parse_text(content: str, title: str = "Untitled", author: str = "") -> Tuple[str, str, List[Dict[str, str]]]:
|
||||
"""
|
||||
Parses plain text or Markdown into readable chapters.
|
||||
Splits by intelligent heading detection (Markdown, numbered sections, keywords)
|
||||
or chunks of ~1000 words.
|
||||
"""
|
||||
content = content.replace("\r\n", "\n").replace("\r", "\n").strip()
|
||||
if not content:
|
||||
return title, author, [{"title": title if title != "Untitled" else "Chapter 1", "content": ""}]
|
||||
|
||||
lines = content.split("\n")
|
||||
first_line = lines[0].strip()
|
||||
|
||||
# Extract title from first line or # Heading / <h1...> if default or generic README
|
||||
if title == "Untitled" or title.lower().startswith("readme"):
|
||||
if first_line.startswith("# "):
|
||||
title = first_line.lstrip("# ").strip()
|
||||
if len(lines) > 1:
|
||||
content = "\n".join(lines[1:]).strip()
|
||||
elif first_line.lower().startswith("title:"):
|
||||
title = first_line.split(":", 1)[1].strip()
|
||||
if len(lines) > 1:
|
||||
content = "\n".join(lines[1:]).strip()
|
||||
else:
|
||||
h1_match = re.search(r"<h1[^>]*>(.*?)</h1>", content[:1500], re.IGNORECASE)
|
||||
if h1_match:
|
||||
detected = re.sub(r"<[^>]+>", "", h1_match.group(1)).strip()
|
||||
if detected:
|
||||
title = detected
|
||||
|
||||
# Extract author from next line if missing
|
||||
if not author:
|
||||
lines = content.split("\n")
|
||||
if lines and lines[0].lower().startswith(("author:", "by:")):
|
||||
author = lines[0].split(":", 1)[1].strip()
|
||||
if len(lines) > 1:
|
||||
content = "\n".join(lines[1:]).strip()
|
||||
|
||||
detected_headings = detect_text_headings(content)
|
||||
chapters: List[Dict[str, str]] = []
|
||||
|
||||
if len(matches) >= 1:
|
||||
if detected_headings:
|
||||
# Pre-chapter intro text
|
||||
intro = content[: matches[0].start()].strip()
|
||||
intro = content[: detected_headings[0]["start"]].strip()
|
||||
if intro and count_words(intro) > 10:
|
||||
chapters.append({"title": "Introduction", "content": intro})
|
||||
is_ru = any(ord(c) > 127 for c in intro[:200])
|
||||
chapters.append({"title": "Введение" if is_ru else "Introduction", "content": intro})
|
||||
|
||||
for i, m in enumerate(matches):
|
||||
h_text = (m.group(1) or m.group(2)).strip()
|
||||
start = m.start()
|
||||
end = matches[i + 1].start() if i + 1 < len(matches) else len(content)
|
||||
for i, h in enumerate(detected_headings):
|
||||
start = h["start"]
|
||||
end = detected_headings[i + 1]["start"] if i + 1 < len(detected_headings) else len(content)
|
||||
body = content[start:end].strip()
|
||||
if body:
|
||||
chapters.append({"title": h_text, "content": body})
|
||||
chapters.append({"title": h["title"], "content": body})
|
||||
|
||||
if not chapters:
|
||||
# No headings detected: split by ~3000 words chunks
|
||||
chunks = split_long_text(content, target_words=3000)
|
||||
if len(chunks) == 1:
|
||||
chapters = [{"title": title if title != "Untitled" else "Chapter 1", "content": chunks[0]}]
|
||||
else:
|
||||
# No headings detected: split by ~1000 words chunks if long enough
|
||||
total_w = count_words(content)
|
||||
if total_w > 1200:
|
||||
chunks = split_long_text(content, target_words=1000)
|
||||
is_ru = any(ord(c) > 127 for c in content[:200])
|
||||
prefix = "Часть" if is_ru else "Chapter"
|
||||
for idx, chunk in enumerate(chunks, start=1):
|
||||
chapters.append({"title": f"Chapter {idx}", "content": chunk})
|
||||
chapters.append({"title": f"{prefix} {idx}", "content": chunk})
|
||||
else:
|
||||
chapters = [{"title": title if title != "Untitled" else "Chapter 1", "content": content}]
|
||||
|
||||
return title, author, chapters
|
||||
|
||||
Reference in new issue
Block a user