refactor(stage-1): eliminate dead parse_text, simplify task progress SQL, remove prompt hacks
This commit is contained in:
1 parent
6c2d94773c
commit
20eef2c684
3 files changed
+19
-95
No files matched your search
@@ -286,44 +286,6 @@ def split_long_text(text: str, target_words: int = 3000) -> List[str]:
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
def parse_text(content: str, title: str = "Untitled", author: str = "") -> Tuple[str, str, List[Dict[str, str]]]:
|
||||
"""
|
||||
Parses plain text or Markdown into readable chapters.
|
||||
Splits by explicit chapter markers (e.g. 'Chapter X', '# Chapter') or chunks of ~3000 words.
|
||||
"""
|
||||
content = content.replace("\r\n", "\n").replace("\r", "\n").strip()
|
||||
if not content:
|
||||
return title, author, [{"title": title if title != "Untitled" else "Chapter 1", "content": ""}]
|
||||
|
||||
lines = content.split("\n")
|
||||
first_line = lines[0].strip()
|
||||
|
||||
# Extract title from first line or # Heading / <h1...> if default or generic README
|
||||
if title == "Untitled" or title.lower().startswith("readme"):
|
||||
if first_line.startswith("# "):
|
||||
title = first_line.lstrip("# ").strip()
|
||||
if len(lines) > 1:
|
||||
content = "\n".join(lines[1:]).strip()
|
||||
elif first_line.lower().startswith("title:"):
|
||||
title = first_line.split(":", 1)[1].strip()
|
||||
if len(lines) > 1:
|
||||
content = "\n".join(lines[1:]).strip()
|
||||
else:
|
||||
h1_match = re.search(r"<h1[^>]*>(.*?)</h1>", content[:1500], re.IGNORECASE)
|
||||
if h1_match:
|
||||
detected = re.sub(r"<[^>]+>", "", h1_match.group(1)).strip()
|
||||
if detected:
|
||||
title = detected
|
||||
|
||||
# Extract author from next line if missing
|
||||
if not author:
|
||||
lines = content.split("\n")
|
||||
if lines and lines[0].lower().startswith(("author:", "by:")):
|
||||
author = lines[0].split(":", 1)[1].strip()
|
||||
if len(lines) > 1:
|
||||
content = "\n".join(lines[1:]).strip()
|
||||
|
||||
def detect_text_headings(content: str) -> List[Dict]:
|
||||
"""
|
||||
Detects section and chapter headings with high accuracy across Markdown,
|
||||
|
||||
Reference in new issue
Block a user