fix: resolve bugs 01 through 08 and set Ollama TTL to 1h
This commit is contained in:
1 parent
a01bf64445
commit
9bbc9473de
14 files changed
+366
-108
No files matched your search
+5
-1
@@ -1182,6 +1182,10 @@ def clean_wiki_html(raw_html: str, article_title: str = "") -> Tuple[List[Dict[s
|
||||
body = re.sub(r'<script\b[^>]*>.*?</script>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<noscript\b[^>]*>.*?</noscript>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# Pre-strip MediaWiki Parsoid metadata attributes (data-mw, data-parsoid, about)
|
||||
# These contain raw template JSON with <ref> tags and literal '>' characters that break tag regexes
|
||||
body = re.sub(r'''\s+(?:data-mw|data-parsoid|about)=(?:"[^"]*"|'[^']*')''', '', body, flags=re.DOTALL)
|
||||
|
||||
# 3. Strip unwanted wiki elements
|
||||
# Remove references: <sup class="reference">...</sup> and bracketed reference markers
|
||||
body = re.sub(r'<sup\b[^>]*class=["\x27][^"\x27]*reference[^"\x27]*["\x27][^>]*>.*?</sup>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
@@ -1229,7 +1233,7 @@ def clean_wiki_html(raw_html: str, article_title: str = "") -> Tuple[List[Dict[s
|
||||
if re.search(r'class=["\x27][^"\x27]*(?:navbox|ambox|metadata|nowraplinks|infobox)[^"\x27]*["\x27]', tbl_html, re.IGNORECASE):
|
||||
return ""
|
||||
cleaned_tbl = re.sub(
|
||||
r'\s+(?:style|bgcolor|align|valign|width|height|border|cellspacing|cellpadding|id)=["\x27][^"\x27]*["\x27]',
|
||||
r'\s+(?:style|bgcolor|align|valign|width|height|border|cellspacing|cellpadding|id)=(?:"[^"]*"|\'[^\']*\')',
|
||||
'',
|
||||
tbl_html,
|
||||
flags=re.IGNORECASE
|
||||
|
||||
Reference in new issue
Block a user