47685f0d34
FastAPI + HTMX Web-App zum Speichern von Links mit automatischer KI-Kategorisierung, Zusammenfassung und semantischer Suche (OpenAI). Mehrbenutzer mit Login, SQLite-Persistenz, Docker/Docker-Compose-Setup. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
60 lines
1.5 KiB
Python
60 lines
1.5 KiB
Python
import re
|
|
|
|
import httpx
|
|
import trafilatura
|
|
|
|
HEADERS = {
|
|
"User-Agent": (
|
|
"Mozilla/5.0 (compatible; LinkVault/1.0; +https://example.local)"
|
|
)
|
|
}
|
|
|
|
|
|
def _download(url: str) -> str:
|
|
try:
|
|
with httpx.Client(
|
|
follow_redirects=True, timeout=15.0, headers=HEADERS
|
|
) as client:
|
|
resp = client.get(url)
|
|
resp.raise_for_status()
|
|
ctype = resp.headers.get("content-type", "")
|
|
if "html" not in ctype and "xml" not in ctype and ctype:
|
|
return ""
|
|
return resp.text
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def fetch(url: str) -> tuple[str, str]:
|
|
"""Lade die Seite und liefere (Titel, Textinhalt). Wirft nie eine Exception."""
|
|
html = _download(url)
|
|
if not html:
|
|
return "", ""
|
|
|
|
title, text = "", ""
|
|
try:
|
|
data = trafilatura.bare_extraction(
|
|
html,
|
|
include_comments=False,
|
|
include_tables=False,
|
|
with_metadata=True,
|
|
)
|
|
if isinstance(data, dict):
|
|
title = data.get("title") or ""
|
|
text = data.get("text") or ""
|
|
except Exception:
|
|
pass
|
|
|
|
if not text:
|
|
try:
|
|
text = trafilatura.extract(html) or ""
|
|
except Exception:
|
|
text = ""
|
|
|
|
if not title:
|
|
m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
|
|
if m:
|
|
title = re.sub(r"\s+", " ", m.group(1)).strip()
|
|
|
|
return title.strip(), (text or "")[:6000]
|