Initial commit: LinkVault – KI-gestützte Link-Sammlung

FastAPI + HTMX Web-App zum Speichern von Links mit automatischer
KI-Kategorisierung, Zusammenfassung und semantischer Suche (OpenAI).
Mehrbenutzer mit Login, SQLite-Persistenz, Docker/Docker-Compose-Setup.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app/scraper.py

@@ -0,0 +1,59 @@
+import re
+
+import httpx
+import trafilatura
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (compatible; LinkVault/1.0; +https://example.local)"
+    )
+}
+
+
+def _download(url: str) -> str:
+    try:
+        with httpx.Client(
+            follow_redirects=True, timeout=15.0, headers=HEADERS
+        ) as client:
+            resp = client.get(url)
+            resp.raise_for_status()
+            ctype = resp.headers.get("content-type", "")
+            if "html" not in ctype and "xml" not in ctype and ctype:
+                return ""
+            return resp.text
+    except Exception:
+        return ""
+
+
+def fetch(url: str) -> tuple[str, str]:
+    """Lade die Seite und liefere (Titel, Textinhalt). Wirft nie eine Exception."""
+    html = _download(url)
+    if not html:
+        return "", ""
+
+    title, text = "", ""
+    try:
+        data = trafilatura.bare_extraction(
+            html,
+            include_comments=False,
+            include_tables=False,
+            with_metadata=True,
+        )
+        if isinstance(data, dict):
+            title = data.get("title") or ""
+            text = data.get("text") or ""
+    except Exception:
+        pass
+
+    if not text:
+        try:
+            text = trafilatura.extract(html) or ""
+        except Exception:
+            text = ""
+
+    if not title:
+        m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
+        if m:
+            title = re.sub(r"\s+", " ", m.group(1)).strip()
+
+    return title.strip(), (text or "")[:6000]