import re

import httpx
import trafilatura

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (compatible; LinkVault/1.0; +https://example.local)"
    )
}


def _download(url: str) -> str:
    try:
        with httpx.Client(
            follow_redirects=True, timeout=15.0, headers=HEADERS
        ) as client:
            resp = client.get(url)
            resp.raise_for_status()
            ctype = resp.headers.get("content-type", "")
            if "html" not in ctype and "xml" not in ctype and ctype:
                return ""
            return resp.text
    except Exception:
        return ""


def fetch(url: str) -> tuple[str, str]:
    """Lade die Seite und liefere (Titel, Textinhalt). Wirft nie eine Exception."""
    html = _download(url)
    if not html:
        return "", ""

    title, text = "", ""
    try:
        data = trafilatura.bare_extraction(
            html,
            include_comments=False,
            include_tables=False,
            with_metadata=True,
        )
        if isinstance(data, dict):
            title = data.get("title") or ""
            text = data.get("text") or ""
    except Exception:
        pass

    if not text:
        try:
            text = trafilatura.extract(html) or ""
        except Exception:
            text = ""

    if not title:
        m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
        if m:
            title = re.sub(r"\s+", " ", m.group(1)).strip()

    return title.strip(), (text or "")[:6000]