#!/usr/bin/env python3
"""Scrape bookmark URLs one at a time through Supadata and clean the Markdown."""

from __future__ import annotations

import html
import json
import os
import re
import sys
import time
from html.parser import HTMLParser
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode, urlparse
from urllib.request import Request, urlopen


ROOT = Path(__file__).resolve().parent
BOOKMARKS = ROOT / "bookmarks_9_14_26.html"
OUTPUT = ROOT / "property-pages"
TARGETS = ROOT / os.environ.get("TARGETS_FILE", "property-targets.txt")
MANIFEST = OUTPUT / "manifest.jsonl"
DELAY_SECONDS = 4.0
MAX_RETRIES = 2


class BookmarkParser(HTMLParser):
    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.entries: list[tuple[str, str]] = []
        self._href: str | None = None
        self._text: list[str] = []

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        if tag.lower() != "a":
            return
        self._href = dict(attrs).get("href")
        self._text = []

    def handle_data(self, data: str) -> None:
        if self._href is not None:
            self._text.append(data)

    def handle_endtag(self, tag: str) -> None:
        if tag.lower() != "a" or self._href is None:
            return
        title = re.sub(r"\s+", " ", html.unescape("".join(self._text))).strip()
        self.entries.append((self._href, title))
        self._href = None
        self._text = []


def slugify(value: str, fallback: str) -> str:
    value = re.sub(r"[^a-zA-Z0-9]+", "-", value).strip("-").lower()
    return value[:100] or fallback


def clean_markdown(content: str, title: str) -> str:
    text = content.replace("\r\n", "\n").replace("\r", "\n")
    text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
    text = re.sub(r"<script\b[^>]*>.*?</script>", "", text, flags=re.I | re.S)
    text = re.sub(r"<style\b[^>]*>.*?</style>", "", text, flags=re.I | re.S)
    text = re.sub(r"!\[([^\]]*)\]\([^)]*\)", r"\1", text)
    text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text)
    text = re.sub(r"<https?://[^>]+>", "", text)
    text = re.sub(r"<[^>]+>", "", text)
    text = re.sub(r"https?://\S+", "", text)
    text = html.unescape(text)
    text = re.sub(r"[ \t]+\n", "\n", text)
    text = re.sub(r"\n{3,}", "\n\n", text)
    text = text.strip()
    if not text:
        text = "No readable page content was returned."
    if not text.startswith("#"):
        text = f"# {title}\n\n{text}"
    return text + "\n"


def load_api_key() -> str:
    key = os.environ.get("SUPADATA_API_KEY")
    if key:
        return key
    env_file = ROOT / ".env"
    for line in env_file.read_text(encoding="utf-8").splitlines():
        if line.startswith("SUPADATA_API_KEY="):
            return line.split("=", 1)[1].strip().strip('"\'')
    raise RuntimeError("SUPADATA_API_KEY was not found")


def scrape(url: str, api_key: str) -> dict:
    query = urlencode({"url": url, "noLinks": "true", "lang": "en"})
    request = Request(
        f"https://api.supadata.ai/v1/web/scrape?{query}",
        headers={"x-api-key": api_key, "Accept": "application/json"},
    )
    with urlopen(request, timeout=120) as response:
        return json.loads(response.read().decode("utf-8"))


def main() -> int:
    if not BOOKMARKS.exists():
        raise RuntimeError(f"Bookmark file not found: {BOOKMARKS.name}")
    api_key = load_api_key()
    parser = BookmarkParser()
    parser.feed(BOOKMARKS.read_text(encoding="utf-8", errors="replace"))
    targets = {
        re.sub(r"\s+", " ", line).strip()
        for line in TARGETS.read_text(encoding="utf-8").splitlines()
        if line.strip()
    }
    parser.entries = [(url, title) for url, title in parser.entries if title in targets]
    OUTPUT.mkdir(exist_ok=True)

    prior: dict[str, dict] = {}
    if MANIFEST.exists():
        for line in MANIFEST.read_text(encoding="utf-8").splitlines():
            if line.strip():
                item = json.loads(line)
                prior[item["index"]] = item

    entries = []
    for href, title in parser.entries:
        scheme = urlparse(href).scheme.lower()
        entries.append((href, title, scheme in {"http", "https"}))

    with MANIFEST.open("a", encoding="utf-8") as manifest:
        for index, (url, bookmark_title, eligible) in enumerate(entries, start=1):
            key = str(index)
            if key in prior and prior[key].get("status") in {"scraped", "skipped"}:
                continue
            if not eligible:
                record = {"index": index, "title": bookmark_title, "url": url, "status": "skipped", "reason": "not-http(s)"}
                manifest.write(json.dumps(record, ensure_ascii=False) + "\n")
                manifest.flush()
                continue

            filename = f"{index:04d}-{slugify(bookmark_title, 'untitled')}.md"
            record = {"index": index, "title": bookmark_title, "url": url, "file": filename}
            print(f"[{index}/{len(entries)}] Scraping {bookmark_title or url}", flush=True)
            last_error = ""
            for attempt in range(MAX_RETRIES + 1):
                try:
                    result = scrape(url, api_key)
                    page_title = (result.get("name") or bookmark_title or url).strip()
                    content = clean_markdown(result.get("content") or "", page_title)
                    (OUTPUT / filename).write_text(content, encoding="utf-8")
                    record.update({"status": "scraped", "characters": len(content)})
                    break
                except (HTTPError, URLError, TimeoutError, json.JSONDecodeError) as exc:
                    last_error = str(exc)
                    if attempt < MAX_RETRIES:
                        time.sleep(10 * (attempt + 1))
                except Exception as exc:  # keep the long run alive for individual page failures
                    last_error = str(exc)
                    break
            else:
                pass
            if record.get("status") != "scraped":
                record.update({"status": "failed", "error": last_error})
                print(f"  failed: {last_error}", file=sys.stderr, flush=True)
            manifest.write(json.dumps(record, ensure_ascii=False) + "\n")
            manifest.flush()
            time.sleep(DELAY_SECONDS)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())