#!/usr/bin/env python3
"""Fetch the final property pages directly, without Supadata."""

import html
import re
import time
from html.parser import HTMLParser
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen

from bs4 import BeautifulSoup

from scrape_bookmarks import BookmarkParser, BOOKMARKS, clean_markdown, slugify


ROOT = Path(__file__).resolve().parent
OUTPUT = ROOT / "property-pages"
TARGETS = ROOT / "last7-targets.txt"
ALL_TARGETS = ROOT / "property-targets.txt"


def fetch(url: str) -> str:
    request = Request(url, headers={"User-Agent": "Mozilla/5.0 (compatible; property-archive/1.0)"})
    with urlopen(request, timeout=90) as response:
        return response.read().decode(response.headers.get_content_charset() or "utf-8", errors="replace")


def html_to_markdown(source: str, title: str) -> str:
    soup = BeautifulSoup(source, "html.parser")
    for node in soup(["script", "style", "noscript", "svg", "form", "nav", "footer", "header"]):
        node.decompose()
    main = soup.find("article") or soup.find("main") or soup.body or soup
    blocks: list[str] = []
    for node in main.find_all(["h1", "h2", "h3", "h4", "p", "li", "blockquote", "pre", "table"]):
        if node.name == "li":
            text = "- " + node.get_text(" ", strip=True)
        elif node.name == "blockquote":
            text = "> " + node.get_text(" ", strip=True)
        elif node.name == "pre":
            text = "```\n" + node.get_text("\n", strip=False).strip() + "\n```"
        elif node.name == "table":
            rows = []
            for tr in node.find_all("tr"):
                cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])]
                if cells:
                    rows.append("| " + " | ".join(cells) + " |")
            text = "\n".join(rows)
            if rows and len(rows) > 0:
                width = len(node.find("tr").find_all(["th", "td"]))
                if width:
                    text += "\n| " + " | ".join(["---"] * width) + " |"
        else:
            level = int(node.name[1]) if node.name.startswith("h") else 0
            prefix = "#" * level + " " if level else ""
            text = prefix + node.get_text(" ", strip=True)
        if text.strip():
            blocks.append(text.strip())
    return clean_markdown("\n\n".join(blocks), title)


def main() -> None:
    parser = BookmarkParser()
    parser.feed(BOOKMARKS.read_text(encoding="utf-8", errors="replace"))
    wanted = {line.strip() for line in TARGETS.read_text(encoding="utf-8").splitlines() if line.strip()}
    all_titles = [line.strip() for line in ALL_TARGETS.read_text(encoding="utf-8").splitlines() if line.strip()]
    entries = {title: url for url, title in parser.entries}
    OUTPUT.mkdir(exist_ok=True)
    for title in TARGETS.read_text(encoding="utf-8").splitlines():
        title = title.strip()
        if not title:
            continue
        url = entries.get(title)
        if not url:
            print(f"missing: {title}")
            continue
        number = all_titles.index(title) + 1
        filename = f"{number:04d}-{slugify(title, 'untitled')}.md"
        print(f"Fetching {title}", flush=True)
        try:
            markdown = html_to_markdown(fetch(url), title)
            (OUTPUT / filename).write_text(markdown, encoding="utf-8")
            print(f"  wrote {filename} ({len(markdown)} characters)", flush=True)
        except (HTTPError, URLError, TimeoutError, UnicodeError) as exc:
            print(f"  failed: {exc}", flush=True)
        time.sleep(5)


if __name__ == "__main__":
    main()