pboProperty Ontology
python90 lines3.6 KB
RawDownload
1#!/usr/bin/env python3
2"""Fetch the final property pages directly, without Supadata."""
3
4import html
5import re
6import time
7from html.parser import HTMLParser
8from pathlib import Path
9from urllib.error import HTTPError, URLError
10from urllib.request import Request, urlopen
11
12from bs4 import BeautifulSoup
13
14from scrape_bookmarks import BookmarkParser, BOOKMARKS, clean_markdown, slugify
15
16
17ROOT = Path(__file__).resolve().parent
18OUTPUT = ROOT / "property-pages"
19TARGETS = ROOT / "last7-targets.txt"
20ALL_TARGETS = ROOT / "property-targets.txt"
21
22
23def fetch(url: str) -> str:
24 request = Request(url, headers={"User-Agent": "Mozilla/5.0 (compatible; property-archive/1.0)"})
25 with urlopen(request, timeout=90) as response:
26 return response.read().decode(response.headers.get_content_charset() or "utf-8", errors="replace")
27
28
29def html_to_markdown(source: str, title: str) -> str:
30 soup = BeautifulSoup(source, "html.parser")
31 for node in soup(["script", "style", "noscript", "svg", "form", "nav", "footer", "header"]):
32 node.decompose()
33 main = soup.find("article") or soup.find("main") or soup.body or soup
34 blocks: list[str] = []
35 for node in main.find_all(["h1", "h2", "h3", "h4", "p", "li", "blockquote", "pre", "table"]):
36 if node.name == "li":
37 text = "- " + node.get_text(" ", strip=True)
38 elif node.name == "blockquote":
39 text = "> " + node.get_text(" ", strip=True)
40 elif node.name == "pre":
41 text = "```\n" + node.get_text("\n", strip=False).strip() + "\n```"
42 elif node.name == "table":
43 rows = []
44 for tr in node.find_all("tr"):
45 cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])]
46 if cells:
47 rows.append("| " + " | ".join(cells) + " |")
48 text = "\n".join(rows)
49 if rows and len(rows) > 0:
50 width = len(node.find("tr").find_all(["th", "td"]))
51 if width:
52 text += "\n| " + " | ".join(["---"] * width) + " |"
53 else:
54 level = int(node.name[1]) if node.name.startswith("h") else 0
55 prefix = "#" * level + " " if level else ""
56 text = prefix + node.get_text(" ", strip=True)
57 if text.strip():
58 blocks.append(text.strip())
59 return clean_markdown("\n\n".join(blocks), title)
60
61
62def main() -> None:
63 parser = BookmarkParser()
64 parser.feed(BOOKMARKS.read_text(encoding="utf-8", errors="replace"))
65 wanted = {line.strip() for line in TARGETS.read_text(encoding="utf-8").splitlines() if line.strip()}
66 all_titles = [line.strip() for line in ALL_TARGETS.read_text(encoding="utf-8").splitlines() if line.strip()]
67 entries = {title: url for url, title in parser.entries}
68 OUTPUT.mkdir(exist_ok=True)
69 for title in TARGETS.read_text(encoding="utf-8").splitlines():
70 title = title.strip()
71 if not title:
72 continue
73 url = entries.get(title)
74 if not url:
75 print(f"missing: {title}")
76 continue
77 number = all_titles.index(title) + 1
78 filename = f"{number:04d}-{slugify(title, 'untitled')}.md"
79 print(f"Fetching {title}", flush=True)
80 try:
81 markdown = html_to_markdown(fetch(url), title)
82 (OUTPUT / filename).write_text(markdown, encoding="utf-8")
83 print(f" wrote {filename} ({len(markdown)} characters)", flush=True)
84 except (HTTPError, URLError, TimeoutError, UnicodeError) as exc:
85 print(f" failed: {exc}", flush=True)
86 time.sleep(5)
87
88
89if __name__ == "__main__":
90 main()