pboProperty Ontology
python169 lines6.3 KB
RawDownload
1#!/usr/bin/env python3
2"""Scrape bookmark URLs one at a time through Supadata and clean the Markdown."""
3
4from __future__ import annotations
5
6import html
7import json
8import os
9import re
10import sys
11import time
12from html.parser import HTMLParser
13from pathlib import Path
14from urllib.error import HTTPError, URLError
15from urllib.parse import urlencode, urlparse
16from urllib.request import Request, urlopen
17
18
19ROOT = Path(__file__).resolve().parent
20BOOKMARKS = ROOT / "bookmarks_9_14_26.html"
21OUTPUT = ROOT / "property-pages"
22TARGETS = ROOT / os.environ.get("TARGETS_FILE", "property-targets.txt")
23MANIFEST = OUTPUT / "manifest.jsonl"
24DELAY_SECONDS = 4.0
25MAX_RETRIES = 2
26
27
28class BookmarkParser(HTMLParser):
29 def __init__(self) -> None:
30 super().__init__(convert_charrefs=True)
31 self.entries: list[tuple[str, str]] = []
32 self._href: str | None = None
33 self._text: list[str] = []
34
35 def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
36 if tag.lower() != "a":
37 return
38 self._href = dict(attrs).get("href")
39 self._text = []
40
41 def handle_data(self, data: str) -> None:
42 if self._href is not None:
43 self._text.append(data)
44
45 def handle_endtag(self, tag: str) -> None:
46 if tag.lower() != "a" or self._href is None:
47 return
48 title = re.sub(r"\s+", " ", html.unescape("".join(self._text))).strip()
49 self.entries.append((self._href, title))
50 self._href = None
51 self._text = []
52
53
54def slugify(value: str, fallback: str) -> str:
55 value = re.sub(r"[^a-zA-Z0-9]+", "-", value).strip("-").lower()
56 return value[:100] or fallback
57
58
59def clean_markdown(content: str, title: str) -> str:
60 text = content.replace("\r\n", "\n").replace("\r", "\n")
61 text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
62 text = re.sub(r"<script\b[^>]*>.*?</script>", "", text, flags=re.I | re.S)
63 text = re.sub(r"<style\b[^>]*>.*?</style>", "", text, flags=re.I | re.S)
64 text = re.sub(r"!\[([^\]]*)\]\([^)]*\)", r"\1", text)
65 text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text)
66 text = re.sub(r"<https?://[^>]+>", "", text)
67 text = re.sub(r"<[^>]+>", "", text)
68 text = re.sub(r"https?://\S+", "", text)
69 text = html.unescape(text)
70 text = re.sub(r"[ \t]+\n", "\n", text)
71 text = re.sub(r"\n{3,}", "\n\n", text)
72 text = text.strip()
73 if not text:
74 text = "No readable page content was returned."
75 if not text.startswith("#"):
76 text = f"# {title}\n\n{text}"
77 return text + "\n"
78
79
80def load_api_key() -> str:
81 key = os.environ.get("SUPADATA_API_KEY")
82 if key:
83 return key
84 env_file = ROOT / ".env"
85 for line in env_file.read_text(encoding="utf-8").splitlines():
86 if line.startswith("SUPADATA_API_KEY="):
87 return line.split("=", 1)[1].strip().strip('"\'')
88 raise RuntimeError("SUPADATA_API_KEY was not found")
89
90
91def scrape(url: str, api_key: str) -> dict:
92 query = urlencode({"url": url, "noLinks": "true", "lang": "en"})
93 request = Request(
94 f"https://api.supadata.ai/v1/web/scrape?{query}",
95 headers={"x-api-key": api_key, "Accept": "application/json"},
96 )
97 with urlopen(request, timeout=120) as response:
98 return json.loads(response.read().decode("utf-8"))
99
100
101def main() -> int:
102 if not BOOKMARKS.exists():
103 raise RuntimeError(f"Bookmark file not found: {BOOKMARKS.name}")
104 api_key = load_api_key()
105 parser = BookmarkParser()
106 parser.feed(BOOKMARKS.read_text(encoding="utf-8", errors="replace"))
107 targets = {
108 re.sub(r"\s+", " ", line).strip()
109 for line in TARGETS.read_text(encoding="utf-8").splitlines()
110 if line.strip()
111 }
112 parser.entries = [(url, title) for url, title in parser.entries if title in targets]
113 OUTPUT.mkdir(exist_ok=True)
114
115 prior: dict[str, dict] = {}
116 if MANIFEST.exists():
117 for line in MANIFEST.read_text(encoding="utf-8").splitlines():
118 if line.strip():
119 item = json.loads(line)
120 prior[item["index"]] = item
121
122 entries = []
123 for href, title in parser.entries:
124 scheme = urlparse(href).scheme.lower()
125 entries.append((href, title, scheme in {"http", "https"}))
126
127 with MANIFEST.open("a", encoding="utf-8") as manifest:
128 for index, (url, bookmark_title, eligible) in enumerate(entries, start=1):
129 key = str(index)
130 if key in prior and prior[key].get("status") in {"scraped", "skipped"}:
131 continue
132 if not eligible:
133 record = {"index": index, "title": bookmark_title, "url": url, "status": "skipped", "reason": "not-http(s)"}
134 manifest.write(json.dumps(record, ensure_ascii=False) + "\n")
135 manifest.flush()
136 continue
137
138 filename = f"{index:04d}-{slugify(bookmark_title, 'untitled')}.md"
139 record = {"index": index, "title": bookmark_title, "url": url, "file": filename}
140 print(f"[{index}/{len(entries)}] Scraping {bookmark_title or url}", flush=True)
141 last_error = ""
142 for attempt in range(MAX_RETRIES + 1):
143 try:
144 result = scrape(url, api_key)
145 page_title = (result.get("name") or bookmark_title or url).strip()
146 content = clean_markdown(result.get("content") or "", page_title)
147 (OUTPUT / filename).write_text(content, encoding="utf-8")
148 record.update({"status": "scraped", "characters": len(content)})
149 break
150 except (HTTPError, URLError, TimeoutError, json.JSONDecodeError) as exc:
151 last_error = str(exc)
152 if attempt < MAX_RETRIES:
153 time.sleep(10 * (attempt + 1))
154 except Exception as exc: # keep the long run alive for individual page failures
155 last_error = str(exc)
156 break
157 else:
158 pass
159 if record.get("status") != "scraped":
160 record.update({"status": "failed", "error": last_error})
161 print(f" failed: {last_error}", file=sys.stderr, flush=True)
162 manifest.write(json.dumps(record, ensure_ascii=False) + "\n")
163 manifest.flush()
164 time.sleep(DELAY_SECONDS)
165 return 0
166
167
168if __name__ == "__main__":
169 raise SystemExit(main())