from pathlib import Path
import csv
import xml.etree.ElementTree as ET
from urllib.parse import urlsplit
raw = Path("sitemap.xml").read_bytes()
if len(raw) > 5_000_000:
raise ValueError("Use a smaller local file")
text = raw.decode("utf-8-sig")
if "<!DOCTYPE" in text.upper() or "<!ENTITY" in text.upper():
raise ValueError("DTD and entity declarations are not accepted")
root = ET.fromstring(text)
ns = {"s": "http://www.sitemaps.org/schemas/sitemap/0.9"}
if root.tag != "{http://www.sitemaps.org/schemas/sitemap/0.9}urlset":
raise ValueError("Expected a namespaced urlset")
rows, seen = [], set()
for item in root.findall("s:url", ns):
url = (item.findtext("s:loc", default="", namespaces=ns)).strip()
try:
u = urlsplit(url)
except ValueError:
continue
if u.scheme not in ("http", "https") or not u.netloc or url in seen:
continue
seen.add(url)
lastmod = item.findtext("s:lastmod", default="", namespaces=ns).strip()
rows.append([url, lastmod])
with open("sitemap_urls.csv", "x", encoding="utf-8-sig", newline="") as f:
writer = csv.writer(f)
writer.writerow(["url", "lastmod_raw"])
for row in rows:
writer.writerow(["'" + v if v.lstrip().startswith(("=", "+", "-", "@")) else v for v in row])