fix(nightly): repoint feed to rss.xml, parse read-time, repair scraper
The FEED table had been frozen since 2026-07-08 — 37 nightly runs that all went green while updating nothing. Both paths were dead at once, which is why it left no trace: FEED_URL requested /feed.xml, which 404s (the site serves /rss.xml), and the HTML fallback then matched zero articles because it looks for a thumbnail alt= attribute and a Title-Case date, and the redesign has neither — no feature images on the index, and uppercase abbreviated months. It looked current only because no post has been published since July 2025, so the four titles it froze with are still the four newest. FEED_URL now points at the real feed, which also carries read-time in the site's own namespace — so the meta column comes from the publisher instead of being scraped back out of a page. The fallback is rewritten against the markup the index actually emits: one `row` div holding an anchor and a `meta` span. Failures stay fail-safe and stop being invisible. A skipped section now emits a GitHub Actions ::warning annotation that shows up in the run summary, plus a count of how many sections did not update. A silent skip and a successful run looked identical, and that is the whole reason this went unnoticed. Verified against the live site: the fallback parses all 25 articles with dates and read-times, and the feed path yields the same four rows the table already shows.
This commit is contained in:
parent
b9ffcc2456
commit
0b31ebf801
2 changed files with 86 additions and 32 deletions
BIN
engine/__pycache__/nightly.cpython-314.pyc
Normal file
BIN
engine/__pycache__/nightly.cpython-314.pyc
Normal file
Binary file not shown.
|
|
@ -27,8 +27,14 @@ README = os.path.join(ROOT, "README.md")
|
|||
STATE_PATH = os.path.join(ROOT, "state", "demon.json")
|
||||
|
||||
SITE = "https://stevenmilanese.com"
|
||||
FEED_URL = f"{SITE}/feed.xml"
|
||||
# The real feed. /feed.xml was a guess that 404'd for weeks while the HTML
|
||||
# fallback silently matched nothing, so the FEED table quietly froze; it now
|
||||
# also 301s here, but point at the canonical URL rather than lean on that.
|
||||
FEED_URL = f"{SITE}/rss.xml"
|
||||
BLOG_URL = f"{SITE}/blog"
|
||||
# The site publishes read-time in its own namespace — valid for strict parsers,
|
||||
# unambiguous for this one.
|
||||
MANUSCRIPT_NS = "https://stevenmilanese.com/ns/manuscript"
|
||||
|
||||
FEATURED_REPOS = [
|
||||
("anthropic-certs", "Verified Anthropic certifications — Claude, API, Claude Code CLI, MCP."),
|
||||
|
|
@ -63,6 +69,11 @@ def replace_section(text, marker, body):
|
|||
|
||||
|
||||
def articles_from_rss():
|
||||
"""Primary source: the site's real feed.
|
||||
|
||||
The feed carries read-time in the site's own namespace, so the meta column
|
||||
comes from the publisher rather than being scraped back out of a page.
|
||||
"""
|
||||
root = ET.fromstring(fetch(FEED_URL))
|
||||
items = []
|
||||
for item in root.iter("item"):
|
||||
|
|
@ -74,47 +85,70 @@ def articles_from_rss():
|
|||
date = datetime.datetime.strptime(pub[:16].strip(), "%a, %d %b %Y")
|
||||
except ValueError:
|
||||
pass
|
||||
mins = (item.findtext(f"{{{MANUSCRIPT_NS}}}readingMinutes") or "").strip()
|
||||
meta = f"{mins} min read" if mins.isdigit() else ""
|
||||
if title and link:
|
||||
items.append({"title": title, "url": link, "date": date, "meta": ""})
|
||||
items.append({"title": title, "url": link, "date": date, "meta": meta})
|
||||
return items
|
||||
|
||||
|
||||
def articles_from_html():
|
||||
"""Fallback: parse the blog index.
|
||||
|
||||
Written against the Manuscript's markup, which is regular enough not to
|
||||
need heuristics — each entry is a `row` div holding one anchor and one
|
||||
`meta` span:
|
||||
|
||||
<div class="row">
|
||||
<a href="/blog/<slug>">Title</a>
|
||||
<span class="meta"> JUL 28, 2025 · 19 MIN </span>
|
||||
</div>
|
||||
|
||||
The previous version looked for a thumbnail `alt=` attribute and a
|
||||
Title-Case date. The redesign has neither — no feature images on the index
|
||||
and uppercase abbreviated months — so it matched nothing and silently
|
||||
returned an empty list for weeks.
|
||||
"""
|
||||
page = fetch(BLOG_URL)
|
||||
skip = ("/blog/tag/", "/blog/author/")
|
||||
anchors = [
|
||||
m for m in re.finditer(r'<a[^>]+href="(/blog/[a-z0-9][a-z0-9-]*)"', page)
|
||||
if not m.group(1).startswith(skip)
|
||||
]
|
||||
seen, articles = set(), []
|
||||
for idx, m in enumerate(anchors):
|
||||
slug = m.group(1)
|
||||
if slug in seen:
|
||||
articles, seen = [], set()
|
||||
|
||||
row_re = re.compile(
|
||||
r'<a[^>]+href="(/blog/[a-z0-9][a-z0-9-]*)"[^>]*>(.*?)</a>'
|
||||
r'.{0,400}?<span[^>]*class="meta"[^>]*>(.*?)</span>',
|
||||
re.S,
|
||||
)
|
||||
for m in row_re.finditer(page):
|
||||
slug, raw_title, raw_meta = m.group(1), m.group(2), m.group(3)
|
||||
if slug.startswith(skip) or slug in seen:
|
||||
continue
|
||||
seen.add(slug)
|
||||
stop = len(page)
|
||||
for nxt in anchors[idx + 1:]:
|
||||
if nxt.group(1) != slug:
|
||||
stop = nxt.start()
|
||||
|
||||
title = htmllib.unescape(re.sub(r"<[^>]+>", "", raw_title)).strip()
|
||||
meta_text = htmllib.unescape(re.sub(r"<[^>]+>", " ", raw_meta))
|
||||
meta_text = re.sub(r"\s+", " ", meta_text).strip()
|
||||
|
||||
# "JUL 28, 2025" — uppercase, abbreviated month.
|
||||
date_m = re.search(r"([A-Za-z]{3,}) (\d{1,2}), (\d{4})", meta_text)
|
||||
if not (title and date_m):
|
||||
continue
|
||||
month, day, year = date_m.groups()
|
||||
date = None
|
||||
for fmt in ("%b %d %Y", "%B %d %Y"):
|
||||
try:
|
||||
date = datetime.datetime.strptime(f"{month.title()} {day} {year}", fmt)
|
||||
break
|
||||
window = page[m.start():stop]
|
||||
title_m = re.search(r'alt="([^"]+)"', window)
|
||||
stripped = re.sub(r"<[^>]+>", " ", window)
|
||||
stripped = htmllib.unescape(re.sub(r"<!--.*?-->", "", stripped))
|
||||
date_m = re.search(r"([A-Z][a-z]+ \d{1,2}, \d{4})", stripped)
|
||||
mins_m = re.search(r"(\d+)\s*min", stripped)
|
||||
views_m = re.search(r"([\d,]+)\s*views", stripped)
|
||||
if not (title_m and date_m):
|
||||
except ValueError:
|
||||
continue
|
||||
if date is None:
|
||||
continue
|
||||
date = datetime.datetime.strptime(date_m.group(1), "%B %d, %Y")
|
||||
meta = f"{mins_m.group(1)} min read" if mins_m else (
|
||||
f"{views_m.group(1)} views" if views_m else ""
|
||||
)
|
||||
|
||||
mins_m = re.search(r"(\d+)\s*MIN", meta_text, re.I)
|
||||
seen.add(slug)
|
||||
articles.append({
|
||||
"title": htmllib.unescape(title_m.group(1)),
|
||||
"title": title,
|
||||
"url": f"{SITE}{slug}",
|
||||
"date": date,
|
||||
"meta": meta,
|
||||
"meta": f"{mins_m.group(1)} min read" if mins_m else "",
|
||||
})
|
||||
return articles
|
||||
|
||||
|
|
@ -124,8 +158,9 @@ def update_feed(text):
|
|||
try:
|
||||
articles = articles_from_rss()
|
||||
except Exception as exc:
|
||||
print(f"feed: rss unusable ({exc}); falling back to html")
|
||||
print(f"::warning title=nightly FEED rss::rss unusable ({exc}); falling back to html")
|
||||
if not articles:
|
||||
print("::warning title=nightly FEED fallback::rss yielded nothing; scraping /blog")
|
||||
articles = articles_from_html()
|
||||
dated = [a for a in articles if a["date"]]
|
||||
dated.sort(key=lambda a: a["date"], reverse=True)
|
||||
|
|
@ -188,22 +223,41 @@ def bump_entropy():
|
|||
)
|
||||
|
||||
|
||||
def warn(label, exc):
|
||||
"""Fail-safe, but never quiet.
|
||||
|
||||
Every section still degrades to 'leave it alone and exit 0' — a broken
|
||||
fetch must not wedge the profile. What changed is visibility: a section
|
||||
that stops updating now announces itself as a GitHub Actions warning
|
||||
annotation, which surfaces in the run summary instead of scrolling past in
|
||||
a log nobody opens. The FEED table sat frozen for 37 days precisely
|
||||
because a silent skip and a successful run looked identical.
|
||||
"""
|
||||
print(f"::warning title=nightly {label} skipped::{label} left untouched — {exc}")
|
||||
print(f"{label}: left untouched ({exc})", file=sys.stderr)
|
||||
|
||||
|
||||
def main():
|
||||
with open(README) as fh:
|
||||
text = fh.read()
|
||||
failures = 0
|
||||
for label, fn in (("FEED", update_feed), ("STATS", update_stats), ("FEATURED", update_featured)):
|
||||
try:
|
||||
text = fn(text)
|
||||
print(f"{label}: updated")
|
||||
except Exception as exc:
|
||||
print(f"{label}: left untouched ({exc})")
|
||||
failures += 1
|
||||
warn(label, exc)
|
||||
with open(README, "w") as fh:
|
||||
fh.write(text)
|
||||
try:
|
||||
bump_entropy()
|
||||
print("ENTROPY: regenerated")
|
||||
except Exception as exc:
|
||||
print(f"ENTROPY: left untouched ({exc})")
|
||||
failures += 1
|
||||
warn("ENTROPY", exc)
|
||||
if failures:
|
||||
print(f"::warning title=nightly incomplete::{failures} of 4 sections did not update")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue