feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler
- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
This commit is contained in:
parent
24b6e0026d
commit
cff1d23071
10 changed files with 569 additions and 4 deletions
42
backend/app/youtube/rss.py
Normal file
42
backend/app/youtube/rss.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
"""Free per-channel RSS feed reader (no API quota). Returns the channel's most recent
|
||||
~15 uploads as lightweight stubs for fast fresh-video detection."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import feedparser
|
||||
import httpx
|
||||
|
||||
RSS_URL = "https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
|
||||
|
||||
|
||||
def fetch_channel_feed(channel_id: str) -> list[dict]:
|
||||
url = RSS_URL.format(channel_id=channel_id)
|
||||
try:
|
||||
resp = httpx.get(url, timeout=20.0, headers={"User-Agent": "Subfeed/1.0"})
|
||||
except httpx.HTTPError:
|
||||
return []
|
||||
if resp.status_code != 200:
|
||||
return []
|
||||
|
||||
parsed = feedparser.parse(resp.content)
|
||||
out: list[dict] = []
|
||||
for entry in parsed.entries:
|
||||
video_id = entry.get("yt_videoid")
|
||||
if not video_id:
|
||||
continue
|
||||
published = None
|
||||
if entry.get("published_parsed"):
|
||||
published = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc)
|
||||
thumbnail = None
|
||||
media = entry.get("media_thumbnail") or []
|
||||
if media:
|
||||
thumbnail = media[0].get("url")
|
||||
out.append(
|
||||
{
|
||||
"id": video_id,
|
||||
"channel_id": channel_id,
|
||||
"title": entry.get("title"),
|
||||
"published_at": published,
|
||||
"thumbnail_url": thumbnail,
|
||||
}
|
||||
)
|
||||
return out
|
||||
Loading…
Add table
Add a link
Reference in a new issue