feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler

- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
  backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
  language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
  quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
This commit is contained in:
npeter83 2026-06-11 01:36:41 +02:00
parent 24b6e0026d
commit cff1d23071
10 changed files with 569 additions and 4 deletions

View file

@ -133,3 +133,29 @@ class YouTubeClient:
)
items.extend(data.get("items", []))
return items
def get_playlist_items(self, playlist_id: str, page_token: str | None = None) -> dict:
"""One page (up to 50) of a playlist's items, most-recent-first for uploads
playlists. Returns the raw response (items + nextPageToken)."""
params = {
"part": "contentDetails,snippet",
"playlistId": playlist_id,
"maxResults": 50,
}
if page_token:
params["pageToken"] = page_token
return self._get("playlistItems", params)
def get_videos(self, video_ids: list[str]) -> list[dict]:
items: list[dict] = []
for batch in _chunks(video_ids, 50):
data = self._get(
"videos",
{
"part": "snippet,contentDetails,statistics,topicDetails,liveStreamingDetails",
"id": ",".join(batch),
"maxResults": 50,
},
)
items.extend(data.get("items", []))
return items

View file

@ -0,0 +1,42 @@
"""Free per-channel RSS feed reader (no API quota). Returns the channel's most recent
~15 uploads as lightweight stubs for fast fresh-video detection."""
from datetime import datetime, timezone
import feedparser
import httpx
RSS_URL = "https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
def fetch_channel_feed(channel_id: str) -> list[dict]:
url = RSS_URL.format(channel_id=channel_id)
try:
resp = httpx.get(url, timeout=20.0, headers={"User-Agent": "Subfeed/1.0"})
except httpx.HTTPError:
return []
if resp.status_code != 200:
return []
parsed = feedparser.parse(resp.content)
out: list[dict] = []
for entry in parsed.entries:
video_id = entry.get("yt_videoid")
if not video_id:
continue
published = None
if entry.get("published_parsed"):
published = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc)
thumbnail = None
media = entry.get("media_thumbnail") or []
if media:
thumbnail = media[0].get("url")
out.append(
{
"id": video_id,
"channel_id": channel_id,
"title": entry.get("title"),
"published_at": published,
"thumbnail_url": thumbnail,
}
)
return out