feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler
- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
This commit is contained in:
parent
24b6e0026d
commit
cff1d23071
10 changed files with 569 additions and 4 deletions
|
|
@ -133,3 +133,29 @@ class YouTubeClient:
|
|||
)
|
||||
items.extend(data.get("items", []))
|
||||
return items
|
||||
|
||||
def get_playlist_items(self, playlist_id: str, page_token: str | None = None) -> dict:
|
||||
"""One page (up to 50) of a playlist's items, most-recent-first for uploads
|
||||
playlists. Returns the raw response (items + nextPageToken)."""
|
||||
params = {
|
||||
"part": "contentDetails,snippet",
|
||||
"playlistId": playlist_id,
|
||||
"maxResults": 50,
|
||||
}
|
||||
if page_token:
|
||||
params["pageToken"] = page_token
|
||||
return self._get("playlistItems", params)
|
||||
|
||||
def get_videos(self, video_ids: list[str]) -> list[dict]:
|
||||
items: list[dict] = []
|
||||
for batch in _chunks(video_ids, 50):
|
||||
data = self._get(
|
||||
"videos",
|
||||
{
|
||||
"part": "snippet,contentDetails,statistics,topicDetails,liveStreamingDetails",
|
||||
"id": ",".join(batch),
|
||||
"maxResults": 50,
|
||||
},
|
||||
)
|
||||
items.extend(data.get("items", []))
|
||||
return items
|
||||
|
|
|
|||
42
backend/app/youtube/rss.py
Normal file
42
backend/app/youtube/rss.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
"""Free per-channel RSS feed reader (no API quota). Returns the channel's most recent
|
||||
~15 uploads as lightweight stubs for fast fresh-video detection."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import feedparser
|
||||
import httpx
|
||||
|
||||
RSS_URL = "https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
|
||||
|
||||
|
||||
def fetch_channel_feed(channel_id: str) -> list[dict]:
|
||||
url = RSS_URL.format(channel_id=channel_id)
|
||||
try:
|
||||
resp = httpx.get(url, timeout=20.0, headers={"User-Agent": "Subfeed/1.0"})
|
||||
except httpx.HTTPError:
|
||||
return []
|
||||
if resp.status_code != 200:
|
||||
return []
|
||||
|
||||
parsed = feedparser.parse(resp.content)
|
||||
out: list[dict] = []
|
||||
for entry in parsed.entries:
|
||||
video_id = entry.get("yt_videoid")
|
||||
if not video_id:
|
||||
continue
|
||||
published = None
|
||||
if entry.get("published_parsed"):
|
||||
published = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc)
|
||||
thumbnail = None
|
||||
media = entry.get("media_thumbnail") or []
|
||||
if media:
|
||||
thumbnail = media[0].get("url")
|
||||
out.append(
|
||||
{
|
||||
"id": video_id,
|
||||
"channel_id": channel_id,
|
||||
"title": entry.get("title"),
|
||||
"published_at": published,
|
||||
"thumbnail_url": thumbnail,
|
||||
}
|
||||
)
|
||||
return out
|
||||
Loading…
Add table
Add a link
Reference in a new issue