feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler
- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
This commit is contained in:
npeter83
parent
24b6e0026d
commit
cff1d23071
10 changed files with 569 additions and 4 deletions
227
backend/app/sync/videos.py
Normal file
227
backend/app/sync/videos.py
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
"""Video ingestion: free RSS detection, recent-first (and deep) backfill from the
|
||||
uploads playlist, and enrichment via videos.list (duration, stats, category, Shorts
|
||||
and livestream classification)."""
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.models import Channel, Video
|
||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
||||
from app.youtube.rss import fetch_channel_feed
|
||||
|
||||
_DURATION_RE = re.compile(
|
||||
r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def parse_iso8601_duration(value: str | None) -> int | None:
|
||||
if not value:
|
||||
return None
|
||||
match = _DURATION_RE.fullmatch(value)
|
||||
if not match:
|
||||
return None
|
||||
days, hours, minutes, seconds = (int(g) if g else 0 for g in match.groups())
|
||||
return days * 86400 + hours * 3600 + minutes * 60 + seconds
|
||||
|
||||
|
||||
def parse_dt(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _insert_stubs(db: Session, rows: list[dict]) -> int:
|
||||
"""Idempotently insert video stubs; returns the number of newly inserted rows.
|
||||
|
||||
Uses RETURNING so the count is exact (ON CONFLICT DO NOTHING leaves rowcount
|
||||
unreliable across drivers). Duplicates within the batch are dropped first so a
|
||||
single INSERT can't raise CardinalityViolation on the conflict target."""
|
||||
if not rows:
|
||||
return 0
|
||||
seen: dict[str, dict] = {}
|
||||
for row in rows:
|
||||
seen[row["id"]] = row
|
||||
stmt = (
|
||||
pg_insert(Video)
|
||||
.values(list(seen.values()))
|
||||
.on_conflict_do_nothing(index_elements=[Video.id])
|
||||
.returning(Video.id)
|
||||
)
|
||||
inserted = len(db.execute(stmt).fetchall())
|
||||
db.commit()
|
||||
return inserted
|
||||
|
||||
|
||||
# --- RSS (free) ---
|
||||
def poll_rss_channel(db: Session, channel: Channel) -> int:
|
||||
entries = fetch_channel_feed(channel.id)
|
||||
inserted = _insert_stubs(db, entries)
|
||||
channel.last_rss_at = _now()
|
||||
db.add(channel)
|
||||
db.commit()
|
||||
return inserted
|
||||
|
||||
|
||||
# --- Backfill (uploads playlist; costs quota) ---
|
||||
def _stub_from_playlist_item(item: dict, channel_id: str) -> dict | None:
|
||||
content = item.get("contentDetails", {})
|
||||
snippet = item.get("snippet", {})
|
||||
video_id = content.get("videoId")
|
||||
if not video_id:
|
||||
return None
|
||||
published = parse_dt(content.get("videoPublishedAt") or snippet.get("publishedAt"))
|
||||
return {
|
||||
"id": video_id,
|
||||
"channel_id": channel_id,
|
||||
"title": snippet.get("title"),
|
||||
"published_at": published,
|
||||
"thumbnail_url": best_thumbnail(snippet.get("thumbnails")),
|
||||
}
|
||||
|
||||
|
||||
def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> int:
|
||||
"""First pass: newest videos up to the configured count or age cutoff. Records a
|
||||
resume cursor for the later deep backfill."""
|
||||
if not channel.uploads_playlist_id:
|
||||
channel.recent_synced_at = _now()
|
||||
db.add(channel)
|
||||
db.commit()
|
||||
return 0
|
||||
|
||||
cutoff = _now() - timedelta(days=settings.backfill_recent_max_days)
|
||||
collected: list[dict] = []
|
||||
page_token: str | None = None
|
||||
next_cursor: str | None = None
|
||||
done = False
|
||||
|
||||
while len(collected) < settings.backfill_recent_max_videos:
|
||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
||||
reached_cutoff = False
|
||||
for item in data.get("items", []):
|
||||
stub = _stub_from_playlist_item(item, channel.id)
|
||||
if stub is None:
|
||||
continue
|
||||
if stub["published_at"] and stub["published_at"] < cutoff:
|
||||
reached_cutoff = True
|
||||
break
|
||||
collected.append(stub)
|
||||
if len(collected) >= settings.backfill_recent_max_videos:
|
||||
break
|
||||
page_token = data.get("nextPageToken")
|
||||
if reached_cutoff or not page_token:
|
||||
next_cursor = page_token
|
||||
done = not page_token
|
||||
break
|
||||
next_cursor = page_token
|
||||
|
||||
channel.recent_synced_at = _now()
|
||||
channel.backfill_cursor = next_cursor
|
||||
channel.backfill_done = done
|
||||
inserted = _insert_stubs(db, collected)
|
||||
db.add(channel)
|
||||
db.commit()
|
||||
return inserted
|
||||
|
||||
|
||||
def backfill_channel_deep(
|
||||
db: Session, yt: YouTubeClient, channel: Channel, max_pages: int = 5
|
||||
) -> int:
|
||||
"""Continue paging older uploads from the stored cursor (bounded per run)."""
|
||||
if channel.backfill_done or not channel.uploads_playlist_id:
|
||||
return 0
|
||||
page_token = channel.backfill_cursor
|
||||
total = 0
|
||||
for _ in range(max_pages):
|
||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
||||
rows = [
|
||||
s
|
||||
for s in (
|
||||
_stub_from_playlist_item(it, channel.id) for it in data.get("items", [])
|
||||
)
|
||||
if s is not None
|
||||
]
|
||||
total += _insert_stubs(db, rows)
|
||||
page_token = data.get("nextPageToken")
|
||||
channel.backfill_cursor = page_token
|
||||
if not page_token:
|
||||
channel.backfill_done = True
|
||||
break
|
||||
db.add(channel)
|
||||
db.commit()
|
||||
return total
|
||||
|
||||
|
||||
# --- Enrichment (videos.list) ---
|
||||
def _live_status(snippet: dict, live_details: dict | None) -> str:
|
||||
broadcast = snippet.get("liveBroadcastContent")
|
||||
if broadcast == "live":
|
||||
return "live"
|
||||
if broadcast == "upcoming":
|
||||
return "upcoming"
|
||||
if live_details and live_details.get("actualEndTime"):
|
||||
return "was_live"
|
||||
return "none"
|
||||
|
||||
|
||||
def apply_video_details(video: Video, item: dict) -> None:
|
||||
snippet = item.get("snippet", {})
|
||||
content = item.get("contentDetails", {})
|
||||
stats = item.get("statistics", {})
|
||||
topics = item.get("topicDetails", {})
|
||||
live = item.get("liveStreamingDetails")
|
||||
|
||||
video.title = snippet.get("title") or video.title
|
||||
video.description = snippet.get("description")
|
||||
if not video.published_at:
|
||||
video.published_at = parse_dt(snippet.get("publishedAt"))
|
||||
video.thumbnail_url = best_thumbnail(snippet.get("thumbnails")) or video.thumbnail_url
|
||||
video.duration_seconds = parse_iso8601_duration(content.get("duration"))
|
||||
video.view_count = int(stats["viewCount"]) if stats.get("viewCount") else None
|
||||
video.like_count = int(stats["likeCount"]) if stats.get("likeCount") else None
|
||||
video.category_id = int(snippet["categoryId"]) if snippet.get("categoryId") else None
|
||||
video.topic_categories = topics.get("topicCategories")
|
||||
video.default_language = snippet.get("defaultLanguage") or snippet.get(
|
||||
"defaultAudioLanguage"
|
||||
)
|
||||
video.live_status = _live_status(snippet, live)
|
||||
video.is_short = bool(
|
||||
video.live_status == "none"
|
||||
and video.duration_seconds
|
||||
and 0 < video.duration_seconds <= settings.shorts_max_seconds
|
||||
)
|
||||
|
||||
|
||||
def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int:
|
||||
limit = limit or settings.enrich_batch_size
|
||||
videos = (
|
||||
db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not videos:
|
||||
return 0
|
||||
items = {it["id"]: it for it in yt.get_videos([v.id for v in videos])}
|
||||
now = _now()
|
||||
enriched = 0
|
||||
for video in videos:
|
||||
item = items.get(video.id)
|
||||
if item is None:
|
||||
# Unavailable (deleted/private): stamp so we don't keep retrying.
|
||||
video.enriched_at = now
|
||||
continue
|
||||
apply_video_details(video, item)
|
||||
video.enriched_at = now
|
||||
enriched += 1
|
||||
db.commit()
|
||||
return enriched
|
||||
Loading…
Add table
Add a link
Reference in a new issue