"""One-off backfill for downloads made before per-asset channel URLs + poster fallbacks existed. Fills `media_assets.uploader_url` (re-extracting metadata for YouTube/Facebook sources, so an auto-detected channel becomes a clickable link) and `poster_path` (an ffmpeg frame for thumbnail-less sources, so the card / watch page / link preview still shows an image). Best-effort and safe to re-run: it only touches assets still missing the field, and only re-fetches YouTube / Facebook sources (a direct media URL like a reddit HLS manifest carries no channel and its signed URL may have expired — those just get a locally-generated poster instead). Run once per environment: docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())" """ import logging from urllib.parse import urlparse import yt_dlp from sqlalchemy import select from app.config import settings from app.db import SessionLocal from app.downloads import edit as editmod from app.downloads import formats, service from app.models import MediaAsset from app.worker import _uploader_url log = logging.getLogger("siftlode.backfill") def _should_refetch(asset: MediaAsset) -> bool: """Only YouTube + Facebook sources have a derivable channel URL worth a network round-trip.""" if asset.source_kind == "youtube": return True if asset.source_kind == "url": host = urlparse(asset.source_ref).netloc.lower() return "facebook.com" in host or "fb.watch" in host return False def _extract_meta(url: str) -> dict | None: """Metadata-only yt-dlp extraction, trying the same client sets as the worker (YouTube needs the POT/player-client strategy even just to extract).""" client_sets = [ c for c in (settings.player_client_list, settings.player_client_fallback_list) if c ] or [None] for clients in client_sets: opts = formats.build_ydl_opts( {}, "%(id)s.%(ext)s", lambda d: None, settings.download_pot_base_url, clients ) opts["skip_download"] = True try: with yt_dlp.YoutubeDL(opts) as ydl: info = ydl.extract_info(url, download=False) if info: return info except Exception as exc: # noqa: BLE001 — best-effort backfill, keep going log.info("meta extract failed for %s (%s): %s", url, clients, str(exc)[:100]) return None def run() -> dict: filled_url = filled_poster = 0 with SessionLocal() as db: assets = ( db.execute( select(MediaAsset).where( MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None) ) ) .scalars() .all() ) for a in assets: if a.uploader_url is None and _should_refetch(a): url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref) info = _extract_meta(url) uu = _uploader_url(info) if info else None if uu: a.uploader_url = uu filled_url += 1 log.info("asset %s uploader_url <- %s", a.id, uu) if a.poster_path is None: # Record a self-hosted poster for every download (the reliable og:image source): # ensure_poster returns the existing .jpg thumbnail sidecar, or cuts a frame. rel = editmod.ensure_poster(a, settings.download_root) if rel: a.poster_path = rel filled_poster += 1 log.info("asset %s poster <- %s", a.id, rel) db.commit() result = { "assets": len(assets), "filled_uploader_url": filled_url, "filled_poster": filled_poster, } log.info("backfill done: %s", result) return result