95 lines
3.7 KiB
Python
95 lines
3.7 KiB
Python
|
"""One-off backfill for downloads made before per-asset channel URLs + poster fallbacks existed.
|
||
|
|
|
||
|
|
Fills `media_assets.uploader_url` (re-extracting metadata for YouTube/Facebook sources, so an
|
||
|
|
auto-detected channel becomes a clickable link) and `poster_path` (an ffmpeg frame for
|
||
|
|
thumbnail-less sources, so the card / watch page / link preview still shows an image). Best-effort
|
||
|
|
and safe to re-run: it only touches assets still missing the field, and only re-fetches YouTube /
|
||
|
|
Facebook sources (a direct media URL like a reddit HLS manifest carries no channel and its signed
|
||
|
|
URL may have expired — those just get a locally-generated poster instead).
|
||
|
|
|
||
|
|
Run once per environment:
|
||
|
|
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||
|
|
"""
|
||
|
|
import logging
|
||
|
|
from urllib.parse import urlparse
|
||
|
|
|
||
|
|
import yt_dlp
|
||
|
|
from sqlalchemy import select
|
||
|
|
|
||
|
|
from app.config import settings
|
||
|
|
from app.db import SessionLocal
|
||
|
|
from app.downloads import edit as editmod
|
||
|
|
from app.downloads import formats, service
|
||
|
|
from app.models import MediaAsset
|
||
|
|
from app.worker import _uploader_url
|
||
|
|
|
||
|
|
log = logging.getLogger("siftlode.backfill")
|
||
|
|
|
||
|
|
|
||
|
|
def _should_refetch(asset: MediaAsset) -> bool:
|
||
|
|
"""Only YouTube + Facebook sources have a derivable channel URL worth a network round-trip."""
|
||
|
|
if asset.source_kind == "youtube":
|
||
|
|
return True
|
||
|
|
if asset.source_kind == "url":
|
||
|
|
host = urlparse(asset.source_ref).netloc.lower()
|
||
|
|
return "facebook.com" in host or "fb.watch" in host
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
def _extract_meta(url: str) -> dict | None:
|
||
|
|
"""Metadata-only yt-dlp extraction, trying the same client sets as the worker (YouTube needs the
|
||
|
|
POT/player-client strategy even just to extract)."""
|
||
|
|
client_sets = [
|
||
|
|
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||
|
|
] or [None]
|
||
|
|
for clients in client_sets:
|
||
|
|
opts = formats.build_ydl_opts(
|
||
|
|
{}, "%(id)s.%(ext)s", lambda d: None, settings.download_pot_base_url, clients
|
||
|
|
)
|
||
|
|
opts["skip_download"] = True
|
||
|
|
try:
|
||
|
|
with yt_dlp.YoutubeDL(opts) as ydl:
|
||
|
|
info = ydl.extract_info(url, download=False)
|
||
|
|
if info:
|
||
|
|
return info
|
||
|
|
except Exception as exc: # noqa: BLE001 — best-effort backfill, keep going
|
||
|
|
log.info("meta extract failed for %s (%s): %s", url, clients, str(exc)[:100])
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def run() -> dict:
|
||
|
|
filled_url = filled_poster = 0
|
||
|
|
with SessionLocal() as db:
|
||
|
|
assets = (
|
||
|
|
db.execute(
|
||
|
|
select(MediaAsset).where(
|
||
|
|
MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)
|
||
|
|
)
|
||
|
|
)
|
||
|
|
.scalars()
|
||
|
|
.all()
|
||
|
|
)
|
||
|
|
for a in assets:
|
||
|
|
if a.uploader_url is None and _should_refetch(a):
|
||
|
|
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
||
|
|
info = _extract_meta(url)
|
||
|
|
uu = _uploader_url(info) if info else None
|
||
|
|
if uu:
|
||
|
|
a.uploader_url = uu
|
||
|
|
filled_url += 1
|
||
|
|
log.info("asset %s uploader_url <- %s", a.id, uu)
|
||
|
|
if a.thumbnail_url is None and a.poster_path is None:
|
||
|
|
rel = editmod.ensure_poster(a, settings.download_root)
|
||
|
|
if rel:
|
||
|
|
a.poster_path = rel
|
||
|
|
filled_poster += 1
|
||
|
|
log.info("asset %s poster <- %s", a.id, rel)
|
||
|
|
db.commit()
|
||
|
|
result = {
|
||
|
|
"assets": len(assets),
|
||
|
|
"filled_uploader_url": filled_url,
|
||
|
|
"filled_poster": filled_poster,
|
||
|
|
}
|
||
|
|
log.info("backfill done: %s", result)
|
||
|
|
return result
|