feat(downloads): clickable channels + poster fallback for thumbnail-less sources
Two visual gaps for non-catalog downloads: - Channel link: YouTube already exposes channel_url; Facebook exposes none but a numeric uploader_id that resolves at facebook.com/<id>. `_uploader_url` derives it so the auto-detected channel renders as a real clickable link. - Poster: a source with no thumbnail (e.g. a direct reddit HLS URL) showed a blank image box. The worker now cuts a representative frame with ffmpeg (`ensure_poster`) into the `<base>.jpg` sidecar and records `poster_path` (migration 0050). The card, the public watch page (<video poster> + og:image), and link previews fall back to it via new authed + public poster endpoints. Adds `app.downloads.backfill` (one-off, re-run-safe) to fill uploader_url (re-extract YouTube/Facebook metadata) and posters for pre-existing downloads.
This commit is contained in:
parent
374ad4ddc4
commit
cb170dfd32
12 changed files with 251 additions and 8 deletions
94
backend/app/downloads/backfill.py
Normal file
94
backend/app/downloads/backfill.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
"""One-off backfill for downloads made before per-asset channel URLs + poster fallbacks existed.
|
||||
|
||||
Fills `media_assets.uploader_url` (re-extracting metadata for YouTube/Facebook sources, so an
|
||||
auto-detected channel becomes a clickable link) and `poster_path` (an ffmpeg frame for
|
||||
thumbnail-less sources, so the card / watch page / link preview still shows an image). Best-effort
|
||||
and safe to re-run: it only touches assets still missing the field, and only re-fetches YouTube /
|
||||
Facebook sources (a direct media URL like a reddit HLS manifest carries no channel and its signed
|
||||
URL may have expired — those just get a locally-generated poster instead).
|
||||
|
||||
Run once per environment:
|
||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||
"""
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, service
|
||||
from app.models import MediaAsset
|
||||
from app.worker import _uploader_url
|
||||
|
||||
log = logging.getLogger("siftlode.backfill")
|
||||
|
||||
|
||||
def _should_refetch(asset: MediaAsset) -> bool:
|
||||
"""Only YouTube + Facebook sources have a derivable channel URL worth a network round-trip."""
|
||||
if asset.source_kind == "youtube":
|
||||
return True
|
||||
if asset.source_kind == "url":
|
||||
host = urlparse(asset.source_ref).netloc.lower()
|
||||
return "facebook.com" in host or "fb.watch" in host
|
||||
return False
|
||||
|
||||
|
||||
def _extract_meta(url: str) -> dict | None:
|
||||
"""Metadata-only yt-dlp extraction, trying the same client sets as the worker (YouTube needs the
|
||||
POT/player-client strategy even just to extract)."""
|
||||
client_sets = [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
for clients in client_sets:
|
||||
opts = formats.build_ydl_opts(
|
||||
{}, "%(id)s.%(ext)s", lambda d: None, settings.download_pot_base_url, clients
|
||||
)
|
||||
opts["skip_download"] = True
|
||||
try:
|
||||
with yt_dlp.YoutubeDL(opts) as ydl:
|
||||
info = ydl.extract_info(url, download=False)
|
||||
if info:
|
||||
return info
|
||||
except Exception as exc: # noqa: BLE001 — best-effort backfill, keep going
|
||||
log.info("meta extract failed for %s (%s): %s", url, clients, str(exc)[:100])
|
||||
return None
|
||||
|
||||
|
||||
def run() -> dict:
|
||||
filled_url = filled_poster = 0
|
||||
with SessionLocal() as db:
|
||||
assets = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for a in assets:
|
||||
if a.uploader_url is None and _should_refetch(a):
|
||||
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
||||
info = _extract_meta(url)
|
||||
uu = _uploader_url(info) if info else None
|
||||
if uu:
|
||||
a.uploader_url = uu
|
||||
filled_url += 1
|
||||
log.info("asset %s uploader_url <- %s", a.id, uu)
|
||||
if a.thumbnail_url is None and a.poster_path is None:
|
||||
rel = editmod.ensure_poster(a, settings.download_root)
|
||||
if rel:
|
||||
a.poster_path = rel
|
||||
filled_poster += 1
|
||||
log.info("asset %s poster <- %s", a.id, rel)
|
||||
db.commit()
|
||||
result = {
|
||||
"assets": len(assets),
|
||||
"filled_uploader_url": filled_url,
|
||||
"filled_poster": filled_poster,
|
||||
}
|
||||
log.info("backfill done: %s", result)
|
||||
return result
|
||||
|
|
@ -298,6 +298,40 @@ def build_storyboard_cmd(src: Path, dest: Path, duration_s: int | None, geom: di
|
|||
]
|
||||
|
||||
|
||||
def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
|
||||
"""Grab one representative frame as a poster JPEG. `-ss` before `-i` seeks fast."""
|
||||
return [
|
||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
||||
"-ss", f"{max(0.0, at_s):.3f}", "-i", str(src),
|
||||
"-frames:v", "1", "-q:v", "3", "-vf", "scale=640:-2", str(dest),
|
||||
]
|
||||
|
||||
|
||||
def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
||||
"""Generate a poster frame for a ready asset that has no thumbnail (e.g. a direct media URL
|
||||
like a reddit HLS manifest, which yt-dlp can't get a thumbnail for). Written as the media's
|
||||
`<base>.jpg` sidecar — so Plex uses it too and `delete_asset_files` cleans it up — and returned
|
||||
as a rel path (or None on failure). Best-effort + time-bounded; a few seconds in (or 10% of a
|
||||
short clip) dodges black intro frames."""
|
||||
if not asset.rel_path:
|
||||
return None
|
||||
root = Path(download_root)
|
||||
src = root / asset.rel_path
|
||||
if not src.exists():
|
||||
return None
|
||||
rel = str(Path(asset.rel_path).with_suffix(".jpg"))
|
||||
dest = root / rel
|
||||
if dest.exists():
|
||||
return rel
|
||||
at = min(3.0, (asset.duration_s or 10) * 0.1)
|
||||
try:
|
||||
subprocess.run(build_poster_cmd(src, dest, at), capture_output=True, timeout=timeout_s, check=True)
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError) as exc:
|
||||
log.info("poster gen skipped for asset %s: %s", asset.id, str(exc)[:120])
|
||||
return None
|
||||
return rel if dest.exists() else None
|
||||
|
||||
|
||||
def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict | None:
|
||||
"""Return the filmstrip meta for a ready source asset, generating the sprite on first use.
|
||||
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ def owner_view(link: DownloadLink, base_url: str = "") -> dict:
|
|||
}
|
||||
|
||||
|
||||
def public_meta(link: DownloadLink, job, asset, file_url: str) -> dict:
|
||||
def public_meta(link: DownloadLink, job, asset, file_url: str, poster_url: str | None = None) -> dict:
|
||||
"""Metadata for the public watch page (only ever returned once access is authorized).
|
||||
|
||||
Resolves the owner's per-download display overrides (custom title/channel) over the asset's
|
||||
|
|
@ -93,4 +93,5 @@ def public_meta(link: DownloadLink, job, asset, file_url: str) -> dict:
|
|||
"height": asset.height,
|
||||
"allow_download": link.allow_download,
|
||||
"file_url": file_url,
|
||||
"poster_url": poster_url,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,7 +50,12 @@ def _watch_tags(token: str, db: Session) -> str | None:
|
|||
return None
|
||||
title = (job.display_name or asset.title or "Video").strip()
|
||||
channel = (job.display_uploader or asset.uploader or "").strip()
|
||||
image = asset.thumbnail_url # already an absolute (youtube/fb) URL, or None
|
||||
# Prefer the source thumbnail; fall back to our generated poster (absolute, token-gated URL) so
|
||||
# even a thumbnail-less video (e.g. a reddit clip) unfurls with an image. (This block only runs
|
||||
# for non-password links, so the poster needs no grant.)
|
||||
image = asset.thumbnail_url or (
|
||||
f"{settings.app_base}/api/public/watch/{token}/poster.jpg" if asset.poster_path else None
|
||||
)
|
||||
tags = [
|
||||
f"<title>{html.escape(title)}</title>",
|
||||
_tag("og:title", title),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue