feat(downloads): clickable channels + poster fallback for thumbnail-less sources

Two visual gaps for non-catalog downloads:
- Channel link: YouTube already exposes channel_url; Facebook exposes none but a
  numeric uploader_id that resolves at facebook.com/<id>. `_uploader_url` derives
  it so the auto-detected channel renders as a real clickable link.
- Poster: a source with no thumbnail (e.g. a direct reddit HLS URL) showed a
  blank image box. The worker now cuts a representative frame with ffmpeg
  (`ensure_poster`) into the `<base>.jpg` sidecar and records `poster_path`
  (migration 0050). The card, the public watch page (<video poster> + og:image),
  and link previews fall back to it via new authed + public poster endpoints.

Adds `app.downloads.backfill` (one-off, re-run-safe) to fill uploader_url
(re-extract YouTube/Facebook metadata) and posters for pre-existing downloads.
This commit is contained in:
npeter83 2026-07-07 22:28:49 +02:00
parent 374ad4ddc4
commit cb170dfd32
12 changed files with 251 additions and 8 deletions

View file

@ -0,0 +1,28 @@
"""local poster path for thumbnail-less downloads
Revision ID: 0050_asset_poster
Revises: 0049_download_display_meta
Create Date: 2026-07-07
Some sources (a direct media URL like a reddit v.redd.it HLS manifest) carry no thumbnail, so the
download shows a blank image box. The worker now extracts a representative frame with ffmpeg and
saves it as a poster sidecar; `poster_path` (relative to download_root) records it so the card /
watch page / link preview can fall back to it. Nullable only set when there was no thumbnail.
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0050_asset_poster"
down_revision: Union[str, None] = "0049_download_display_meta"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("media_assets", sa.Column("poster_path", sa.Text(), nullable=True))
def downgrade() -> None:
op.drop_column("media_assets", "poster_path")

View file

@ -0,0 +1,94 @@
"""One-off backfill for downloads made before per-asset channel URLs + poster fallbacks existed.
Fills `media_assets.uploader_url` (re-extracting metadata for YouTube/Facebook sources, so an
auto-detected channel becomes a clickable link) and `poster_path` (an ffmpeg frame for
thumbnail-less sources, so the card / watch page / link preview still shows an image). Best-effort
and safe to re-run: it only touches assets still missing the field, and only re-fetches YouTube /
Facebook sources (a direct media URL like a reddit HLS manifest carries no channel and its signed
URL may have expired those just get a locally-generated poster instead).
Run once per environment:
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
"""
import logging
from urllib.parse import urlparse
import yt_dlp
from sqlalchemy import select
from app.config import settings
from app.db import SessionLocal
from app.downloads import edit as editmod
from app.downloads import formats, service
from app.models import MediaAsset
from app.worker import _uploader_url
log = logging.getLogger("siftlode.backfill")
def _should_refetch(asset: MediaAsset) -> bool:
"""Only YouTube + Facebook sources have a derivable channel URL worth a network round-trip."""
if asset.source_kind == "youtube":
return True
if asset.source_kind == "url":
host = urlparse(asset.source_ref).netloc.lower()
return "facebook.com" in host or "fb.watch" in host
return False
def _extract_meta(url: str) -> dict | None:
"""Metadata-only yt-dlp extraction, trying the same client sets as the worker (YouTube needs the
POT/player-client strategy even just to extract)."""
client_sets = [
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
] or [None]
for clients in client_sets:
opts = formats.build_ydl_opts(
{}, "%(id)s.%(ext)s", lambda d: None, settings.download_pot_base_url, clients
)
opts["skip_download"] = True
try:
with yt_dlp.YoutubeDL(opts) as ydl:
info = ydl.extract_info(url, download=False)
if info:
return info
except Exception as exc: # noqa: BLE001 — best-effort backfill, keep going
log.info("meta extract failed for %s (%s): %s", url, clients, str(exc)[:100])
return None
def run() -> dict:
filled_url = filled_poster = 0
with SessionLocal() as db:
assets = (
db.execute(
select(MediaAsset).where(
MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)
)
)
.scalars()
.all()
)
for a in assets:
if a.uploader_url is None and _should_refetch(a):
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
info = _extract_meta(url)
uu = _uploader_url(info) if info else None
if uu:
a.uploader_url = uu
filled_url += 1
log.info("asset %s uploader_url <- %s", a.id, uu)
if a.thumbnail_url is None and a.poster_path is None:
rel = editmod.ensure_poster(a, settings.download_root)
if rel:
a.poster_path = rel
filled_poster += 1
log.info("asset %s poster <- %s", a.id, rel)
db.commit()
result = {
"assets": len(assets),
"filled_uploader_url": filled_url,
"filled_poster": filled_poster,
}
log.info("backfill done: %s", result)
return result

View file

@ -298,6 +298,40 @@ def build_storyboard_cmd(src: Path, dest: Path, duration_s: int | None, geom: di
]
def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
"""Grab one representative frame as a poster JPEG. `-ss` before `-i` seeks fast."""
return [
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
"-ss", f"{max(0.0, at_s):.3f}", "-i", str(src),
"-frames:v", "1", "-q:v", "3", "-vf", "scale=640:-2", str(dest),
]
def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
"""Generate a poster frame for a ready asset that has no thumbnail (e.g. a direct media URL
like a reddit HLS manifest, which yt-dlp can't get a thumbnail for). Written as the media's
`<base>.jpg` sidecar so Plex uses it too and `delete_asset_files` cleans it up and returned
as a rel path (or None on failure). Best-effort + time-bounded; a few seconds in (or 10% of a
short clip) dodges black intro frames."""
if not asset.rel_path:
return None
root = Path(download_root)
src = root / asset.rel_path
if not src.exists():
return None
rel = str(Path(asset.rel_path).with_suffix(".jpg"))
dest = root / rel
if dest.exists():
return rel
at = min(3.0, (asset.duration_s or 10) * 0.1)
try:
subprocess.run(build_poster_cmd(src, dest, at), capture_output=True, timeout=timeout_s, check=True)
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError) as exc:
log.info("poster gen skipped for asset %s: %s", asset.id, str(exc)[:120])
return None
return rel if dest.exists() else None
def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict | None:
"""Return the filmstrip meta for a ready source asset, generating the sprite on first use.

View file

@ -68,7 +68,7 @@ def owner_view(link: DownloadLink, base_url: str = "") -> dict:
}
def public_meta(link: DownloadLink, job, asset, file_url: str) -> dict:
def public_meta(link: DownloadLink, job, asset, file_url: str, poster_url: str | None = None) -> dict:
"""Metadata for the public watch page (only ever returned once access is authorized).
Resolves the owner's per-download display overrides (custom title/channel) over the asset's
@ -93,4 +93,5 @@ def public_meta(link: DownloadLink, job, asset, file_url: str) -> dict:
"height": asset.height,
"allow_download": link.allow_download,
"file_url": file_url,
"poster_url": poster_url,
}

View file

@ -50,7 +50,12 @@ def _watch_tags(token: str, db: Session) -> str | None:
return None
title = (job.display_name or asset.title or "Video").strip()
channel = (job.display_uploader or asset.uploader or "").strip()
image = asset.thumbnail_url # already an absolute (youtube/fb) URL, or None
# Prefer the source thumbnail; fall back to our generated poster (absolute, token-gated URL) so
# even a thumbnail-less video (e.g. a reddit clip) unfurls with an image. (This block only runs
# for non-password links, so the poster needs no grant.)
image = asset.thumbnail_url or (
f"{settings.app_base}/api/public/watch/{token}/poster.jpg" if asset.poster_path else None
)
tags = [
f"<title>{html.escape(title)}</title>",
_tag("og:title", title),

View file

@ -722,6 +722,9 @@ class MediaAsset(Base, TimestampMixin):
) # "pending" | "ready" | "error"
rel_path: Mapped[str | None] = mapped_column(Text) # relative to download_root
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
# card / watch page / link preview can still show an image. Relative to download_root.
poster_path: Mapped[str | None] = mapped_column(Text)
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
duration_s: Mapped[int | None] = mapped_column(Integer)
width: Mapped[int | None] = mapped_column(Integer)

View file

@ -97,6 +97,10 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
"source_kind": job.source_kind,
"source_ref": job.source_ref,
"source_url": _reference_url(job, asset),
# A locally-generated poster to fall back to when the source had no thumbnail.
"poster_url": f"/api/downloads/{job.id}/poster.jpg"
if (asset is not None and asset.poster_path)
else None,
"profile_id": job.profile_id,
"spec": job.profile_snapshot,
"display_name": job.display_name,
@ -588,6 +592,22 @@ def storyboard_meta(
}
@router.get("/{job_id}/poster.jpg")
def poster_image(
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
):
"""The locally-generated poster frame for a download whose source had no thumbnail."""
job = _accessible_job(db, user, job_id)
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or not asset.poster_path:
raise HTTPException(status_code=404, detail="No poster.")
path = storage.abs_path(settings.download_root, asset.poster_path).resolve()
root = Path(settings.download_root).resolve()
if root not in path.parents or not path.exists():
raise HTTPException(status_code=404, detail="No poster.")
return FileResponse(path, media_type="image/jpeg")
@router.get("/{job_id}/storyboard.jpg")
def storyboard_image(
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)

View file

@ -58,6 +58,13 @@ def _file_url(token: str, grant: str | None = None) -> str:
return f"{base}?g={grant}" if grant else base
def _poster_url(token: str, asset, grant: str | None = None) -> str | None:
if not asset.poster_path:
return None
base = f"/api/public/watch/{token}/poster.jpg"
return f"{base}?g={grant}" if grant else base
@router.get("/watch/{token}")
def watch_meta(token: str, db: Session = Depends(get_db)) -> dict:
"""Public metadata for the watch page. For a password link, returns only `needs_password`
@ -68,7 +75,10 @@ def watch_meta(token: str, db: Session = Depends(get_db)) -> dict:
job, asset = _ready_target(db, link)
link.view_count += 1
db.commit()
return {"needs_password": False, **linksmod.public_meta(link, job, asset, _file_url(token))}
return {
"needs_password": False,
**linksmod.public_meta(link, job, asset, _file_url(token), _poster_url(token, asset)),
}
@router.post("/watch/{token}/unlock")
@ -80,14 +90,22 @@ def watch_unlock(token: str, payload: dict, db: Session = Depends(get_db)) -> di
if not link.password_hash:
# Not a password link — nothing to unlock; just hand back the plain meta.
job, asset = _ready_target(db, link)
return {"needs_password": False, **linksmod.public_meta(link, job, asset, _file_url(token))}
return {
"needs_password": False,
**linksmod.public_meta(link, job, asset, _file_url(token), _poster_url(token, asset)),
}
if not verify_password((payload.get("password") or ""), link.password_hash):
raise HTTPException(status_code=403, detail="Wrong password.")
job, asset = _ready_target(db, link)
link.view_count += 1
db.commit()
grant = linksmod.make_grant(token)
return {"needs_password": False, **linksmod.public_meta(link, job, asset, _file_url(token, grant))}
return {
"needs_password": False,
**linksmod.public_meta(
link, job, asset, _file_url(token, grant), _poster_url(token, asset, grant)
),
}
@router.get("/watch/{token}/file")
@ -111,3 +129,20 @@ def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)):
disposition = "attachment" if link.allow_download else "inline"
# Starlette's FileResponse honours the Range header (206) for seeking/scrubbing.
return FileResponse(path, filename=filename, content_disposition_type=disposition)
@router.get("/watch/{token}/poster.jpg")
def watch_poster(token: str, g: str | None = None, db: Session = Depends(get_db)):
"""The generated poster frame for a shared thumbnail-less video (used as the page/video poster
and the link-preview image). Password links require a valid grant, like the file."""
link = _link_or_404(db, token)
if link.password_hash and not linksmod.check_grant(token, g):
raise HTTPException(status_code=403, detail="This link needs a password.")
job, asset = _ready_target(db, link)
if not asset.poster_path:
raise HTTPException(status_code=404, detail="No poster.")
path = storage.abs_path(settings.download_root, asset.poster_path).resolve()
root = Path(settings.download_root).resolve()
if root not in path.parents or not path.exists():
raise HTTPException(status_code=404, detail="No poster.")
return FileResponse(path, media_type="image/jpeg")

View file

@ -209,6 +209,21 @@ def _finish_from_cache(job_id: int, asset_id: int) -> None:
db.commit()
def _uploader_url(info: dict) -> str | None:
"""The channel/uploader page URL for the auto-clickable channel line. YouTube gives a
`channel_url`/`uploader_url` directly; Facebook gives neither but exposes a numeric
`uploader_id` (the page id) that resolves at facebook.com/<id>. Returns None for sources with
no derivable channel page (e.g. reddit / a raw media URL)."""
url = info.get("channel_url") or info.get("uploader_url")
if url:
return url
uid = info.get("uploader_id")
tag = f"{info.get('extractor_key') or info.get('extractor') or ''} {info.get('webpage_url') or ''}".lower()
if uid and "facebook" in tag:
return f"https://www.facebook.com/{uid}"
return None
def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
"""Populate an asset's display metadata from yt-dlp's info_dict the first time it's seen
(only if not already filled at enqueue from our catalog) so an ad-hoc URL's row shows a
@ -219,7 +234,7 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
return
asset.title = normalize_title(info.get("title"))
asset.uploader = info.get("uploader") or info.get("channel")
asset.uploader_url = info.get("channel_url") or info.get("uploader_url")
asset.uploader_url = _uploader_url(info)
asset.thumbnail_url = info.get("thumbnail")
asset.duration_s = int(info["duration"]) if info.get("duration") else None
asset.upload_date = _parse_upload_date(info.get("upload_date"))
@ -410,9 +425,13 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
asset.title = meta.title
asset.uploader = meta.uploader
asset.uploader_url = info.get("channel_url") or info.get("uploader_url")
asset.uploader_url = _uploader_url(info)
asset.upload_date = meta.upload_date
asset.thumbnail_url = meta.thumbnail_url
# No thumbnail from the source (e.g. a direct media URL) → cut a poster frame from
# the file so the card / watch page / link preview still shows an image.
if not meta.thumbnail_url:
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
asset.source_webpage_url = info.get("webpage_url")
asset.nfo_written = nfo_ok
asset.error = None