feat(downloads): clickable channels + poster fallback for thumbnail-less sources
Two visual gaps for non-catalog downloads: - Channel link: YouTube already exposes channel_url; Facebook exposes none but a numeric uploader_id that resolves at facebook.com/<id>. `_uploader_url` derives it so the auto-detected channel renders as a real clickable link. - Poster: a source with no thumbnail (e.g. a direct reddit HLS URL) showed a blank image box. The worker now cuts a representative frame with ffmpeg (`ensure_poster`) into the `<base>.jpg` sidecar and records `poster_path` (migration 0050). The card, the public watch page (<video poster> + og:image), and link previews fall back to it via new authed + public poster endpoints. Adds `app.downloads.backfill` (one-off, re-run-safe) to fill uploader_url (re-extract YouTube/Facebook metadata) and posters for pre-existing downloads.
This commit is contained in:
parent
374ad4ddc4
commit
cb170dfd32
12 changed files with 251 additions and 8 deletions
28
backend/alembic/versions/0050_asset_poster.py
Normal file
28
backend/alembic/versions/0050_asset_poster.py
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
"""local poster path for thumbnail-less downloads
|
||||
|
||||
Revision ID: 0050_asset_poster
|
||||
Revises: 0049_download_display_meta
|
||||
Create Date: 2026-07-07
|
||||
|
||||
Some sources (a direct media URL like a reddit v.redd.it HLS manifest) carry no thumbnail, so the
|
||||
download shows a blank image box. The worker now extracts a representative frame with ffmpeg and
|
||||
saves it as a poster sidecar; `poster_path` (relative to download_root) records it so the card /
|
||||
watch page / link preview can fall back to it. Nullable — only set when there was no thumbnail.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0050_asset_poster"
|
||||
down_revision: Union[str, None] = "0049_download_display_meta"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("media_assets", sa.Column("poster_path", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("media_assets", "poster_path")
|
||||
94
backend/app/downloads/backfill.py
Normal file
94
backend/app/downloads/backfill.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
"""One-off backfill for downloads made before per-asset channel URLs + poster fallbacks existed.
|
||||
|
||||
Fills `media_assets.uploader_url` (re-extracting metadata for YouTube/Facebook sources, so an
|
||||
auto-detected channel becomes a clickable link) and `poster_path` (an ffmpeg frame for
|
||||
thumbnail-less sources, so the card / watch page / link preview still shows an image). Best-effort
|
||||
and safe to re-run: it only touches assets still missing the field, and only re-fetches YouTube /
|
||||
Facebook sources (a direct media URL like a reddit HLS manifest carries no channel and its signed
|
||||
URL may have expired — those just get a locally-generated poster instead).
|
||||
|
||||
Run once per environment:
|
||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||
"""
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, service
|
||||
from app.models import MediaAsset
|
||||
from app.worker import _uploader_url
|
||||
|
||||
log = logging.getLogger("siftlode.backfill")
|
||||
|
||||
|
||||
def _should_refetch(asset: MediaAsset) -> bool:
|
||||
"""Only YouTube + Facebook sources have a derivable channel URL worth a network round-trip."""
|
||||
if asset.source_kind == "youtube":
|
||||
return True
|
||||
if asset.source_kind == "url":
|
||||
host = urlparse(asset.source_ref).netloc.lower()
|
||||
return "facebook.com" in host or "fb.watch" in host
|
||||
return False
|
||||
|
||||
|
||||
def _extract_meta(url: str) -> dict | None:
|
||||
"""Metadata-only yt-dlp extraction, trying the same client sets as the worker (YouTube needs the
|
||||
POT/player-client strategy even just to extract)."""
|
||||
client_sets = [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
for clients in client_sets:
|
||||
opts = formats.build_ydl_opts(
|
||||
{}, "%(id)s.%(ext)s", lambda d: None, settings.download_pot_base_url, clients
|
||||
)
|
||||
opts["skip_download"] = True
|
||||
try:
|
||||
with yt_dlp.YoutubeDL(opts) as ydl:
|
||||
info = ydl.extract_info(url, download=False)
|
||||
if info:
|
||||
return info
|
||||
except Exception as exc: # noqa: BLE001 — best-effort backfill, keep going
|
||||
log.info("meta extract failed for %s (%s): %s", url, clients, str(exc)[:100])
|
||||
return None
|
||||
|
||||
|
||||
def run() -> dict:
|
||||
filled_url = filled_poster = 0
|
||||
with SessionLocal() as db:
|
||||
assets = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for a in assets:
|
||||
if a.uploader_url is None and _should_refetch(a):
|
||||
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
||||
info = _extract_meta(url)
|
||||
uu = _uploader_url(info) if info else None
|
||||
if uu:
|
||||
a.uploader_url = uu
|
||||
filled_url += 1
|
||||
log.info("asset %s uploader_url <- %s", a.id, uu)
|
||||
if a.thumbnail_url is None and a.poster_path is None:
|
||||
rel = editmod.ensure_poster(a, settings.download_root)
|
||||
if rel:
|
||||
a.poster_path = rel
|
||||
filled_poster += 1
|
||||
log.info("asset %s poster <- %s", a.id, rel)
|
||||
db.commit()
|
||||
result = {
|
||||
"assets": len(assets),
|
||||
"filled_uploader_url": filled_url,
|
||||
"filled_poster": filled_poster,
|
||||
}
|
||||
log.info("backfill done: %s", result)
|
||||
return result
|
||||
|
|
@ -298,6 +298,40 @@ def build_storyboard_cmd(src: Path, dest: Path, duration_s: int | None, geom: di
|
|||
]
|
||||
|
||||
|
||||
def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
|
||||
"""Grab one representative frame as a poster JPEG. `-ss` before `-i` seeks fast."""
|
||||
return [
|
||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
||||
"-ss", f"{max(0.0, at_s):.3f}", "-i", str(src),
|
||||
"-frames:v", "1", "-q:v", "3", "-vf", "scale=640:-2", str(dest),
|
||||
]
|
||||
|
||||
|
||||
def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
||||
"""Generate a poster frame for a ready asset that has no thumbnail (e.g. a direct media URL
|
||||
like a reddit HLS manifest, which yt-dlp can't get a thumbnail for). Written as the media's
|
||||
`<base>.jpg` sidecar — so Plex uses it too and `delete_asset_files` cleans it up — and returned
|
||||
as a rel path (or None on failure). Best-effort + time-bounded; a few seconds in (or 10% of a
|
||||
short clip) dodges black intro frames."""
|
||||
if not asset.rel_path:
|
||||
return None
|
||||
root = Path(download_root)
|
||||
src = root / asset.rel_path
|
||||
if not src.exists():
|
||||
return None
|
||||
rel = str(Path(asset.rel_path).with_suffix(".jpg"))
|
||||
dest = root / rel
|
||||
if dest.exists():
|
||||
return rel
|
||||
at = min(3.0, (asset.duration_s or 10) * 0.1)
|
||||
try:
|
||||
subprocess.run(build_poster_cmd(src, dest, at), capture_output=True, timeout=timeout_s, check=True)
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError) as exc:
|
||||
log.info("poster gen skipped for asset %s: %s", asset.id, str(exc)[:120])
|
||||
return None
|
||||
return rel if dest.exists() else None
|
||||
|
||||
|
||||
def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict | None:
|
||||
"""Return the filmstrip meta for a ready source asset, generating the sprite on first use.
|
||||
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ def owner_view(link: DownloadLink, base_url: str = "") -> dict:
|
|||
}
|
||||
|
||||
|
||||
def public_meta(link: DownloadLink, job, asset, file_url: str) -> dict:
|
||||
def public_meta(link: DownloadLink, job, asset, file_url: str, poster_url: str | None = None) -> dict:
|
||||
"""Metadata for the public watch page (only ever returned once access is authorized).
|
||||
|
||||
Resolves the owner's per-download display overrides (custom title/channel) over the asset's
|
||||
|
|
@ -93,4 +93,5 @@ def public_meta(link: DownloadLink, job, asset, file_url: str) -> dict:
|
|||
"height": asset.height,
|
||||
"allow_download": link.allow_download,
|
||||
"file_url": file_url,
|
||||
"poster_url": poster_url,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,7 +50,12 @@ def _watch_tags(token: str, db: Session) -> str | None:
|
|||
return None
|
||||
title = (job.display_name or asset.title or "Video").strip()
|
||||
channel = (job.display_uploader or asset.uploader or "").strip()
|
||||
image = asset.thumbnail_url # already an absolute (youtube/fb) URL, or None
|
||||
# Prefer the source thumbnail; fall back to our generated poster (absolute, token-gated URL) so
|
||||
# even a thumbnail-less video (e.g. a reddit clip) unfurls with an image. (This block only runs
|
||||
# for non-password links, so the poster needs no grant.)
|
||||
image = asset.thumbnail_url or (
|
||||
f"{settings.app_base}/api/public/watch/{token}/poster.jpg" if asset.poster_path else None
|
||||
)
|
||||
tags = [
|
||||
f"<title>{html.escape(title)}</title>",
|
||||
_tag("og:title", title),
|
||||
|
|
|
|||
|
|
@ -722,6 +722,9 @@ class MediaAsset(Base, TimestampMixin):
|
|||
) # "pending" | "ready" | "error"
|
||||
rel_path: Mapped[str | None] = mapped_column(Text) # relative to download_root
|
||||
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
|
||||
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
|
||||
# card / watch page / link preview can still show an image. Relative to download_root.
|
||||
poster_path: Mapped[str | None] = mapped_column(Text)
|
||||
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
||||
duration_s: Mapped[int | None] = mapped_column(Integer)
|
||||
width: Mapped[int | None] = mapped_column(Integer)
|
||||
|
|
|
|||
|
|
@ -97,6 +97,10 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
|||
"source_kind": job.source_kind,
|
||||
"source_ref": job.source_ref,
|
||||
"source_url": _reference_url(job, asset),
|
||||
# A locally-generated poster to fall back to when the source had no thumbnail.
|
||||
"poster_url": f"/api/downloads/{job.id}/poster.jpg"
|
||||
if (asset is not None and asset.poster_path)
|
||||
else None,
|
||||
"profile_id": job.profile_id,
|
||||
"spec": job.profile_snapshot,
|
||||
"display_name": job.display_name,
|
||||
|
|
@ -588,6 +592,22 @@ def storyboard_meta(
|
|||
}
|
||||
|
||||
|
||||
@router.get("/{job_id}/poster.jpg")
|
||||
def poster_image(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
):
|
||||
"""The locally-generated poster frame for a download whose source had no thumbnail."""
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or not asset.poster_path:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
path = storage.abs_path(settings.download_root, asset.poster_path).resolve()
|
||||
root = Path(settings.download_root).resolve()
|
||||
if root not in path.parents or not path.exists():
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
||||
|
||||
@router.get("/{job_id}/storyboard.jpg")
|
||||
def storyboard_image(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
|
|
|
|||
|
|
@ -58,6 +58,13 @@ def _file_url(token: str, grant: str | None = None) -> str:
|
|||
return f"{base}?g={grant}" if grant else base
|
||||
|
||||
|
||||
def _poster_url(token: str, asset, grant: str | None = None) -> str | None:
|
||||
if not asset.poster_path:
|
||||
return None
|
||||
base = f"/api/public/watch/{token}/poster.jpg"
|
||||
return f"{base}?g={grant}" if grant else base
|
||||
|
||||
|
||||
@router.get("/watch/{token}")
|
||||
def watch_meta(token: str, db: Session = Depends(get_db)) -> dict:
|
||||
"""Public metadata for the watch page. For a password link, returns only `needs_password`
|
||||
|
|
@ -68,7 +75,10 @@ def watch_meta(token: str, db: Session = Depends(get_db)) -> dict:
|
|||
job, asset = _ready_target(db, link)
|
||||
link.view_count += 1
|
||||
db.commit()
|
||||
return {"needs_password": False, **linksmod.public_meta(link, job, asset, _file_url(token))}
|
||||
return {
|
||||
"needs_password": False,
|
||||
**linksmod.public_meta(link, job, asset, _file_url(token), _poster_url(token, asset)),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/watch/{token}/unlock")
|
||||
|
|
@ -80,14 +90,22 @@ def watch_unlock(token: str, payload: dict, db: Session = Depends(get_db)) -> di
|
|||
if not link.password_hash:
|
||||
# Not a password link — nothing to unlock; just hand back the plain meta.
|
||||
job, asset = _ready_target(db, link)
|
||||
return {"needs_password": False, **linksmod.public_meta(link, job, asset, _file_url(token))}
|
||||
return {
|
||||
"needs_password": False,
|
||||
**linksmod.public_meta(link, job, asset, _file_url(token), _poster_url(token, asset)),
|
||||
}
|
||||
if not verify_password((payload.get("password") or ""), link.password_hash):
|
||||
raise HTTPException(status_code=403, detail="Wrong password.")
|
||||
job, asset = _ready_target(db, link)
|
||||
link.view_count += 1
|
||||
db.commit()
|
||||
grant = linksmod.make_grant(token)
|
||||
return {"needs_password": False, **linksmod.public_meta(link, job, asset, _file_url(token, grant))}
|
||||
return {
|
||||
"needs_password": False,
|
||||
**linksmod.public_meta(
|
||||
link, job, asset, _file_url(token, grant), _poster_url(token, asset, grant)
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/watch/{token}/file")
|
||||
|
|
@ -111,3 +129,20 @@ def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)):
|
|||
disposition = "attachment" if link.allow_download else "inline"
|
||||
# Starlette's FileResponse honours the Range header (206) for seeking/scrubbing.
|
||||
return FileResponse(path, filename=filename, content_disposition_type=disposition)
|
||||
|
||||
|
||||
@router.get("/watch/{token}/poster.jpg")
|
||||
def watch_poster(token: str, g: str | None = None, db: Session = Depends(get_db)):
|
||||
"""The generated poster frame for a shared thumbnail-less video (used as the page/video poster
|
||||
and the link-preview image). Password links require a valid grant, like the file."""
|
||||
link = _link_or_404(db, token)
|
||||
if link.password_hash and not linksmod.check_grant(token, g):
|
||||
raise HTTPException(status_code=403, detail="This link needs a password.")
|
||||
job, asset = _ready_target(db, link)
|
||||
if not asset.poster_path:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
path = storage.abs_path(settings.download_root, asset.poster_path).resolve()
|
||||
root = Path(settings.download_root).resolve()
|
||||
if root not in path.parents or not path.exists():
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
|
|
|||
|
|
@ -209,6 +209,21 @@ def _finish_from_cache(job_id: int, asset_id: int) -> None:
|
|||
db.commit()
|
||||
|
||||
|
||||
def _uploader_url(info: dict) -> str | None:
|
||||
"""The channel/uploader page URL for the auto-clickable channel line. YouTube gives a
|
||||
`channel_url`/`uploader_url` directly; Facebook gives neither but exposes a numeric
|
||||
`uploader_id` (the page id) that resolves at facebook.com/<id>. Returns None for sources with
|
||||
no derivable channel page (e.g. reddit / a raw media URL)."""
|
||||
url = info.get("channel_url") or info.get("uploader_url")
|
||||
if url:
|
||||
return url
|
||||
uid = info.get("uploader_id")
|
||||
tag = f"{info.get('extractor_key') or info.get('extractor') or ''} {info.get('webpage_url') or ''}".lower()
|
||||
if uid and "facebook" in tag:
|
||||
return f"https://www.facebook.com/{uid}"
|
||||
return None
|
||||
|
||||
|
||||
def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
||||
"""Populate an asset's display metadata from yt-dlp's info_dict the first time it's seen
|
||||
(only if not already filled at enqueue from our catalog) — so an ad-hoc URL's row shows a
|
||||
|
|
@ -219,7 +234,7 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
|||
return
|
||||
asset.title = normalize_title(info.get("title"))
|
||||
asset.uploader = info.get("uploader") or info.get("channel")
|
||||
asset.uploader_url = info.get("channel_url") or info.get("uploader_url")
|
||||
asset.uploader_url = _uploader_url(info)
|
||||
asset.thumbnail_url = info.get("thumbnail")
|
||||
asset.duration_s = int(info["duration"]) if info.get("duration") else None
|
||||
asset.upload_date = _parse_upload_date(info.get("upload_date"))
|
||||
|
|
@ -410,9 +425,13 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
|||
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||
asset.title = meta.title
|
||||
asset.uploader = meta.uploader
|
||||
asset.uploader_url = info.get("channel_url") or info.get("uploader_url")
|
||||
asset.uploader_url = _uploader_url(info)
|
||||
asset.upload_date = meta.upload_date
|
||||
asset.thumbnail_url = meta.thumbnail_url
|
||||
# No thumbnail from the source (e.g. a direct media URL) → cut a poster frame from
|
||||
# the file so the card / watch page / link preview still shows an image.
|
||||
if not meta.thumbnail_url:
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||
asset.source_webpage_url = info.get("webpage_url")
|
||||
asset.nfo_written = nfo_ok
|
||||
asset.error = None
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue