merge: Download Center Phase 1 (yt-dlp) into dev

yt-dlp Download Center epic Phase 1: worker+queue, cache/dedup, presets +
profile editor, per-user quota, Plex layout + .nfo/poster, sharing,
retention/GC, admin storage, title normalization, YouTube bot-detection
mitigation (PO-token sidecar + force-ipv4 + Deno). Not yet released to prod.
This commit is contained in:
npeter83 2026-07-04 00:30:25 +02:00
commit c0b138cd9a
41 changed files with 4184 additions and 4 deletions

4
.gitignore vendored
View file

@ -21,6 +21,10 @@ frontend/dist/
*.sql.gz
backups/
# Download center: locally downloaded media (the DOWNLOAD_ROOT bind mount at the repo root).
# Anchored with a leading slash so it does NOT also ignore the backend/app/downloads package.
/downloads/
# OS / editor
.DS_Store
Thumbs.db

View file

@ -34,6 +34,18 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
WORKDIR /app
# ffmpeg: merge separate video+audio streams + postprocessing. Deno: the JavaScript runtime
# yt-dlp uses to solve YouTube's "n" signature challenge for the web player clients (required
# alongside the PO token, else only storyboards are offered). yt-dlp auto-detects deno on PATH.
ARG DENO_VERSION=v2.9.1
RUN apt-get update \
&& apt-get install -y --no-install-recommends ffmpeg unzip curl ca-certificates \
&& curl -fsSL "https://github.com/denoland/deno/releases/download/${DENO_VERSION}/deno-x86_64-unknown-linux-gnu.zip" -o /tmp/deno.zip \
&& unzip -o /tmp/deno.zip -d /usr/local/bin \
&& chmod +x /usr/local/bin/deno \
&& rm /tmp/deno.zip \
&& rm -rf /var/lib/apt/lists/*
COPY backend/requirements.txt .
RUN pip install --upgrade pip && pip install -r requirements.txt

View file

@ -0,0 +1,143 @@
"""download center core schema (yt-dlp)
Revision ID: 0036_download_center
Revises: 0035_saved_views
Create Date: 2026-07-02
The download center's data layer. `media_assets` is the shared file cache (one row per unique
source+format signature); `download_jobs` is the per-user request over it (worker-claimed via
FOR UPDATE SKIP LOCKED, live progress written back for polling); `download_profiles` are the
reusable format presets (builtins seeded in the next migration); `download_shares` grant a file
to another user; `download_quotas` hold admin-set per-user limits (default via sysconfig).
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0036_download_center"
down_revision: Union[str, None] = "0035_saved_views"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"media_assets",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("source_kind", sa.String(length=16), nullable=False),
sa.Column("source_ref", sa.String(length=512), nullable=False),
sa.Column("format_sig", sa.String(length=64), nullable=False),
sa.Column("status", sa.String(length=16), server_default="pending", nullable=False),
sa.Column("rel_path", sa.Text(), nullable=True),
sa.Column("storyboard_path", sa.Text(), nullable=True),
sa.Column("size_bytes", sa.BigInteger(), nullable=True),
sa.Column("duration_s", sa.Integer(), nullable=True),
sa.Column("width", sa.Integer(), nullable=True),
sa.Column("height", sa.Integer(), nullable=True),
sa.Column("container", sa.String(length=16), nullable=True),
sa.Column("vcodec", sa.String(length=32), nullable=True),
sa.Column("acodec", sa.String(length=32), nullable=True),
sa.Column("nfo_written", sa.Boolean(), server_default="false", nullable=False),
sa.Column("title", sa.Text(), nullable=True),
sa.Column("uploader", sa.String(length=255), nullable=True),
sa.Column("upload_date", sa.Date(), nullable=True),
sa.Column("thumbnail_url", sa.Text(), nullable=True),
sa.Column("error", sa.Text(), nullable=True),
sa.Column("ref_count", sa.Integer(), server_default="0", nullable=False),
sa.Column("last_access_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"),
)
op.create_index("ix_media_assets_status", "media_assets", ["status"])
op.create_table(
"download_profiles",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("user_id", sa.Integer(), nullable=True),
sa.Column("name", sa.String(length=80), nullable=False),
sa.Column("spec", sa.JSON(), nullable=False),
sa.Column("is_builtin", sa.Boolean(), server_default="false", nullable=False),
sa.Column("sort_order", sa.Integer(), server_default="0", nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_download_profiles_user_id", "download_profiles", ["user_id"])
op.create_table(
"download_jobs",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("user_id", sa.Integer(), nullable=False),
sa.Column("asset_id", sa.Integer(), nullable=True),
sa.Column("source_kind", sa.String(length=16), nullable=False),
sa.Column("source_ref", sa.String(length=512), nullable=False),
sa.Column("profile_id", sa.Integer(), nullable=True),
sa.Column("profile_snapshot", sa.JSON(), nullable=False),
sa.Column("display_name", sa.String(length=255), nullable=True),
sa.Column("status", sa.String(length=16), server_default="queued", nullable=False),
sa.Column("progress", sa.Integer(), server_default="0", nullable=False),
sa.Column("speed_bps", sa.BigInteger(), nullable=True),
sa.Column("eta_s", sa.Integer(), nullable=True),
sa.Column("phase", sa.String(length=32), nullable=True),
sa.Column("error", sa.Text(), nullable=True),
sa.Column("queue_pos", sa.Integer(), server_default="0", nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["asset_id"], ["media_assets.id"], ondelete="SET NULL"),
sa.ForeignKeyConstraint(["profile_id"], ["download_profiles.id"], ondelete="SET NULL"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_download_jobs_user_id", "download_jobs", ["user_id"])
op.create_index("ix_download_jobs_asset_id", "download_jobs", ["asset_id"])
op.create_index("ix_download_jobs_status", "download_jobs", ["status"])
op.create_index("ix_download_jobs_claim", "download_jobs", ["status", "queue_pos"])
op.create_table(
"download_shares",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("job_id", sa.Integer(), nullable=False),
sa.Column("from_user_id", sa.Integer(), nullable=False),
sa.Column("to_user_id", sa.Integer(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(["job_id"], ["download_jobs.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["from_user_id"], ["users.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["to_user_id"], ["users.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),
)
op.create_index("ix_download_shares_job_id", "download_shares", ["job_id"])
op.create_index("ix_download_shares_from_user_id", "download_shares", ["from_user_id"])
op.create_index("ix_download_shares_to_user_id", "download_shares", ["to_user_id"])
op.create_table(
"download_quotas",
sa.Column("user_id", sa.Integer(), nullable=False),
sa.Column("max_bytes", sa.BigInteger(), nullable=False),
sa.Column("max_concurrent", sa.Integer(), nullable=False),
sa.Column("max_jobs", sa.Integer(), nullable=False),
sa.Column("unlimited", sa.Boolean(), server_default="false", nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("user_id"),
)
def downgrade() -> None:
op.drop_table("download_quotas")
op.drop_index("ix_download_shares_to_user_id", table_name="download_shares")
op.drop_index("ix_download_shares_from_user_id", table_name="download_shares")
op.drop_index("ix_download_shares_job_id", table_name="download_shares")
op.drop_table("download_shares")
op.drop_index("ix_download_jobs_claim", table_name="download_jobs")
op.drop_index("ix_download_jobs_status", table_name="download_jobs")
op.drop_index("ix_download_jobs_asset_id", table_name="download_jobs")
op.drop_index("ix_download_jobs_user_id", table_name="download_jobs")
op.drop_table("download_jobs")
op.drop_index("ix_download_profiles_user_id", table_name="download_profiles")
op.drop_table("download_profiles")
op.drop_index("ix_media_assets_status", table_name="media_assets")
op.drop_table("media_assets")

View file

@ -0,0 +1,67 @@
"""seed builtin download profiles (presets)
Revision ID: 0037_dl_builtin_profiles
Revises: 0036_download_center
Create Date: 2026-07-02
Seeds the shipped download presets (user_id NULL, is_builtin=true). `spec` is the format
definition consumed by app.downloads.formats: mode ("av" audio+video / "v" video-only /
"a" audio-only), max_height (null = best), container, audio_format, vcodec preference, and
the embed_*/sponsorblock postprocessor flags. Names are English (the UI default); the frontend
localizes builtin labels by a stable key derived from the name.
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0037_dl_builtin_profiles"
down_revision: Union[str, None] = "0036_download_center"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _spec(mode, height=None, container=None, audio_format=None, vcodec=None):
return {
"mode": mode,
"max_height": height,
"container": container,
"audio_format": audio_format,
"vcodec": vcodec,
"embed_subs": False,
"embed_chapters": True,
"embed_thumbnail": True,
"sponsorblock": False,
}
_BUILTINS = [
("Best (MP4)", _spec("av", None, "mp4"), 10),
("1080p (MP4)", _spec("av", 1080, "mp4"), 20),
("720p (MP4)", _spec("av", 720, "mp4"), 30),
("Audio (M4A)", _spec("a", None, None, "m4a"), 40),
("Audio (MP3)", _spec("a", None, None, "mp3"), 50),
("Video only (MP4)", _spec("v", None, "mp4"), 60),
]
def upgrade() -> None:
profiles = sa.table(
"download_profiles",
sa.column("user_id", sa.Integer),
sa.column("name", sa.String),
sa.column("spec", sa.JSON),
sa.column("is_builtin", sa.Boolean),
sa.column("sort_order", sa.Integer),
)
op.bulk_insert(
profiles,
[
{"user_id": None, "name": name, "spec": spec, "is_builtin": True, "sort_order": order}
for name, spec, order in _BUILTINS
],
)
def downgrade() -> None:
op.execute("DELETE FROM download_profiles WHERE is_builtin = true")

View file

@ -0,0 +1,29 @@
"""media_assets.gc_notified flag for one-shot pre-expiry warnings
Revision ID: 0038_asset_gc_notified
Revises: 0037_dl_builtin_profiles
Create Date: 2026-07-03
The download GC warns each holder once, a grace window before an asset's TTL expiry; this flag
records that so subsequent GC runs don't re-notify until the asset is deleted.
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0038_asset_gc_notified"
down_revision: Union[str, None] = "0037_dl_builtin_profiles"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"media_assets",
sa.Column("gc_notified", sa.Boolean(), server_default="false", nullable=False),
)
def downgrade() -> None:
op.drop_column("media_assets", "gc_notified")

View file

@ -0,0 +1,49 @@
"""normalize video titles for display; keep the raw one in original_title
Revision ID: 0039_title_normalize
Revises: 0038_asset_gc_notified
Create Date: 2026-07-03
Adds videos.original_title (the raw YouTube title) and rewrites videos.title to a normalized,
display-friendly form (emoji stripped, ALL-CAPS de-shouted, trailing SEO hashtags removed
see app.titles). Reversible: original_title preserves the source, and title can be re-derived.
The generated search_vector column regenerates automatically as each title is updated.
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
from app.titles import normalize_title
revision: str = "0039_title_normalize"
down_revision: Union[str, None] = "0038_asset_gc_notified"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
_BATCH = 2000
def upgrade() -> None:
op.add_column("videos", sa.Column("original_title", sa.Text(), nullable=True))
conn = op.get_bind()
# Preserve the raw title first (fast, set-based).
conn.execute(sa.text("UPDATE videos SET original_title = title WHERE title IS NOT NULL"))
# Then rewrite title to the normalized form, batched (each update regenerates its FTS vector).
rows = conn.execute(
sa.text("SELECT id, original_title FROM videos WHERE original_title IS NOT NULL")
).fetchall()
changes = []
for vid, raw in rows:
norm = normalize_title(raw)
if norm != raw:
changes.append({"i": vid, "t": norm})
stmt = sa.text("UPDATE videos SET title = :t WHERE id = :i")
for start in range(0, len(changes), _BATCH):
conn.execute(stmt, changes[start : start + _BATCH])
def downgrade() -> None:
conn = op.get_bind()
conn.execute(sa.text("UPDATE videos SET title = original_title WHERE original_title IS NOT NULL"))
op.drop_column("videos", "original_title")

View file

@ -0,0 +1,72 @@
"""reset builtin download profiles: no embedding, 1080p default
Revision ID: 0040_reset_profiles
Revises: 0039_title_normalize
Create Date: 2026-07-03
Redefines the shipped presets: embedding (thumbnail/chapters/subs) is OFF (it rewrites the whole
file expensive on large videos and the Plex .nfo/poster sidecars already carry the metadata),
and the default is now 1080p (Best stays available but no longer first). Re-seeds the builtin
rows so both fresh installs (0037 then this) and existing DBs converge on the same set. Existing
jobs are unaffected (they snapshot their spec; profile_id just SET NULLs if it pointed at a
replaced row).
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0040_reset_profiles"
down_revision: Union[str, None] = "0039_title_normalize"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _spec(mode, height=None, container=None, audio_format=None):
return {
"mode": mode,
"max_height": height,
"container": container,
"audio_format": audio_format,
"vcodec": None,
"embed_subs": False,
"embed_chapters": False,
"embed_thumbnail": False,
"sponsorblock": False,
}
# Default first (1080p), then Best, then the rest.
_BUILTINS = [
("1080p (MP4)", _spec("av", 1080, "mp4"), 10),
("Best (MP4)", _spec("av", None, "mp4"), 20),
("720p (MP4)", _spec("av", 720, "mp4"), 30),
("Audio (M4A)", _spec("a", None, None, "m4a"), 40),
("Audio (MP3)", _spec("a", None, None, "mp3"), 50),
("Video only (MP4)", _spec("v", None, "mp4"), 60),
]
_profiles = sa.table(
"download_profiles",
sa.column("user_id", sa.Integer),
sa.column("name", sa.String),
sa.column("spec", sa.JSON),
sa.column("is_builtin", sa.Boolean),
sa.column("sort_order", sa.Integer),
)
def upgrade() -> None:
op.execute("DELETE FROM download_profiles WHERE is_builtin = true")
op.bulk_insert(
_profiles,
[
{"user_id": None, "name": n, "spec": s, "is_builtin": True, "sort_order": o}
for n, s, o in _BUILTINS
],
)
def downgrade() -> None:
# No-op: the previous builtin set was seeded by 0037; not worth reconstructing.
pass

View file

@ -683,7 +683,10 @@ def resolved_user_id(request: Request) -> tuple[int | None, bool]:
"""
default_id = request.session.get("user_id")
wallet = request.session.get("account_ids") or []
hdr = request.headers.get(ACTIVE_ACCOUNT_HEADER)
# Header for normal XHR; ?account= for contexts that can't set headers (WebSocket, and a
# plain <a> file download). Both are wallet-gated below, so neither can impersonate an
# account that never signed into this browser.
hdr = request.headers.get(ACTIVE_ACCOUNT_HEADER) or request.query_params.get("account")
if hdr:
try:
hid = int(hdr)

View file

@ -131,6 +131,57 @@ class Settings(BaseSettings):
# ("was_live") are real watchable content and stay visible.
feed_default_hidden_live: str = "live,upcoming"
# --- Download center (yt-dlp) ---
# Root directory (inside the container) for downloaded media — a Docker volume/bind mount.
# Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it.
download_root: str = "/downloads"
# Whether THIS process runs the download worker loop. On for the dedicated worker
# container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled.
worker_enabled: bool = False
# Base URL of the bgutil PO-token provider sidecar. When set, the download worker asks it
# for YouTube Proof-of-Origin tokens (bypasses bot-detection + unlocks high-quality
# formats). Empty disables it (yt-dlp still works, but may hit "confirm you're not a bot").
download_pot_base_url: str = "http://bgutil-pot:4416"
# yt-dlp player clients (comma-separated). PRIMARY = android_vr: high-quality DASH, no PO
# token / JS runtime needed, reliable for normal use. On a bot/DRM/format failure (YouTube's
# per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the
# worker retries with the FALLBACK set — the POT-capable web clients (web needs the token +
# Deno) plus android for audio. Both admin-tunable when YouTube shifts what works.
download_player_clients: str = "android_vr"
download_player_clients_fallback: str = "web_safari,web,android"
@staticmethod
def _clients(raw: str) -> list[str]:
return [c.strip() for c in raw.split(",") if c.strip()]
@property
def player_client_list(self) -> list[str]:
return self._clients(self.download_player_clients)
@property
def player_client_fallback_list(self) -> list[str]:
return self._clients(self.download_player_clients_fallback)
# How many downloads the worker runs concurrently (claimed via FOR UPDATE SKIP LOCKED).
download_worker_concurrency: int = 2
# How often (empty pending queue) the worker polls for a new job, in seconds.
download_poll_seconds: int = 5
# Total cache size cap across ALL users (bytes). When >0 and exceeded, the GC evicts the
# least-recently-accessed ready assets until under it. 0 = unlimited (disk is the limit).
download_total_max_bytes: int = 0
# Default per-user quota (admin-tunable; a per-user download_quotas row overrides).
download_default_max_bytes: int = 5_368_709_120 # 5 GiB
download_default_max_concurrent: int = 2
download_default_max_jobs: int = 50
# Retention: an asset expires this many days after its last access; the GC job then
# deletes it (once no job references it) after an additional grace window.
download_retention_days: int = 30
download_gc_grace_days: int = 2
download_gc_minutes: int = 360
# On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat".
download_layout: str = "plex"
# Default SponsorBlock (skip sponsor segments) for new custom profiles.
sponsorblock_default: bool = False
@property
def allowed_email_set(self) -> set[str]:
return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()}

View file

View file

@ -0,0 +1,179 @@
"""Format profiles → yt-dlp options, and the cache signature.
A profile `spec` (stored on DownloadProfile / snapshotted on DownloadJob) is the single source
of truth for what file gets produced. `format_sig` hashes exactly the fields that change the
OUTPUT BYTES, so two users picking equivalent settings share one cached MediaAsset.
`build_ydl_opts` turns a spec into a yt-dlp options dict (format selector + postprocessors).
spec shape (all keys optional; defaults applied by `normalize`):
mode "av" (audio+video) | "v" (video-only) | "a" (audio-only)
max_height int | null (null = best)
container "mp4" | "mkv" | "webm" | null (merge/remux target for av/v)
audio_format "m4a" | "mp3" | "opus" | null (extract target for mode "a")
vcodec "h264" | "vp9" | "av1" | null (preference, best-effort)
embed_subs bool embed_chapters bool embed_thumbnail bool sponsorblock bool
"""
import hashlib
import json
# Fields that affect the produced file — and therefore the cache identity.
_SIG_KEYS = (
"mode",
"max_height",
"container",
"audio_format",
"vcodec",
"embed_subs",
"embed_chapters",
"embed_thumbnail",
"sponsorblock",
)
_DEFAULTS = {
"mode": "av",
"max_height": None,
"container": "mp4",
"audio_format": "m4a",
"vcodec": None,
# Embedding is OFF by default: it rewrites the whole file (doubles I/O — a 10 GB video
# gets a 10 GB temp copy), and the Plex `.nfo` + poster sidecars already carry the
# metadata/thumbnail. Users who want self-contained files opt in via a custom profile.
"embed_subs": False,
"embed_chapters": False,
"embed_thumbnail": False,
"sponsorblock": False,
}
_SPONSORBLOCK_CATEGORIES = ["sponsor", "selfpromo", "interaction"]
_VCODEC_SORT = {"h264": "vcodec:h264", "vp9": "vcodec:vp9", "av1": "vcodec:av01"}
def normalize(spec: dict) -> dict:
"""Fill defaults + coerce types so signature and yt-dlp opts are deterministic."""
s = dict(_DEFAULTS)
for k in _DEFAULTS:
if spec.get(k) is not None:
s[k] = spec[k]
if s["mode"] not in ("av", "v", "a"):
s["mode"] = "av"
if s["max_height"] is not None:
s["max_height"] = int(s["max_height"])
for b in ("embed_subs", "embed_chapters", "embed_thumbnail", "sponsorblock"):
s[b] = bool(s[b])
# Audio-only files can't embed subtitles/thumbnails as video; keep the flags meaningful.
if s["mode"] == "a":
s["embed_subs"] = False
return s
def format_sig(spec: dict) -> str:
"""Stable short hash of the output-affecting spec — the MediaAsset cache key component."""
s = normalize(spec)
payload = json.dumps({k: s[k] for k in _SIG_KEYS}, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(payload.encode()).hexdigest()[:24]
def target_ext(spec: dict) -> str:
"""Best-effort final extension for the produced file (also used to name the staging output)."""
s = normalize(spec)
if s["mode"] == "a":
return s["audio_format"] or "m4a"
return s["container"] or "mp4"
def build_ydl_opts(
spec: dict,
outtmpl: str,
progress_hook,
pot_base_url: str | None = None,
player_clients: list[str] | None = None,
) -> dict:
"""Assemble yt-dlp options for a single download to `outtmpl` (a full path template).
`pot_base_url` points the bgutil PO-token provider plugin at the sidecar server;
`player_clients` picks which YouTube clients to use (POT-capable ones like web use the token)."""
s = normalize(spec)
h = s["max_height"]
opts: dict = {
"outtmpl": outtmpl,
"noplaylist": True,
"quiet": True,
"no_warnings": True,
"noprogress": True,
"writethumbnail": True, # kept for the poster.jpg sidecar (and embed, if requested)
"progress_hooks": [progress_hook],
"postprocessors": [],
# Force IPv4: containers often have no working IPv6 route, but googlevideo CDN hosts
# advertise AAAA records, so ffmpeg/the downloader would try IPv6 and fail with
# "[Errno -5] No address associated with hostname". Binding to the IPv4 any-address
# makes every connection IPv4-only. (yt-dlp's --force-ipv4 does the same thing.)
"source_address": "0.0.0.0",
# Self-heal transient network blips instead of failing the whole job.
"retries": 10,
"fragment_retries": 10,
"extractor_retries": 3,
"retry_sleep": {"http": 5, "fragment": 5, "extractor": 5},
"socket_timeout": 30,
}
# Extractor args: which YouTube clients to use + where the bgutil PO-token sidecar lives
# (== --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…"). Values
# are lists in the Python API.
ea: dict = {}
if player_clients:
ea["youtube"] = {"player_client": player_clients}
if pot_base_url:
ea["youtubepot-bgutilhttp"] = {"base_url": [pot_base_url]}
if ea:
opts["extractor_args"] = ea
if s["mode"] == "a":
opts["format"] = "bestaudio/best"
opts["postprocessors"].append(
{
"key": "FFmpegExtractAudio",
"preferredcodec": s["audio_format"] or "m4a",
"preferredquality": "0",
}
)
else:
hcap = f"[height<=?{h}]" if h else ""
if s["mode"] == "v":
opts["format"] = f"bestvideo{hcap}/best{hcap}"
else: # av
opts["format"] = f"bestvideo{hcap}+bestaudio/best{hcap}"
if s["container"]:
opts["merge_output_format"] = s["container"]
sort = []
if h:
sort.append(f"res:{h}")
if s["vcodec"] in _VCODEC_SORT:
sort.append(_VCODEC_SORT[s["vcodec"]])
if sort:
opts["format_sort"] = sort
# SponsorBlock first (marks segments), then strip them from the media + chapters.
if s["sponsorblock"]:
opts["postprocessors"].append(
{"key": "SponsorBlock", "categories": _SPONSORBLOCK_CATEGORIES, "when": "after_filter"}
)
opts["postprocessors"].append(
{"key": "ModifyChapters", "remove_sponsor_segments": _SPONSORBLOCK_CATEGORIES}
)
if s["embed_subs"] and s["mode"] != "a":
opts["writesubtitles"] = True
opts["subtitleslangs"] = ["en.*", "hu.*", "de.*"]
opts["postprocessors"].append({"key": "FFmpegEmbedSubtitle"})
if s["embed_chapters"]:
opts["postprocessors"].append(
{"key": "FFmpegMetadata", "add_chapters": True, "add_metadata": True}
)
# Normalize the written thumbnail to jpg so we always have a `<name>.jpg` poster sidecar.
# `already_have_thumbnail` keeps that file on disk after embedding (default deletes it).
opts["postprocessors"].append({"key": "FFmpegThumbnailsConvertor", "format": "jpg"})
if s["embed_thumbnail"]:
opts["postprocessors"].append(
{"key": "EmbedThumbnail", "already_have_thumbnail": True}
)
return opts

119
backend/app/downloads/gc.py Normal file
View file

@ -0,0 +1,119 @@
"""Download retention garbage collector (a scheduler job).
Three passes each run:
1. Pre-expiry warning for ready assets expiring within the grace window, notify each holder
once (gc_notified flag) so a download isn't silently deleted.
2. Expiry deletion delete assets past their TTL (files + row). The FK SET NULL nulls the
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
3. LRU eviction if a total cache cap is set and exceeded, delete least-recently-accessed
ready assets until back under it.
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work no YouTube quota.
"""
import logging
from datetime import datetime, timedelta, timezone
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app import sysconfig
from app.config import settings
from app.downloads import storage
from app.models import DownloadJob, MediaAsset
from app.notifications import create_notification
log = logging.getLogger("siftlode.downloads.gc")
def _holders(db: Session, asset_id: int) -> list[int]:
return list(
db.execute(
select(DownloadJob.user_id)
.where(DownloadJob.asset_id == asset_id)
.distinct()
).scalars()
)
def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None:
title = asset.title or "download"
for uid in _holders(db, asset.id):
create_notification(
db,
user_id=uid,
type=ntype,
title=title, # English fallback; the client renders a translated message from data
data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref},
commit=False,
)
def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
if asset.rel_path:
storage.delete_asset_files(settings.download_root, asset.rel_path)
_notify_holders(db, asset, ntype)
db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL
def run_download_gc(db: Session) -> dict:
now = datetime.now(timezone.utc)
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
warned = expired = evicted = 0
# 1. Pre-expiry warnings.
warn_before = now + timedelta(days=grace_days)
soon = db.execute(
select(MediaAsset).where(
MediaAsset.status == "ready",
MediaAsset.gc_notified.is_(False),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at > now,
MediaAsset.expires_at <= warn_before,
)
).scalars().all()
for asset in soon:
_notify_holders(db, asset, "download_expiring")
asset.gc_notified = True
warned += 1
if soon:
db.commit()
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
dead = db.execute(
select(MediaAsset).where(
MediaAsset.status.in_(("ready", "error")),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at <= now,
)
).scalars().all()
for asset in dead:
_delete_asset(db, asset, "download_expired")
expired += 1
if dead:
db.commit()
# 3. LRU eviction over the total cache cap.
cap = sysconfig.get_int(db, "download_total_max_bytes")
if cap > 0:
total = db.execute(
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
MediaAsset.status == "ready"
)
).scalar() or 0
if total > cap:
candidates = db.execute(
select(MediaAsset)
.where(MediaAsset.status == "ready")
.order_by(MediaAsset.last_access_at.asc().nulls_first())
).scalars().all()
for asset in candidates:
if total <= cap:
break
total -= asset.size_bytes or 0
_delete_asset(db, asset, "download_evicted")
evicted += 1
db.commit()
result = {"warned": warned, "expired": expired, "evicted": evicted}
log.info("download_gc %s", result)
return result

View file

@ -0,0 +1,129 @@
"""Per-user download limits + footprint accounting.
A user's *footprint* is the total size of the ready assets their library references (cache hits
count it's "how much disk you're responsible for", not "how much you personally downloaded").
Limits come from a per-user DownloadQuota row if the admin set one, else the sysconfig defaults.
`unlimited` (e.g. the admin's own account) bypasses the byte cap.
Enforcement split: max_jobs + max_bytes are checked at enqueue (they bound holdings); per-user
max_concurrent is enforced by the worker at claim time (it bounds simultaneous downloads).
"""
from dataclasses import dataclass
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app import sysconfig
from app.models import DownloadJob, DownloadQuota, MediaAsset
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
# (error, canceled) don't count.
_HOLDING = ("queued", "running", "paused", "done")
_RUNNING = ("queued", "running")
@dataclass
class QuotaLimits:
max_bytes: int
max_concurrent: int
max_jobs: int
unlimited: bool
class QuotaExceeded(Exception):
"""Raised by check_enqueue; `reason` is one of max_jobs|max_bytes (i18n key on the client)."""
def __init__(self, reason: str, limit: int, current: int):
self.reason = reason
self.limit = limit
self.current = current
super().__init__(f"download quota exceeded: {reason} ({current}/{limit})")
def resolve(db: Session, user_id: int) -> QuotaLimits:
row = db.get(DownloadQuota, user_id)
if row is not None:
return QuotaLimits(row.max_bytes, row.max_concurrent, row.max_jobs, row.unlimited)
return QuotaLimits(
max_bytes=sysconfig.get_int(db, "download_default_max_bytes"),
max_concurrent=sysconfig.get_int(db, "download_default_max_concurrent"),
max_jobs=sysconfig.get_int(db, "download_default_max_jobs"),
unlimited=False,
)
def footprint(db: Session, user_id: int) -> int:
"""Total bytes of the distinct ready assets this user's active jobs reference."""
held_assets = (
select(DownloadJob.asset_id)
.where(
DownloadJob.user_id == user_id,
DownloadJob.status.in_(_HOLDING),
DownloadJob.asset_id.is_not(None),
)
.distinct()
)
return int(
db.execute(
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
MediaAsset.id.in_(held_assets), MediaAsset.status == "ready"
)
).scalar()
or 0
)
def _count(db: Session, user_id: int, states) -> int:
return (
db.execute(
select(func.count())
.select_from(DownloadJob)
.where(DownloadJob.user_id == user_id, DownloadJob.status.in_(states))
).scalar()
or 0
)
def active_jobs(db: Session, user_id: int) -> int:
return _count(db, user_id, _HOLDING)
def running_jobs(db: Session, user_id: int) -> int:
return _count(db, user_id, _RUNNING)
def at_concurrency_limit(db: Session, user_id: int) -> bool:
"""Worker-side gate: is this user already downloading their max? Unlimited users never are."""
lim = resolve(db, user_id)
if lim.unlimited:
return False
running = _count(db, user_id, ("running",))
return running >= lim.max_concurrent
def check_enqueue(db: Session, user_id: int) -> None:
"""Raise QuotaExceeded if the user can't take another job (holdings caps). Concurrency is
enforced later by the worker, so a big queue is allowed; it just drains max_concurrent-wide."""
lim = resolve(db, user_id)
if lim.unlimited:
return
n = active_jobs(db, user_id)
if n >= lim.max_jobs:
raise QuotaExceeded("max_jobs", lim.max_jobs, n)
used = footprint(db, user_id)
if used >= lim.max_bytes:
raise QuotaExceeded("max_bytes", lim.max_bytes, used)
def usage(db: Session, user_id: int) -> dict:
"""Compact usage snapshot for the UI (footprint + counts vs limits)."""
lim = resolve(db, user_id)
return {
"footprint_bytes": footprint(db, user_id),
"active_jobs": active_jobs(db, user_id),
"running_jobs": running_jobs(db, user_id),
"max_bytes": lim.max_bytes,
"max_concurrent": lim.max_concurrent,
"max_jobs": lim.max_jobs,
"unlimited": lim.unlimited,
}

View file

@ -0,0 +1,123 @@
"""Enqueue + cache resolution for the download center (the per-user layer over MediaAsset).
`enqueue` is the single entry point used by the API: it resolves (or creates) the shared
MediaAsset for the requested source+format, links a per-user DownloadJob to it, and on a
cache hit (asset already ready) marks the job done instantly with no re-download. The worker
handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
"""
from datetime import datetime, timezone
from sqlalchemy import func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.downloads import formats, quota
from app.models import Channel, DownloadJob, MediaAsset, Video
def source_url(source_kind: str, source_ref: str) -> str:
"""The URL yt-dlp should fetch. YouTube stores the 11-char id; ad-hoc stores the URL."""
if source_kind == "youtube":
return f"https://www.youtube.com/watch?v={source_ref}"
return source_ref
def get_or_create_asset(
db: Session, source_kind: str, source_ref: str, format_sig: str
) -> MediaAsset:
"""Fetch the cache row for this (source, format) or create a fresh `pending` one.
Races on the unique cache key are resolved by re-selecting after an IntegrityError, so two
simultaneous enqueues of the same video+format converge on one shared asset."""
stmt = select(MediaAsset).where(
MediaAsset.source_kind == source_kind,
MediaAsset.source_ref == source_ref,
MediaAsset.format_sig == format_sig,
)
asset = db.execute(stmt).scalar_one_or_none()
if asset is not None:
return asset
asset = MediaAsset(
source_kind=source_kind,
source_ref=source_ref,
format_sig=format_sig,
status="pending",
ref_count=0,
)
db.add(asset)
try:
db.flush()
except IntegrityError:
db.rollback()
asset = db.execute(stmt).scalar_one()
return asset
def populate_from_catalog(db: Session, asset: MediaAsset) -> bool:
"""Fill an asset's display metadata (title/uploader/thumbnail/date/duration) from our own
catalog when the source is a known YouTube video so a queued/downloading row shows the
real title + thumbnail immediately instead of the bare id. 0 network cost. Returns True if
it found a catalog match."""
if asset.source_kind != "youtube" or asset.title:
return False
video = db.get(Video, asset.source_ref)
if video is None:
return False
asset.title = video.title
asset.thumbnail_url = video.thumbnail_url
asset.duration_s = video.duration_seconds
if video.published_at:
asset.upload_date = video.published_at.date()
channel = db.get(Channel, video.channel_id)
if channel is not None:
asset.uploader = channel.title
return True
def _next_queue_pos(db: Session) -> int:
return (db.execute(select(func.coalesce(func.max(DownloadJob.queue_pos), 0))).scalar() or 0) + 1
def enqueue(
db: Session,
user_id: int,
source_kind: str,
source_ref: str,
spec: dict,
profile_id: int | None = None,
display_name: str | None = None,
) -> DownloadJob:
"""Queue a download for `user_id`. Returns the DownloadJob (already `done` on a cache hit).
Raises quota.QuotaExceeded if the user is at their job-count or footprint cap."""
quota.check_enqueue(db, user_id)
spec = formats.normalize(spec)
sig = formats.format_sig(spec)
asset = get_or_create_asset(db, source_kind, source_ref, sig)
# Show a real title + thumbnail the moment it's queued (feed downloads are catalog videos).
populate_from_catalog(db, asset)
job = DownloadJob(
user_id=user_id,
asset_id=asset.id,
source_kind=source_kind,
source_ref=source_ref,
profile_id=profile_id,
profile_snapshot=spec,
display_name=display_name or (asset.title if asset.status == "ready" else None),
status="queued",
queue_pos=_next_queue_pos(db),
)
db.add(job)
asset.ref_count = (asset.ref_count or 0) + 1
# Cache hit: the file already exists — no worker round-trip needed.
if asset.status == "ready":
job.status = "done"
job.progress = 100
asset.last_access_at = datetime.now(timezone.utc)
db.commit()
db.refresh(job)
return job

View file

@ -0,0 +1,154 @@
"""On-disk layout for downloaded media — Plex-friendly tree + .nfo/poster sidecars.
The physical filename is system-decided and bound to the source id (never renamed; the user's
custom name is display-only, applied at local-download time). Layout is admin-configurable:
plex: {Channel}/Season {Year}/{Channel} - {YYYY-MM-DD} - {title} [{id}].{ext}
flat: {title} [{id}].{ext}
Plus a Kodi/Plex-readable `<basename>.nfo` (episodedetails) and `<basename>.jpg` thumbnail, so
a Plex library using a metadata agent picks up rich info. Channel = "show", video = episode,
date-based the community-standard mapping for YouTube-in-Plex.
"""
import re
import shutil
import unicodedata
from dataclasses import dataclass
from datetime import date
from pathlib import Path
from xml.sax.saxutils import escape
# Reserve room for the " [id].ext" suffix + parent dirs within the 255-byte component limit.
_MAX_TITLE = 120
# Filesystem-illegal on Windows/Plex + control chars.
_ILLEGAL = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
# Unicode categories we drop entirely: emoji/pictographs (So), modifier symbols/skin-tones
# (Sk), and every control/format/private/surrogate/unassigned char (C*). Letters (incl.
# accented á/ö/ü/ß), digits, and ordinary punctuation are KEPT — Siftlode is trilingual, so
# ASCII-folding would mangle Hungarian/German titles.
_DROP_CATEGORIES = {"So", "Sk", "Cc", "Cf", "Cs", "Co", "Cn"}
@dataclass
class MediaMeta:
video_id: str
title: str
uploader: str
upload_date: date | None
duration_s: int | None = None
description: str | None = None
thumbnail_url: str | None = None
def sanitize(name: str, limit: int = _MAX_TITLE) -> str:
"""Turn an arbitrary (emoji-laden, clickbait) title into a clean, space-free path token.
Steps: NFKC-normalize (fold fancy width/compat forms) drop emoji/symbols/control chars
strip filesystem-illegal chars collapse whitespace and runs of the same punctuation
join words with underscores trim junk punctuation from the ends length-cap. Accented
letters and digits survive; the result never contains spaces or path separators."""
text = unicodedata.normalize("NFKC", name or "")
text = "".join(
ch for ch in text if unicodedata.category(ch) not in _DROP_CATEGORIES
)
text = _ILLEGAL.sub("", text)
# Collapse repeated punctuation ("!!!", "???", "-----") to a single mark.
text = re.sub(r"([^\w\s])\1{1,}", r"\1", text)
# Whitespace → single underscore (no spaces in generated names).
text = re.sub(r"\s+", "_", text.strip())
# Tidy separator pile-ups and trim junk from the ends.
text = re.sub(r"_{2,}", "_", text).strip("_.-·—–|,;:!?ّ ")
if len(text) > limit:
text = text[:limit].rstrip("_.-")
return text or "untitled"
def display_filename(name: str) -> str:
"""User-facing download filename (Content-Disposition): strip emoji/symbols/control +
filesystem-illegal chars, but KEEP spaces, accents, and ordinary punctuation unlike the
underscore-joined on-disk name, this is the readable name the user saves to their device."""
text = unicodedata.normalize("NFKC", name or "")
text = "".join(ch for ch in text if unicodedata.category(ch) not in _DROP_CATEGORIES)
text = _ILLEGAL.sub("", text)
text = re.sub(r"\s+", " ", text).strip(" .")
return text or "download"
def rel_path(meta: MediaMeta, ext: str, layout: str = "plex") -> str:
"""Path of the media file relative to DOWNLOAD_ROOT (forward slashes)."""
title = sanitize(meta.title)
vid = meta.video_id
if layout == "flat":
return f"{title}_[{vid}].{ext}"
channel = sanitize(meta.uploader or "Unknown_Channel", 80)
year = meta.upload_date.year if meta.upload_date else 0
d = meta.upload_date.isoformat() if meta.upload_date else "0000-00-00"
epname = f"{channel}_-_{d}_-_{title}_[{vid}].{ext}"
return f"{channel}/Season_{year}/{epname}"
def abs_path(download_root: str, rel: str) -> Path:
return Path(download_root) / rel
def place_file(staging_file: Path, download_root: str, rel: str) -> None:
"""Move a completed staging file into its final location under DOWNLOAD_ROOT."""
dest = abs_path(download_root, rel)
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(staging_file), str(dest))
def _nfo_xml(meta: MediaMeta) -> str:
aired = meta.upload_date.isoformat() if meta.upload_date else ""
year = meta.upload_date.year if meta.upload_date else ""
parts = [
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>',
"<episodedetails>",
f" <title>{escape(meta.title or '')}</title>",
f" <showtitle>{escape(meta.uploader or '')}</showtitle>",
f" <studio>{escape(meta.uploader or '')}</studio>",
f" <aired>{aired}</aired>",
f" <premiered>{aired}</premiered>",
f" <year>{year}</year>",
f" <plot>{escape(meta.description or '')}</plot>",
f" <uniqueid type=\"youtube\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
]
if meta.duration_s:
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
parts.append("</episodedetails>")
return "\n".join(parts)
def write_sidecars(
download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None
) -> bool:
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
media = abs_path(download_root, rel)
base = media.with_suffix("")
try:
base.with_suffix(".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
if thumb_src and thumb_src.exists():
shutil.copyfile(str(thumb_src), str(base.with_suffix(".jpg")))
return True
except OSError:
return False
def delete_asset_files(download_root: str, rel: str) -> None:
"""Remove the media file and its sidecars; prune now-empty parent dirs (best effort)."""
media = abs_path(download_root, rel)
base = media.with_suffix("")
for p in (media, base.with_suffix(".nfo"), base.with_suffix(".jpg")):
try:
p.unlink()
except OSError:
pass
# Prune empty Season/Channel dirs up to (but not including) the root.
root = Path(download_root).resolve()
parent = media.parent
while parent != root and root in parent.parents:
try:
parent.rmdir() # only succeeds if empty
except OSError:
break
parent = parent.parent

View file

@ -31,6 +31,7 @@ from app.routes import (
admin,
channels,
config as config_routes,
downloads,
feed,
health,
me,
@ -122,6 +123,8 @@ app.include_router(messages.router)
app.include_router(channels.router)
app.include_router(playlists.router)
app.include_router(saved_views.router)
app.include_router(downloads.router)
app.include_router(downloads.admin_router)
app.include_router(admin.router)
app.include_router(config_routes.router)
app.include_router(scheduler_routes.router)

View file

@ -8,6 +8,7 @@ from sqlalchemy import (
DateTime,
Float,
ForeignKey,
Index,
Integer,
JSON,
String,
@ -240,7 +241,8 @@ class Video(Base, TimestampMixin):
channel_id: Mapped[str] = mapped_column(
ForeignKey("channels.id", ondelete="CASCADE"), index=True
)
title: Mapped[str | None] = mapped_column(Text)
title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles)
original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved
description: Mapped[str | None] = mapped_column(Text)
published_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), index=True
@ -688,6 +690,164 @@ class Message(Base):
)
class MediaAsset(Base, TimestampMixin):
"""A physical downloaded media file — the SHARED cache layer of the download center.
Mirrors the app's "shared catalog, private per-user state" model: one row per unique
``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests
the same video with a format that hashes to the same `format_sig`, their `download_job`
links to this ready asset instantly no re-download, no cost. Only the SOURCE download is
shared; per-user edited derivatives (phase 2) are never cached here.
`rel_path` is relative to `settings.download_root` (the mount), so the physical filename is
system-decided and bound to the source id never renamed (the user's custom name is display
metadata on `download_job`, applied only when downloading to their own machine). The naming
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here)
plus `last_access_at`/`expires_at` drive retention + LRU eviction in the GC job."""
__tablename__ = "media_assets"
__table_args__ = (
UniqueConstraint(
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
),
)
id: Mapped[int] = mapped_column(primary_key=True)
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc
format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec
status: Mapped[str] = mapped_column(
String(16), default="pending", server_default="pending", index=True
) # "pending" | "ready" | "error"
rel_path: Mapped[str | None] = mapped_column(Text) # relative to download_root
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
duration_s: Mapped[int | None] = mapped_column(Integer)
width: Mapped[int | None] = mapped_column(Integer)
height: Mapped[int | None] = mapped_column(Integer)
container: Mapped[str | None] = mapped_column(String(16))
vcodec: Mapped[str | None] = mapped_column(String(32))
acodec: Mapped[str | None] = mapped_column(String(32))
nfo_written: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
# Set once the GC has warned holders of the upcoming expiry, so it doesn't warn every run.
gc_notified: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
title: Mapped[str | None] = mapped_column(Text)
uploader: Mapped[str | None] = mapped_column(String(255))
upload_date: Mapped[date | None] = mapped_column(Date)
thumbnail_url: Mapped[str | None] = mapped_column(Text)
error: Mapped[str | None] = mapped_column(Text)
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class DownloadProfile(Base, TimestampMixin):
"""A named download preset — a reusable format profile.
`user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, );
a user's saved custom profiles carry their `user_id`. `spec` is the format definition
(mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/
thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache
`format_sig`."""
__tablename__ = "download_profiles"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
) # NULL for a builtin preset
name: Mapped[str] = mapped_column(String(80))
spec: Mapped[dict] = mapped_column(JSON)
is_builtin: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
"""A user's request to download one source — the per-user layer over `MediaAsset`.
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery the
worker is a separate process and can't reach the in-process WS registry). `profile_snapshot`
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
`display_name` is the user's own name for the file (display + the Content-Disposition on
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
(SET NULL if the asset is GC'd — the job then shows as expired)."""
__tablename__ = "download_jobs"
__table_args__ = (Index("ix_download_jobs_claim", "status", "queue_pos"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
asset_id: Mapped[int | None] = mapped_column(
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
)
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
source_ref: Mapped[str] = mapped_column(String(512))
profile_id: Mapped[int | None] = mapped_column(
ForeignKey("download_profiles.id", ondelete="SET NULL")
)
profile_snapshot: Mapped[dict] = mapped_column(JSON)
display_name: Mapped[str | None] = mapped_column(String(255))
status: Mapped[str] = mapped_column(
String(16), default="queued", server_default="queued", index=True
) # queued | running | paused | done | error | canceled
progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
speed_bps: Mapped[int | None] = mapped_column(BigInteger)
eta_s: Mapped[int | None] = mapped_column(Integer)
phase: Mapped[str | None] = mapped_column(String(32))
error: Mapped[str | None] = mapped_column(Text)
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
class DownloadShare(Base, TimestampMixin):
"""A user→user grant: `to_user` may access `from_user`'s downloaded file (private by
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
__tablename__ = "download_shares"
__table_args__ = (
UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),
)
id: Mapped[int] = mapped_column(primary_key=True)
job_id: Mapped[int] = mapped_column(
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
)
from_user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
to_user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
class DownloadQuota(Base, UpdatedAtMixin):
"""Per-user download limits (admin-managed). A row exists only when an admin customizes a
user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the
byte cap entirely (e.g. for the admin's own account)."""
__tablename__ = "download_quotas"
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
)
max_bytes: Mapped[int] = mapped_column(BigInteger)
max_concurrent: Mapped[int] = mapped_column(Integer)
max_jobs: Mapped[int] = mapped_column(Integer)
unlimited: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
class MessageKey(Base, TimestampMixin):
"""A user's end-to-end-encryption key material for direct messaging (phase 2).

View file

@ -0,0 +1,659 @@
"""Download center HTTP API — the per-user surface over the worker/cache/quota layers.
All routes are behind `require_human` (the shared demo account can't spend server disk / needs a
real identity). Admin routes add `admin_user`. Live job state is polled by the frontend (the
worker is a separate process); this module never downloads it enqueues, manages job state,
serves finished files (range-aware, with the user's custom display name), and shares them.
"""
import re
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import parse_qs, urlparse
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi.responses import FileResponse
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app.auth import admin_user, require_human
from app.config import settings
from app.db import get_db
from app.downloads import quota, service, storage
from app.models import (
DownloadJob,
DownloadProfile,
DownloadQuota,
DownloadShare,
MediaAsset,
User,
)
router = APIRouter(prefix="/api/downloads", tags=["downloads"])
admin_router = APIRouter(prefix="/api/admin/downloads", tags=["admin-downloads"])
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
# --- source + serialization ----------------------------------------------------------------
def resolve_source(raw: str) -> tuple[str, str]:
"""Map a user-supplied id/URL to (source_kind, source_ref). YouTube is normalized to its
11-char id (so the cache dedups across watch/youtu.be/shorts URLs); anything else is kept
as a raw URL for yt-dlp's generic extractor."""
raw = (raw or "").strip()
if not raw:
raise HTTPException(status_code=400, detail="A video link or id is required.")
if _VIDEO_ID.match(raw):
return "youtube", raw
if raw.startswith(("http://", "https://")):
u = urlparse(raw)
host = u.netloc.lower()
if "youtube.com" in host:
if u.path.startswith(("/shorts/", "/embed/", "/live/")):
vid = u.path.split("/")[2] if len(u.path.split("/")) > 2 else ""
if _VIDEO_ID.match(vid):
return "youtube", vid
qs = parse_qs(u.query).get("v", [])
if qs and _VIDEO_ID.match(qs[0]):
return "youtube", qs[0]
if "youtu.be" in host:
vid = u.path.lstrip("/").split("/")[0]
if _VIDEO_ID.match(vid):
return "youtube", vid
return "url", raw
raise HTTPException(status_code=400, detail="That doesn't look like a valid video link.")
def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
ready = asset is not None and asset.status == "ready" and bool(asset.rel_path)
return {
"id": job.id,
"status": job.status,
"progress": job.progress,
"speed_bps": job.speed_bps,
"eta_s": job.eta_s,
"phase": job.phase,
"error": job.error,
"queue_pos": job.queue_pos,
"created_at": job.created_at.isoformat() if job.created_at else None,
"source_kind": job.source_kind,
"source_ref": job.source_ref,
"profile_id": job.profile_id,
"spec": job.profile_snapshot,
"display_name": job.display_name,
"can_download": ready and job.status == "done",
"expired": job.asset_id is None and job.status == "done",
"asset": None
if asset is None
else {
"id": asset.id,
"status": asset.status,
"title": asset.title,
"uploader": asset.uploader,
"upload_date": asset.upload_date.isoformat() if asset.upload_date else None,
"duration_s": asset.duration_s,
"size_bytes": asset.size_bytes,
"width": asset.width,
"height": asset.height,
"container": asset.container,
"thumbnail_url": asset.thumbnail_url,
"expires_at": asset.expires_at.isoformat() if asset.expires_at else None,
},
}
def _assets_for(db: Session, jobs: list[DownloadJob]) -> dict[int, MediaAsset]:
ids = {j.asset_id for j in jobs if j.asset_id}
if not ids:
return {}
rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars()
return {a.id: a for a in rows}
def _own_job(db: Session, user: User, job_id: int) -> DownloadJob:
job = db.get(DownloadJob, job_id)
if job is None or job.user_id != user.id:
raise HTTPException(status_code=404, detail="Unknown download")
return job
# --- profiles ------------------------------------------------------------------------------
def _serialize_profile(p: DownloadProfile) -> dict:
return {
"id": p.id,
"name": p.name,
"spec": p.spec,
"is_builtin": p.is_builtin,
"sort_order": p.sort_order,
}
@router.get("/profiles")
def list_profiles(
user: User = Depends(require_human), db: Session = Depends(get_db)
) -> list[dict]:
rows = (
db.execute(
select(DownloadProfile)
.where(
(DownloadProfile.is_builtin.is_(True))
| (DownloadProfile.user_id == user.id)
)
.order_by(DownloadProfile.is_builtin.desc(), DownloadProfile.sort_order, DownloadProfile.id)
)
.scalars()
.all()
)
return [_serialize_profile(p) for p in rows]
@router.post("/profiles")
def create_profile(
payload: dict, user: User = Depends(require_human), db: Session = Depends(get_db)
) -> dict:
name = (payload.get("name") or "").strip()
spec = payload.get("spec")
if not name:
raise HTTPException(status_code=400, detail="A profile name is required.")
if not isinstance(spec, dict):
raise HTTPException(status_code=400, detail="spec must be an object")
maxpos = db.execute(
select(func.coalesce(func.max(DownloadProfile.sort_order), 100)).where(
DownloadProfile.user_id == user.id
)
).scalar_one()
p = DownloadProfile(
user_id=user.id, name=name[:80], spec=spec, is_builtin=False, sort_order=maxpos + 1
)
db.add(p)
db.commit()
return _serialize_profile(p)
@router.patch("/profiles/{profile_id}")
def update_profile(
profile_id: int,
payload: dict,
user: User = Depends(require_human),
db: Session = Depends(get_db),
) -> dict:
p = db.get(DownloadProfile, profile_id)
if p is None or p.is_builtin or p.user_id != user.id:
raise HTTPException(status_code=404, detail="Unknown profile")
if "name" in payload:
name = (payload.get("name") or "").strip()
if not name:
raise HTTPException(status_code=400, detail="A profile name is required.")
p.name = name[:80]
if "spec" in payload:
if not isinstance(payload["spec"], dict):
raise HTTPException(status_code=400, detail="spec must be an object")
p.spec = payload["spec"]
db.commit()
return _serialize_profile(p)
@router.delete("/profiles/{profile_id}")
def delete_profile(
profile_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
) -> dict:
p = db.get(DownloadProfile, profile_id)
if p is None or p.is_builtin or p.user_id != user.id:
raise HTTPException(status_code=404, detail="Unknown profile")
db.delete(p)
db.commit()
return {"deleted": profile_id}
def _resolve_spec(db: Session, user: User, payload: dict) -> tuple[dict, int | None]:
"""Pick the format spec for an enqueue: an explicit profile_id (builtin or the user's own),
an inline spec, or the first builtin as a fallback."""
pid = payload.get("profile_id")
if pid is not None:
p = db.get(DownloadProfile, pid)
if p is None or (not p.is_builtin and p.user_id != user.id):
raise HTTPException(status_code=404, detail="Unknown profile")
return p.spec, p.id
if isinstance(payload.get("spec"), dict):
return payload["spec"], None
fallback = db.execute(
select(DownloadProfile)
.where(DownloadProfile.is_builtin.is_(True))
.order_by(DownloadProfile.sort_order)
.limit(1)
).scalar_one_or_none()
if fallback is None:
raise HTTPException(status_code=400, detail="No download profile available.")
return fallback.spec, fallback.id
# --- enqueue + list + manage ---------------------------------------------------------------
@router.post("")
def enqueue_download(
payload: dict, user: User = Depends(require_human), db: Session = Depends(get_db)
) -> dict:
source_kind, source_ref = resolve_source(payload.get("source") or "")
spec, profile_id = _resolve_spec(db, user, payload)
display_name = (payload.get("display_name") or "").strip() or None
try:
job = service.enqueue(
db, user.id, source_kind, source_ref, spec, profile_id, display_name
)
except quota.QuotaExceeded as e:
raise HTTPException(status_code=422, detail=_quota_message(e))
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
return _serialize(job, asset)
def _quota_message(e: quota.QuotaExceeded) -> str:
if e.reason == "max_jobs":
return f"You've reached your download limit ({e.limit} items). Remove some first."
if e.reason == "max_bytes":
gb = e.limit / 1_073_741_824
return f"You've used your download storage quota ({gb:.1f} GB). Free up space first."
return "Download quota exceeded."
@router.get("")
def list_downloads(
user: User = Depends(require_human), db: Session = Depends(get_db)
) -> list[dict]:
jobs = (
db.execute(
select(DownloadJob)
.where(DownloadJob.user_id == user.id)
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
)
.scalars()
.all()
)
assets = _assets_for(db, jobs)
return [_serialize(j, assets.get(j.asset_id)) for j in jobs]
@router.get("/usage")
def my_usage(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
return quota.usage(db, user.id)
# Priority when a source has several jobs (e.g. a done one plus a fresh queued one).
_INDEX_RANK = {"done": 4, "running": 3, "paused": 2, "queued": 1}
@router.get("/index")
def my_download_index(
user: User = Depends(require_human), db: Session = Depends(get_db)
) -> dict:
"""source_ref -> best active status, for the feed's per-card "downloaded / queued" badge.
Small + cheap so the feed can poll it without loading the full job list."""
rows = db.execute(
select(DownloadJob.source_ref, DownloadJob.status).where(
DownloadJob.user_id == user.id,
DownloadJob.status.in_(("queued", "running", "paused", "done")),
)
).all()
out: dict[str, str] = {}
for ref, status in rows:
if _INDEX_RANK.get(status, 0) > _INDEX_RANK.get(out.get(ref, ""), 0):
out[ref] = status
return out
@router.get("/shared")
def shared_with_me(
user: User = Depends(require_human), db: Session = Depends(get_db)
) -> list[dict]:
jobs = (
db.execute(
select(DownloadJob)
.join(DownloadShare, DownloadShare.job_id == DownloadJob.id)
.where(DownloadShare.to_user_id == user.id)
.order_by(DownloadShare.created_at.desc())
)
.scalars()
.all()
)
assets = _assets_for(db, jobs)
out = []
for j in jobs:
data = _serialize(j, assets.get(j.asset_id))
data["shared"] = True
out.append(data)
return out
@router.patch("/{job_id}")
def rename_download(
job_id: int,
payload: dict,
user: User = Depends(require_human),
db: Session = Depends(get_db),
) -> dict:
job = _own_job(db, user, job_id)
name = (payload.get("display_name") or "").strip()
# Display name only — the on-disk file keeps its system-decided, id-bound name.
job.display_name = name[:255] or None
db.commit()
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
return _serialize(job, asset)
def _set_status(db: Session, job: DownloadJob, status: str) -> None:
job.status = status
db.commit()
@router.post("/{job_id}/pause")
def pause_download(
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
) -> dict:
job = _own_job(db, user, job_id)
if job.status in ("queued", "running"):
_set_status(db, job, "paused") # a running worker aborts cooperatively via the hook
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
return _serialize(job, asset)
@router.post("/{job_id}/resume")
def resume_download(
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
) -> dict:
job = _own_job(db, user, job_id)
if job.status in ("paused", "error"):
job.progress = 0
job.error = None
# If the shared asset itself failed, reset it to pending so the worker actually
# re-downloads — otherwise the requeued job would instantly re-inherit the asset's
# stale error (the worker short-circuits a job whose asset is already errored).
if job.asset_id:
asset = db.get(MediaAsset, job.asset_id)
if asset and asset.status == "error":
asset.status = "pending"
asset.error = None
_set_status(db, job, "queued")
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
return _serialize(job, asset)
@router.post("/{job_id}/cancel")
def cancel_download(
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
) -> dict:
job = _own_job(db, user, job_id)
if job.status not in ("done", "canceled"):
_release_asset(db, job) # release while the job still counts as holding
job.status = "canceled"
db.commit()
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
return _serialize(job, asset)
@router.delete("/{job_id}")
def delete_download(
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
) -> dict:
job = _own_job(db, user, job_id)
_release_asset(db, job)
db.delete(job)
db.commit()
return {"deleted": job_id}
def _release_asset(db: Session, job: DownloadJob) -> None:
"""Drop this job's hold on its asset when it leaves a holding state. Once NO job holds the
asset anymore, delete the file + row immediately a deleted download should free its disk,
and the shared cache only needs to span *overlapping* holders (a later re-add just downloads
again). Idempotent: a job that's already left holding (canceled/error) is a no-op, so delete
after cancel doesn't double-release."""
if not job.asset_id or job.status not in ("queued", "running", "paused", "done"):
return
asset = db.get(MediaAsset, job.asset_id)
if asset is None:
return
asset.ref_count = max(0, (asset.ref_count or 0) - 1)
if asset.ref_count == 0 and asset.status == "ready":
if asset.rel_path:
storage.delete_asset_files(settings.download_root, asset.rel_path)
db.delete(asset)
# --- file download (range-aware, custom display name) --------------------------------------
def _clean_basename(name: str) -> str:
# Emoji/symbol-free but keeps spaces + accents (a readable device filename).
return storage.display_filename(name)
def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
job = db.get(DownloadJob, job_id)
if job is None:
raise HTTPException(status_code=404, detail="Unknown download")
if job.user_id == user.id:
return job
shared = db.execute(
select(DownloadShare.id).where(
DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
)
).first()
if shared is None:
raise HTTPException(status_code=404, detail="Unknown download")
return job
@router.get("/{job_id}/file")
def download_file(
job_id: int,
request: Request,
user: User = Depends(require_human),
db: Session = Depends(get_db),
):
job = _accessible_job(db, user, job_id)
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or asset.status != "ready" or not asset.rel_path:
raise HTTPException(status_code=409, detail="This download isn't ready.")
path = storage.abs_path(settings.download_root, asset.rel_path).resolve()
root = Path(settings.download_root).resolve()
if root not in path.parents or not path.exists():
raise HTTPException(status_code=404, detail="File is no longer available.")
ext = asset.container or path.suffix.lstrip(".")
base = _clean_basename(job.display_name or asset.title or asset.source_ref)
if base.lower().endswith(f".{ext.lower()}"):
base = base[: -(len(ext) + 1)]
filename = f"{base}.{ext}"
asset.last_access_at = datetime.now(timezone.utc)
db.commit()
# Starlette's FileResponse honours the Range header (206 partial content) for seek/resume.
return FileResponse(path, filename=filename)
# --- sharing -------------------------------------------------------------------------------
@router.post("/{job_id}/share")
def share_download(
job_id: int,
payload: dict,
user: User = Depends(require_human),
db: Session = Depends(get_db),
) -> dict:
job = _own_job(db, user, job_id)
if job.status != "done" or job.asset_id is None:
raise HTTPException(status_code=409, detail="Only a finished download can be shared.")
target = (payload.get("email") or "").strip().lower()
if not target:
raise HTTPException(status_code=400, detail="A recipient email is required.")
recipient = db.execute(
select(User).where(func.lower(User.email) == target)
).scalar_one_or_none()
if recipient is None:
raise HTTPException(status_code=404, detail="No user with that email.")
if recipient.id == user.id:
raise HTTPException(status_code=400, detail="That's already your download.")
exists = db.execute(
select(DownloadShare.id).where(
DownloadShare.job_id == job_id, DownloadShare.to_user_id == recipient.id
)
).first()
if exists is None:
db.add(
DownloadShare(job_id=job_id, from_user_id=user.id, to_user_id=recipient.id)
)
from app.notifications import create_notification
create_notification(
db,
user_id=recipient.id,
type="download_shared",
title=(job.display_name or "A download") + " was shared with you",
data={"kind": "download_shared", "from": user.email, "job_id": job_id},
commit=False,
)
db.commit()
return {"shared_with": recipient.email}
@router.delete("/{job_id}/share/{email}")
def unshare_download(
job_id: int,
email: str,
user: User = Depends(require_human),
db: Session = Depends(get_db),
) -> dict:
job = _own_job(db, user, job_id)
recipient = db.execute(
select(User).where(func.lower(User.email) == email.strip().lower())
).scalar_one_or_none()
if recipient is not None:
row = db.execute(
select(DownloadShare).where(
DownloadShare.job_id == job_id,
DownloadShare.to_user_id == recipient.id,
)
).scalar_one_or_none()
if row is not None:
db.delete(row)
db.commit()
return {"ok": True}
# --- admin ---------------------------------------------------------------------------------
@admin_router.get("")
def admin_list_downloads(
admin: User = Depends(admin_user), db: Session = Depends(get_db)
) -> list[dict]:
jobs = (
db.execute(
select(DownloadJob).order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc()).limit(500)
)
.scalars()
.all()
)
assets = _assets_for(db, jobs)
emails = {
u.id: u.email
for u in db.execute(
select(User).where(User.id.in_({j.user_id for j in jobs}))
).scalars()
}
out = []
for j in jobs:
data = _serialize(j, assets.get(j.asset_id))
data["user_email"] = emails.get(j.user_id)
out.append(data)
return out
@admin_router.get("/storage")
def admin_storage(
admin: User = Depends(admin_user), db: Session = Depends(get_db)
) -> dict:
ready = db.execute(
select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
MediaAsset.status == "ready"
)
).one()
total_assets = db.execute(select(func.count()).select_from(MediaAsset)).scalar()
per_user = db.execute(
select(DownloadJob.user_id, func.count(func.distinct(DownloadJob.asset_id)))
.where(DownloadJob.asset_id.is_not(None))
.group_by(DownloadJob.user_id)
).all()
emails = {
u.id: u.email
for u in db.execute(
select(User).where(User.id.in_([uid for uid, _ in per_user]))
).scalars()
}
return {
"ready_files": ready[0],
"total_bytes": int(ready[1]),
"total_assets": total_assets,
"total_cap_bytes": settings.download_total_max_bytes,
"per_user": [
{"user_id": uid, "email": emails.get(uid), "footprint_bytes": quota.footprint(db, uid)}
for uid, _ in per_user
],
}
@admin_router.get("/quota/{user_id}")
def admin_get_quota(
user_id: int, admin: User = Depends(admin_user), db: Session = Depends(get_db)
) -> dict:
lim = quota.resolve(db, user_id)
row = db.get(DownloadQuota, user_id)
return {
"user_id": user_id,
"custom": row is not None,
"max_bytes": lim.max_bytes,
"max_concurrent": lim.max_concurrent,
"max_jobs": lim.max_jobs,
"unlimited": lim.unlimited,
}
@admin_router.put("/quota/{user_id}")
def admin_set_quota(
user_id: int,
payload: dict,
admin: User = Depends(admin_user),
db: Session = Depends(get_db),
) -> dict:
if db.get(User, user_id) is None:
raise HTTPException(status_code=404, detail="Unknown user")
cur = quota.resolve(db, user_id)
row = db.get(DownloadQuota, user_id)
if row is None:
row = DownloadQuota(
user_id=user_id,
max_bytes=cur.max_bytes,
max_concurrent=cur.max_concurrent,
max_jobs=cur.max_jobs,
unlimited=cur.unlimited,
)
db.add(row)
if "max_bytes" in payload:
row.max_bytes = max(0, int(payload["max_bytes"]))
if "max_concurrent" in payload:
row.max_concurrent = max(1, int(payload["max_concurrent"]))
if "max_jobs" in payload:
row.max_jobs = max(1, int(payload["max_jobs"]))
if "unlimited" in payload:
row.unlimited = bool(payload["unlimited"])
db.commit()
return admin_get_quota(user_id, admin, db)
@admin_router.delete("/quota/{user_id}")
def admin_reset_quota(
user_id: int, admin: User = Depends(admin_user), db: Session = Depends(get_db)
) -> dict:
row = db.get(DownloadQuota, user_id)
if row is not None:
db.delete(row)
db.commit()
return admin_get_quota(user_id, admin, db)

View file

@ -12,6 +12,7 @@ from sqlalchemy import select
from app import progress, quota
from app.config import settings
from app.db import SessionLocal
from app.downloads.gc import run_download_gc
from app.models import SchedulerSetting
from app.notifications import create_notification
from app.state import is_sync_paused
@ -59,6 +60,7 @@ JOB_INTERVALS: dict[str, int] = {
"maintenance": settings.maintenance_interval_minutes,
"demo_reset": settings.demo_reset_minutes,
"explore_cleanup": settings.explore_cleanup_minutes,
"download_gc": settings.download_gc_minutes,
}
# Sane bounds for an admin-set interval (minutes).
@ -276,6 +278,12 @@ def _explore_cleanup_job() -> None:
_job("explore_cleanup", purge_ephemeral)
def _download_gc_job() -> None:
# Retention GC for the download center: pre-expiry warnings, TTL deletion, LRU eviction.
# Pure disk/DB work (no YouTube quota).
_job("download_gc", run_download_gc)
# job_id -> wrapper. The single source of truth for which jobs exist and how to run one,
# shared by start_scheduler (recurring registration) and trigger_job (manual "run now").
JOB_FUNCS: dict[str, Callable[[], None]] = {
@ -289,6 +297,7 @@ JOB_FUNCS: dict[str, Callable[[], None]] = {
"maintenance": _maintenance_job,
"demo_reset": _demo_reset_job,
"explore_cleanup": _explore_cleanup_job,
"download_gc": _download_gc_job,
}

View file

@ -11,6 +11,7 @@ from sqlalchemy.orm import Session
from app import progress, sysconfig
from app.models import Channel, Video
from app.titles import normalize_title
from app.youtube.client import YouTubeClient, best_thumbnail
from app.youtube.rss import fetch_channel_feed
from app.youtube.shorts import make_client, probe_is_short
@ -53,6 +54,11 @@ def _insert_stubs(db: Session, rows: list[dict]) -> int:
return 0
seen: dict[str, dict] = {}
for row in rows:
# Normalize the title at first insert too (RSS/backfill stubs), so a brand-new video
# never flashes its raw title before enrichment refreshes it.
raw = row.get("title")
if raw:
row = {**row, "original_title": raw, "title": normalize_title(raw)}
seen[row["id"]] = row
stmt = (
pg_insert(Video)
@ -225,7 +231,11 @@ def apply_video_details(video: Video, item: dict) -> None:
live = item.get("liveStreamingDetails")
status = item.get("status", {})
video.title = snippet.get("title") or video.title
raw_title = snippet.get("title")
if raw_title:
# Store the raw title and a normalized one for display (feed/search/downloads).
video.original_title = raw_title
video.title = normalize_title(raw_title)
video.description = snippet.get("description")
if not video.published_at:
video.published_at = parse_dt(snippet.get("publishedAt"))

View file

@ -66,6 +66,16 @@ SPECS: tuple[ConfigSpec, ...] = (
ConfigSpec("google_client_secret", "str", "google", "google_client_secret", secret=True),
# --- Access / registration ---
ConfigSpec("allow_registration", "bool", "access", "allow_registration"),
# --- Download center (yt-dlp) — per-user defaults + retention/GC + layout ---
ConfigSpec("download_total_max_bytes", "int", "downloads", "download_total_max_bytes", min=0),
ConfigSpec("download_default_max_bytes", "int", "downloads", "download_default_max_bytes", min=0),
ConfigSpec("download_default_max_concurrent", "int", "downloads", "download_default_max_concurrent", min=1, max=20),
ConfigSpec("download_default_max_jobs", "int", "downloads", "download_default_max_jobs", min=1, max=100_000),
ConfigSpec("download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650),
ConfigSpec("download_gc_grace_days", "int", "downloads", "download_gc_grace_days", min=0, max=3_650),
ConfigSpec("download_layout", "str", "downloads", "download_layout"),
ConfigSpec("download_worker_concurrency", "int", "downloads", "download_worker_concurrency", min=1, max=16),
ConfigSpec("sponsorblock_default", "bool", "downloads", "sponsorblock_default"),
)
_BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS}

80
backend/app/titles.py Normal file
View file

@ -0,0 +1,80 @@
"""Video title normalization — clean the noisy YouTube titles for display + storage.
Applied where a video's title is written (enrichment) and where a download's title comes from
yt-dlp, so the feed, search, channel pages, and the download center all show tidy titles. The
raw title is preserved in `Video.original_title` so this is reversible / re-derivable.
Rules (user-approved "option b"):
1. Drop emoji / pictographs / symbols / control chars (keep accents HU/DE/).
2. Strip trailing SEO hashtag clusters (word hashtags at the end); a numeric "#3" episode
marker survives.
3. De-shout ALL-CAPS words, context-aware:
- If the title is *mostly shouting* ( half the multi-letter words are all-caps), every
all-caps word is Title-cased (short shouted words like CAR/WAR/ÉN get fixed too), and
the first letter is capitalized. Known acronyms (PS, AI, USA, PC) and words with
digits (PS5, 3D) are kept; function words (to/the/és/az) are lowercased.
- Otherwise only long all-caps words (4 letters) are Title-cased, so a lone acronym in
an otherwise normal title is left alone.
4. Collapse repeated punctuation (!!! !) and whitespace.
"""
import re
import unicodedata
_DROP_CATEGORIES = {"So", "Sk", "Cc", "Cf", "Cs", "Co", "Cn"}
_LETTER_RUN = re.compile(r"[^\W\d_]+", re.UNICODE)
_TRAILING_HASHTAGS = re.compile(r"(?:\s+#[^\s#]*[^\W\d\s_][^\s#]*)+\s*$", re.UNICODE)
_REPEAT_PUNCT = re.compile(r"([!?.,\-])\1{1,}")
# Short function words → lowercased when the title is de-shouted (proper title-case style).
_SHORT_LOWER = {
"to", "the", "and", "of", "a", "an", "in", "on", "at", "for", "or", "vs", "the",
"és", "az", "egy", "meg", "nem", "hogy", "de", "ha", "der", "die", "das", "und", "von",
}
# Kept as-is even when everything else is de-shouted (common acronyms; compared lowercased).
_ACRONYMS = {
"ps", "ai", "pc", "tv", "hd", "4k", "8k", "uk", "eu", "us", "usa", "gta", "rpg", "fps",
"diy", "vr", "ar", "id", "ok", "faq", "nasa", "fbi", "cia", "ceo", "amd", "pdf", "usb",
"gps", "api", "dj", "mc", "suv", "gpu", "cpu", "ssd", "hdmi", "led", "ufo", "dna", "nba",
"nfl", "asmr", "pov", "diy", "wtf", "lol", "rtx", "gtx", "ios", "mmo", "vip", "hp",
}
def _titlecase(w: str) -> str:
return w[0].upper() + w[1:].lower()
def normalize_title(raw: str | None) -> str | None:
"""Return the cleaned title, or the input unchanged for empty/whitespace."""
if not raw or not raw.strip():
return raw
t = unicodedata.normalize("NFKC", raw)
t = "".join(ch for ch in t if unicodedata.category(ch) not in _DROP_CATEGORIES)
t = _TRAILING_HASHTAGS.sub("", t)
multi = [r for r in _LETTER_RUN.findall(t) if len(r) >= 2]
caps = [r for r in multi if r.isupper()]
shouting = len(caps) >= 2 and len(caps) >= 0.5 * len(multi)
def repl(m: re.Match) -> str:
w = m.group(0)
if len(w) < 2 or not w.isupper():
return w
low = w.lower()
if low in _ACRONYMS:
return w
if low in _SHORT_LOWER:
return low
if shouting or len(w) >= 4:
return _titlecase(w)
return w # short all-caps in a non-shouting title → likely an acronym, keep
t = _LETTER_RUN.sub(repl, t)
t = _REPEAT_PUNCT.sub(r"\1", t)
t = re.sub(r"\s+", " ", t).strip()
if shouting: # ensure the first letter is capitalized (a leading function word got lowered)
for i, ch in enumerate(t):
if ch.isalpha():
t = t[:i] + ch.upper() + t[i + 1:]
break
return t or raw

477
backend/app/worker.py Normal file
View file

@ -0,0 +1,477 @@
"""Download worker process entry point.
Runs in a DEDICATED container (`command: python -m app.worker`, `WORKER_ENABLED=1`), separate
from the API so long yt-dlp downloads + ffmpeg postprocessing never block web requests.
Design:
* N loop threads (download_worker_concurrency) each claim a `queued` DownloadJob with
FOR UPDATE SKIP LOCKED and process it.
* The shared MediaAsset is the dedup unit. A job whose asset is already `ready` finishes
instantly (no re-download); a fresh asset is claimed pendingdownloading by exactly one
worker (a DB compare-and-set), so two jobs for the same video+format never double-download.
* Live progress is written to the job row (short-lived sessions, throttled ~1/s) the
frontend polls it; the worker can't reach the API's in-process WebSocket registry.
* Pause/cancel are cooperative: the progress hook re-reads the job status and aborts the
yt-dlp download if it's no longer `running`.
* Crash-safe: on startup, orphaned `running` jobs are requeued and `downloading` assets reset.
"""
import logging
import shutil
import signal
import threading
import time
from datetime import date, datetime, timedelta, timezone
from pathlib import Path
import yt_dlp
from sqlalchemy import text
from app.config import settings
from app.db import SessionLocal
from app.downloads import formats, quota, service, storage
from app.models import DownloadJob, MediaAsset
from app.titles import normalize_title
log = logging.getLogger("siftlode.worker")
_stop = threading.Event()
class _Aborted(Exception):
"""Raised inside the progress hook when the job was paused/canceled mid-download."""
def _handle_signal(signum, frame):
_stop.set()
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
def _set_job(job_id: int, **fields) -> None:
with SessionLocal() as db:
job = db.get(DownloadJob, job_id)
if job is None:
return
for k, v in fields.items():
setattr(job, k, v)
db.commit()
def _job_status(job_id: int) -> str | None:
with SessionLocal() as db:
return db.execute(
text("SELECT status FROM download_jobs WHERE id = :id"), {"id": job_id}
).scalar()
def _parse_upload_date(raw) -> date | None:
if not raw:
return None
try:
return datetime.strptime(str(raw), "%Y%m%d").date()
except ValueError:
return None
# --- claim + recovery -----------------------------------------------------------------------
def _recover_orphans() -> None:
"""Reset state left behind by a previous crash so nothing is stuck."""
with SessionLocal() as db:
db.execute(text("UPDATE download_jobs SET status='queued' WHERE status='running'"))
db.execute(text("UPDATE media_assets SET status='pending' WHERE status='downloading'"))
db.commit()
def _claim_job() -> int | None:
"""Claim the oldest queued job whose user is under their per-user concurrency limit.
Fetches a window of locked candidates (FOR UPDATE SKIP LOCKED) and picks the first eligible
one, so one user flooding the queue can't starve others beyond their max_concurrent."""
with SessionLocal() as db:
rows = db.execute(
text(
"""
SELECT id, user_id FROM download_jobs
WHERE status='queued'
ORDER BY queue_pos, id
FOR UPDATE SKIP LOCKED
LIMIT 50
"""
)
).fetchall()
chosen = next(
(jid for jid, uid in rows if not quota.at_concurrency_limit(db, uid)), None
)
if chosen is None:
db.commit() # release the row locks; nothing eligible right now
return None
db.execute(
text(
"UPDATE download_jobs SET status='running', phase='starting', updated_at=now() "
"WHERE id = :id"
),
{"id": chosen},
)
db.commit()
return chosen
def _claim_asset(asset_id: int) -> bool:
"""Compare-and-set pending→downloading. True if THIS worker won the right to download."""
with SessionLocal() as db:
row = db.execute(
text(
"UPDATE media_assets SET status='downloading' "
"WHERE id = :id AND status='pending' RETURNING id"
),
{"id": asset_id},
).fetchone()
db.commit()
return row is not None
# --- processing -----------------------------------------------------------------------------
def _process_job(job_id: int) -> None:
with SessionLocal() as db:
job = db.get(DownloadJob, job_id)
if job is None:
return
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
spec = dict(job.profile_snapshot or {})
source_kind, source_ref = job.source_kind, job.source_ref
asset_id = asset.id if asset else None
asset_status = asset.status if asset else None
if asset is None:
_set_job(job_id, status="error", error="No media asset", phase=None)
return
if asset_status == "ready":
_finish_from_cache(job_id, asset_id)
return
if asset_status == "error":
_set_job(job_id, status="error", error=asset.error or "Download failed", phase=None)
return
if asset_status == "downloading":
# Another worker owns this asset; wait and let the job be reclaimed later.
_set_job(job_id, status="queued", phase="waiting")
time.sleep(2)
return
# asset_status == "pending": try to win the download.
if not _claim_asset(asset_id):
_set_job(job_id, status="queued", phase="waiting")
time.sleep(2)
return
_download(job_id, asset_id, source_kind, source_ref, spec)
def _finish_from_cache(job_id: int, asset_id: int) -> None:
with SessionLocal() as db:
job = db.get(DownloadJob, job_id)
asset = db.get(MediaAsset, asset_id)
if job and asset:
job.status = "done"
job.progress = 100
job.phase = None
if not job.display_name:
job.display_name = asset.title
asset.last_access_at = datetime.now(timezone.utc)
db.commit()
def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
"""Populate an asset's display metadata from yt-dlp's info_dict the first time it's seen
(only if not already filled at enqueue from our catalog) so an ad-hoc URL's row shows a
real title + thumbnail while it downloads, not just the URL. Zero extra network cost."""
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
if asset is None or asset.title:
return
asset.title = normalize_title(info.get("title"))
asset.uploader = info.get("uploader") or info.get("channel")
asset.thumbnail_url = info.get("thumbnail")
asset.duration_s = int(info["duration"]) if info.get("duration") else None
asset.upload_date = _parse_upload_date(info.get("upload_date"))
db.commit()
def _make_progress_hook(job_id: int, asset_id: int):
state = {"last_db": 0.0, "last_check": 0.0, "meta_done": False}
def hook(d: dict) -> None:
now = time.monotonic()
# Cooperative pause/cancel: if the job is no longer 'running', abort the download.
if now - state["last_check"] >= 2.0:
state["last_check"] = now
if _job_status(job_id) not in ("running", None):
raise _Aborted
if not state["meta_done"] and d.get("info_dict"):
state["meta_done"] = True
try:
_fill_asset_meta_early(asset_id, d["info_dict"])
except Exception: # noqa: BLE001 — metadata is best-effort, never fail a download
pass
if d.get("status") != "downloading":
return
if now - state["last_db"] < 1.0:
return
state["last_db"] = now
# yt-dlp downloads the video and audio streams SEPARATELY (each 0→100%), then merges.
# Label the phase so the user understands the bar resetting between streams instead of
# seeing a mysterious 95%→5% jump.
info = d.get("info_dict") or {}
vcodec = info.get("vcodec") or "none"
acodec = info.get("acodec") or "none"
if vcodec != "none" and acodec == "none":
phase = "video"
elif acodec != "none" and vcodec == "none":
phase = "audio"
else:
phase = "media"
total = d.get("total_bytes") or d.get("total_bytes_estimate") or 0
done = d.get("downloaded_bytes") or 0
pct = int(done * 100 / total) if total else 0
_set_job(
job_id,
progress=min(pct, 99),
speed_bps=int(d["speed"]) if d.get("speed") else None,
eta_s=int(d["eta"]) if d.get("eta") else None,
phase=phase,
)
return hook
def _pp_phase(pp: str) -> str:
"""Map a yt-dlp postprocessor class name to a user-facing phase label. ffmpeg post-steps
aren't byte-progress operations (no %), so the UI shows the step name + an indeterminate
pulse this makes the long merge/finalize legible instead of a silent 'Processing'."""
if "Merg" in pp:
return "merging"
if "ExtractAudio" in pp:
return "audio_extract"
if "Thumbnail" in pp:
return "thumbnail"
if "SponsorBlock" in pp or "ModifyChapters" in pp:
return "sponsorblock"
if "Metadata" in pp:
return "metadata"
return "processing"
def _make_pp_hook(job_id: int):
"""Postprocessor hook: surface the current ffmpeg step so the bar shows a concrete label
(Merging / Embedding thumbnail / ) instead of freezing at 100% after the streams finish."""
def hook(d: dict) -> None:
if d.get("status") in ("started", "processing"):
_set_job(job_id, phase=_pp_phase(d.get("postprocessor") or ""), speed_bps=None, eta_s=None)
return hook
def _staging_dir(asset_id: int) -> Path:
return Path(settings.download_root) / ".staging" / f"asset-{asset_id}"
def _find_thumbnail(staging: Path, media: Path) -> Path | None:
for p in staging.iterdir():
if p != media and p.suffix.lower() in (".jpg", ".jpeg", ".png", ".webp"):
return p
return None
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
_RETRIABLE_MARKERS = (
"not a bot",
"DRM protected",
"Requested format is not available",
"Only images are available",
"Sign in to confirm",
)
def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, staging: Path):
"""Download via yt-dlp, trying the primary player clients then the POT-backed fallback set
on a bot/DRM/format failure. Returns the info dict of the successful attempt."""
client_sets = [
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
] or [None]
outtmpl = str(staging / "%(id)s.%(ext)s")
last_exc: Exception | None = None
for i, clients in enumerate(client_sets):
opts = formats.build_ydl_opts(
spec, outtmpl, _make_progress_hook(job_id, asset_id),
settings.download_pot_base_url, clients,
)
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
try:
with yt_dlp.YoutubeDL(opts) as ydl:
return ydl.extract_info(url, download=True)
except yt_dlp.utils.DownloadError as exc:
last_exc = exc
retriable = any(m in str(exc) for m in _RETRIABLE_MARKERS)
if i < len(client_sets) - 1 and retriable:
log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
for p in list(staging.iterdir()): # drop partials before the retry
try:
p.unlink()
except OSError:
pass
_set_job(job_id, progress=0, phase="downloading")
continue
raise
raise last_exc # unreachable (loop either returns or raises)
def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict) -> None:
staging = _staging_dir(asset_id)
try:
if staging.exists():
shutil.rmtree(staging, ignore_errors=True)
staging.mkdir(parents=True, exist_ok=True)
_set_job(job_id, phase="downloading", progress=0)
url = service.source_url(source_kind, source_ref)
info = _extract_with_fallback(job_id, asset_id, url, spec, staging)
req = (info.get("requested_downloads") or [{}])[0]
produced = Path(req.get("filepath") or "")
if not produced.exists():
# Fallback: the single media file left in staging (ignore the thumbnail).
cands = [p for p in staging.iterdir() if p.suffix.lower() not in (".jpg", ".jpeg", ".png", ".webp")]
if not cands:
raise RuntimeError("yt-dlp produced no output file")
produced = cands[0]
ext = produced.suffix.lstrip(".").lower()
meta = storage.MediaMeta(
video_id=info.get("id") or source_ref,
title=normalize_title(info.get("title")) or source_ref,
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
upload_date=_parse_upload_date(info.get("upload_date")),
duration_s=int(info["duration"]) if info.get("duration") else None,
description=info.get("description"),
thumbnail_url=info.get("thumbnail"),
)
rel = storage.rel_path(meta, ext, settings.download_layout)
thumb = _find_thumbnail(staging, produced)
storage.place_file(produced, settings.download_root, rel)
nfo_ok = storage.write_sidecars(settings.download_root, rel, meta, thumb)
final = storage.abs_path(settings.download_root, rel)
size = final.stat().st_size if final.exists() else None
expires = datetime.now(timezone.utc) + timedelta(days=settings.download_retention_days)
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
if asset:
asset.status = "ready"
asset.rel_path = rel
asset.size_bytes = size
asset.duration_s = meta.duration_s
asset.width = int(info["width"]) if info.get("width") else None
asset.height = int(info["height"]) if info.get("height") else None
asset.container = ext
asset.vcodec = info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
asset.title = meta.title
asset.uploader = meta.uploader
asset.upload_date = meta.upload_date
asset.thumbnail_url = meta.thumbnail_url
asset.nfo_written = nfo_ok
asset.error = None
asset.last_access_at = datetime.now(timezone.utc)
asset.expires_at = expires
job = db.get(DownloadJob, job_id)
if job:
job.status = "done"
job.progress = 100
job.phase = None
job.speed_bps = None
job.eta_s = None
if not job.display_name:
job.display_name = meta.title
db.commit()
log.info("job %s done: %s", job_id, rel)
except _Aborted:
# Paused/canceled mid-download: free the asset for a later retry; keep the job's status
# (the route set it to paused/canceled). ref_count is adjusted by the route.
_reset_asset_pending(asset_id)
log.info("job %s aborted (pause/cancel)", job_id)
except Exception as exc: # noqa: BLE001 — surface any failure to the user, keep worker alive
msg = str(exc)[:500]
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
if asset:
asset.status = "error"
asset.error = msg
job = db.get(DownloadJob, job_id)
if job:
job.status = "error"
job.error = msg
job.phase = None
db.commit()
log.warning("job %s failed: %s", job_id, msg)
finally:
shutil.rmtree(staging, ignore_errors=True)
def _reset_asset_pending(asset_id: int) -> None:
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
if asset and asset.status == "downloading":
asset.status = "pending"
db.commit()
# --- loop -----------------------------------------------------------------------------------
def _worker_loop(idx: int) -> None:
while not _stop.is_set():
try:
job_id = _claim_job()
except Exception: # noqa: BLE001
log.exception("claim failed")
_stop.wait(settings.download_poll_seconds)
continue
if job_id is None:
_stop.wait(settings.download_poll_seconds)
continue
try:
_process_job(job_id)
except Exception: # noqa: BLE001
log.exception("processing job %s crashed", job_id)
_set_job(job_id, status="error", error="Worker error", phase=None)
def main() -> None:
logging.basicConfig(level=logging.INFO)
signal.signal(signal.SIGTERM, _handle_signal)
signal.signal(signal.SIGINT, _handle_signal)
if not settings.worker_enabled:
log.info("WORKER_ENABLED is off; worker exiting (nothing to do).")
return
_recover_orphans()
n = max(1, settings.download_worker_concurrency)
log.info("Download worker started (concurrency=%d, root=%s).", n, settings.download_root)
threads = [threading.Thread(target=_worker_loop, args=(i,), daemon=True) for i in range(n)]
for t in threads:
t.start()
while not _stop.is_set():
_stop.wait(1.0)
log.info("Download worker stopping…")
for t in threads:
t.join(timeout=5)
if __name__ == "__main__":
main()

View file

@ -13,3 +13,12 @@ py3langid>=0.3,<1.0
itsdangerous>=2.1,<3.0
cryptography>=46.0.7,<47
argon2-cffi>=23.1,<26
# Download center: yt-dlp does the extraction/download; ffmpeg (OS-level in the Dockerfile)
# does the merge + postprocessing. The [default] extra pulls yt-dlp-ejs (the challenge-solver
# scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so
# keep the floor loose and bump it when extraction breaks.
yt-dlp[default]>=2025.5.22
# PO Token provider plugin: talks to the bgutil sidecar (brainicism/bgutil-ytdlp-pot-provider)
# to mint YouTube Proof-of-Origin tokens on demand, bypassing "confirm you're not a bot" and
# unlocking high-quality formats. Pin to match the sidecar image tag (versions must align).
bgutil-ytdlp-pot-provider==1.3.1

View file

@ -43,6 +43,9 @@ services:
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode}
# This instance owns its scheduler now (its own DB, so no double-write concern).
SCHEDULER_ENABLED: "true"
# Download center: the API serves finished files (it does NOT run the worker loop).
DOWNLOAD_ROOT: /downloads
WORKER_ENABLED: "false"
depends_on:
db:
condition: service_healthy
@ -51,6 +54,49 @@ services:
- apparmor:unconfined
ports:
- "${APP_PORT:-8080}:8000"
volumes:
# Downloaded media lives on the host so you can inspect the generated Plex tree.
# Gitignored. The worker writes here; the API reads it to serve local downloads.
- ./downloads:/downloads
restart: unless-stopped
# Dedicated download worker: same image, runs the yt-dlp job loop instead of the API.
# It shares the DB (job queue) and the downloads mount with the API.
worker:
build:
context: .
dockerfile: Dockerfile
args:
APP_VERSION: ${APP_VERSION:-dev}
GIT_SHA: ${GIT_SHA:-unknown}
BUILD_DATE: ${BUILD_DATE:-}
command: ["python", "-m", "app.worker"]
env_file: .env
environment:
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode}
# The worker must not also run the scheduler; the API owns that.
SCHEDULER_ENABLED: "false"
DOWNLOAD_ROOT: /downloads
WORKER_ENABLED: "true"
depends_on:
db:
condition: service_healthy
bgutil-pot:
condition: service_started
security_opt:
- apparmor:unconfined
volumes:
- ./downloads:/downloads
restart: unless-stopped
# PO-token provider: mints YouTube Proof-of-Origin tokens on demand so the worker bypasses
# bot-detection and gets high-quality formats. The worker reaches it at http://bgutil-pot:4416
# (DOWNLOAD_POT_BASE_URL). Pin the tag to match the plugin version in requirements.txt.
bgutil-pot:
image: brainicism/bgutil-ytdlp-pot-provider:1.3.1
init: true
security_opt:
- apparmor:unconfined
restart: unless-stopped
volumes:

View file

@ -57,6 +57,8 @@ import AdminUsers, { focusAccessRequestsTab } from "./components/AdminUsers";
import SettingsPanel from "./components/SettingsPanel";
import NotificationsPanel from "./components/NotificationsPanel";
import Messages from "./components/Messages";
import DownloadCenter from "./components/DownloadCenter";
import { setNavigator } from "./lib/nav";
import ChatDock from "./components/ChatDock";
import OnboardingWizard from "./components/OnboardingWizard";
import { shouldAutoOpenOnboarding } from "./lib/onboarding";
@ -279,6 +281,8 @@ export default function App() {
}
go();
}
// Expose the current setPage to decoupled callers (download toast "View", etc.).
useEffect(() => setNavigator(setPage));
function setSidebarLayout(next: SidebarLayout) {
setSidebarLayoutState(next);
@ -728,6 +732,8 @@ export default function App() {
<div className="p-4 max-w-3xl w-full mx-auto">
<Messages meId={meQuery.data!.id} />
</div>
) : page === "downloads" && !meQuery.data!.is_demo ? (
<DownloadCenter me={meQuery.data!} />
) : page === "settings" ? (
<SettingsPanel
me={meQuery.data!}

View file

@ -0,0 +1,66 @@
import { useState } from "react";
import { useTranslation } from "react-i18next";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { Download } from "lucide-react";
import clsx from "clsx";
import DownloadDialog from "./DownloadDialog";
import { api, type Me } from "../lib/api";
// Self-contained download affordance for a video card / the player. Hidden for the demo account
// (downloads spend server disk + need a real identity). Reflects the video's current state from
// the shared per-user download index (one polled query shared across every card).
export default function DownloadButton({
videoId,
title,
className,
}: {
videoId: string;
title?: string | null;
className?: string;
}) {
const { t } = useTranslation();
const qc = useQueryClient();
const me = qc.getQueryData<Me>(["me"]);
const isDemo = !!me?.is_demo;
const indexQ = useQuery({
queryKey: ["download-index"],
queryFn: api.downloadIndex,
enabled: !isDemo,
staleTime: 10000,
});
const [open, setOpen] = useState(false);
if (isDemo) return null;
const status = indexQ.data?.[videoId];
const downloaded = status === "done";
const inQueue = status === "queued" || status === "running" || status === "paused";
const label = downloaded
? t("downloads.button.downloaded")
: inQueue
? t("downloads.button.queued")
: t("downloads.button.label");
return (
<>
<button
onClick={(e) => {
e.preventDefault();
e.stopPropagation();
setOpen(true);
}}
title={label}
className={clsx(
className ?? "p-1.5 rounded-md hover:bg-surface text-muted hover:text-fg",
(downloaded || inQueue) && "text-accent"
)}
>
<Download className={clsx("w-4 h-4", inQueue && "animate-pulse")} />
</button>
{open && (
<DownloadDialog source={videoId} title={title} onClose={() => setOpen(false)} />
)}
</>
);
}

View file

@ -0,0 +1,531 @@
import { useState } from "react";
import { useTranslation } from "react-i18next";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import {
Ban,
Download,
Pause,
Play,
Plus,
Settings2,
Share2,
Pencil,
Trash2,
} from "lucide-react";
import clsx from "clsx";
import Tabs, { usePersistedTab } from "./Tabs";
import Modal from "./Modal";
import DownloadDialog from "./DownloadDialog";
import ProfileEditor from "./ProfileEditor";
import { useConfirm } from "./ConfirmProvider";
import { useLiveQuery } from "../lib/useLiveQuery";
import { api, type DownloadJob, type Me } from "../lib/api";
import { notify } from "../lib/notifications";
import { formatBytes, formatDuration, formatEta, formatSpeed } from "../lib/format";
const inputCls =
"w-full rounded-lg bg-surface border border-border px-3 py-2 text-sm focus:outline-none focus:ring-2 focus:ring-accent/40";
const GiB = 1024 ** 3;
const STATUS_CLS: Record<string, string> = {
queued: "text-muted",
running: "text-accent",
paused: "text-amber-400",
done: "text-emerald-400",
error: "text-red-400",
canceled: "text-muted",
};
// Byte-progress phases (show a % bar) vs ffmpeg post-steps (no %, indeterminate pulse).
const DOWNLOAD_PHASES = ["video", "audio"];
const PHASE_KEYS = [
"video", "audio", "merging", "audio_extract", "thumbnail", "sponsorblock", "metadata", "processing",
];
function StatusPill({ job }: { job: DownloadJob }) {
const { t } = useTranslation();
// While downloading, show the concrete phase (video / audio / merging) so the user isn't
// confused by the bar resetting between the separate video and audio streams.
if (job.status === "running" && job.phase && PHASE_KEYS.includes(job.phase)) {
return <span className="text-xs font-medium text-accent">{t(`downloads.phase.${job.phase}`)}</span>;
}
const key = job.expired ? "expired" : job.status;
return (
<span className={clsx("text-xs font-medium", STATUS_CLS[job.status] ?? "text-muted")}>
{t(`downloads.status.${key}`)}
</span>
);
}
function Thumb({ job }: { job: DownloadJob }) {
const url = job.asset?.thumbnail_url;
return (
<div className="w-28 aspect-video shrink-0 rounded-lg overflow-hidden bg-surface border border-border">
{url ? <img src={url} alt="" loading="lazy" className="w-full h-full object-cover" /> : null}
</div>
);
}
function jobTitle(job: DownloadJob): string {
return job.display_name || job.asset?.title || job.source_ref;
}
function IconBtn({
onClick,
title,
danger,
children,
}: {
onClick: () => void;
title: string;
danger?: boolean;
children: React.ReactNode;
}) {
return (
<button
onClick={onClick}
title={title}
className={clsx(
"p-1.5 rounded-md hover:bg-surface text-muted transition",
danger ? "hover:text-red-400" : "hover:text-fg"
)}
>
{children}
</button>
);
}
function DownloadRow({
job,
children,
}: {
job: DownloadJob;
children?: React.ReactNode;
}) {
const { t } = useTranslation();
const running = job.status === "running";
return (
<div className="flex gap-3 p-2.5 rounded-xl bg-card/40 hover:bg-card transition">
<Thumb job={job} />
<div className="min-w-0 flex-1">
<div className="font-medium leading-snug line-clamp-1">{jobTitle(job)}</div>
<div className="text-xs text-muted truncate mt-0.5">
{job.asset?.uploader || job.source_ref}
</div>
<div className="flex items-center gap-2 mt-1 text-xs">
<StatusPill job={job} />
{job.asset?.size_bytes ? <span className="text-muted">· {formatBytes(job.asset.size_bytes)}</span> : null}
{job.asset?.duration_s ? <span className="text-muted">· {formatDuration(job.asset.duration_s)}</span> : null}
{job.error ? <span className="text-red-400 truncate">· {job.error}</span> : null}
</div>
{running && (
<div className="mt-1.5">
{job.phase && !DOWNLOAD_PHASES.includes(job.phase) ? (
// ffmpeg post-steps aren't byte-progress — show an indeterminate pulse, no %.
<div className="h-1.5 rounded-full bg-surface overflow-hidden">
<div className="h-full w-full bg-accent/60 animate-pulse" />
</div>
) : (
<>
<div className="h-1.5 rounded-full bg-surface overflow-hidden">
<div className="h-full bg-accent transition-all" style={{ width: `${job.progress}%` }} />
</div>
<div className="text-[11px] text-muted mt-0.5 flex gap-2">
<span>{job.progress}%</span>
{job.speed_bps ? <span>{formatSpeed(job.speed_bps)}</span> : null}
{job.eta_s ? <span>{formatEta(job.eta_s)}</span> : null}
</div>
</>
)}
</div>
)}
</div>
<div className="flex items-start gap-0.5 shrink-0">{children}</div>
</div>
);
}
// --- Rename + Share small modals ------------------------------------------------------------
function RenameModal({ job, onClose }: { job: DownloadJob; onClose: () => void }) {
const { t } = useTranslation();
const qc = useQueryClient();
const [name, setName] = useState(jobTitle(job));
const save = useMutation({
mutationFn: () => api.renameDownload(job.id, name.trim()),
onSuccess: () => {
qc.invalidateQueries({ queryKey: ["downloads"] });
onClose();
},
});
return (
<Modal title={t("downloads.rename.title")} onClose={onClose}>
<label className="block text-sm font-medium mb-1">{t("downloads.rename.label")}</label>
<input value={name} onChange={(e) => setName(e.target.value)} className={inputCls} autoFocus />
<p className="text-xs text-muted mt-2 mb-4">{t("downloads.rename.hint")}</p>
<div className="flex justify-end">
<button
onClick={() => save.mutate()}
disabled={!name.trim() || save.isPending}
className="px-3 py-1.5 rounded-lg text-sm font-medium bg-accent text-accent-fg hover:opacity-90 disabled:opacity-50 transition"
>
{t("downloads.rename.save")}
</button>
</div>
</Modal>
);
}
function ShareModal({ job, onClose }: { job: DownloadJob; onClose: () => void }) {
const { t } = useTranslation();
const [email, setEmail] = useState("");
const share = useMutation({
mutationFn: () => api.shareDownload(job.id, email.trim()),
onSuccess: (r) => {
notify({ level: "success", message: t("downloads.share.done", { email: r.shared_with }) });
onClose();
},
});
return (
<Modal title={t("downloads.share.title")} onClose={onClose}>
<label className="block text-sm font-medium mb-1">{t("downloads.share.label")}</label>
<input
value={email}
onChange={(e) => setEmail(e.target.value)}
placeholder={t("downloads.share.placeholder")}
className={inputCls}
autoFocus
/>
<div className="flex justify-end mt-4">
<button
onClick={() => share.mutate()}
disabled={!email.trim() || share.isPending}
className="px-3 py-1.5 rounded-lg text-sm font-medium bg-accent text-accent-fg hover:opacity-90 disabled:opacity-50 transition"
>
{t("downloads.share.submit")}
</button>
</div>
</Modal>
);
}
// --- Usage bar ------------------------------------------------------------------------------
function UsageBar() {
const { t } = useTranslation();
const usage = useQuery({ queryKey: ["download-usage"], queryFn: api.downloadUsage });
const u = usage.data;
if (!u) return null;
const pct = u.unlimited || !u.max_bytes ? 0 : Math.min(100, (u.footprint_bytes / u.max_bytes) * 100);
return (
<div className="rounded-xl bg-card/40 p-3 mb-4">
<div className="flex justify-between text-sm mb-1.5">
<span className="font-medium">{t("downloads.usage.title")}</span>
<span className="text-muted">
{formatBytes(u.footprint_bytes)}
{u.unlimited ? "" : ` / ${formatBytes(u.max_bytes)}`} ·{" "}
{u.unlimited ? t("downloads.usage.unlimited") : t("downloads.usage.items", { used: u.active_jobs, max: u.max_jobs })}
</span>
</div>
{!u.unlimited && (
<div className="h-2 rounded-full bg-surface overflow-hidden">
<div className={clsx("h-full transition-all", pct > 90 ? "bg-red-400" : "bg-accent")} style={{ width: `${pct}%` }} />
</div>
)}
</div>
);
}
// --- Admin system tab -----------------------------------------------------------------------
function QuotaModal({ userId, email, onClose }: { userId: number; email: string; onClose: () => void }) {
const { t } = useTranslation();
const qc = useQueryClient();
const q = useQuery({ queryKey: ["dl-quota", userId], queryFn: () => api.adminGetDownloadQuota(userId) });
const [form, setForm] = useState<{ gb: number; jobs: number; conc: number; unlimited: boolean } | null>(null);
const cur = q.data;
const state = form ?? (cur ? { gb: Math.round((cur.max_bytes / GiB) * 10) / 10, jobs: cur.max_jobs, conc: cur.max_concurrent, unlimited: cur.unlimited } : null);
const done = () => {
qc.invalidateQueries({ queryKey: ["dl-quota", userId] });
qc.invalidateQueries({ queryKey: ["admin-storage"] });
onClose();
};
const save = useMutation({
mutationFn: () =>
api.adminSetDownloadQuota(userId, {
max_bytes: Math.round((state!.gb) * GiB),
max_jobs: state!.jobs,
max_concurrent: state!.conc,
unlimited: state!.unlimited,
}),
onSuccess: done,
});
const reset = useMutation({ mutationFn: () => api.adminResetDownloadQuota(userId), onSuccess: done });
if (!state) return null;
const upd = (p: Partial<typeof state>) => setForm({ ...state, ...p });
return (
<Modal title={t("downloads.admin.quotaTitle", { email })} onClose={onClose}>
<div className="space-y-3">
<div className="grid grid-cols-3 gap-3">
<label className="text-sm">
{t("downloads.admin.maxBytes")}
<input type="number" value={state.gb} onChange={(e) => upd({ gb: Number(e.target.value) })} className={inputCls} disabled={state.unlimited} />
</label>
<label className="text-sm">
{t("downloads.admin.maxJobs")}
<input type="number" value={state.jobs} onChange={(e) => upd({ jobs: Number(e.target.value) })} className={inputCls} />
</label>
<label className="text-sm">
{t("downloads.admin.maxConcurrent")}
<input type="number" value={state.conc} onChange={(e) => upd({ conc: Number(e.target.value) })} className={inputCls} />
</label>
</div>
<label className="flex items-center gap-2 text-sm cursor-pointer">
<input type="checkbox" checked={state.unlimited} onChange={(e) => upd({ unlimited: e.target.checked })} />
{t("downloads.admin.unlimited")}
</label>
<div className="flex justify-between pt-1">
<button onClick={() => reset.mutate()} className="px-3 py-1.5 rounded-lg text-sm text-muted hover:text-fg hover:bg-surface transition">
{t("downloads.admin.reset")}
</button>
<button onClick={() => save.mutate()} disabled={save.isPending} className="px-3 py-1.5 rounded-lg text-sm font-medium bg-accent text-accent-fg hover:opacity-90 disabled:opacity-50 transition">
{t("downloads.admin.save")}
</button>
</div>
</div>
</Modal>
);
}
function AdminSystem() {
const { t } = useTranslation();
const storage = useLiveQuery(["admin-storage"], api.adminDownloadStorage, { intervalMs: 5000 });
const jobs = useLiveQuery(["admin-downloads"], api.adminDownloads, { intervalMs: 5000 });
const [quotaFor, setQuotaFor] = useState<{ id: number; email: string } | null>(null);
const s = storage.data;
return (
<div>
{s && (
<div className="grid grid-cols-2 sm:grid-cols-4 gap-3 mb-5">
{[
[t("downloads.admin.readyFiles"), String(s.ready_files)],
[t("downloads.admin.totalSize"), formatBytes(s.total_bytes)],
[t("downloads.admin.cap"), s.total_cap_bytes ? formatBytes(s.total_cap_bytes) : t("downloads.admin.noCap")],
].map(([label, val]) => (
<div key={label} className="rounded-xl bg-card/40 p-3">
<div className="text-xs text-muted">{label}</div>
<div className="text-lg font-semibold mt-0.5">{val}</div>
</div>
))}
</div>
)}
{s && s.per_user.length > 0 && (
<div className="mb-6">
<div className="text-sm font-medium mb-2">{t("downloads.admin.perUser")}</div>
<div className="space-y-1">
{s.per_user.map((u) => (
<div key={u.user_id} className="flex items-center gap-2 text-sm px-3 py-1.5 rounded-lg bg-card/40">
<span className="flex-1 truncate">{u.email}</span>
<span className="text-muted">{formatBytes(u.footprint_bytes)}</span>
<button
onClick={() => setQuotaFor({ id: u.user_id, email: u.email ?? "" })}
className="p-1 rounded hover:bg-surface text-muted hover:text-fg"
title={t("downloads.admin.editQuota")}
>
<Settings2 className="w-4 h-4" />
</button>
</div>
))}
</div>
</div>
)}
<div className="text-sm font-medium mb-2">{t("downloads.tabs.system")}</div>
<div className="space-y-1.5">
{(jobs.data ?? []).map((j) => (
<div key={j.id} className="flex items-center gap-3 text-sm px-3 py-1.5 rounded-lg bg-card/40">
<span className="flex-1 truncate">{jobTitle(j)}</span>
<span className="text-muted truncate w-40">{j.user_email}</span>
<StatusPill job={j} />
<span className="text-muted w-16 text-right">{j.asset?.size_bytes ? formatBytes(j.asset.size_bytes) : ""}</span>
</div>
))}
{jobs.data && jobs.data.length === 0 && <div className="text-sm text-muted py-6 text-center">{t("downloads.empty.admin")}</div>}
</div>
{quotaFor && <QuotaModal userId={quotaFor.id} email={quotaFor.email} onClose={() => setQuotaFor(null)} />}
</div>
);
}
// --- Main -----------------------------------------------------------------------------------
export default function DownloadCenter({ me }: { me: Me }) {
const { t } = useTranslation();
const qc = useQueryClient();
const confirm = useConfirm();
const [tab, setTab] = usePersistedTab("siftlode.downloadTab", "queue");
const [addUrl, setAddUrl] = useState("");
const [addSource, setAddSource] = useState<string | null>(null);
const [manage, setManage] = useState(false);
const [renameJob, setRenameJob] = useState<DownloadJob | null>(null);
const [shareJob, setShareJob] = useState<DownloadJob | null>(null);
const jobsQ = useLiveQuery(["downloads"], api.downloads, { intervalMs: 2000 });
const sharedQ = useQuery({ queryKey: ["downloads-shared"], queryFn: api.downloadsShared, enabled: tab === "shared" });
const jobs = jobsQ.data ?? [];
const queue = jobs.filter((j) => ["queued", "running", "paused", "error"].includes(j.status));
const library = jobs.filter((j) => j.status === "done");
const invalidate = () => {
qc.invalidateQueries({ queryKey: ["downloads"] });
qc.invalidateQueries({ queryKey: ["download-index"] });
qc.invalidateQueries({ queryKey: ["download-usage"] });
};
const act = useMutation({
mutationFn: ({ id, op }: { id: number; op: "pause" | "resume" | "cancel" | "delete" }) =>
op === "pause" ? api.pauseDownload(id)
: op === "resume" ? api.resumeDownload(id)
: op === "cancel" ? api.cancelDownload(id)
: api.deleteDownload(id),
onSuccess: invalidate,
});
const confirmThen = async (title: string, body: string, fn: () => void) => {
if (await confirm({ title, message: body, confirmLabel: title, danger: true })) fn();
};
const tabs = [
{ id: "queue", label: t("downloads.tabs.queue"), badge: queue.length },
{ id: "done", label: t("downloads.tabs.done"), badge: library.length },
{ id: "shared", label: t("downloads.tabs.shared") },
...(me.role === "admin" ? [{ id: "system", label: t("downloads.tabs.system") }] : []),
];
const saveBtn = (job: DownloadJob) => (
<a
href={api.downloadFileUrl(job.id)}
title={t("downloads.actions.download")}
className="p-1.5 rounded-md hover:bg-surface text-muted hover:text-fg transition"
>
<Download className="w-4 h-4" />
</a>
);
return (
<div className="p-4 max-w-4xl w-full mx-auto">
<div className="flex items-center justify-between gap-3 mb-1">
<h1 className="text-xl font-semibold">{t("downloads.page.title")}</h1>
<button
onClick={() => setManage(true)}
className="inline-flex items-center gap-1.5 px-2.5 py-1.5 rounded-lg text-sm text-muted hover:text-fg hover:bg-surface transition"
>
<Settings2 className="w-4 h-4" /> {t("downloads.page.manage")}
</button>
</div>
<p className="text-sm text-muted mb-4">{t("downloads.page.subtitle")}</p>
<Tabs tabs={tabs} active={tab} onChange={setTab} />
{tab === "queue" && (
<>
<div className="flex gap-2 mb-4">
<input
value={addUrl}
onChange={(e) => setAddUrl(e.target.value)}
onKeyDown={(e) => e.key === "Enter" && addUrl.trim() && setAddSource(addUrl.trim())}
placeholder={t("downloads.page.addUrl")}
className={inputCls}
/>
<button
onClick={() => addUrl.trim() && setAddSource(addUrl.trim())}
className="inline-flex items-center gap-1.5 px-3 py-1.5 rounded-lg text-sm font-medium bg-accent text-accent-fg hover:opacity-90 transition shrink-0"
>
<Plus className="w-4 h-4" /> {t("downloads.page.add")}
</button>
</div>
<div className="space-y-2">
{queue.map((job) => (
<DownloadRow key={job.id} job={job}>
{(job.status === "queued" || job.status === "running") && (
<IconBtn onClick={() => act.mutate({ id: job.id, op: "pause" })} title={t("downloads.actions.pause")}>
<Pause className="w-4 h-4" />
</IconBtn>
)}
{(job.status === "paused" || job.status === "error") && (
<IconBtn onClick={() => act.mutate({ id: job.id, op: "resume" })} title={t(job.status === "error" ? "downloads.actions.retry" : "downloads.actions.resume")}>
<Play className="w-4 h-4" />
</IconBtn>
)}
<IconBtn
onClick={() => confirmThen(t("downloads.confirm.cancelTitle"), t("downloads.confirm.cancelBody"), () => act.mutate({ id: job.id, op: "cancel" }))}
title={t("downloads.actions.cancel")}
danger
>
<Ban className="w-4 h-4" />
</IconBtn>
</DownloadRow>
))}
{queue.length === 0 && <div className="text-sm text-muted py-10 text-center">{t("downloads.empty.queue")}</div>}
</div>
</>
)}
{tab === "done" && (
<>
<UsageBar />
<div className="space-y-2">
{library.map((job) => (
<DownloadRow key={job.id} job={job}>
{saveBtn(job)}
<IconBtn onClick={() => setRenameJob(job)} title={t("downloads.actions.rename")}>
<Pencil className="w-4 h-4" />
</IconBtn>
<IconBtn onClick={() => setShareJob(job)} title={t("downloads.actions.share")}>
<Share2 className="w-4 h-4" />
</IconBtn>
<IconBtn
onClick={() => confirmThen(t("downloads.confirm.deleteTitle"), t("downloads.confirm.deleteBody"), () => act.mutate({ id: job.id, op: "delete" }))}
title={t("downloads.actions.delete")}
danger
>
<Trash2 className="w-4 h-4" />
</IconBtn>
</DownloadRow>
))}
{library.length === 0 && <div className="text-sm text-muted py-10 text-center">{t("downloads.empty.done")}</div>}
</div>
</>
)}
{tab === "shared" && (
<div className="space-y-2">
{(sharedQ.data ?? []).map((job) => (
<DownloadRow key={job.id} job={job}>
{job.can_download && saveBtn(job)}
</DownloadRow>
))}
{sharedQ.data && sharedQ.data.length === 0 && (
<div className="text-sm text-muted py-10 text-center">{t("downloads.empty.shared")}</div>
)}
</div>
)}
{tab === "system" && me.role === "admin" && <AdminSystem />}
{addSource && (
<DownloadDialog
source={addSource}
onClose={() => {
setAddSource(null);
setAddUrl("");
}}
/>
)}
{manage && <ProfileEditor onClose={() => setManage(false)} />}
{renameJob && <RenameModal job={renameJob} onClose={() => setRenameJob(null)} />}
{shareJob && <ShareModal job={shareJob} onClose={() => setShareJob(null)} />}
</div>
);
}

View file

@ -0,0 +1,117 @@
import { useState } from "react";
import { useTranslation } from "react-i18next";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import Modal from "./Modal";
import ProfileEditor from "./ProfileEditor";
import { api } from "../lib/api";
import { notify } from "../lib/notifications";
import { navigateTo } from "../lib/nav";
const inputCls =
"w-full rounded-lg bg-surface border border-border px-3 py-2 text-sm focus:outline-none focus:ring-2 focus:ring-accent/40";
// Preset picker shown when the user hits Download on a card / in the player, or adds a URL from
// the Download Center. Enqueues via the API and points the user at the Download Center.
export default function DownloadDialog({
source,
title,
onClose,
}: {
source: string;
title?: string | null;
onClose: () => void;
}) {
const { t } = useTranslation();
const qc = useQueryClient();
const profilesQ = useQuery({ queryKey: ["download-profiles"], queryFn: api.downloadProfiles });
const [profileId, setProfileId] = useState<number | null>(null);
const [name, setName] = useState("");
const [busy, setBusy] = useState(false);
const [manage, setManage] = useState(false);
const profiles = profilesQ.data ?? [];
const chosen = profileId ?? profiles[0]?.id ?? null;
const submit = async () => {
if (!chosen || busy) return;
setBusy(true);
try {
await api.enqueueDownload({
source,
profile_id: chosen,
display_name: name.trim() || undefined,
});
qc.invalidateQueries({ queryKey: ["downloads"] });
qc.invalidateQueries({ queryKey: ["download-index"] });
qc.invalidateQueries({ queryKey: ["download-usage"] });
notify({
level: "success",
message: t("downloads.dialog.added"),
action: { label: t("downloads.dialog.view"), onClick: () => navigateTo("downloads") },
});
onClose();
} catch {
// Non-quiet request: the global error dialog already surfaced the reason (e.g. quota).
setBusy(false);
}
};
return (
<Modal title={t("downloads.dialog.title")} onClose={onClose}>
{title && <p className="text-sm text-muted mb-4 line-clamp-2">{title}</p>}
<label className="block text-sm font-medium mb-1">{t("downloads.dialog.profile")}</label>
<select
value={chosen ?? ""}
onChange={(e) => setProfileId(Number(e.target.value))}
className={inputCls}
>
{profiles.map((p) => (
<option key={p.id} value={p.id}>
{p.name}
</option>
))}
</select>
<button
type="button"
onClick={() => setManage(true)}
className="text-xs text-accent hover:underline mt-1 mb-4"
>
{t("downloads.dialog.manage")}
</button>
<label className="block text-sm font-medium mb-1">{t("downloads.dialog.nameLabel")}</label>
<input
value={name}
onChange={(e) => setName(e.target.value)}
placeholder={t("downloads.dialog.namePlaceholder")}
className={`${inputCls} mb-5`}
/>
<div className="flex justify-end gap-2">
<button
onClick={onClose}
className="px-3 py-1.5 rounded-lg text-sm text-muted hover:text-fg hover:bg-surface transition"
>
{t("downloads.actions.cancel")}
</button>
<button
onClick={submit}
disabled={busy || !chosen}
className="px-3 py-1.5 rounded-lg text-sm font-medium bg-accent text-accent-fg hover:opacity-90 disabled:opacity-50 transition"
>
{t("downloads.dialog.submit")}
</button>
</div>
{manage && (
<ProfileEditor
onClose={() => {
setManage(false);
profilesQ.refetch();
}}
/>
)}
</Modal>
);
}

View file

@ -89,7 +89,9 @@ export default function Header({
? t("inbox.navLabel")
: page === "messages"
? t("messages.navLabel")
: t("header.channelManager")}
: page === "downloads"
? t("downloads.navLabel")
: t("header.channelManager")}
</div>
)}
</header>

View file

@ -8,6 +8,7 @@ import {
Bell,
ChevronLeft,
ChevronRight,
Download,
Home,
Info,
ListVideo,
@ -147,6 +148,13 @@ export default function NavSidebar({
{ intervalMs: 30000, enabled: !me.is_demo }
);
const msgUnread = msgUnreadQuery.data?.count ?? 0;
// Download center badge = items still working (queued/running/paused). Reuses the same
// lightweight per-user index the feed cards poll, so no extra request shape.
const dlIndexQuery = useLiveQuery(["download-index"], api.downloadIndex, {
intervalMs: 30000,
enabled: !me.is_demo,
});
const dlActive = Object.values(dlIndexQuery.data ?? {}).filter((s) => s !== "done").length;
type NavItem = { page: Page; icon: typeof Home; label: string; badge?: number };
// User-facing content modules vs. admin/system modules, separated by a divider in the rail.
@ -159,6 +167,11 @@ export default function NavSidebar({
...(me.is_demo
? []
: [{ page: "messages" as Page, icon: MessageSquare, label: t("messages.navLabel"), badge: msgUnread }]),
// Download center — YouTube → server → your device. Hidden for the shared demo account
// (spends server disk + needs a real identity).
...(me.is_demo
? []
: [{ page: "downloads" as Page, icon: Download, label: t("downloads.navLabel"), badge: dlActive }]),
// Per-user sync status + your own API usage (admins get an extra system-wide tab inside).
{ page: "stats", icon: BarChart3, label: t("header.account.stats") },
];

View file

@ -5,6 +5,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query";
import { AlertTriangle, ArrowLeft, Check, CheckCheck, ExternalLink, SkipBack, SkipForward, Volume2, VolumeX, X } from "lucide-react";
import Avatar from "./Avatar";
import AddToPlaylist from "./AddToPlaylist";
import DownloadButton from "./DownloadButton";
import { api, type Video } from "../lib/api";
import { channelYouTubeUrl, formatDuration, formatViews, relativeTime } from "../lib/format";
import { renderDescription } from "../lib/descriptionLinks";
@ -630,6 +631,13 @@ export default function PlayerModal({
className="ml-auto shrink-0 inline-flex items-center justify-center w-9 h-9 rounded-lg text-muted hover:text-fg hover:bg-surface border border-border transition"
/>
)}
{!navigated && (
<DownloadButton
videoId={active.id}
title={active.title}
className="shrink-0 inline-flex items-center justify-center w-9 h-9 rounded-lg text-muted hover:text-fg hover:bg-surface border border-border transition"
/>
)}
{!navigated && (
<button
onClick={() => setWatched(!watched)}

View file

@ -0,0 +1,215 @@
import { useState } from "react";
import { useTranslation } from "react-i18next";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { Lock, Pencil, Plus, Trash2 } from "lucide-react";
import Modal from "./Modal";
import { useConfirm } from "./ConfirmProvider";
import { api, type DownloadProfile, type DownloadSpec } from "../lib/api";
const inputCls =
"w-full rounded-lg bg-surface border border-border px-3 py-2 text-sm focus:outline-none focus:ring-2 focus:ring-accent/40";
const HEIGHTS = [null, 2160, 1440, 1080, 720, 480, 360];
const BLANK: DownloadSpec = {
mode: "av",
max_height: 1080,
container: "mp4",
audio_format: "m4a",
vcodec: null,
embed_subs: false,
embed_chapters: true,
embed_thumbnail: true,
sponsorblock: false,
};
function Check({
label,
checked,
onChange,
}: {
label: string;
checked: boolean;
onChange: (v: boolean) => void;
}) {
return (
<label className="flex items-center gap-2 text-sm cursor-pointer select-none">
<input type="checkbox" checked={checked} onChange={(e) => onChange(e.target.checked)} />
{label}
</label>
);
}
export default function ProfileEditor({ onClose }: { onClose: () => void }) {
const { t } = useTranslation();
const qc = useQueryClient();
const confirm = useConfirm();
const profilesQ = useQuery({ queryKey: ["download-profiles"], queryFn: api.downloadProfiles });
const profiles = profilesQ.data ?? [];
const builtins = profiles.filter((p) => p.is_builtin);
const mine = profiles.filter((p) => !p.is_builtin);
const [editing, setEditing] = useState<DownloadProfile | null>(null);
const [name, setName] = useState("");
const [spec, setSpec] = useState<DownloadSpec>(BLANK);
const [formOpen, setFormOpen] = useState(false);
const invalidate = () => qc.invalidateQueries({ queryKey: ["download-profiles"] });
const startNew = () => {
setEditing(null);
setName("");
setSpec(BLANK);
setFormOpen(true);
};
const startEdit = (p: DownloadProfile) => {
setEditing(p);
setName(p.name);
setSpec({ ...BLANK, ...p.spec });
setFormOpen(true);
};
const save = useMutation({
mutationFn: () =>
editing
? api.updateDownloadProfile(editing.id, { name, spec })
: api.createDownloadProfile({ name, spec }),
onSuccess: () => {
invalidate();
setFormOpen(false);
},
});
const del = useMutation({
mutationFn: (id: number) => api.deleteDownloadProfile(id),
onSuccess: invalidate,
});
const onDelete = async (p: DownloadProfile) => {
if (await confirm({ title: t("downloads.profiles.delete"), message: t("downloads.profiles.deleteConfirm", { name: p.name }), confirmLabel: t("downloads.profiles.delete"), danger: true })) {
del.mutate(p.id);
}
};
const patch = (p: Partial<DownloadSpec>) => setSpec((s) => ({ ...s, ...p }));
const set = (k: keyof DownloadSpec) => (e: React.ChangeEvent<HTMLSelectElement>) =>
patch({ [k]: e.target.value === "" ? null : e.target.value } as Partial<DownloadSpec>);
return (
<Modal title={t("downloads.profiles.title")} onClose={onClose} maxWidth="max-w-xl">
{/* Existing formats */}
<div className="space-y-1.5 mb-4">
<div className="text-xs uppercase tracking-wide text-muted">{t("downloads.profiles.builtin")}</div>
{builtins.map((p) => (
<div key={p.id} className="flex items-center gap-2 text-sm px-3 py-1.5 rounded-lg bg-surface/60">
<Lock className="w-3.5 h-3.5 text-muted shrink-0" />
<span className="flex-1 truncate">{p.name}</span>
</div>
))}
{mine.length > 0 && (
<div className="text-xs uppercase tracking-wide text-muted pt-2">{t("downloads.profiles.yours")}</div>
)}
{mine.map((p) => (
<div key={p.id} className="flex items-center gap-2 text-sm px-3 py-1.5 rounded-lg bg-surface/60">
<span className="flex-1 truncate">{p.name}</span>
<button onClick={() => startEdit(p)} title={t("downloads.actions.rename")} className="p-1 rounded hover:bg-surface text-muted hover:text-fg">
<Pencil className="w-3.5 h-3.5" />
</button>
<button onClick={() => onDelete(p)} title={t("downloads.profiles.delete")} className="p-1 rounded hover:bg-surface text-muted hover:text-red-400">
<Trash2 className="w-3.5 h-3.5" />
</button>
</div>
))}
</div>
{!formOpen ? (
<button
onClick={startNew}
className="inline-flex items-center gap-1.5 px-3 py-1.5 rounded-lg text-sm font-medium bg-accent text-accent-fg hover:opacity-90 transition"
>
<Plus className="w-4 h-4" /> {t("downloads.profiles.new")}
</button>
) : (
<div className="rounded-xl border border-border p-4 space-y-3">
<div>
<label className="block text-sm font-medium mb-1">{t("downloads.profiles.name")}</label>
<input value={name} onChange={(e) => setName(e.target.value)} placeholder={t("downloads.profiles.namePlaceholder")} className={inputCls} />
</div>
<div className="grid grid-cols-2 gap-3">
<div>
<label className="block text-sm font-medium mb-1">{t("downloads.profiles.mode")}</label>
<select value={spec.mode} onChange={set("mode")} className={inputCls}>
<option value="av">{t("downloads.profiles.modeAv")}</option>
<option value="v">{t("downloads.profiles.modeV")}</option>
<option value="a">{t("downloads.profiles.modeA")}</option>
</select>
</div>
{spec.mode !== "a" ? (
<div>
<label className="block text-sm font-medium mb-1">{t("downloads.profiles.quality")}</label>
<select
value={spec.max_height ?? ""}
onChange={(e) => patch({ max_height: e.target.value === "" ? null : Number(e.target.value) })}
className={inputCls}
>
{HEIGHTS.map((h) => (
<option key={h ?? "best"} value={h ?? ""}>
{h ? `${h}p` : t("downloads.profiles.best")}
</option>
))}
</select>
</div>
) : (
<div>
<label className="block text-sm font-medium mb-1">{t("downloads.profiles.audioFormat")}</label>
<select value={spec.audio_format ?? "m4a"} onChange={set("audio_format")} className={inputCls}>
<option value="m4a">M4A</option>
<option value="mp3">MP3</option>
<option value="opus">Opus</option>
</select>
</div>
)}
{spec.mode !== "a" && (
<>
<div>
<label className="block text-sm font-medium mb-1">{t("downloads.profiles.container")}</label>
<select value={spec.container ?? "mp4"} onChange={set("container")} className={inputCls}>
<option value="mp4">MP4</option>
<option value="mkv">MKV</option>
<option value="webm">WebM</option>
</select>
</div>
<div>
<label className="block text-sm font-medium mb-1">{t("downloads.profiles.vcodec")}</label>
<select value={spec.vcodec ?? ""} onChange={set("vcodec")} className={inputCls}>
<option value="">{t("downloads.profiles.any")}</option>
<option value="h264">H.264</option>
<option value="vp9">VP9</option>
<option value="av1">AV1</option>
</select>
</div>
</>
)}
</div>
<div className="space-y-1.5 pt-1">
{spec.mode !== "a" && (
<Check label={t("downloads.profiles.embedSubs")} checked={spec.embed_subs} onChange={(v) => patch({ embed_subs: v })} />
)}
<Check label={t("downloads.profiles.embedChapters")} checked={spec.embed_chapters} onChange={(v) => patch({ embed_chapters: v })} />
<Check label={t("downloads.profiles.embedThumbnail")} checked={spec.embed_thumbnail} onChange={(v) => patch({ embed_thumbnail: v })} />
<Check label={t("downloads.profiles.sponsorblock")} checked={spec.sponsorblock} onChange={(v) => patch({ sponsorblock: v })} />
</div>
<div className="flex justify-end gap-2 pt-1">
<button onClick={() => setFormOpen(false)} className="px-3 py-1.5 rounded-lg text-sm text-muted hover:text-fg hover:bg-surface transition">
{t("downloads.actions.cancel")}
</button>
<button
onClick={() => save.mutate()}
disabled={!name.trim() || save.isPending}
className="px-3 py-1.5 rounded-lg text-sm font-medium bg-accent text-accent-fg hover:opacity-90 disabled:opacity-50 transition"
>
{editing ? t("downloads.profiles.save") : t("downloads.profiles.create")}
</button>
</div>
</div>
)}
</Modal>
);
}

View file

@ -11,6 +11,7 @@ import {
} from "lucide-react";
import Avatar from "./Avatar";
import AddToPlaylist from "./AddToPlaylist";
import DownloadButton from "./DownloadButton";
import clsx from "clsx";
import type { Video } from "../lib/api";
import { formatDuration, formatViews, relativeTime } from "../lib/format";
@ -67,6 +68,7 @@ function Actions({
<Bookmark className="w-4 h-4" />
</button>
<AddToPlaylist videoId={video.id} />
<DownloadButton videoId={video.id} title={video.title} />
<button
onClick={act("hidden")}
title={video.status === "hidden" ? t("card.unhide") : t("card.hide")}

View file

@ -0,0 +1,146 @@
{
"navLabel": "Downloads",
"button": {
"label": "Herunterladen",
"queued": "In Warteschlange",
"downloaded": "Heruntergeladen"
},
"dialog": {
"title": "Video herunterladen",
"profile": "Format",
"nameLabel": "Dateiname (optional)",
"namePlaceholder": "Leer lassen, um den Videotitel zu verwenden",
"submit": "Zu Downloads hinzufügen",
"added": "Zu Downloads hinzugefügt",
"view": "Öffnen",
"manage": "Formate verwalten"
},
"page": {
"title": "Downloads",
"subtitle": "Videos auf den Server laden und dann auf dein Gerät speichern.",
"addUrl": "YouTube-Link oder Video-ID einfügen…",
"add": "Hinzufügen",
"manage": "Formate verwalten"
},
"tabs": {
"queue": "Warteschlange",
"done": "Bibliothek",
"shared": "Mit mir geteilt",
"system": "System"
},
"status": {
"queued": "Wartet",
"running": "Lädt",
"paused": "Pausiert",
"done": "Fertig",
"error": "Fehlgeschlagen",
"canceled": "Abgebrochen",
"expired": "Abgelaufen"
},
"phase": {
"video": "Video wird geladen",
"audio": "Audio wird geladen",
"merging": "Zusammenführen",
"audio_extract": "Audio wird extrahiert",
"thumbnail": "Vorschaubild einbetten",
"sponsorblock": "Sponsoren entfernen",
"metadata": "Metadaten schreiben",
"processing": "Verarbeitung"
},
"actions": {
"pause": "Pause",
"resume": "Fortsetzen",
"cancel": "Abbrechen",
"delete": "Entfernen",
"retry": "Erneut",
"download": "Auf mein Gerät speichern",
"rename": "Umbenennen",
"share": "Teilen"
},
"empty": {
"queue": "Die Warteschlange ist leer.",
"done": "Noch keine Downloads.",
"shared": "Es wurde nichts mit dir geteilt.",
"admin": "Auf dieser Instanz gibt es noch keine Downloads."
},
"usage": {
"title": "Dein Speicher",
"items": "{{used}} / {{max}} Elemente",
"unlimited": "Unbegrenzt"
},
"rename": {
"title": "Download umbenennen",
"label": "Anzeigename",
"hint": "Ändert nur den angezeigten und beim Speichern verwendeten Namen — die gespeicherte Datei behält ihren Namen.",
"save": "Speichern"
},
"share": {
"title": "Download teilen",
"label": "E-Mail des Empfängers",
"placeholder": "user@example.com",
"submit": "Teilen",
"done": "Geteilt mit {{email}}"
},
"confirm": {
"cancelTitle": "Download abbrechen?",
"cancelBody": "Dies stoppt den Download und entfernt ihn aus deiner Warteschlange.",
"deleteTitle": "Download entfernen?",
"deleteBody": "Dies entfernt ihn aus deiner Liste. Die Datei kann bis zum Ablauf für andere Nutzer im Cache bleiben."
},
"profiles": {
"manage": "Formate verwalten",
"title": "Download-Formate",
"builtin": "Integriert",
"yours": "Deine Formate",
"new": "Neues Format",
"name": "Name",
"namePlaceholder": "Mein Format",
"mode": "Inhalt",
"modeAv": "Video + Audio",
"modeV": "Nur Video",
"modeA": "Nur Audio",
"quality": "Max. Qualität",
"best": "Beste",
"container": "Container",
"audioFormat": "Audioformat",
"vcodec": "Video-Codec",
"any": "Beliebig",
"embedSubs": "Untertitel einbetten",
"embedChapters": "Kapitel einbetten",
"embedThumbnail": "Vorschaubild einbetten",
"sponsorblock": "Sponsor-Segmente überspringen (SponsorBlock)",
"save": "Speichern",
"create": "Erstellen",
"delete": "Löschen",
"deleteConfirm": "Das Format „{{name}}“ löschen?"
},
"admin": {
"storageTitle": "Speicher",
"readyFiles": "Fertige Dateien",
"totalSize": "Gesamtgröße",
"cap": "Cache-Limit",
"noCap": "kein Limit",
"perUser": "Speicher pro Nutzer",
"user": "Nutzer",
"footprint": "Belegung",
"editQuota": "Kontingent bearbeiten",
"quotaTitle": "Download-Kontingent — {{email}}",
"maxBytes": "Speicherlimit (GB)",
"maxJobs": "Max. Downloads",
"maxConcurrent": "Max. gleichzeitig",
"unlimited": "Unbegrenzt (Speicherlimit umgehen)",
"reset": "Auf Standard zurücksetzen",
"save": "Speichern",
"custom": "individuell",
"default": "Standard"
},
"cols": {
"title": "Titel",
"channel": "Kanal",
"format": "Format",
"size": "Größe",
"status": "Status",
"added": "Hinzugefügt",
"user": "Nutzer"
}
}

View file

@ -0,0 +1,146 @@
{
"navLabel": "Downloads",
"button": {
"label": "Download",
"queued": "In queue",
"downloaded": "Downloaded"
},
"dialog": {
"title": "Download video",
"profile": "Format",
"nameLabel": "File name (optional)",
"namePlaceholder": "Leave blank to use the video title",
"submit": "Add to downloads",
"added": "Added to downloads",
"view": "View",
"manage": "Manage formats"
},
"page": {
"title": "Downloads",
"subtitle": "Download videos to the server, then save them to your device.",
"addUrl": "Paste a YouTube link or video id…",
"add": "Add",
"manage": "Manage formats"
},
"tabs": {
"queue": "Queue",
"done": "Library",
"shared": "Shared with me",
"system": "System"
},
"status": {
"queued": "Queued",
"running": "Downloading",
"paused": "Paused",
"done": "Ready",
"error": "Failed",
"canceled": "Canceled",
"expired": "Expired"
},
"phase": {
"video": "Downloading video",
"audio": "Downloading audio",
"merging": "Merging",
"audio_extract": "Extracting audio",
"thumbnail": "Embedding thumbnail",
"sponsorblock": "Removing sponsors",
"metadata": "Writing metadata",
"processing": "Processing"
},
"actions": {
"pause": "Pause",
"resume": "Resume",
"cancel": "Cancel",
"delete": "Remove",
"retry": "Retry",
"download": "Save to my device",
"rename": "Rename",
"share": "Share"
},
"empty": {
"queue": "Nothing in the queue.",
"done": "No downloads yet.",
"shared": "Nothing has been shared with you.",
"admin": "No downloads on this instance yet."
},
"usage": {
"title": "Your storage",
"items": "{{used}} / {{max}} items",
"unlimited": "Unlimited"
},
"rename": {
"title": "Rename download",
"label": "Display name",
"hint": "Only changes the name you see and download with — the stored file keeps its own name.",
"save": "Save"
},
"share": {
"title": "Share download",
"label": "Recipient email",
"placeholder": "user@example.com",
"submit": "Share",
"done": "Shared with {{email}}"
},
"confirm": {
"cancelTitle": "Cancel download?",
"cancelBody": "This stops the download and removes it from your queue.",
"deleteTitle": "Remove download?",
"deleteBody": "This removes it from your list. The file may stay cached for other users until it expires."
},
"profiles": {
"manage": "Manage formats",
"title": "Download formats",
"builtin": "Built-in",
"yours": "Your formats",
"new": "New format",
"name": "Name",
"namePlaceholder": "My format",
"mode": "Content",
"modeAv": "Video + audio",
"modeV": "Video only",
"modeA": "Audio only",
"quality": "Max quality",
"best": "Best",
"container": "Container",
"audioFormat": "Audio format",
"vcodec": "Video codec",
"any": "Any",
"embedSubs": "Embed subtitles",
"embedChapters": "Embed chapters",
"embedThumbnail": "Embed thumbnail",
"sponsorblock": "Skip sponsor segments (SponsorBlock)",
"save": "Save",
"create": "Create",
"delete": "Delete",
"deleteConfirm": "Delete the format “{{name}}”?"
},
"admin": {
"storageTitle": "Storage",
"readyFiles": "Ready files",
"totalSize": "Total size",
"cap": "Cache cap",
"noCap": "no cap",
"perUser": "Per-user footprint",
"user": "User",
"footprint": "Footprint",
"editQuota": "Edit quota",
"quotaTitle": "Download quota — {{email}}",
"maxBytes": "Storage limit (GB)",
"maxJobs": "Max downloads",
"maxConcurrent": "Max concurrent",
"unlimited": "Unlimited (bypass storage limit)",
"reset": "Reset to default",
"save": "Save",
"custom": "custom",
"default": "default"
},
"cols": {
"title": "Title",
"channel": "Channel",
"format": "Format",
"size": "Size",
"status": "Status",
"added": "Added",
"user": "User"
}
}

View file

@ -0,0 +1,146 @@
{
"navLabel": "Letöltések",
"button": {
"label": "Letöltés",
"queued": "Sorban",
"downloaded": "Letöltve"
},
"dialog": {
"title": "Videó letöltése",
"profile": "Formátum",
"nameLabel": "Fájlnév (opcionális)",
"namePlaceholder": "Üresen hagyva a videó címét használja",
"submit": "Hozzáadás a letöltésekhez",
"added": "Hozzáadva a letöltésekhez",
"view": "Megnyitás",
"manage": "Formátumok kezelése"
},
"page": {
"title": "Letöltések",
"subtitle": "Töltsd le a videókat a szerverre, majd mentsd a saját gépedre.",
"addUrl": "Illessz be egy YouTube linket vagy videó azonosítót…",
"add": "Hozzáad",
"manage": "Formátumok kezelése"
},
"tabs": {
"queue": "Sor",
"done": "Könyvtár",
"shared": "Velem megosztva",
"system": "Rendszer"
},
"status": {
"queued": "Várakozik",
"running": "Letöltés",
"paused": "Szüneteltetve",
"done": "Kész",
"error": "Sikertelen",
"canceled": "Megszakítva",
"expired": "Lejárt"
},
"phase": {
"video": "Videósáv letöltése",
"audio": "Hangsáv letöltése",
"merging": "Összefűzés",
"audio_extract": "Hang kinyerése",
"thumbnail": "Bélyegkép beágyazása",
"sponsorblock": "Szponzorok eltávolítása",
"metadata": "Metaadat írása",
"processing": "Feldolgozás"
},
"actions": {
"pause": "Szünet",
"resume": "Folytatás",
"cancel": "Megszakítás",
"delete": "Eltávolítás",
"retry": "Újra",
"download": "Mentés a gépemre",
"rename": "Átnevezés",
"share": "Megosztás"
},
"empty": {
"queue": "A sor üres.",
"done": "Még nincs letöltés.",
"shared": "Még nem osztottak meg veled semmit.",
"admin": "Ezen a példányon még nincs letöltés."
},
"usage": {
"title": "Tárhelyed",
"items": "{{used}} / {{max}} elem",
"unlimited": "Korlátlan"
},
"rename": {
"title": "Letöltés átnevezése",
"label": "Megjelenített név",
"hint": "Csak a megjelenített és letöltéskor használt nevet módosítja — a tárolt fájl neve nem változik.",
"save": "Mentés"
},
"share": {
"title": "Letöltés megosztása",
"label": "Címzett e-mail címe",
"placeholder": "user@example.com",
"submit": "Megosztás",
"done": "Megosztva vele: {{email}}"
},
"confirm": {
"cancelTitle": "Megszakítod a letöltést?",
"cancelBody": "Ez leállítja a letöltést és eltávolítja a sorból.",
"deleteTitle": "Eltávolítod a letöltést?",
"deleteBody": "Ez eltávolítja a listádról. A fájl a lejáratáig még a gyorsítótárban maradhat mások számára."
},
"profiles": {
"manage": "Formátumok kezelése",
"title": "Letöltési formátumok",
"builtin": "Beépített",
"yours": "Saját formátumaid",
"new": "Új formátum",
"name": "Név",
"namePlaceholder": "Saját formátum",
"mode": "Tartalom",
"modeAv": "Videó + hang",
"modeV": "Csak videó",
"modeA": "Csak hang",
"quality": "Max. minőség",
"best": "Legjobb",
"container": "Konténer",
"audioFormat": "Hangformátum",
"vcodec": "Videó codec",
"any": "Bármelyik",
"embedSubs": "Feliratok beágyazása",
"embedChapters": "Fejezetek beágyazása",
"embedThumbnail": "Bélyegkép beágyazása",
"sponsorblock": "Szponzorált szakaszok kihagyása (SponsorBlock)",
"save": "Mentés",
"create": "Létrehozás",
"delete": "Törlés",
"deleteConfirm": "Törlöd a(z) „{{name}}” formátumot?"
},
"admin": {
"storageTitle": "Tárhely",
"readyFiles": "Kész fájlok",
"totalSize": "Teljes méret",
"cap": "Gyorsítótár-korlát",
"noCap": "nincs korlát",
"perUser": "Felhasználónkénti tárhelyhasználat",
"user": "Felhasználó",
"footprint": "Használat",
"editQuota": "Kvóta szerkesztése",
"quotaTitle": "Letöltési kvóta — {{email}}",
"maxBytes": "Tárhelykorlát (GB)",
"maxJobs": "Max. letöltés",
"maxConcurrent": "Max. egyidejű",
"unlimited": "Korlátlan (tárhelykorlát kikapcsolása)",
"reset": "Visszaállítás alapértékre",
"save": "Mentés",
"custom": "egyedi",
"default": "alapértelmezett"
},
"cols": {
"title": "Cím",
"channel": "Csatorna",
"format": "Formátum",
"size": "Méret",
"status": "Állapot",
"added": "Hozzáadva",
"user": "Felhasználó"
}
}

View file

@ -623,6 +623,99 @@ export interface AdminUserRow {
created_at: string | null;
}
// --- download center ---
export interface DownloadSpec {
mode: "av" | "v" | "a"; // audio+video | video-only | audio-only
max_height: number | null;
container: string | null;
audio_format: string | null;
vcodec: string | null;
embed_subs: boolean;
embed_chapters: boolean;
embed_thumbnail: boolean;
sponsorblock: boolean;
}
export interface DownloadProfile {
id: number;
name: string;
spec: DownloadSpec;
is_builtin: boolean;
sort_order: number;
}
export interface DownloadAsset {
id: number;
status: string;
title: string | null;
uploader: string | null;
upload_date: string | null;
duration_s: number | null;
size_bytes: number | null;
width: number | null;
height: number | null;
container: string | null;
thumbnail_url: string | null;
expires_at: string | null;
}
export type DownloadStatus =
| "queued"
| "running"
| "paused"
| "done"
| "error"
| "canceled";
export interface DownloadJob {
id: number;
status: DownloadStatus;
progress: number;
speed_bps: number | null;
eta_s: number | null;
phase: string | null;
error: string | null;
queue_pos: number;
created_at: string | null;
source_kind: string;
source_ref: string;
profile_id: number | null;
spec: DownloadSpec;
display_name: string | null;
can_download: boolean;
expired: boolean;
asset: DownloadAsset | null;
shared?: boolean;
user_email?: string;
}
export interface DownloadUsage {
footprint_bytes: number;
active_jobs: number;
running_jobs: number;
max_bytes: number;
max_concurrent: number;
max_jobs: number;
unlimited: boolean;
}
export interface AdminStorage {
ready_files: number;
total_bytes: number;
total_assets: number;
total_cap_bytes: number;
per_user: { user_id: number; email: string | null; footprint_bytes: number }[];
}
export interface AdminDownloadQuota {
user_id: number;
custom: boolean;
max_bytes: number;
max_concurrent: number;
max_jobs: number;
unlimited: boolean;
}
export const api = {
me: (): Promise<Me> => req("/api/me"),
accounts: (): Promise<Account[]> => req("/api/me/accounts"),
@ -914,6 +1007,63 @@ export const api = {
updateTag: (id: number, patch: { name?: string; color?: string }) =>
req(`/api/tags/${id}`, { method: "PATCH", body: JSON.stringify(patch) }),
deleteTag: (id: number) => req(`/api/tags/${id}`, { method: "DELETE" }),
// --- download center ---
downloadProfiles: (): Promise<DownloadProfile[]> => req("/api/downloads/profiles"),
createDownloadProfile: (p: { name: string; spec: DownloadSpec }): Promise<DownloadProfile> =>
req("/api/downloads/profiles", { method: "POST", body: JSON.stringify(p) }),
updateDownloadProfile: (
id: number,
patch: { name?: string; spec?: DownloadSpec }
): Promise<DownloadProfile> =>
req(`/api/downloads/profiles/${id}`, { method: "PATCH", body: JSON.stringify(patch) }),
deleteDownloadProfile: (id: number) =>
req(`/api/downloads/profiles/${id}`, { method: "DELETE" }),
enqueueDownload: (body: {
source: string;
profile_id?: number;
spec?: DownloadSpec;
display_name?: string;
}): Promise<DownloadJob> =>
req("/api/downloads", { method: "POST", body: JSON.stringify(body) }),
downloads: (): Promise<DownloadJob[]> => req("/api/downloads"),
downloadsShared: (): Promise<DownloadJob[]> => req("/api/downloads/shared"),
downloadUsage: (): Promise<DownloadUsage> => req("/api/downloads/usage"),
downloadIndex: (): Promise<Record<string, DownloadStatus>> => req("/api/downloads/index"),
pauseDownload: (id: number): Promise<DownloadJob> =>
req(`/api/downloads/${id}/pause`, { method: "POST" }, { idempotent: true }),
resumeDownload: (id: number): Promise<DownloadJob> =>
req(`/api/downloads/${id}/resume`, { method: "POST" }, { idempotent: true }),
cancelDownload: (id: number): Promise<DownloadJob> =>
req(`/api/downloads/${id}/cancel`, { method: "POST" }, { idempotent: true }),
deleteDownload: (id: number) => req(`/api/downloads/${id}`, { method: "DELETE" }),
renameDownload: (id: number, display_name: string): Promise<DownloadJob> =>
req(`/api/downloads/${id}`, { method: "PATCH", body: JSON.stringify({ display_name }) }),
shareDownload: (id: number, email: string): Promise<{ shared_with: string }> =>
req(`/api/downloads/${id}/share`, { method: "POST", body: JSON.stringify({ email }) }),
unshareDownload: (id: number, email: string) =>
req(`/api/downloads/${id}/share/${encodeURIComponent(email)}`, { method: "DELETE" }),
// A plain <a> navigation can't send the X-Siftlode-Account header, so pass the active
// account via ?account= (same wallet-gated selection the WebSocket uses) — otherwise the
// download resolves to the session-default account and 404s for a per-tab account's file.
downloadFileUrl: (id: number): string => {
const a = getActiveAccount();
return a != null ? `/api/downloads/${id}/file?account=${a}` : `/api/downloads/${id}/file`;
},
// admin
adminDownloads: (): Promise<DownloadJob[]> => req("/api/admin/downloads"),
adminDownloadStorage: (): Promise<AdminStorage> => req("/api/admin/downloads/storage"),
adminGetDownloadQuota: (userId: number): Promise<AdminDownloadQuota> =>
req(`/api/admin/downloads/quota/${userId}`),
adminSetDownloadQuota: (
userId: number,
patch: Partial<Pick<AdminDownloadQuota, "max_bytes" | "max_concurrent" | "max_jobs" | "unlimited">>
): Promise<AdminDownloadQuota> =>
req(`/api/admin/downloads/quota/${userId}`, { method: "PUT", body: JSON.stringify(patch) }),
adminResetDownloadQuota: (userId: number): Promise<AdminDownloadQuota> =>
req(`/api/admin/downloads/quota/${userId}`, { method: "DELETE" }),
};
export { HttpError };

View file

@ -5,6 +5,26 @@ export function relativeTime(iso: string | null): string {
return relativeFromMs(new Date(iso).getTime());
}
/** Human file size, binary units (1024). e.g. 501747 -> "490 KB", 5368709120 -> "5.0 GB". */
export function formatBytes(bytes: number | null | undefined): string {
const n = Number(bytes);
if (!n || n < 0) return "0 B";
const units = ["B", "KB", "MB", "GB", "TB"];
let i = 0;
let v = n;
while (v >= 1024 && i < units.length - 1) {
v /= 1024;
i++;
}
return `${v >= 100 || i === 0 ? Math.round(v) : v.toFixed(1)} ${units[i]}`;
}
/** Download speed from bytes/sec, e.g. "2.4 MB/s". */
export function formatSpeed(bps: number | null | undefined): string {
if (!bps) return "";
return `${formatBytes(bps)}/s`;
}
/** Relative-time label from an epoch-millis timestamp (the ms-based half of relativeTime, for
* client-side notifications whose timestamps are already numbers). One implementation, one
* set of `time.*` strings shared instead of re-implemented per component. */

14
frontend/src/lib/nav.ts Normal file
View file

@ -0,0 +1,14 @@
import type { Page } from "./urlState";
// Tiny decoupled navigator so deep components (a feed card's download toast, the Download
// Center) can switch pages without threading setPage through the whole tree. App registers
// its setPage on mount; navigateTo is a no-op until then.
let _navigate: ((p: Page) => void) | null = null;
export function setNavigator(fn: (p: Page) => void): void {
_navigate = fn;
}
export function navigateTo(p: Page): void {
_navigate?.(p);
}

View file

@ -100,6 +100,7 @@ export const PAGES = [
"users",
"notifications",
"messages",
"downloads",
] as const;
export type Page = (typeof PAGES)[number];