feat(downloads): M1 — schema, config, worker container, deps
- models: MediaAsset (shared file cache, unique on source+format_sig), DownloadJob (per-user request), DownloadProfile (presets), DownloadShare (ACL), DownloadQuota - migrations 0036 (5 tables) + 0037 (6 builtin presets) - sysconfig 'downloads' group (per-user defaults, retention/GC, layout, worker concurrency) - config: DOWNLOAD_ROOT / WORKER_ENABLED env + download defaults - deps: yt-dlp in requirements, ffmpeg in Dockerfile runtime stage - worker: dedicated container (python -m app.worker), thin idle entry (loop lands in M2) - localdev compose: worker service + downloads bind-mount; gitignore downloads/
This commit is contained in:
npeter83
parent
9ca90f515c
commit
317750e491
10 changed files with 496 additions and 0 deletions
|
|
@ -131,6 +131,31 @@ class Settings(BaseSettings):
|
|||
# ("was_live") are real watchable content and stay visible.
|
||||
feed_default_hidden_live: str = "live,upcoming"
|
||||
|
||||
# --- Download center (yt-dlp) ---
|
||||
# Root directory (inside the container) for downloaded media — a Docker volume/bind mount.
|
||||
# Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it.
|
||||
download_root: str = "/downloads"
|
||||
# Whether THIS process runs the download worker loop. On for the dedicated worker
|
||||
# container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled.
|
||||
worker_enabled: bool = False
|
||||
# How many downloads the worker runs concurrently (claimed via FOR UPDATE SKIP LOCKED).
|
||||
download_worker_concurrency: int = 2
|
||||
# How often (empty pending queue) the worker polls for a new job, in seconds.
|
||||
download_poll_seconds: int = 5
|
||||
# Default per-user quota (admin-tunable; a per-user download_quotas row overrides).
|
||||
download_default_max_bytes: int = 5_368_709_120 # 5 GiB
|
||||
download_default_max_concurrent: int = 2
|
||||
download_default_max_jobs: int = 50
|
||||
# Retention: an asset expires this many days after its last access; the GC job then
|
||||
# deletes it (once no job references it) after an additional grace window.
|
||||
download_retention_days: int = 30
|
||||
download_gc_grace_days: int = 2
|
||||
download_gc_minutes: int = 360
|
||||
# On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat".
|
||||
download_layout: str = "plex"
|
||||
# Default SponsorBlock (skip sponsor segments) for new custom profiles.
|
||||
sponsorblock_default: bool = False
|
||||
|
||||
@property
|
||||
def allowed_email_set(self) -> set[str]:
|
||||
return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from sqlalchemy import (
|
|||
DateTime,
|
||||
Float,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
String,
|
||||
|
|
@ -688,6 +689,160 @@ class Message(Base):
|
|||
)
|
||||
|
||||
|
||||
class MediaAsset(Base, TimestampMixin):
|
||||
"""A physical downloaded media file — the SHARED cache layer of the download center.
|
||||
|
||||
Mirrors the app's "shared catalog, private per-user state" model: one row per unique
|
||||
``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests
|
||||
the same video with a format that hashes to the same `format_sig`, their `download_job`
|
||||
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
|
||||
shared; per-user edited derivatives (phase 2) are never cached here.
|
||||
|
||||
`rel_path` is relative to `settings.download_root` (the mount), so the physical filename is
|
||||
system-decided and bound to the source id — never renamed (the user's custom name is display
|
||||
metadata on `download_job`, applied only when downloading to their own machine). The naming
|
||||
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
|
||||
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here)
|
||||
plus `last_access_at`/`expires_at` drive retention + LRU eviction in the GC job."""
|
||||
|
||||
__tablename__ = "media_assets"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||
source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc
|
||||
format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), default="pending", server_default="pending", index=True
|
||||
) # "pending" | "ready" | "error"
|
||||
rel_path: Mapped[str | None] = mapped_column(Text) # relative to download_root
|
||||
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
|
||||
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
||||
duration_s: Mapped[int | None] = mapped_column(Integer)
|
||||
width: Mapped[int | None] = mapped_column(Integer)
|
||||
height: Mapped[int | None] = mapped_column(Integer)
|
||||
container: Mapped[str | None] = mapped_column(String(16))
|
||||
vcodec: Mapped[str | None] = mapped_column(String(32))
|
||||
acodec: Mapped[str | None] = mapped_column(String(32))
|
||||
nfo_written: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false"
|
||||
)
|
||||
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
|
||||
title: Mapped[str | None] = mapped_column(Text)
|
||||
uploader: Mapped[str | None] = mapped_column(String(255))
|
||||
upload_date: Mapped[date | None] = mapped_column(Date)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(Text)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class DownloadProfile(Base, TimestampMixin):
|
||||
"""A named download preset — a reusable format profile.
|
||||
|
||||
`user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …);
|
||||
a user's saved custom profiles carry their `user_id`. `spec` is the format definition
|
||||
(mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/
|
||||
thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache
|
||||
`format_sig`."""
|
||||
|
||||
__tablename__ = "download_profiles"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
) # NULL for a builtin preset
|
||||
name: Mapped[str] = mapped_column(String(80))
|
||||
spec: Mapped[dict] = mapped_column(JSON)
|
||||
is_builtin: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false"
|
||||
)
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's request to download one source — the per-user layer over `MediaAsset`.
|
||||
|
||||
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
|
||||
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery — the
|
||||
worker is a separate process and can't reach the in-process WS registry). `profile_snapshot`
|
||||
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
|
||||
`display_name` is the user's own name for the file (display + the Content-Disposition on
|
||||
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
|
||||
(SET NULL if the asset is GC'd — the job then shows as expired)."""
|
||||
|
||||
__tablename__ = "download_jobs"
|
||||
__table_args__ = (Index("ix_download_jobs_claim", "status", "queue_pos"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
asset_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||
source_ref: Mapped[str] = mapped_column(String(512))
|
||||
profile_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_profiles.id", ondelete="SET NULL")
|
||||
)
|
||||
profile_snapshot: Mapped[dict] = mapped_column(JSON)
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), default="queued", server_default="queued", index=True
|
||||
) # queued | running | paused | done | error | canceled
|
||||
progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
speed_bps: Mapped[int | None] = mapped_column(BigInteger)
|
||||
eta_s: Mapped[int | None] = mapped_column(Integer)
|
||||
phase: Mapped[str | None] = mapped_column(String(32))
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class DownloadShare(Base, TimestampMixin):
|
||||
"""A user→user grant: `to_user` may access `from_user`'s downloaded file (private by
|
||||
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
|
||||
|
||||
__tablename__ = "download_shares"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
job_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
from_user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
to_user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
|
||||
|
||||
class DownloadQuota(Base, UpdatedAtMixin):
|
||||
"""Per-user download limits (admin-managed). A row exists only when an admin customizes a
|
||||
user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the
|
||||
byte cap entirely (e.g. for the admin's own account)."""
|
||||
|
||||
__tablename__ = "download_quotas"
|
||||
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
max_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
max_concurrent: Mapped[int] = mapped_column(Integer)
|
||||
max_jobs: Mapped[int] = mapped_column(Integer)
|
||||
unlimited: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false"
|
||||
)
|
||||
|
||||
|
||||
class MessageKey(Base, TimestampMixin):
|
||||
"""A user's end-to-end-encryption key material for direct messaging (phase 2).
|
||||
|
||||
|
|
|
|||
|
|
@ -66,6 +66,15 @@ SPECS: tuple[ConfigSpec, ...] = (
|
|||
ConfigSpec("google_client_secret", "str", "google", "google_client_secret", secret=True),
|
||||
# --- Access / registration ---
|
||||
ConfigSpec("allow_registration", "bool", "access", "allow_registration"),
|
||||
# --- Download center (yt-dlp) — per-user defaults + retention/GC + layout ---
|
||||
ConfigSpec("download_default_max_bytes", "int", "downloads", "download_default_max_bytes", min=0),
|
||||
ConfigSpec("download_default_max_concurrent", "int", "downloads", "download_default_max_concurrent", min=1, max=20),
|
||||
ConfigSpec("download_default_max_jobs", "int", "downloads", "download_default_max_jobs", min=1, max=100_000),
|
||||
ConfigSpec("download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650),
|
||||
ConfigSpec("download_gc_grace_days", "int", "downloads", "download_gc_grace_days", min=0, max=3_650),
|
||||
ConfigSpec("download_layout", "str", "downloads", "download_layout"),
|
||||
ConfigSpec("download_worker_concurrency", "int", "downloads", "download_worker_concurrency", min=1, max=16),
|
||||
ConfigSpec("sponsorblock_default", "bool", "downloads", "sponsorblock_default"),
|
||||
)
|
||||
|
||||
_BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS}
|
||||
|
|
|
|||
50
backend/app/worker.py
Normal file
50
backend/app/worker.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
"""Download worker process entry point.
|
||||
|
||||
Runs in a DEDICATED container (`command: python -m app.worker`, `WORKER_ENABLED=1`), separate
|
||||
from the API so long yt-dlp downloads + ffmpeg postprocessing never block web requests. It
|
||||
claims `queued` download jobs from the DB with FOR UPDATE SKIP LOCKED and runs them, writing
|
||||
live progress back to the job row (the frontend polls it — the worker is a separate process and
|
||||
can't reach the API's in-process WebSocket registry).
|
||||
|
||||
M1: thin, idle entry point (keeps the container healthy). The claim loop + yt-dlp integration
|
||||
land in M2.
|
||||
"""
|
||||
import logging
|
||||
import signal
|
||||
import time
|
||||
|
||||
from app.config import settings
|
||||
|
||||
log = logging.getLogger("siftlode.worker")
|
||||
|
||||
_stop = False
|
||||
|
||||
|
||||
def _handle_signal(signum, frame):
|
||||
global _stop
|
||||
_stop = True
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
signal.signal(signal.SIGTERM, _handle_signal)
|
||||
signal.signal(signal.SIGINT, _handle_signal)
|
||||
|
||||
if not settings.worker_enabled:
|
||||
log.info("WORKER_ENABLED is off; worker exiting (nothing to do).")
|
||||
return
|
||||
|
||||
log.info(
|
||||
"Download worker started (concurrency=%d, root=%s). Claim loop lands in M2.",
|
||||
settings.download_worker_concurrency,
|
||||
settings.download_root,
|
||||
)
|
||||
while not _stop:
|
||||
# M2: claim queued jobs (FOR UPDATE SKIP LOCKED) and download them.
|
||||
time.sleep(settings.download_poll_seconds)
|
||||
|
||||
log.info("Download worker stopped.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue