From 317750e4916094a19fc229597b60e069fd75a039 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Thu, 2 Jul 2026 23:57:40 +0200 Subject: [PATCH] =?UTF-8?q?feat(downloads):=20M1=20=E2=80=94=20schema,=20c?= =?UTF-8?q?onfig,=20worker=20container,=20deps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - models: MediaAsset (shared file cache, unique on source+format_sig), DownloadJob (per-user request), DownloadProfile (presets), DownloadShare (ACL), DownloadQuota - migrations 0036 (5 tables) + 0037 (6 builtin presets) - sysconfig 'downloads' group (per-user defaults, retention/GC, layout, worker concurrency) - config: DOWNLOAD_ROOT / WORKER_ENABLED env + download defaults - deps: yt-dlp in requirements, ffmpeg in Dockerfile runtime stage - worker: dedicated container (python -m app.worker), thin idle entry (loop lands in M2) - localdev compose: worker service + downloads bind-mount; gitignore downloads/ --- .gitignore | 3 + Dockerfile | 6 + .../alembic/versions/0036_download_center.py | 143 ++++++++++++++++ .../0037_download_builtin_profiles.py | 67 ++++++++ backend/app/config.py | 25 +++ backend/app/models.py | 155 ++++++++++++++++++ backend/app/sysconfig.py | 9 + backend/app/worker.py | 50 ++++++ backend/requirements.txt | 4 + docker-compose.localdev.yml | 34 ++++ 10 files changed, 496 insertions(+) create mode 100644 backend/alembic/versions/0036_download_center.py create mode 100644 backend/alembic/versions/0037_download_builtin_profiles.py create mode 100644 backend/app/worker.py diff --git a/.gitignore b/.gitignore index 38f6d4f..1afa1fc 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,9 @@ frontend/dist/ *.sql.gz backups/ +# Download center: locally downloaded media (the DOWNLOAD_ROOT bind mount) +downloads/ + # OS / editor .DS_Store Thumbs.db diff --git a/Dockerfile b/Dockerfile index d5aa7fd..188bd05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,12 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ WORKDIR /app +# ffmpeg is required by the download center (yt-dlp) to merge separate video+audio streams +# and run postprocessing (audio extract, embed thumbnail/chapters/subs, SponsorBlock). +RUN apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg \ + && rm -rf /var/lib/apt/lists/* + COPY backend/requirements.txt . RUN pip install --upgrade pip && pip install -r requirements.txt diff --git a/backend/alembic/versions/0036_download_center.py b/backend/alembic/versions/0036_download_center.py new file mode 100644 index 0000000..b9f0677 --- /dev/null +++ b/backend/alembic/versions/0036_download_center.py @@ -0,0 +1,143 @@ +"""download center core schema (yt-dlp) + +Revision ID: 0036_download_center +Revises: 0035_saved_views +Create Date: 2026-07-02 + +The download center's data layer. `media_assets` is the shared file cache (one row per unique +source+format signature); `download_jobs` is the per-user request over it (worker-claimed via +FOR UPDATE SKIP LOCKED, live progress written back for polling); `download_profiles` are the +reusable format presets (builtins seeded in the next migration); `download_shares` grant a file +to another user; `download_quotas` hold admin-set per-user limits (default via sysconfig). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0036_download_center" +down_revision: Union[str, None] = "0035_saved_views" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "media_assets", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("source_kind", sa.String(length=16), nullable=False), + sa.Column("source_ref", sa.String(length=512), nullable=False), + sa.Column("format_sig", sa.String(length=64), nullable=False), + sa.Column("status", sa.String(length=16), server_default="pending", nullable=False), + sa.Column("rel_path", sa.Text(), nullable=True), + sa.Column("storyboard_path", sa.Text(), nullable=True), + sa.Column("size_bytes", sa.BigInteger(), nullable=True), + sa.Column("duration_s", sa.Integer(), nullable=True), + sa.Column("width", sa.Integer(), nullable=True), + sa.Column("height", sa.Integer(), nullable=True), + sa.Column("container", sa.String(length=16), nullable=True), + sa.Column("vcodec", sa.String(length=32), nullable=True), + sa.Column("acodec", sa.String(length=32), nullable=True), + sa.Column("nfo_written", sa.Boolean(), server_default="false", nullable=False), + sa.Column("title", sa.Text(), nullable=True), + sa.Column("uploader", sa.String(length=255), nullable=True), + sa.Column("upload_date", sa.Date(), nullable=True), + sa.Column("thumbnail_url", sa.Text(), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("ref_count", sa.Integer(), server_default="0", nullable=False), + sa.Column("last_access_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"), + ) + op.create_index("ix_media_assets_status", "media_assets", ["status"]) + + op.create_table( + "download_profiles", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=True), + sa.Column("name", sa.String(length=80), nullable=False), + sa.Column("spec", sa.JSON(), nullable=False), + sa.Column("is_builtin", sa.Boolean(), server_default="false", nullable=False), + sa.Column("sort_order", sa.Integer(), server_default="0", nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_download_profiles_user_id", "download_profiles", ["user_id"]) + + op.create_table( + "download_jobs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("asset_id", sa.Integer(), nullable=True), + sa.Column("source_kind", sa.String(length=16), nullable=False), + sa.Column("source_ref", sa.String(length=512), nullable=False), + sa.Column("profile_id", sa.Integer(), nullable=True), + sa.Column("profile_snapshot", sa.JSON(), nullable=False), + sa.Column("display_name", sa.String(length=255), nullable=True), + sa.Column("status", sa.String(length=16), server_default="queued", nullable=False), + sa.Column("progress", sa.Integer(), server_default="0", nullable=False), + sa.Column("speed_bps", sa.BigInteger(), nullable=True), + sa.Column("eta_s", sa.Integer(), nullable=True), + sa.Column("phase", sa.String(length=32), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("queue_pos", sa.Integer(), server_default="0", nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["asset_id"], ["media_assets.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["profile_id"], ["download_profiles.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_download_jobs_user_id", "download_jobs", ["user_id"]) + op.create_index("ix_download_jobs_asset_id", "download_jobs", ["asset_id"]) + op.create_index("ix_download_jobs_status", "download_jobs", ["status"]) + op.create_index("ix_download_jobs_claim", "download_jobs", ["status", "queue_pos"]) + + op.create_table( + "download_shares", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("job_id", sa.Integer(), nullable=False), + sa.Column("from_user_id", sa.Integer(), nullable=False), + sa.Column("to_user_id", sa.Integer(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["job_id"], ["download_jobs.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["from_user_id"], ["users.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["to_user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("job_id", "to_user_id", name="uq_download_share"), + ) + op.create_index("ix_download_shares_job_id", "download_shares", ["job_id"]) + op.create_index("ix_download_shares_from_user_id", "download_shares", ["from_user_id"]) + op.create_index("ix_download_shares_to_user_id", "download_shares", ["to_user_id"]) + + op.create_table( + "download_quotas", + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("max_bytes", sa.BigInteger(), nullable=False), + sa.Column("max_concurrent", sa.Integer(), nullable=False), + sa.Column("max_jobs", sa.Integer(), nullable=False), + sa.Column("unlimited", sa.Boolean(), server_default="false", nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("user_id"), + ) + + +def downgrade() -> None: + op.drop_table("download_quotas") + op.drop_index("ix_download_shares_to_user_id", table_name="download_shares") + op.drop_index("ix_download_shares_from_user_id", table_name="download_shares") + op.drop_index("ix_download_shares_job_id", table_name="download_shares") + op.drop_table("download_shares") + op.drop_index("ix_download_jobs_claim", table_name="download_jobs") + op.drop_index("ix_download_jobs_status", table_name="download_jobs") + op.drop_index("ix_download_jobs_asset_id", table_name="download_jobs") + op.drop_index("ix_download_jobs_user_id", table_name="download_jobs") + op.drop_table("download_jobs") + op.drop_index("ix_download_profiles_user_id", table_name="download_profiles") + op.drop_table("download_profiles") + op.drop_index("ix_media_assets_status", table_name="media_assets") + op.drop_table("media_assets") diff --git a/backend/alembic/versions/0037_download_builtin_profiles.py b/backend/alembic/versions/0037_download_builtin_profiles.py new file mode 100644 index 0000000..d8742a0 --- /dev/null +++ b/backend/alembic/versions/0037_download_builtin_profiles.py @@ -0,0 +1,67 @@ +"""seed builtin download profiles (presets) + +Revision ID: 0037_dl_builtin_profiles +Revises: 0036_download_center +Create Date: 2026-07-02 + +Seeds the shipped download presets (user_id NULL, is_builtin=true). `spec` is the format +definition consumed by app.downloads.formats: mode ("av" audio+video / "v" video-only / +"a" audio-only), max_height (null = best), container, audio_format, vcodec preference, and +the embed_*/sponsorblock postprocessor flags. Names are English (the UI default); the frontend +localizes builtin labels by a stable key derived from the name. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0037_dl_builtin_profiles" +down_revision: Union[str, None] = "0036_download_center" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _spec(mode, height=None, container=None, audio_format=None, vcodec=None): + return { + "mode": mode, + "max_height": height, + "container": container, + "audio_format": audio_format, + "vcodec": vcodec, + "embed_subs": False, + "embed_chapters": True, + "embed_thumbnail": True, + "sponsorblock": False, + } + + +_BUILTINS = [ + ("Best (MP4)", _spec("av", None, "mp4"), 10), + ("1080p (MP4)", _spec("av", 1080, "mp4"), 20), + ("720p (MP4)", _spec("av", 720, "mp4"), 30), + ("Audio (M4A)", _spec("a", None, None, "m4a"), 40), + ("Audio (MP3)", _spec("a", None, None, "mp3"), 50), + ("Video only (MP4)", _spec("v", None, "mp4"), 60), +] + + +def upgrade() -> None: + profiles = sa.table( + "download_profiles", + sa.column("user_id", sa.Integer), + sa.column("name", sa.String), + sa.column("spec", sa.JSON), + sa.column("is_builtin", sa.Boolean), + sa.column("sort_order", sa.Integer), + ) + op.bulk_insert( + profiles, + [ + {"user_id": None, "name": name, "spec": spec, "is_builtin": True, "sort_order": order} + for name, spec, order in _BUILTINS + ], + ) + + +def downgrade() -> None: + op.execute("DELETE FROM download_profiles WHERE is_builtin = true") diff --git a/backend/app/config.py b/backend/app/config.py index 938cfcb..cb44539 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -131,6 +131,31 @@ class Settings(BaseSettings): # ("was_live") are real watchable content and stay visible. feed_default_hidden_live: str = "live,upcoming" + # --- Download center (yt-dlp) --- + # Root directory (inside the container) for downloaded media — a Docker volume/bind mount. + # Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it. + download_root: str = "/downloads" + # Whether THIS process runs the download worker loop. On for the dedicated worker + # container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled. + worker_enabled: bool = False + # How many downloads the worker runs concurrently (claimed via FOR UPDATE SKIP LOCKED). + download_worker_concurrency: int = 2 + # How often (empty pending queue) the worker polls for a new job, in seconds. + download_poll_seconds: int = 5 + # Default per-user quota (admin-tunable; a per-user download_quotas row overrides). + download_default_max_bytes: int = 5_368_709_120 # 5 GiB + download_default_max_concurrent: int = 2 + download_default_max_jobs: int = 50 + # Retention: an asset expires this many days after its last access; the GC job then + # deletes it (once no job references it) after an additional grace window. + download_retention_days: int = 30 + download_gc_grace_days: int = 2 + download_gc_minutes: int = 360 + # On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat". + download_layout: str = "plex" + # Default SponsorBlock (skip sponsor segments) for new custom profiles. + sponsorblock_default: bool = False + @property def allowed_email_set(self) -> set[str]: return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()} diff --git a/backend/app/models.py b/backend/app/models.py index 3160514..be145e0 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -8,6 +8,7 @@ from sqlalchemy import ( DateTime, Float, ForeignKey, + Index, Integer, JSON, String, @@ -688,6 +689,160 @@ class Message(Base): ) +class MediaAsset(Base, TimestampMixin): + """A physical downloaded media file — the SHARED cache layer of the download center. + + Mirrors the app's "shared catalog, private per-user state" model: one row per unique + ``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests + the same video with a format that hashes to the same `format_sig`, their `download_job` + links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is + shared; per-user edited derivatives (phase 2) are never cached here. + + `rel_path` is relative to `settings.download_root` (the mount), so the physical filename is + system-decided and bound to the source id — never renamed (the user's custom name is display + metadata on `download_job`, applied only when downloading to their own machine). The naming + fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built + uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) + plus `last_access_at`/`expires_at` drive retention + LRU eviction in the GC job.""" + + __tablename__ = "media_assets" + __table_args__ = ( + UniqueConstraint( + "source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key" + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url" + source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc + format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec + status: Mapped[str] = mapped_column( + String(16), default="pending", server_default="pending", index=True + ) # "pending" | "ready" | "error" + rel_path: Mapped[str | None] = mapped_column(Text) # relative to download_root + storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic + size_bytes: Mapped[int | None] = mapped_column(BigInteger) + duration_s: Mapped[int | None] = mapped_column(Integer) + width: Mapped[int | None] = mapped_column(Integer) + height: Mapped[int | None] = mapped_column(Integer) + container: Mapped[str | None] = mapped_column(String(16)) + vcodec: Mapped[str | None] = mapped_column(String(32)) + acodec: Mapped[str | None] = mapped_column(String(32)) + nfo_written: Mapped[bool] = mapped_column( + Boolean, default=False, server_default="false" + ) + # Denormalized naming/metadata for the Plex tree + ad-hoc display. + title: Mapped[str | None] = mapped_column(Text) + uploader: Mapped[str | None] = mapped_column(String(255)) + upload_date: Mapped[date | None] = mapped_column(Date) + thumbnail_url: Mapped[str | None] = mapped_column(Text) + error: Mapped[str | None] = mapped_column(Text) + ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + +class DownloadProfile(Base, TimestampMixin): + """A named download preset — a reusable format profile. + + `user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …); + a user's saved custom profiles carry their `user_id`. `spec` is the format definition + (mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/ + thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache + `format_sig`.""" + + __tablename__ = "download_profiles" + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) # NULL for a builtin preset + name: Mapped[str] = mapped_column(String(80)) + spec: Mapped[dict] = mapped_column(JSON) + is_builtin: Mapped[bool] = mapped_column( + Boolean, default=False, server_default="false" + ) + sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + + +class DownloadJob(Base, TimestampMixin, UpdatedAtMixin): + """A user's request to download one source — the per-user layer over `MediaAsset`. + + The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live + `progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery — the + worker is a separate process and can't reach the in-process WS registry). `profile_snapshot` + freezes the spec at enqueue so later edits/deletion of the profile don't change this job. + `display_name` is the user's own name for the file (display + the Content-Disposition on + local download); the on-disk asset is never renamed. `asset_id` links to the shared file + (SET NULL if the asset is GC'd — the job then shows as expired).""" + + __tablename__ = "download_jobs" + __table_args__ = (Index("ix_download_jobs_claim", "status", "queue_pos"),) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + asset_id: Mapped[int | None] = mapped_column( + ForeignKey("media_assets.id", ondelete="SET NULL"), index=True + ) + source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url" + source_ref: Mapped[str] = mapped_column(String(512)) + profile_id: Mapped[int | None] = mapped_column( + ForeignKey("download_profiles.id", ondelete="SET NULL") + ) + profile_snapshot: Mapped[dict] = mapped_column(JSON) + display_name: Mapped[str | None] = mapped_column(String(255)) + status: Mapped[str] = mapped_column( + String(16), default="queued", server_default="queued", index=True + ) # queued | running | paused | done | error | canceled + progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + speed_bps: Mapped[int | None] = mapped_column(BigInteger) + eta_s: Mapped[int | None] = mapped_column(Integer) + phase: Mapped[str | None] = mapped_column(String(32)) + error: Mapped[str | None] = mapped_column(Text) + queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + + +class DownloadShare(Base, TimestampMixin): + """A user→user grant: `to_user` may access `from_user`'s downloaded file (private by + default). Since assets are already deduped/shared on disk, a share is just an ACL row.""" + + __tablename__ = "download_shares" + __table_args__ = ( + UniqueConstraint("job_id", "to_user_id", name="uq_download_share"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + job_id: Mapped[int] = mapped_column( + ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True + ) + from_user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + to_user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + + +class DownloadQuota(Base, UpdatedAtMixin): + """Per-user download limits (admin-managed). A row exists only when an admin customizes a + user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the + byte cap entirely (e.g. for the admin's own account).""" + + __tablename__ = "download_quotas" + + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), primary_key=True + ) + max_bytes: Mapped[int] = mapped_column(BigInteger) + max_concurrent: Mapped[int] = mapped_column(Integer) + max_jobs: Mapped[int] = mapped_column(Integer) + unlimited: Mapped[bool] = mapped_column( + Boolean, default=False, server_default="false" + ) + + class MessageKey(Base, TimestampMixin): """A user's end-to-end-encryption key material for direct messaging (phase 2). diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index ff396c7..1febd37 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -66,6 +66,15 @@ SPECS: tuple[ConfigSpec, ...] = ( ConfigSpec("google_client_secret", "str", "google", "google_client_secret", secret=True), # --- Access / registration --- ConfigSpec("allow_registration", "bool", "access", "allow_registration"), + # --- Download center (yt-dlp) — per-user defaults + retention/GC + layout --- + ConfigSpec("download_default_max_bytes", "int", "downloads", "download_default_max_bytes", min=0), + ConfigSpec("download_default_max_concurrent", "int", "downloads", "download_default_max_concurrent", min=1, max=20), + ConfigSpec("download_default_max_jobs", "int", "downloads", "download_default_max_jobs", min=1, max=100_000), + ConfigSpec("download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650), + ConfigSpec("download_gc_grace_days", "int", "downloads", "download_gc_grace_days", min=0, max=3_650), + ConfigSpec("download_layout", "str", "downloads", "download_layout"), + ConfigSpec("download_worker_concurrency", "int", "downloads", "download_worker_concurrency", min=1, max=16), + ConfigSpec("sponsorblock_default", "bool", "downloads", "sponsorblock_default"), ) _BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS} diff --git a/backend/app/worker.py b/backend/app/worker.py new file mode 100644 index 0000000..1b7564f --- /dev/null +++ b/backend/app/worker.py @@ -0,0 +1,50 @@ +"""Download worker process entry point. + +Runs in a DEDICATED container (`command: python -m app.worker`, `WORKER_ENABLED=1`), separate +from the API so long yt-dlp downloads + ffmpeg postprocessing never block web requests. It +claims `queued` download jobs from the DB with FOR UPDATE SKIP LOCKED and runs them, writing +live progress back to the job row (the frontend polls it — the worker is a separate process and +can't reach the API's in-process WebSocket registry). + +M1: thin, idle entry point (keeps the container healthy). The claim loop + yt-dlp integration +land in M2. +""" +import logging +import signal +import time + +from app.config import settings + +log = logging.getLogger("siftlode.worker") + +_stop = False + + +def _handle_signal(signum, frame): + global _stop + _stop = True + + +def main() -> None: + logging.basicConfig(level=logging.INFO) + signal.signal(signal.SIGTERM, _handle_signal) + signal.signal(signal.SIGINT, _handle_signal) + + if not settings.worker_enabled: + log.info("WORKER_ENABLED is off; worker exiting (nothing to do).") + return + + log.info( + "Download worker started (concurrency=%d, root=%s). Claim loop lands in M2.", + settings.download_worker_concurrency, + settings.download_root, + ) + while not _stop: + # M2: claim queued jobs (FOR UPDATE SKIP LOCKED) and download them. + time.sleep(settings.download_poll_seconds) + + log.info("Download worker stopped.") + + +if __name__ == "__main__": + main() diff --git a/backend/requirements.txt b/backend/requirements.txt index 67f14ad..2e293ce 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -13,3 +13,7 @@ py3langid>=0.3,<1.0 itsdangerous>=2.1,<3.0 cryptography>=46.0.7,<47 argon2-cffi>=23.1,<26 +# Download center: yt-dlp does the extraction/download; ffmpeg (installed at the OS level in +# the Dockerfile) does the merge + postprocessing. yt-dlp is release-often (YouTube changes), +# so keep the floor loose and bump it when extraction breaks. +yt-dlp>=2025.1 diff --git a/docker-compose.localdev.yml b/docker-compose.localdev.yml index 37260aa..2cac4c0 100644 --- a/docker-compose.localdev.yml +++ b/docker-compose.localdev.yml @@ -43,6 +43,9 @@ services: DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode} # This instance owns its scheduler now (its own DB, so no double-write concern). SCHEDULER_ENABLED: "true" + # Download center: the API serves finished files (it does NOT run the worker loop). + DOWNLOAD_ROOT: /downloads + WORKER_ENABLED: "false" depends_on: db: condition: service_healthy @@ -51,6 +54,37 @@ services: - apparmor:unconfined ports: - "${APP_PORT:-8080}:8000" + volumes: + # Downloaded media lives on the host so you can inspect the generated Plex tree. + # Gitignored. The worker writes here; the API reads it to serve local downloads. + - ./downloads:/downloads + restart: unless-stopped + + # Dedicated download worker: same image, runs the yt-dlp job loop instead of the API. + # It shares the DB (job queue) and the downloads mount with the API. + worker: + build: + context: . + dockerfile: Dockerfile + args: + APP_VERSION: ${APP_VERSION:-dev} + GIT_SHA: ${GIT_SHA:-unknown} + BUILD_DATE: ${BUILD_DATE:-} + command: ["python", "-m", "app.worker"] + env_file: .env + environment: + DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode} + # The worker must not also run the scheduler; the API owns that. + SCHEDULER_ENABLED: "false" + DOWNLOAD_ROOT: /downloads + WORKER_ENABLED: "true" + depends_on: + db: + condition: service_healthy + security_opt: + - apparmor:unconfined + volumes: + - ./downloads:/downloads restart: unless-stopped volumes: