feat(downloads): M1 — schema, config, worker container, deps
- models: MediaAsset (shared file cache, unique on source+format_sig), DownloadJob (per-user request), DownloadProfile (presets), DownloadShare (ACL), DownloadQuota - migrations 0036 (5 tables) + 0037 (6 builtin presets) - sysconfig 'downloads' group (per-user defaults, retention/GC, layout, worker concurrency) - config: DOWNLOAD_ROOT / WORKER_ENABLED env + download defaults - deps: yt-dlp in requirements, ffmpeg in Dockerfile runtime stage - worker: dedicated container (python -m app.worker), thin idle entry (loop lands in M2) - localdev compose: worker service + downloads bind-mount; gitignore downloads/
This commit is contained in:
npeter83
parent
9ca90f515c
commit
317750e491
10 changed files with 496 additions and 0 deletions
143
backend/alembic/versions/0036_download_center.py
Normal file
143
backend/alembic/versions/0036_download_center.py
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
"""download center core schema (yt-dlp)
|
||||
|
||||
Revision ID: 0036_download_center
|
||||
Revises: 0035_saved_views
|
||||
Create Date: 2026-07-02
|
||||
|
||||
The download center's data layer. `media_assets` is the shared file cache (one row per unique
|
||||
source+format signature); `download_jobs` is the per-user request over it (worker-claimed via
|
||||
FOR UPDATE SKIP LOCKED, live progress written back for polling); `download_profiles` are the
|
||||
reusable format presets (builtins seeded in the next migration); `download_shares` grant a file
|
||||
to another user; `download_quotas` hold admin-set per-user limits (default via sysconfig).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0036_download_center"
|
||||
down_revision: Union[str, None] = "0035_saved_views"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"media_assets",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_kind", sa.String(length=16), nullable=False),
|
||||
sa.Column("source_ref", sa.String(length=512), nullable=False),
|
||||
sa.Column("format_sig", sa.String(length=64), nullable=False),
|
||||
sa.Column("status", sa.String(length=16), server_default="pending", nullable=False),
|
||||
sa.Column("rel_path", sa.Text(), nullable=True),
|
||||
sa.Column("storyboard_path", sa.Text(), nullable=True),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=True),
|
||||
sa.Column("duration_s", sa.Integer(), nullable=True),
|
||||
sa.Column("width", sa.Integer(), nullable=True),
|
||||
sa.Column("height", sa.Integer(), nullable=True),
|
||||
sa.Column("container", sa.String(length=16), nullable=True),
|
||||
sa.Column("vcodec", sa.String(length=32), nullable=True),
|
||||
sa.Column("acodec", sa.String(length=32), nullable=True),
|
||||
sa.Column("nfo_written", sa.Boolean(), server_default="false", nullable=False),
|
||||
sa.Column("title", sa.Text(), nullable=True),
|
||||
sa.Column("uploader", sa.String(length=255), nullable=True),
|
||||
sa.Column("upload_date", sa.Date(), nullable=True),
|
||||
sa.Column("thumbnail_url", sa.Text(), nullable=True),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("ref_count", sa.Integer(), server_default="0", nullable=False),
|
||||
sa.Column("last_access_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"),
|
||||
)
|
||||
op.create_index("ix_media_assets_status", "media_assets", ["status"])
|
||||
|
||||
op.create_table(
|
||||
"download_profiles",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("user_id", sa.Integer(), nullable=True),
|
||||
sa.Column("name", sa.String(length=80), nullable=False),
|
||||
sa.Column("spec", sa.JSON(), nullable=False),
|
||||
sa.Column("is_builtin", sa.Boolean(), server_default="false", nullable=False),
|
||||
sa.Column("sort_order", sa.Integer(), server_default="0", nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index("ix_download_profiles_user_id", "download_profiles", ["user_id"])
|
||||
|
||||
op.create_table(
|
||||
"download_jobs",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("user_id", sa.Integer(), nullable=False),
|
||||
sa.Column("asset_id", sa.Integer(), nullable=True),
|
||||
sa.Column("source_kind", sa.String(length=16), nullable=False),
|
||||
sa.Column("source_ref", sa.String(length=512), nullable=False),
|
||||
sa.Column("profile_id", sa.Integer(), nullable=True),
|
||||
sa.Column("profile_snapshot", sa.JSON(), nullable=False),
|
||||
sa.Column("display_name", sa.String(length=255), nullable=True),
|
||||
sa.Column("status", sa.String(length=16), server_default="queued", nullable=False),
|
||||
sa.Column("progress", sa.Integer(), server_default="0", nullable=False),
|
||||
sa.Column("speed_bps", sa.BigInteger(), nullable=True),
|
||||
sa.Column("eta_s", sa.Integer(), nullable=True),
|
||||
sa.Column("phase", sa.String(length=32), nullable=True),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("queue_pos", sa.Integer(), server_default="0", nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
||||
sa.ForeignKeyConstraint(["asset_id"], ["media_assets.id"], ondelete="SET NULL"),
|
||||
sa.ForeignKeyConstraint(["profile_id"], ["download_profiles.id"], ondelete="SET NULL"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index("ix_download_jobs_user_id", "download_jobs", ["user_id"])
|
||||
op.create_index("ix_download_jobs_asset_id", "download_jobs", ["asset_id"])
|
||||
op.create_index("ix_download_jobs_status", "download_jobs", ["status"])
|
||||
op.create_index("ix_download_jobs_claim", "download_jobs", ["status", "queue_pos"])
|
||||
|
||||
op.create_table(
|
||||
"download_shares",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("job_id", sa.Integer(), nullable=False),
|
||||
sa.Column("from_user_id", sa.Integer(), nullable=False),
|
||||
sa.Column("to_user_id", sa.Integer(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(["job_id"], ["download_jobs.id"], ondelete="CASCADE"),
|
||||
sa.ForeignKeyConstraint(["from_user_id"], ["users.id"], ondelete="CASCADE"),
|
||||
sa.ForeignKeyConstraint(["to_user_id"], ["users.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),
|
||||
)
|
||||
op.create_index("ix_download_shares_job_id", "download_shares", ["job_id"])
|
||||
op.create_index("ix_download_shares_from_user_id", "download_shares", ["from_user_id"])
|
||||
op.create_index("ix_download_shares_to_user_id", "download_shares", ["to_user_id"])
|
||||
|
||||
op.create_table(
|
||||
"download_quotas",
|
||||
sa.Column("user_id", sa.Integer(), nullable=False),
|
||||
sa.Column("max_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("max_concurrent", sa.Integer(), nullable=False),
|
||||
sa.Column("max_jobs", sa.Integer(), nullable=False),
|
||||
sa.Column("unlimited", sa.Boolean(), server_default="false", nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("user_id"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("download_quotas")
|
||||
op.drop_index("ix_download_shares_to_user_id", table_name="download_shares")
|
||||
op.drop_index("ix_download_shares_from_user_id", table_name="download_shares")
|
||||
op.drop_index("ix_download_shares_job_id", table_name="download_shares")
|
||||
op.drop_table("download_shares")
|
||||
op.drop_index("ix_download_jobs_claim", table_name="download_jobs")
|
||||
op.drop_index("ix_download_jobs_status", table_name="download_jobs")
|
||||
op.drop_index("ix_download_jobs_asset_id", table_name="download_jobs")
|
||||
op.drop_index("ix_download_jobs_user_id", table_name="download_jobs")
|
||||
op.drop_table("download_jobs")
|
||||
op.drop_index("ix_download_profiles_user_id", table_name="download_profiles")
|
||||
op.drop_table("download_profiles")
|
||||
op.drop_index("ix_media_assets_status", table_name="media_assets")
|
||||
op.drop_table("media_assets")
|
||||
67
backend/alembic/versions/0037_download_builtin_profiles.py
Normal file
67
backend/alembic/versions/0037_download_builtin_profiles.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
"""seed builtin download profiles (presets)
|
||||
|
||||
Revision ID: 0037_dl_builtin_profiles
|
||||
Revises: 0036_download_center
|
||||
Create Date: 2026-07-02
|
||||
|
||||
Seeds the shipped download presets (user_id NULL, is_builtin=true). `spec` is the format
|
||||
definition consumed by app.downloads.formats: mode ("av" audio+video / "v" video-only /
|
||||
"a" audio-only), max_height (null = best), container, audio_format, vcodec preference, and
|
||||
the embed_*/sponsorblock postprocessor flags. Names are English (the UI default); the frontend
|
||||
localizes builtin labels by a stable key derived from the name.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0037_dl_builtin_profiles"
|
||||
down_revision: Union[str, None] = "0036_download_center"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def _spec(mode, height=None, container=None, audio_format=None, vcodec=None):
|
||||
return {
|
||||
"mode": mode,
|
||||
"max_height": height,
|
||||
"container": container,
|
||||
"audio_format": audio_format,
|
||||
"vcodec": vcodec,
|
||||
"embed_subs": False,
|
||||
"embed_chapters": True,
|
||||
"embed_thumbnail": True,
|
||||
"sponsorblock": False,
|
||||
}
|
||||
|
||||
|
||||
_BUILTINS = [
|
||||
("Best (MP4)", _spec("av", None, "mp4"), 10),
|
||||
("1080p (MP4)", _spec("av", 1080, "mp4"), 20),
|
||||
("720p (MP4)", _spec("av", 720, "mp4"), 30),
|
||||
("Audio (M4A)", _spec("a", None, None, "m4a"), 40),
|
||||
("Audio (MP3)", _spec("a", None, None, "mp3"), 50),
|
||||
("Video only (MP4)", _spec("v", None, "mp4"), 60),
|
||||
]
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
profiles = sa.table(
|
||||
"download_profiles",
|
||||
sa.column("user_id", sa.Integer),
|
||||
sa.column("name", sa.String),
|
||||
sa.column("spec", sa.JSON),
|
||||
sa.column("is_builtin", sa.Boolean),
|
||||
sa.column("sort_order", sa.Integer),
|
||||
)
|
||||
op.bulk_insert(
|
||||
profiles,
|
||||
[
|
||||
{"user_id": None, "name": name, "spec": spec, "is_builtin": True, "sort_order": order}
|
||||
for name, spec, order in _BUILTINS
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DELETE FROM download_profiles WHERE is_builtin = true")
|
||||
Loading…
Add table
Add a link
Reference in a new issue