diff --git a/.gitignore b/.gitignore index 1afa1fc..6710e89 100644 --- a/.gitignore +++ b/.gitignore @@ -21,8 +21,9 @@ frontend/dist/ *.sql.gz backups/ -# Download center: locally downloaded media (the DOWNLOAD_ROOT bind mount) -downloads/ +# Download center: locally downloaded media (the DOWNLOAD_ROOT bind mount at the repo root). +# Anchored with a leading slash so it does NOT also ignore the backend/app/downloads package. +/downloads/ # OS / editor .DS_Store diff --git a/backend/app/downloads/__init__.py b/backend/app/downloads/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/downloads/formats.py b/backend/app/downloads/formats.py new file mode 100644 index 0000000..6360ca3 --- /dev/null +++ b/backend/app/downloads/formats.py @@ -0,0 +1,145 @@ +"""Format profiles → yt-dlp options, and the cache signature. + +A profile `spec` (stored on DownloadProfile / snapshotted on DownloadJob) is the single source +of truth for what file gets produced. `format_sig` hashes exactly the fields that change the +OUTPUT BYTES, so two users picking equivalent settings share one cached MediaAsset. +`build_ydl_opts` turns a spec into a yt-dlp options dict (format selector + postprocessors). + +spec shape (all keys optional; defaults applied by `normalize`): + mode "av" (audio+video) | "v" (video-only) | "a" (audio-only) + max_height int | null (null = best) + container "mp4" | "mkv" | "webm" | null (merge/remux target for av/v) + audio_format "m4a" | "mp3" | "opus" | null (extract target for mode "a") + vcodec "h264" | "vp9" | "av1" | null (preference, best-effort) + embed_subs bool embed_chapters bool embed_thumbnail bool sponsorblock bool +""" +import hashlib +import json + +# Fields that affect the produced file — and therefore the cache identity. +_SIG_KEYS = ( + "mode", + "max_height", + "container", + "audio_format", + "vcodec", + "embed_subs", + "embed_chapters", + "embed_thumbnail", + "sponsorblock", +) + +_DEFAULTS = { + "mode": "av", + "max_height": None, + "container": "mp4", + "audio_format": "m4a", + "vcodec": None, + "embed_subs": False, + "embed_chapters": True, + "embed_thumbnail": True, + "sponsorblock": False, +} + +_SPONSORBLOCK_CATEGORIES = ["sponsor", "selfpromo", "interaction"] +_VCODEC_SORT = {"h264": "vcodec:h264", "vp9": "vcodec:vp9", "av1": "vcodec:av01"} + + +def normalize(spec: dict) -> dict: + """Fill defaults + coerce types so signature and yt-dlp opts are deterministic.""" + s = dict(_DEFAULTS) + for k in _DEFAULTS: + if spec.get(k) is not None: + s[k] = spec[k] + if s["mode"] not in ("av", "v", "a"): + s["mode"] = "av" + if s["max_height"] is not None: + s["max_height"] = int(s["max_height"]) + for b in ("embed_subs", "embed_chapters", "embed_thumbnail", "sponsorblock"): + s[b] = bool(s[b]) + # Audio-only files can't embed subtitles/thumbnails as video; keep the flags meaningful. + if s["mode"] == "a": + s["embed_subs"] = False + return s + + +def format_sig(spec: dict) -> str: + """Stable short hash of the output-affecting spec — the MediaAsset cache key component.""" + s = normalize(spec) + payload = json.dumps({k: s[k] for k in _SIG_KEYS}, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode()).hexdigest()[:24] + + +def target_ext(spec: dict) -> str: + """Best-effort final extension for the produced file (also used to name the staging output).""" + s = normalize(spec) + if s["mode"] == "a": + return s["audio_format"] or "m4a" + return s["container"] or "mp4" + + +def build_ydl_opts(spec: dict, outtmpl: str, progress_hook) -> dict: + """Assemble yt-dlp options for a single download to `outtmpl` (a full path template).""" + s = normalize(spec) + h = s["max_height"] + opts: dict = { + "outtmpl": outtmpl, + "noplaylist": True, + "quiet": True, + "no_warnings": True, + "noprogress": True, + "writethumbnail": True, # kept for the poster.jpg sidecar (and embed, if requested) + "progress_hooks": [progress_hook], + "postprocessors": [], + } + + if s["mode"] == "a": + opts["format"] = "bestaudio/best" + opts["postprocessors"].append( + { + "key": "FFmpegExtractAudio", + "preferredcodec": s["audio_format"] or "m4a", + "preferredquality": "0", + } + ) + else: + hcap = f"[height<=?{h}]" if h else "" + if s["mode"] == "v": + opts["format"] = f"bestvideo{hcap}/best{hcap}" + else: # av + opts["format"] = f"bestvideo{hcap}+bestaudio/best{hcap}" + if s["container"]: + opts["merge_output_format"] = s["container"] + sort = [] + if h: + sort.append(f"res:{h}") + if s["vcodec"] in _VCODEC_SORT: + sort.append(_VCODEC_SORT[s["vcodec"]]) + if sort: + opts["format_sort"] = sort + + # SponsorBlock first (marks segments), then strip them from the media + chapters. + if s["sponsorblock"]: + opts["postprocessors"].append( + {"key": "SponsorBlock", "categories": _SPONSORBLOCK_CATEGORIES, "when": "after_filter"} + ) + opts["postprocessors"].append( + {"key": "ModifyChapters", "remove_sponsor_segments": _SPONSORBLOCK_CATEGORIES} + ) + if s["embed_subs"] and s["mode"] != "a": + opts["writesubtitles"] = True + opts["subtitleslangs"] = ["en.*", "hu.*", "de.*"] + opts["postprocessors"].append({"key": "FFmpegEmbedSubtitle"}) + if s["embed_chapters"]: + opts["postprocessors"].append( + {"key": "FFmpegMetadata", "add_chapters": True, "add_metadata": True} + ) + # Normalize the written thumbnail to jpg so we always have a `.jpg` poster sidecar. + # `already_have_thumbnail` keeps that file on disk after embedding (default deletes it). + opts["postprocessors"].append({"key": "FFmpegThumbnailsConvertor", "format": "jpg"}) + if s["embed_thumbnail"]: + opts["postprocessors"].append( + {"key": "EmbedThumbnail", "already_have_thumbnail": True} + ) + + return opts diff --git a/backend/app/downloads/service.py b/backend/app/downloads/service.py new file mode 100644 index 0000000..e832d6b --- /dev/null +++ b/backend/app/downloads/service.py @@ -0,0 +1,97 @@ +"""Enqueue + cache resolution for the download center (the per-user layer over MediaAsset). + +`enqueue` is the single entry point used by the API: it resolves (or creates) the shared +MediaAsset for the requested source+format, links a per-user DownloadJob to it, and — on a +cache hit (asset already ready) — marks the job done instantly with no re-download. The worker +handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and +the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping. +""" +from datetime import datetime, timezone + +from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from app.downloads import formats +from app.models import DownloadJob, MediaAsset + + +def source_url(source_kind: str, source_ref: str) -> str: + """The URL yt-dlp should fetch. YouTube stores the 11-char id; ad-hoc stores the URL.""" + if source_kind == "youtube": + return f"https://www.youtube.com/watch?v={source_ref}" + return source_ref + + +def get_or_create_asset( + db: Session, source_kind: str, source_ref: str, format_sig: str +) -> MediaAsset: + """Fetch the cache row for this (source, format) or create a fresh `pending` one. + + Races on the unique cache key are resolved by re-selecting after an IntegrityError, so two + simultaneous enqueues of the same video+format converge on one shared asset.""" + stmt = select(MediaAsset).where( + MediaAsset.source_kind == source_kind, + MediaAsset.source_ref == source_ref, + MediaAsset.format_sig == format_sig, + ) + asset = db.execute(stmt).scalar_one_or_none() + if asset is not None: + return asset + asset = MediaAsset( + source_kind=source_kind, + source_ref=source_ref, + format_sig=format_sig, + status="pending", + ref_count=0, + ) + db.add(asset) + try: + db.flush() + except IntegrityError: + db.rollback() + asset = db.execute(stmt).scalar_one() + return asset + + +def _next_queue_pos(db: Session) -> int: + return (db.execute(select(func.coalesce(func.max(DownloadJob.queue_pos), 0))).scalar() or 0) + 1 + + +def enqueue( + db: Session, + user_id: int, + source_kind: str, + source_ref: str, + spec: dict, + profile_id: int | None = None, + display_name: str | None = None, +) -> DownloadJob: + """Queue a download for `user_id`. Returns the DownloadJob (already `done` on a cache hit).""" + spec = formats.normalize(spec) + sig = formats.format_sig(spec) + asset = get_or_create_asset(db, source_kind, source_ref, sig) + + job = DownloadJob( + user_id=user_id, + asset_id=asset.id, + source_kind=source_kind, + source_ref=source_ref, + profile_id=profile_id, + profile_snapshot=spec, + display_name=display_name or (asset.title if asset.status == "ready" else None), + status="queued", + queue_pos=_next_queue_pos(db), + ) + db.add(job) + asset.ref_count = (asset.ref_count or 0) + 1 + + # Cache hit: the file already exists — no worker round-trip needed. + if asset.status == "ready": + job.status = "done" + job.progress = 100 + asset.last_access_at = datetime.now(timezone.utc) + + db.commit() + db.refresh(job) + return job diff --git a/backend/app/downloads/storage.py b/backend/app/downloads/storage.py new file mode 100644 index 0000000..cf94a28 --- /dev/null +++ b/backend/app/downloads/storage.py @@ -0,0 +1,143 @@ +"""On-disk layout for downloaded media — Plex-friendly tree + .nfo/poster sidecars. + +The physical filename is system-decided and bound to the source id (never renamed; the user's +custom name is display-only, applied at local-download time). Layout is admin-configurable: + + plex: {Channel}/Season {Year}/{Channel} - {YYYY-MM-DD} - {title} [{id}].{ext} + flat: {title} [{id}].{ext} + +Plus a Kodi/Plex-readable `.nfo` (episodedetails) and `.jpg` thumbnail, so +a Plex library using a metadata agent picks up rich info. Channel = "show", video = episode, +date-based — the community-standard mapping for YouTube-in-Plex. +""" +import re +import shutil +import unicodedata +from dataclasses import dataclass +from datetime import date +from pathlib import Path +from xml.sax.saxutils import escape + +# Reserve room for the " [id].ext" suffix + parent dirs within the 255-byte component limit. +_MAX_TITLE = 120 +# Filesystem-illegal on Windows/Plex + control chars. +_ILLEGAL = re.compile(r'[<>:"/\\|?*\x00-\x1f]') +# Unicode categories we drop entirely: emoji/pictographs (So), modifier symbols/skin-tones +# (Sk), and every control/format/private/surrogate/unassigned char (C*). Letters (incl. +# accented á/ö/ü/ß), digits, and ordinary punctuation are KEPT — Siftlode is trilingual, so +# ASCII-folding would mangle Hungarian/German titles. +_DROP_CATEGORIES = {"So", "Sk", "Cc", "Cf", "Cs", "Co", "Cn"} + + +@dataclass +class MediaMeta: + video_id: str + title: str + uploader: str + upload_date: date | None + duration_s: int | None = None + description: str | None = None + thumbnail_url: str | None = None + + +def sanitize(name: str, limit: int = _MAX_TITLE) -> str: + """Turn an arbitrary (emoji-laden, clickbait) title into a clean, space-free path token. + + Steps: NFKC-normalize (fold fancy width/compat forms) → drop emoji/symbols/control chars → + strip filesystem-illegal chars → collapse whitespace and runs of the same punctuation → + join words with underscores → trim junk punctuation from the ends → length-cap. Accented + letters and digits survive; the result never contains spaces or path separators.""" + text = unicodedata.normalize("NFKC", name or "") + text = "".join( + ch for ch in text if unicodedata.category(ch) not in _DROP_CATEGORIES + ) + text = _ILLEGAL.sub("", text) + # Collapse repeated punctuation ("!!!", "???", "-----") to a single mark. + text = re.sub(r"([^\w\s])\1{1,}", r"\1", text) + # Whitespace → single underscore (no spaces in generated names). + text = re.sub(r"\s+", "_", text.strip()) + # Tidy separator pile-ups and trim junk from the ends. + text = re.sub(r"_{2,}", "_", text).strip("_.-·—–|,;:!?ّ ") + if len(text) > limit: + text = text[:limit].rstrip("_.-") + return text or "untitled" + + +def rel_path(meta: MediaMeta, ext: str, layout: str = "plex") -> str: + """Path of the media file relative to DOWNLOAD_ROOT (forward slashes).""" + title = sanitize(meta.title) + vid = meta.video_id + if layout == "flat": + return f"{title}_[{vid}].{ext}" + channel = sanitize(meta.uploader or "Unknown_Channel", 80) + year = meta.upload_date.year if meta.upload_date else 0 + d = meta.upload_date.isoformat() if meta.upload_date else "0000-00-00" + epname = f"{channel}_-_{d}_-_{title}_[{vid}].{ext}" + return f"{channel}/Season_{year}/{epname}" + + +def abs_path(download_root: str, rel: str) -> Path: + return Path(download_root) / rel + + +def place_file(staging_file: Path, download_root: str, rel: str) -> None: + """Move a completed staging file into its final location under DOWNLOAD_ROOT.""" + dest = abs_path(download_root, rel) + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(staging_file), str(dest)) + + +def _nfo_xml(meta: MediaMeta) -> str: + aired = meta.upload_date.isoformat() if meta.upload_date else "" + year = meta.upload_date.year if meta.upload_date else "" + parts = [ + '', + "", + f" {escape(meta.title or '')}", + f" {escape(meta.uploader or '')}", + f" {escape(meta.uploader or '')}", + f" {aired}", + f" {aired}", + f" {year}", + f" {escape(meta.description or '')}", + f" {escape(meta.video_id)}", + ] + if meta.duration_s: + parts.append(f" {round(meta.duration_s / 60)}") + parts.append("") + return "\n".join(parts) + + +def write_sidecars( + download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None +) -> bool: + """Write `.nfo` + `.jpg` next to the media file. Returns True if written.""" + media = abs_path(download_root, rel) + base = media.with_suffix("") + try: + base.with_suffix(".nfo").write_text(_nfo_xml(meta), encoding="utf-8") + if thumb_src and thumb_src.exists(): + shutil.copyfile(str(thumb_src), str(base.with_suffix(".jpg"))) + return True + except OSError: + return False + + +def delete_asset_files(download_root: str, rel: str) -> None: + """Remove the media file and its sidecars; prune now-empty parent dirs (best effort).""" + media = abs_path(download_root, rel) + base = media.with_suffix("") + for p in (media, base.with_suffix(".nfo"), base.with_suffix(".jpg")): + try: + p.unlink() + except OSError: + pass + # Prune empty Season/Channel dirs up to (but not including) the root. + root = Path(download_root).resolve() + parent = media.parent + while parent != root and root in parent.parents: + try: + parent.rmdir() # only succeeds if empty + except OSError: + break + parent = parent.parent