fix(downloads): add the downloads package (was wrongly gitignored) + robust sanitizer
The .gitignore 'downloads/' pattern for the DOWNLOAD_ROOT bind mount also matched the
backend/app/downloads source package, so formats.py/storage.py/service.py never got
committed with M2. Anchor the ignore to '/downloads/' (repo root only) and add the package.
storage.sanitize() now handles arbitrarily messy titles (emoji, ZWJ, fullwidth, clickbait
punctuation): NFKC-normalize, drop emoji/symbol/control unicode categories, collapse
repeated punctuation, underscore-join words -> space-free paths. Accents (HU/DE) preserved,
no ASCII folding. Plex layout uses _-_ separators + Season_{year}.
This commit is contained in:
parent
1ae4333922
commit
7f9847c2c2
5 changed files with 388 additions and 2 deletions
0
backend/app/downloads/__init__.py
Normal file
0
backend/app/downloads/__init__.py
Normal file
145
backend/app/downloads/formats.py
Normal file
145
backend/app/downloads/formats.py
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
"""Format profiles → yt-dlp options, and the cache signature.
|
||||
|
||||
A profile `spec` (stored on DownloadProfile / snapshotted on DownloadJob) is the single source
|
||||
of truth for what file gets produced. `format_sig` hashes exactly the fields that change the
|
||||
OUTPUT BYTES, so two users picking equivalent settings share one cached MediaAsset.
|
||||
`build_ydl_opts` turns a spec into a yt-dlp options dict (format selector + postprocessors).
|
||||
|
||||
spec shape (all keys optional; defaults applied by `normalize`):
|
||||
mode "av" (audio+video) | "v" (video-only) | "a" (audio-only)
|
||||
max_height int | null (null = best)
|
||||
container "mp4" | "mkv" | "webm" | null (merge/remux target for av/v)
|
||||
audio_format "m4a" | "mp3" | "opus" | null (extract target for mode "a")
|
||||
vcodec "h264" | "vp9" | "av1" | null (preference, best-effort)
|
||||
embed_subs bool embed_chapters bool embed_thumbnail bool sponsorblock bool
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
# Fields that affect the produced file — and therefore the cache identity.
|
||||
_SIG_KEYS = (
|
||||
"mode",
|
||||
"max_height",
|
||||
"container",
|
||||
"audio_format",
|
||||
"vcodec",
|
||||
"embed_subs",
|
||||
"embed_chapters",
|
||||
"embed_thumbnail",
|
||||
"sponsorblock",
|
||||
)
|
||||
|
||||
_DEFAULTS = {
|
||||
"mode": "av",
|
||||
"max_height": None,
|
||||
"container": "mp4",
|
||||
"audio_format": "m4a",
|
||||
"vcodec": None,
|
||||
"embed_subs": False,
|
||||
"embed_chapters": True,
|
||||
"embed_thumbnail": True,
|
||||
"sponsorblock": False,
|
||||
}
|
||||
|
||||
_SPONSORBLOCK_CATEGORIES = ["sponsor", "selfpromo", "interaction"]
|
||||
_VCODEC_SORT = {"h264": "vcodec:h264", "vp9": "vcodec:vp9", "av1": "vcodec:av01"}
|
||||
|
||||
|
||||
def normalize(spec: dict) -> dict:
|
||||
"""Fill defaults + coerce types so signature and yt-dlp opts are deterministic."""
|
||||
s = dict(_DEFAULTS)
|
||||
for k in _DEFAULTS:
|
||||
if spec.get(k) is not None:
|
||||
s[k] = spec[k]
|
||||
if s["mode"] not in ("av", "v", "a"):
|
||||
s["mode"] = "av"
|
||||
if s["max_height"] is not None:
|
||||
s["max_height"] = int(s["max_height"])
|
||||
for b in ("embed_subs", "embed_chapters", "embed_thumbnail", "sponsorblock"):
|
||||
s[b] = bool(s[b])
|
||||
# Audio-only files can't embed subtitles/thumbnails as video; keep the flags meaningful.
|
||||
if s["mode"] == "a":
|
||||
s["embed_subs"] = False
|
||||
return s
|
||||
|
||||
|
||||
def format_sig(spec: dict) -> str:
|
||||
"""Stable short hash of the output-affecting spec — the MediaAsset cache key component."""
|
||||
s = normalize(spec)
|
||||
payload = json.dumps({k: s[k] for k in _SIG_KEYS}, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(payload.encode()).hexdigest()[:24]
|
||||
|
||||
|
||||
def target_ext(spec: dict) -> str:
|
||||
"""Best-effort final extension for the produced file (also used to name the staging output)."""
|
||||
s = normalize(spec)
|
||||
if s["mode"] == "a":
|
||||
return s["audio_format"] or "m4a"
|
||||
return s["container"] or "mp4"
|
||||
|
||||
|
||||
def build_ydl_opts(spec: dict, outtmpl: str, progress_hook) -> dict:
|
||||
"""Assemble yt-dlp options for a single download to `outtmpl` (a full path template)."""
|
||||
s = normalize(spec)
|
||||
h = s["max_height"]
|
||||
opts: dict = {
|
||||
"outtmpl": outtmpl,
|
||||
"noplaylist": True,
|
||||
"quiet": True,
|
||||
"no_warnings": True,
|
||||
"noprogress": True,
|
||||
"writethumbnail": True, # kept for the poster.jpg sidecar (and embed, if requested)
|
||||
"progress_hooks": [progress_hook],
|
||||
"postprocessors": [],
|
||||
}
|
||||
|
||||
if s["mode"] == "a":
|
||||
opts["format"] = "bestaudio/best"
|
||||
opts["postprocessors"].append(
|
||||
{
|
||||
"key": "FFmpegExtractAudio",
|
||||
"preferredcodec": s["audio_format"] or "m4a",
|
||||
"preferredquality": "0",
|
||||
}
|
||||
)
|
||||
else:
|
||||
hcap = f"[height<=?{h}]" if h else ""
|
||||
if s["mode"] == "v":
|
||||
opts["format"] = f"bestvideo{hcap}/best{hcap}"
|
||||
else: # av
|
||||
opts["format"] = f"bestvideo{hcap}+bestaudio/best{hcap}"
|
||||
if s["container"]:
|
||||
opts["merge_output_format"] = s["container"]
|
||||
sort = []
|
||||
if h:
|
||||
sort.append(f"res:{h}")
|
||||
if s["vcodec"] in _VCODEC_SORT:
|
||||
sort.append(_VCODEC_SORT[s["vcodec"]])
|
||||
if sort:
|
||||
opts["format_sort"] = sort
|
||||
|
||||
# SponsorBlock first (marks segments), then strip them from the media + chapters.
|
||||
if s["sponsorblock"]:
|
||||
opts["postprocessors"].append(
|
||||
{"key": "SponsorBlock", "categories": _SPONSORBLOCK_CATEGORIES, "when": "after_filter"}
|
||||
)
|
||||
opts["postprocessors"].append(
|
||||
{"key": "ModifyChapters", "remove_sponsor_segments": _SPONSORBLOCK_CATEGORIES}
|
||||
)
|
||||
if s["embed_subs"] and s["mode"] != "a":
|
||||
opts["writesubtitles"] = True
|
||||
opts["subtitleslangs"] = ["en.*", "hu.*", "de.*"]
|
||||
opts["postprocessors"].append({"key": "FFmpegEmbedSubtitle"})
|
||||
if s["embed_chapters"]:
|
||||
opts["postprocessors"].append(
|
||||
{"key": "FFmpegMetadata", "add_chapters": True, "add_metadata": True}
|
||||
)
|
||||
# Normalize the written thumbnail to jpg so we always have a `<name>.jpg` poster sidecar.
|
||||
# `already_have_thumbnail` keeps that file on disk after embedding (default deletes it).
|
||||
opts["postprocessors"].append({"key": "FFmpegThumbnailsConvertor", "format": "jpg"})
|
||||
if s["embed_thumbnail"]:
|
||||
opts["postprocessors"].append(
|
||||
{"key": "EmbedThumbnail", "already_have_thumbnail": True}
|
||||
)
|
||||
|
||||
return opts
|
||||
97
backend/app/downloads/service.py
Normal file
97
backend/app/downloads/service.py
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
"""Enqueue + cache resolution for the download center (the per-user layer over MediaAsset).
|
||||
|
||||
`enqueue` is the single entry point used by the API: it resolves (or creates) the shared
|
||||
MediaAsset for the requested source+format, links a per-user DownloadJob to it, and — on a
|
||||
cache hit (asset already ready) — marks the job done instantly with no re-download. The worker
|
||||
handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and
|
||||
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.downloads import formats
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
def source_url(source_kind: str, source_ref: str) -> str:
|
||||
"""The URL yt-dlp should fetch. YouTube stores the 11-char id; ad-hoc stores the URL."""
|
||||
if source_kind == "youtube":
|
||||
return f"https://www.youtube.com/watch?v={source_ref}"
|
||||
return source_ref
|
||||
|
||||
|
||||
def get_or_create_asset(
|
||||
db: Session, source_kind: str, source_ref: str, format_sig: str
|
||||
) -> MediaAsset:
|
||||
"""Fetch the cache row for this (source, format) or create a fresh `pending` one.
|
||||
|
||||
Races on the unique cache key are resolved by re-selecting after an IntegrityError, so two
|
||||
simultaneous enqueues of the same video+format converge on one shared asset."""
|
||||
stmt = select(MediaAsset).where(
|
||||
MediaAsset.source_kind == source_kind,
|
||||
MediaAsset.source_ref == source_ref,
|
||||
MediaAsset.format_sig == format_sig,
|
||||
)
|
||||
asset = db.execute(stmt).scalar_one_or_none()
|
||||
if asset is not None:
|
||||
return asset
|
||||
asset = MediaAsset(
|
||||
source_kind=source_kind,
|
||||
source_ref=source_ref,
|
||||
format_sig=format_sig,
|
||||
status="pending",
|
||||
ref_count=0,
|
||||
)
|
||||
db.add(asset)
|
||||
try:
|
||||
db.flush()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
asset = db.execute(stmt).scalar_one()
|
||||
return asset
|
||||
|
||||
|
||||
def _next_queue_pos(db: Session) -> int:
|
||||
return (db.execute(select(func.coalesce(func.max(DownloadJob.queue_pos), 0))).scalar() or 0) + 1
|
||||
|
||||
|
||||
def enqueue(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
source_kind: str,
|
||||
source_ref: str,
|
||||
spec: dict,
|
||||
profile_id: int | None = None,
|
||||
display_name: str | None = None,
|
||||
) -> DownloadJob:
|
||||
"""Queue a download for `user_id`. Returns the DownloadJob (already `done` on a cache hit)."""
|
||||
spec = formats.normalize(spec)
|
||||
sig = formats.format_sig(spec)
|
||||
asset = get_or_create_asset(db, source_kind, source_ref, sig)
|
||||
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
asset_id=asset.id,
|
||||
source_kind=source_kind,
|
||||
source_ref=source_ref,
|
||||
profile_id=profile_id,
|
||||
profile_snapshot=spec,
|
||||
display_name=display_name or (asset.title if asset.status == "ready" else None),
|
||||
status="queued",
|
||||
queue_pos=_next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
asset.ref_count = (asset.ref_count or 0) + 1
|
||||
|
||||
# Cache hit: the file already exists — no worker round-trip needed.
|
||||
if asset.status == "ready":
|
||||
job.status = "done"
|
||||
job.progress = 100
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
return job
|
||||
143
backend/app/downloads/storage.py
Normal file
143
backend/app/downloads/storage.py
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
"""On-disk layout for downloaded media — Plex-friendly tree + .nfo/poster sidecars.
|
||||
|
||||
The physical filename is system-decided and bound to the source id (never renamed; the user's
|
||||
custom name is display-only, applied at local-download time). Layout is admin-configurable:
|
||||
|
||||
plex: {Channel}/Season {Year}/{Channel} - {YYYY-MM-DD} - {title} [{id}].{ext}
|
||||
flat: {title} [{id}].{ext}
|
||||
|
||||
Plus a Kodi/Plex-readable `<basename>.nfo` (episodedetails) and `<basename>.jpg` thumbnail, so
|
||||
a Plex library using a metadata agent picks up rich info. Channel = "show", video = episode,
|
||||
date-based — the community-standard mapping for YouTube-in-Plex.
|
||||
"""
|
||||
import re
|
||||
import shutil
|
||||
import unicodedata
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
# Reserve room for the " [id].ext" suffix + parent dirs within the 255-byte component limit.
|
||||
_MAX_TITLE = 120
|
||||
# Filesystem-illegal on Windows/Plex + control chars.
|
||||
_ILLEGAL = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
|
||||
# Unicode categories we drop entirely: emoji/pictographs (So), modifier symbols/skin-tones
|
||||
# (Sk), and every control/format/private/surrogate/unassigned char (C*). Letters (incl.
|
||||
# accented á/ö/ü/ß), digits, and ordinary punctuation are KEPT — Siftlode is trilingual, so
|
||||
# ASCII-folding would mangle Hungarian/German titles.
|
||||
_DROP_CATEGORIES = {"So", "Sk", "Cc", "Cf", "Cs", "Co", "Cn"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaMeta:
|
||||
video_id: str
|
||||
title: str
|
||||
uploader: str
|
||||
upload_date: date | None
|
||||
duration_s: int | None = None
|
||||
description: str | None = None
|
||||
thumbnail_url: str | None = None
|
||||
|
||||
|
||||
def sanitize(name: str, limit: int = _MAX_TITLE) -> str:
|
||||
"""Turn an arbitrary (emoji-laden, clickbait) title into a clean, space-free path token.
|
||||
|
||||
Steps: NFKC-normalize (fold fancy width/compat forms) → drop emoji/symbols/control chars →
|
||||
strip filesystem-illegal chars → collapse whitespace and runs of the same punctuation →
|
||||
join words with underscores → trim junk punctuation from the ends → length-cap. Accented
|
||||
letters and digits survive; the result never contains spaces or path separators."""
|
||||
text = unicodedata.normalize("NFKC", name or "")
|
||||
text = "".join(
|
||||
ch for ch in text if unicodedata.category(ch) not in _DROP_CATEGORIES
|
||||
)
|
||||
text = _ILLEGAL.sub("", text)
|
||||
# Collapse repeated punctuation ("!!!", "???", "-----") to a single mark.
|
||||
text = re.sub(r"([^\w\s])\1{1,}", r"\1", text)
|
||||
# Whitespace → single underscore (no spaces in generated names).
|
||||
text = re.sub(r"\s+", "_", text.strip())
|
||||
# Tidy separator pile-ups and trim junk from the ends.
|
||||
text = re.sub(r"_{2,}", "_", text).strip("_.-·—–|,;:!?ّ ")
|
||||
if len(text) > limit:
|
||||
text = text[:limit].rstrip("_.-")
|
||||
return text or "untitled"
|
||||
|
||||
|
||||
def rel_path(meta: MediaMeta, ext: str, layout: str = "plex") -> str:
|
||||
"""Path of the media file relative to DOWNLOAD_ROOT (forward slashes)."""
|
||||
title = sanitize(meta.title)
|
||||
vid = meta.video_id
|
||||
if layout == "flat":
|
||||
return f"{title}_[{vid}].{ext}"
|
||||
channel = sanitize(meta.uploader or "Unknown_Channel", 80)
|
||||
year = meta.upload_date.year if meta.upload_date else 0
|
||||
d = meta.upload_date.isoformat() if meta.upload_date else "0000-00-00"
|
||||
epname = f"{channel}_-_{d}_-_{title}_[{vid}].{ext}"
|
||||
return f"{channel}/Season_{year}/{epname}"
|
||||
|
||||
|
||||
def abs_path(download_root: str, rel: str) -> Path:
|
||||
return Path(download_root) / rel
|
||||
|
||||
|
||||
def place_file(staging_file: Path, download_root: str, rel: str) -> None:
|
||||
"""Move a completed staging file into its final location under DOWNLOAD_ROOT."""
|
||||
dest = abs_path(download_root, rel)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(staging_file), str(dest))
|
||||
|
||||
|
||||
def _nfo_xml(meta: MediaMeta) -> str:
|
||||
aired = meta.upload_date.isoformat() if meta.upload_date else ""
|
||||
year = meta.upload_date.year if meta.upload_date else ""
|
||||
parts = [
|
||||
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>',
|
||||
"<episodedetails>",
|
||||
f" <title>{escape(meta.title or '')}</title>",
|
||||
f" <showtitle>{escape(meta.uploader or '')}</showtitle>",
|
||||
f" <studio>{escape(meta.uploader or '')}</studio>",
|
||||
f" <aired>{aired}</aired>",
|
||||
f" <premiered>{aired}</premiered>",
|
||||
f" <year>{year}</year>",
|
||||
f" <plot>{escape(meta.description or '')}</plot>",
|
||||
f" <uniqueid type=\"youtube\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
|
||||
]
|
||||
if meta.duration_s:
|
||||
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
|
||||
parts.append("</episodedetails>")
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def write_sidecars(
|
||||
download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None
|
||||
) -> bool:
|
||||
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
|
||||
media = abs_path(download_root, rel)
|
||||
base = media.with_suffix("")
|
||||
try:
|
||||
base.with_suffix(".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
|
||||
if thumb_src and thumb_src.exists():
|
||||
shutil.copyfile(str(thumb_src), str(base.with_suffix(".jpg")))
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def delete_asset_files(download_root: str, rel: str) -> None:
|
||||
"""Remove the media file and its sidecars; prune now-empty parent dirs (best effort)."""
|
||||
media = abs_path(download_root, rel)
|
||||
base = media.with_suffix("")
|
||||
for p in (media, base.with_suffix(".nfo"), base.with_suffix(".jpg")):
|
||||
try:
|
||||
p.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
# Prune empty Season/Channel dirs up to (but not including) the root.
|
||||
root = Path(download_root).resolve()
|
||||
parent = media.parent
|
||||
while parent != root and root in parent.parents:
|
||||
try:
|
||||
parent.rmdir() # only succeeds if empty
|
||||
except OSError:
|
||||
break
|
||||
parent = parent.parent
|
||||
Loading…
Add table
Add a link
Reference in a new issue