From 8588fa104bb5dc6b2320b54579a29c765902009d Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 3 Jul 2026 23:01:09 +0200 Subject: [PATCH] fix(downloads): android_vr primary + POT web fallback (fixes intermittent DRM/bot errors) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit YouTube returns flaky per-client responses: the web clients intermittently mislabel normal videos as 'DRM protected' (like tv did), and android_vr can get bot-flagged under heavy load. Neither client alone is reliable. So: - PRIMARY player client = android_vr: high-quality DASH (399+251), no PO token / Deno needed, reliable for normal use - on a bot/DRM/format failure the worker retries with the FALLBACK set (web_safari,web,android + POT sidecar + Deno) — bypasses bot-detection when android_vr is flagged - both client sets admin-tunable (DOWNLOAD_PLAYER_CLIENTS[_FALLBACK]) Verified: the previously DRM-failing video now downloads steadily via android_vr, no DRM error. --- backend/app/config.py | 21 ++++++++++++---- backend/app/worker.py | 58 +++++++++++++++++++++++++++++++++---------- 2 files changed, 61 insertions(+), 18 deletions(-) diff --git a/backend/app/config.py b/backend/app/config.py index 6a39add..224b11a 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -142,14 +142,25 @@ class Settings(BaseSettings): # for YouTube Proof-of-Origin tokens (bypasses bot-detection + unlocks high-quality # formats). Empty disables it (yt-dlp still works, but may hit "confirm you're not a bot"). download_pot_base_url: str = "http://bgutil-pot:4416" - # yt-dlp player clients to use (comma-separated). POT-capable clients (web_safari/web) must - # be used for the PO token to actually apply; the default clients (android_vr) don't request - # one. Empty = yt-dlp's own default. Admin-tunable when YouTube shifts what works. - download_player_clients: str = "web_safari,web,android" + # yt-dlp player clients (comma-separated). PRIMARY = android_vr: high-quality DASH, no PO + # token / JS runtime needed, reliable for normal use. On a bot/DRM/format failure (YouTube's + # per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the + # worker retries with the FALLBACK set — the POT-capable web clients (web needs the token + + # Deno) plus android for audio. Both admin-tunable when YouTube shifts what works. + download_player_clients: str = "android_vr" + download_player_clients_fallback: str = "web_safari,web,android" + + @staticmethod + def _clients(raw: str) -> list[str]: + return [c.strip() for c in raw.split(",") if c.strip()] @property def player_client_list(self) -> list[str]: - return [c.strip() for c in self.download_player_clients.split(",") if c.strip()] + return self._clients(self.download_player_clients) + + @property + def player_client_fallback_list(self) -> list[str]: + return self._clients(self.download_player_clients_fallback) # How many downloads the worker runs concurrently (claimed via FOR UPDATE SKIP LOCKED). download_worker_concurrency: int = 2 # How often (empty pending queue) the worker polls for a new job, in seconds. diff --git a/backend/app/worker.py b/backend/app/worker.py index 57db81e..4633e5e 100644 --- a/backend/app/worker.py +++ b/backend/app/worker.py @@ -285,6 +285,49 @@ def _find_thumbnail(staging: Path, media: Path) -> Path | None: return None +# Errors worth retrying with a different player-client set (YouTube per-client flakiness). +_RETRIABLE_MARKERS = ( + "not a bot", + "DRM protected", + "Requested format is not available", + "Only images are available", + "Sign in to confirm", +) + + +def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, staging: Path): + """Download via yt-dlp, trying the primary player clients then the POT-backed fallback set + on a bot/DRM/format failure. Returns the info dict of the successful attempt.""" + client_sets = [ + c for c in (settings.player_client_list, settings.player_client_fallback_list) if c + ] or [None] + outtmpl = str(staging / "%(id)s.%(ext)s") + last_exc: Exception | None = None + for i, clients in enumerate(client_sets): + opts = formats.build_ydl_opts( + spec, outtmpl, _make_progress_hook(job_id, asset_id), + settings.download_pot_base_url, clients, + ) + opts["postprocessor_hooks"] = [_make_pp_hook(job_id)] + try: + with yt_dlp.YoutubeDL(opts) as ydl: + return ydl.extract_info(url, download=True) + except yt_dlp.utils.DownloadError as exc: + last_exc = exc + retriable = any(m in str(exc) for m in _RETRIABLE_MARKERS) + if i < len(client_sets) - 1 and retriable: + log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70]) + for p in list(staging.iterdir()): # drop partials before the retry + try: + p.unlink() + except OSError: + pass + _set_job(job_id, progress=0, phase="downloading") + continue + raise + raise last_exc # unreachable (loop either returns or raises) + + def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict) -> None: staging = _staging_dir(asset_id) try: @@ -292,20 +335,9 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe shutil.rmtree(staging, ignore_errors=True) staging.mkdir(parents=True, exist_ok=True) - outtmpl = str(staging / "%(id)s.%(ext)s") - opts = formats.build_ydl_opts( - spec, - outtmpl, - _make_progress_hook(job_id, asset_id), - settings.download_pot_base_url, - settings.player_client_list, - ) - opts["postprocessor_hooks"] = [_make_pp_hook(job_id)] - url = service.source_url(source_kind, source_ref) - _set_job(job_id, phase="downloading", progress=0) - with yt_dlp.YoutubeDL(opts) as ydl: - info = ydl.extract_info(url, download=True) + url = service.source_url(source_kind, source_ref) + info = _extract_with_fallback(job_id, asset_id, url, spec, staging) req = (info.get("requested_downloads") or [{}])[0] produced = Path(req.get("filepath") or "")