fix(downloads): android_vr primary + POT web fallback (fixes intermittent DRM/bot errors)
YouTube returns flaky per-client responses: the web clients intermittently mislabel normal videos as 'DRM protected' (like tv did), and android_vr can get bot-flagged under heavy load. Neither client alone is reliable. So: - PRIMARY player client = android_vr: high-quality DASH (399+251), no PO token / Deno needed, reliable for normal use - on a bot/DRM/format failure the worker retries with the FALLBACK set (web_safari,web,android + POT sidecar + Deno) — bypasses bot-detection when android_vr is flagged - both client sets admin-tunable (DOWNLOAD_PLAYER_CLIENTS[_FALLBACK]) Verified: the previously DRM-failing video now downloads steadily via android_vr, no DRM error.
This commit is contained in:
parent
21c5508889
commit
8588fa104b
2 changed files with 61 additions and 18 deletions
|
|
@ -142,14 +142,25 @@ class Settings(BaseSettings):
|
|||
# for YouTube Proof-of-Origin tokens (bypasses bot-detection + unlocks high-quality
|
||||
# formats). Empty disables it (yt-dlp still works, but may hit "confirm you're not a bot").
|
||||
download_pot_base_url: str = "http://bgutil-pot:4416"
|
||||
# yt-dlp player clients to use (comma-separated). POT-capable clients (web_safari/web) must
|
||||
# be used for the PO token to actually apply; the default clients (android_vr) don't request
|
||||
# one. Empty = yt-dlp's own default. Admin-tunable when YouTube shifts what works.
|
||||
download_player_clients: str = "web_safari,web,android"
|
||||
# yt-dlp player clients (comma-separated). PRIMARY = android_vr: high-quality DASH, no PO
|
||||
# token / JS runtime needed, reliable for normal use. On a bot/DRM/format failure (YouTube's
|
||||
# per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the
|
||||
# worker retries with the FALLBACK set — the POT-capable web clients (web needs the token +
|
||||
# Deno) plus android for audio. Both admin-tunable when YouTube shifts what works.
|
||||
download_player_clients: str = "android_vr"
|
||||
download_player_clients_fallback: str = "web_safari,web,android"
|
||||
|
||||
@staticmethod
|
||||
def _clients(raw: str) -> list[str]:
|
||||
return [c.strip() for c in raw.split(",") if c.strip()]
|
||||
|
||||
@property
|
||||
def player_client_list(self) -> list[str]:
|
||||
return [c.strip() for c in self.download_player_clients.split(",") if c.strip()]
|
||||
return self._clients(self.download_player_clients)
|
||||
|
||||
@property
|
||||
def player_client_fallback_list(self) -> list[str]:
|
||||
return self._clients(self.download_player_clients_fallback)
|
||||
# How many downloads the worker runs concurrently (claimed via FOR UPDATE SKIP LOCKED).
|
||||
download_worker_concurrency: int = 2
|
||||
# How often (empty pending queue) the worker polls for a new job, in seconds.
|
||||
|
|
|
|||
|
|
@ -285,6 +285,49 @@ def _find_thumbnail(staging: Path, media: Path) -> Path | None:
|
|||
return None
|
||||
|
||||
|
||||
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
|
||||
_RETRIABLE_MARKERS = (
|
||||
"not a bot",
|
||||
"DRM protected",
|
||||
"Requested format is not available",
|
||||
"Only images are available",
|
||||
"Sign in to confirm",
|
||||
)
|
||||
|
||||
|
||||
def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, staging: Path):
|
||||
"""Download via yt-dlp, trying the primary player clients then the POT-backed fallback set
|
||||
on a bot/DRM/format failure. Returns the info dict of the successful attempt."""
|
||||
client_sets = [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||
last_exc: Exception | None = None
|
||||
for i, clients in enumerate(client_sets):
|
||||
opts = formats.build_ydl_opts(
|
||||
spec, outtmpl, _make_progress_hook(job_id, asset_id),
|
||||
settings.download_pot_base_url, clients,
|
||||
)
|
||||
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||
try:
|
||||
with yt_dlp.YoutubeDL(opts) as ydl:
|
||||
return ydl.extract_info(url, download=True)
|
||||
except yt_dlp.utils.DownloadError as exc:
|
||||
last_exc = exc
|
||||
retriable = any(m in str(exc) for m in _RETRIABLE_MARKERS)
|
||||
if i < len(client_sets) - 1 and retriable:
|
||||
log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
|
||||
for p in list(staging.iterdir()): # drop partials before the retry
|
||||
try:
|
||||
p.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
_set_job(job_id, progress=0, phase="downloading")
|
||||
continue
|
||||
raise
|
||||
raise last_exc # unreachable (loop either returns or raises)
|
||||
|
||||
|
||||
def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict) -> None:
|
||||
staging = _staging_dir(asset_id)
|
||||
try:
|
||||
|
|
@ -292,20 +335,9 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
|||
shutil.rmtree(staging, ignore_errors=True)
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||
opts = formats.build_ydl_opts(
|
||||
spec,
|
||||
outtmpl,
|
||||
_make_progress_hook(job_id, asset_id),
|
||||
settings.download_pot_base_url,
|
||||
settings.player_client_list,
|
||||
)
|
||||
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||
url = service.source_url(source_kind, source_ref)
|
||||
|
||||
_set_job(job_id, phase="downloading", progress=0)
|
||||
with yt_dlp.YoutubeDL(opts) as ydl:
|
||||
info = ydl.extract_info(url, download=True)
|
||||
url = service.source_url(source_kind, source_ref)
|
||||
info = _extract_with_fallback(job_id, asset_id, url, spec, staging)
|
||||
|
||||
req = (info.get("requested_downloads") or [{}])[0]
|
||||
produced = Path(req.get("filepath") or "")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue