From 3c622fd44c7fbbdb7d718638f4e440966d5380ba Mon Sep 17 00:00:00 2001 From: npeter83 Date: Wed, 8 Jul 2026 22:25:03 +0200 Subject: [PATCH] feat(plex): multi-rendition audio (client-side switch) + reliable resume-on-F5 - Multi-audio items now ship every audio track as an HLS rendition in one session (stream.py var_stream_map -> master.m3u8), so hls.js switches audio CLIENT-SIDE with no ffmpeg restart, same timeline, no gap/drift. /session gains ?multi=1 (forces HLS even for direct-playable multi-audio files); K is probed from the video variant seg. - Restore the selected audio track on AUDIO_TRACKS_UPDATED, not MANIFEST_PARSED (the renditions aren't parsed yet on MANIFEST_PARSED, so the set was dropped -> after F5 the UI showed the restored track but playback stayed on the default). - Resume position now survives F5: a pagehide keepalive beacon (plexProgressBeacon) saves the current position on reload/close/navigate, since React effect cleanup does not run on a full reload; seekTo writes the target into absRef immediately so a save right after a seek is accurate. --- backend/app/plex/stream.py | 112 +++++++++++++++++-------- backend/app/routes/plex.py | 17 ++-- frontend/src/components/PlexPlayer.tsx | 45 +++++++++- frontend/src/lib/api.ts | 24 +++++- 4 files changed, 155 insertions(+), 43 deletions(-) diff --git a/backend/app/plex/stream.py b/backend/app/plex/stream.py index e4242fe..129e050 100644 --- a/backend/app/plex/stream.py +++ b/backend/app/plex/stream.py @@ -63,40 +63,77 @@ def _seek_opts(start_s: float) -> list[str]: return ["-noaccurate_seek", "-ss", f"{start_s:.3f}"] if start_s > 0 else [] -def _ffmpeg_cmd(src: Path, start_s: float, out_dir: Path, audio_ord: int | None, aoff: float = 0.0) -> list[str]: - ao = audio_ord if audio_ord is not None else 0 +_HLS_TAIL = [ + "-f", "hls", + "-hls_time", str(_SEG_SECONDS), + "-hls_list_size", "0", + # EVENT (not VOD): ffmpeg writes/appends the playlist AS segments complete, so playback can + # start immediately from this session's offset. VOD only writes the playlist at the end. The + # full seekbar comes from our known duration + the seek-restart model, not from the playlist. + "-hls_playlist_type", "event", + "-hls_segment_type", "mpegts", + "-hls_flags", "independent_segments+temp_file", +] + + +def _probe_audio_langs(src: Path) -> list[str]: + """One entry per audio stream (its language tag, or "" if untagged). len() = audio-track count. + Used to build the multi-rendition var_stream_map. Empty on probe failure → single-audio path.""" + try: + out = subprocess.run( + ["ffprobe", "-v", "error", "-select_streams", "a", "-show_entries", "stream_tags=language", + "-of", "csv=p=0", str(src)], + capture_output=True, text=True, timeout=10, + ) + return [ln.strip() for ln in out.stdout.splitlines()] + except (subprocess.SubprocessError, OSError): + return [] + + +def _ffmpeg_cmd( + src: Path, start_s: float, out_dir: Path, audio_ord: int | None, + aoff: float = 0.0, audio_langs: list[str] | None = None, +) -> list[str]: args = ["ffmpeg", "-nostdin", "-loglevel", "error"] # Keep ORIGINAL timestamps on the output so the first segment carries the true absolute PTS of the # keyframe ffmpeg actually seeked to. hls.js still zero-bases playback to that PTS; we read it back # (see _probe_first_pts) to learn the real content offset. Without -copyts the segment PTS is # rewritten toward 0 and the true offset is unknowable, which caused the ~seconds subtitle lead. args += _seek_opts(start_s) + ["-copyts", "-i", str(src)] + ai = 0 # input index the audio maps come from if aoff: # A/V-sync offset: read the file a SECOND time for audio only, with `-itsoffset` shifting the - # audio PTS by ±aoff relative to the (input-0) video. Only when the user set a non-zero offset - # (default single-input path is unchanged, no double read). Maps audio from input 1 then. + # audio PTS by ±aoff relative to the (input-0) video. Maps audio from input 1 then. args += ["-itsoffset", f"{aoff:.3f}"] + _seek_opts(start_s) + ["-copyts", "-i", str(src)] - amap = f"1:a:{ao}?" + ai = 1 + # Video always stream-copied; audio → AAC. Subtitles are NOT muxed here — they're separate WebVTT + # tracks (GET /subtitle) the browser overlays, so choosing a subtitle doesn't restart this session. + if audio_langs: + # MULTI-RENDITION: map EVERY audio track as an alternate HLS rendition in one group, so hls.js + # switches audio CLIENT-SIDE on the same timeline (no session restart, no drift) — the premium + # audio-switch path. Produces master.m3u8 + stream_%v.m3u8 (v0=video, v1..=audio) + seg_%v_%d.ts. + args += ["-map", "0:v:0"] + for i in range(len(audio_langs)): + args += ["-map", f"{ai}:a:{i}?"] + args += ["-sn", "-c:v", "copy", "-c:a", "aac", "-ac", "2", "-b:a", "192k"] + var = ["v:0,agroup:aud"] + for i, lang in enumerate(audio_langs): + entry = f"a:{i},agroup:aud,name:a{i}" + if lang: + entry += f",language:{lang}" + if i == 0: + entry += ",default:yes" + var.append(entry) + args += _HLS_TAIL + [ + "-master_pl_name", "master.m3u8", + "-var_stream_map", " ".join(var), + "-hls_segment_filename", str(out_dir / "seg_%v_%d.ts"), str(out_dir / "stream_%v.m3u8"), + ] else: - amap = f"0:a:{ao}?" - # Video always stream-copied; the selected audio track (default = first) is transcoded to AAC. - # Subtitles are NOT muxed here — they're served as standalone WebVTT tracks (GET /subtitle) that - # the browser overlays, so choosing a subtitle no longer restarts this session (and external - # sidecar subs, which aren't in the file, work — the old `-map 0:s` broke on those). - args += ["-map", "0:v:0", "-map", amap, "-sn"] - args += ["-c:v", "copy", "-c:a", "aac", "-ac", "2", "-b:a", "192k"] - args += [ - "-f", "hls", - "-hls_time", str(_SEG_SECONDS), - "-hls_list_size", "0", - # EVENT (not VOD): ffmpeg writes/appends the playlist AS segments complete, so playback can - # start immediately from this session's offset. VOD only writes the playlist at the end. The - # full seekbar comes from our known duration + the seek-restart model, not from the playlist. - "-hls_playlist_type", "event", - "-hls_segment_type", "mpegts", - "-hls_flags", "independent_segments+temp_file", - ] - args += ["-hls_segment_filename", str(out_dir / "seg_%d.ts"), str(out_dir / "index.m3u8")] + ao = audio_ord if audio_ord is not None else 0 + args += ["-map", "0:v:0", "-map", f"{ai}:a:{ao}?", "-sn"] + args += ["-c:v", "copy", "-c:a", "aac", "-ac", "2", "-b:a", "192k"] + args += _HLS_TAIL + ["-hls_segment_filename", str(out_dir / "seg_%d.ts"), str(out_dir / "index.m3u8")] return args @@ -146,43 +183,48 @@ def start_session( start_s: float, audio_ord: int | None = None, aoff: float = 0.0, + multi: bool = False, ) -> HlsSession | None: - """(Re)start the HLS remux for an item at the given offset, with an optional selected audio track - (ordinal among audio streams). Subtitles are separate WebVTT tracks (not muxed here). Returns - None if the local file can't be read.""" + """(Re)start the HLS remux for an item at the given offset. `multi` (item has >1 audio track) maps + every audio track as an HLS rendition so the client switches audio without a restart; otherwise a + single audio track (`audio_ord`) is muxed. Subtitles are separate WebVTT tracks (not muxed here). + Returns None if the local file can't be read.""" src = paths.local_media_path(db, item.file_path) if src is None: return None key = item.rating_key start_s = max(0.0, float(start_s)) + audio_langs = _probe_audio_langs(src) if multi else [] + is_multi = len(audio_langs) > 1 # only worth a master playlist when there really are ≥2 tracks with _lock: old = _sessions.pop(key, None) if old is not None: _kill(old) - directory = _HLS_ROOT / f"{key}_{int(start_s)}_{audio_ord}_{aoff:+.2f}" + tag = "multi" if is_multi else str(audio_ord) + directory = _HLS_ROOT / f"{key}_{int(start_s)}_{tag}_{aoff:+.2f}" shutil.rmtree(directory, ignore_errors=True) directory.mkdir(parents=True, exist_ok=True) proc = subprocess.Popen( - _ffmpeg_cmd(src, start_s, directory, audio_ord, aoff), + _ffmpeg_cmd(src, start_s, directory, audio_ord, aoff, audio_langs if is_multi else None), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) - s = HlsSession(key, directory, proc, start_s, "index.m3u8") + s = HlsSession(key, directory, proc, start_s, "master.m3u8" if is_multi else "index.m3u8") _sessions[key] = s _enforce_cap() - # Learn the REAL start offset K from the first segment (outside the lock — this blocks on ffmpeg - # producing seg_0, and we must not hold up other sessions). The client reads `media_start_s`, so + # Learn the REAL start offset K from the first VIDEO segment (outside the lock — this blocks on + # ffmpeg producing it, and we must not hold up other sessions). The client reads `media_start_s`, so # its absolute clock + subtitle shift key off the true keyframe, not the requested offset. On # timeout/probe failure we keep the provisional requested offset (old behaviour, small drift). if start_s > 0: - seg0 = directory / "seg_0.ts" + seg0 = directory / ("seg_0_0.ts" if is_multi else "seg_0.ts") # multi: video variant = v0 if wait_for(seg0, timeout=25.0): k = _probe_first_pts(seg0) if k is not None and k >= 0: s.media_start_s = k log.info( - "plex hls session start key=%s req=%.1f real=%.3f audio=%s", - key, start_s, s.media_start_s, audio_ord, + "plex hls session start key=%s req=%.1f real=%.3f multi=%s audio=%s", + key, start_s, s.media_start_s, is_multi, audio_ord, ) return s diff --git a/backend/app/routes/plex.py b/backend/app/routes/plex.py index bf1fa55..1f63726 100644 --- a/backend/app/routes/plex.py +++ b/backend/app/routes/plex.py @@ -1499,6 +1499,7 @@ def stream_session( start: float = 0.0, audio: int | None = None, aoff: float = 0.0, + multi: bool = False, user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: @@ -1510,11 +1511,13 @@ def stream_session( it = _item_or_404(db, rating_key) if it.playable == "transcode": raise HTTPException(status_code=501, detail="This format needs transcoding (a later phase).") - if it.playable == "direct" and audio is None and not aoff: + # `multi` (item has ≥2 audio tracks) forces the HLS path so ALL audio tracks ship as renditions and + # the client can switch audio without a restart — even for otherwise direct-playable files. + if it.playable == "direct" and audio is None and not aoff and not multi: return {"mode": "direct", "url": f"/api/plex/stream/{it.rating_key}/file", "start_s": 0.0} if plex_paths.local_media_path(db, it.file_path) is None: raise HTTPException(status_code=404, detail="Media file not found on disk") - s = plex_stream.start_session(db, it, start, audio, aoff) + s = plex_stream.start_session(db, it, start, audio, aoff, multi) if s is None: raise HTTPException(status_code=404, detail="Media file not found on disk") # Report the REAL media start (the keyframe ffmpeg landed on, which hls.js zero-bases to), not the @@ -1562,11 +1565,15 @@ def stream_hls_file( raise HTTPException(status_code=404, detail="Not found") s = plex_stream.current_session(str(rating_key)) if s is None: - # No session yet: only the entry playlist for a remux item may auto-start one at 0. + # No session yet (e.g. reaped): the ENTRY playlist may auto-start one at 0 — master.m3u8 for a + # multi-audio item, index.m3u8 for a single-audio remux item. it = _item_or_404(db, rating_key) - if filename != "index.m3u8" or it.playable != "remux": + if filename == "master.m3u8": + s = plex_stream.start_session(db, it, 0.0, multi=True) + elif filename == "index.m3u8" and it.playable == "remux": + s = plex_stream.start_session(db, it, 0.0) + else: raise HTTPException(status_code=404, detail="No active playback session") - s = plex_stream.start_session(db, it, 0.0) if s is None: raise HTTPException(status_code=404, detail="Media file not found on disk") path = s.dir / filename diff --git a/frontend/src/components/PlexPlayer.tsx b/frontend/src/components/PlexPlayer.tsx index 3873449..0463506 100644 --- a/frontend/src/components/PlexPlayer.tsx +++ b/frontend/src/components/PlexPlayer.tsx @@ -133,6 +133,10 @@ export default function PlexPlayer({ itemId, onClose, queue }: Props) { const wrapRef = useRef(null); const sessionStartRef = useRef(0); // absolute offset the current HLS/direct session begins at const durationRef = useRef(0); + // ≥2 audio tracks → multi-rendition HLS: all tracks ship in one session and hls.js switches audio + // client-side (no restart). Ref so the stable loadSession/changeAudio callbacks read it fresh. + const multiAudioRef = useRef(false); + multiAudioRef.current = (detail?.audio_streams.length ?? 0) > 1; const [playing, setPlaying] = useState(false); const [abs, setAbs] = useState(0); // absolute current time @@ -213,7 +217,10 @@ export default function PlexPlayer({ itemId, onClose, queue }: Props) { setLoadError(null); let sess; try { - sess = await api.plexSession(id, startAt, audioRef.current, aoffRef.current); + // Multi-audio → request the multi-rendition session (no single `audio` ordinal); the track is + // selected client-side after the manifest loads. + const m = multiAudioRef.current; + sess = await api.plexSession(id, startAt, m ? null : audioRef.current, aoffRef.current, m); } catch (e: any) { // Surface WHY playback can't start instead of an eternal spinner. 404 = the physical file // isn't reachable (missing media bind-mount or a wrong plex_path_map); 501 = the codec @@ -247,6 +254,21 @@ export default function PlexPlayer({ itemId, onClose, queue }: Props) { setReady(true); if (aliveRef.current) video.play().catch(() => {}); }); + // Restore the selected audio track ONCE hls.js has parsed the renditions — doing it on + // MANIFEST_PARSED was too early (the track list wasn't populated, so the set was dropped and + // playback stayed on the default track while the UI showed the restored selection). + hls.on(Hls.Events.AUDIO_TRACKS_UPDATED, () => { + if (multiAudioRef.current) { + const want = audioRef.current ?? 0; + if (hls.audioTrack !== want) { + try { + hls.audioTrack = want; + } catch { + /* out of range — ignore */ + } + } + } + }); // A fatal HLS error (e.g. the remux session died / the file became unreadable) would // otherwise leave the spinner up forever — surface it instead. hls.on(Hls.Events.ERROR, (_e, data) => { @@ -353,8 +375,17 @@ export default function PlexPlayer({ itemId, onClose, queue }: Props) { api.plexProgress(id, Math.floor(absRef.current), durationRef.current).catch(() => {}); }; const iv = window.setInterval(flush, 10000); + // Save on page unload too (F5/close/navigate): React effect cleanup does NOT run on a full reload, + // so without this the resume point lags up to 10s — e.g. a seek right before F5 was lost. Uses a + // keepalive beacon so the POST survives the unload. + const onHide = () => { + if (absRef.current > 0 && durationRef.current > 0) + api.plexProgressBeacon(id, Math.floor(absRef.current), durationRef.current); + }; + window.addEventListener("pagehide", onHide); return () => { window.clearInterval(iv); + window.removeEventListener("pagehide", onHide); flush(); qc.invalidateQueries({ queryKey: ["plex-browse"] }); qc.invalidateQueries({ queryKey: ["plex-show"] }); @@ -368,6 +399,7 @@ export default function PlexPlayer({ itemId, onClose, queue }: Props) { if (!video) return; const dur = durationRef.current || target; const T = Math.max(0, Math.min(target, dur - 1)); + absRef.current = T; // reflect the target immediately so a save/beacon right after a seek is accurate const rel = T - sessionStartRef.current; if (rel >= 0 && rel <= (video.seekable.length ? video.seekable.end(0) : 0) + 0.5) { video.currentTime = rel; @@ -461,7 +493,16 @@ export default function PlexPlayer({ itemId, onClose, queue }: Props) { setAudioOrd(ord); // Remember the LANGUAGE (not the ord) so F5 / the next item restores this choice. Menu stays open. patchPrefs({ audioLang: ord == null ? "" : (detail?.audio_streams.find((s) => s.ord === ord)?.language ?? "") }); - loadSession(absRef.current); + if (multiAudioRef.current && hlsRef.current) { + // Multi-rendition: switch the audio track CLIENT-SIDE — instant, same timeline, no restart. + try { + hlsRef.current.audioTrack = ord ?? 0; + } catch { + /* ignore */ + } + } else { + loadSession(absRef.current); // single-audio: re-map the track server-side (session restart) + } }, [loadSession, detail, patchPrefs], ); diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 1979b4b..51ff4ff 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1218,10 +1218,11 @@ export const api = { req(`/api/plex/show/${encodeURIComponent(id)}`), plexItem: (id: string): Promise => req(`/api/plex/item/${encodeURIComponent(id)}`), - plexSession: (id: string, start = 0, audio?: number | null, aoff = 0): Promise => { + plexSession: (id: string, start = 0, audio?: number | null, aoff = 0, multi = false): Promise => { const p = new URLSearchParams({ start: String(Math.max(0, Math.floor(start))) }); if (audio != null) p.set("audio", String(audio)); if (aoff) p.set("aoff", String(aoff)); + if (multi) p.set("multi", "1"); // ≥2 audio tracks → multi-rendition HLS (client-side audio switch) return req(`/api/plex/stream/${encodeURIComponent(id)}/session?${p.toString()}`, { method: "POST" }); }, // WebVTT URL for a text subtitle track (used directly as a ). Same-origin → cookie-authed. @@ -1234,6 +1235,27 @@ export const api = { method: "POST", body: JSON.stringify({ position_seconds, duration_seconds }), }), + // Fire-and-forget progress save that SURVIVES page unload (F5/close/navigate): a normal fetch is + // cancelled on unload and React effect cleanup doesn't run on a full reload, so the last position + // (esp. right after a seek) would be lost. `keepalive` lets the POST complete during unload; we + // replicate req()'s credentials + account header (no retry — there's no page left to retry on). + plexProgressBeacon: (id: string, position_seconds: number, duration_seconds: number): void => { + const active = getActiveAccount(); + try { + void fetch(`/api/plex/item/${encodeURIComponent(id)}/progress`, { + method: "POST", + keepalive: true, + credentials: "include", + headers: { + "Content-Type": "application/json", + ...(active != null ? { [ACTIVE_ACCOUNT_HEADER]: String(active) } : {}), + }, + body: JSON.stringify({ position_seconds, duration_seconds }), + }).catch(() => {}); + } catch { + /* ignore */ + } + }, plexSetState: (id: string, status: "new" | "watched" | "hidden"): Promise => req(`/api/plex/item/${encodeURIComponent(id)}/state`, { method: "POST",