From 0a0703b769f67badbafebc84a9d88c38bfc771b0 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Wed, 8 Jul 2026 21:35:38 +0200 Subject: [PATCH 01/11] feat(plex): rework player timing (copyts) + player settings & personalization Streaming / subtitle sync (the core fix): - stream.py: add -copyts so HLS segments carry the true absolute PTS, and measure the real keyframe start K from seg_0 (ffprobe) -> return it as the session start. Fixes the seconds-long subtitle lead caused by using the requested seek offset instead of the keyframe ffmpeg actually lands on with video stream-copy. - Add -noaccurate_seek so the re-encoded audio starts at the same keyframe as the video (was starting (X-K)s later -> seconds of silence after each seek/audio switch). - Compensate the fixed ~1.0s lag hls.js introduces for non-zero-start copyts streams, folded into the session start so the clock, seeking and the subtitle shift are all content-accurate. - /subtitle gains an offset param; _shift_vtt shifts absolute cues onto the session's zero-based clock and DROPS fully-past cue blocks (collapsing them to 0->0 made every past cue active at currentTime 0 on resume -> a pile-up until playback advanced). Audio: - /session + stream.py gain an audio A/V-sync offset (-itsoffset, full +/-, second input only when non-zero). Player settings & personalization (per-account, persisted -> survive F5): - storage.ts: useAccountPersistedObject (per-account JSON prefs blob). - PlexPlayer: volume/mute, audio+subtitle language (index-based match, fixes the F5 audio-revert), sync offsets, seek steps, subtitle style, auto-hide, play intent. - Hotkeys A (cycle audio) / S (cycle subtitle), mouse-wheel volume, Ctrl+arrow fine seek, per-user plain/fine seek-step + auto-hide toggle. - Subtitle appearance: size / colour / vertical position / background via ::cue + line. - UI: split into a Tracks quick-menu + a tabbed gear panel (Sync | Playback | Subtitle); both dismiss on outside-click; edge-aware control tooltips. - i18n en/hu/de for all new strings. --- backend/app/plex/stream.py | 79 +++- backend/app/routes/plex.py | 74 +++- frontend/src/components/PlexPlayer.tsx | 573 +++++++++++++++++++++---- frontend/src/i18n/locales/de/plex.json | 21 +- frontend/src/i18n/locales/en/plex.json | 21 +- frontend/src/i18n/locales/hu/plex.json | 21 +- frontend/src/lib/api.ts | 9 +- frontend/src/lib/storage.ts | 23 +- 8 files changed, 707 insertions(+), 114 deletions(-) diff --git a/backend/app/plex/stream.py b/backend/app/plex/stream.py index bf16f46..e4242fe 100644 --- a/backend/app/plex/stream.py +++ b/backend/app/plex/stream.py @@ -43,21 +43,47 @@ class HlsSession: self.key = key self.dir = directory self.proc = proc - self.start_s = start_s + self.start_s = start_s # the REQUESTED seek offset + # The REAL media start: with `-ss X -c:v copy` ffmpeg can only cut at the nearest earlier + # keyframe K<=X, and hls.js zero-bases the timeline to K (so video.currentTime=0 is K, not X). + # We measure K from the first segment's absolute PTS (kept absolute by `-copyts`) and report + # THAT as the session start, so the client's absolute clock (start + currentTime) and the + # subtitle cue-shift both key off the true content position — no drift. Provisional = X until + # the first segment is probed. + self.media_start_s = start_s self.entry = entry # the playlist filename hls.js should load (master.m3u8 with subs, else index.m3u8) self.last_access = time.time() -def _ffmpeg_cmd(src: Path, start_s: float, out_dir: Path, audio_ord: int | None) -> list[str]: +def _seek_opts(start_s: float) -> list[str]: + # -noaccurate_seek: with `-ss` before -i, video (stream-copy) backs up to the keyframe K<=X, + # but the re-encoded audio is normally trimmed to the exact X — so audio starts (X-K) AFTER + # video → seconds of silence at the start of every seek/audio-switch. noaccurate_seek makes + # audio ALSO start at K, aligned with the video (K is our real clock anchor anyway). + return ["-noaccurate_seek", "-ss", f"{start_s:.3f}"] if start_s > 0 else [] + + +def _ffmpeg_cmd(src: Path, start_s: float, out_dir: Path, audio_ord: int | None, aoff: float = 0.0) -> list[str]: + ao = audio_ord if audio_ord is not None else 0 args = ["ffmpeg", "-nostdin", "-loglevel", "error"] - if start_s > 0: - args += ["-ss", f"{start_s:.3f}"] # fast keyframe seek before -i - args += ["-i", str(src)] + # Keep ORIGINAL timestamps on the output so the first segment carries the true absolute PTS of the + # keyframe ffmpeg actually seeked to. hls.js still zero-bases playback to that PTS; we read it back + # (see _probe_first_pts) to learn the real content offset. Without -copyts the segment PTS is + # rewritten toward 0 and the true offset is unknowable, which caused the ~seconds subtitle lead. + args += _seek_opts(start_s) + ["-copyts", "-i", str(src)] + if aoff: + # A/V-sync offset: read the file a SECOND time for audio only, with `-itsoffset` shifting the + # audio PTS by ±aoff relative to the (input-0) video. Only when the user set a non-zero offset + # (default single-input path is unchanged, no double read). Maps audio from input 1 then. + args += ["-itsoffset", f"{aoff:.3f}"] + _seek_opts(start_s) + ["-copyts", "-i", str(src)] + amap = f"1:a:{ao}?" + else: + amap = f"0:a:{ao}?" # Video always stream-copied; the selected audio track (default = first) is transcoded to AAC. # Subtitles are NOT muxed here — they're served as standalone WebVTT tracks (GET /subtitle) that # the browser overlays, so choosing a subtitle no longer restarts this session (and external # sidecar subs, which aren't in the file, work — the old `-map 0:s` broke on those). - args += ["-map", "0:v:0", "-map", f"0:a:{audio_ord if audio_ord is not None else 0}?", "-sn"] + args += ["-map", "0:v:0", "-map", amap, "-sn"] args += ["-c:v", "copy", "-c:a", "aac", "-ac", "2", "-b:a", "192k"] args += [ "-f", "hls", @@ -74,6 +100,25 @@ def _ffmpeg_cmd(src: Path, start_s: float, out_dir: Path, audio_ord: int | None) return args +def _probe_first_pts(seg: Path) -> float | None: + """Read the first video packet's absolute PTS from a finished HLS segment (`-copyts` kept it + absolute). This is the true content position of the segment's first frame — the value hls.js + zero-bases the timeline to. Returns None on any failure (caller falls back to the requested offset).""" + try: + out = subprocess.run( + ["ffprobe", "-v", "error", "-select_streams", "v:0", "-read_intervals", "%+#1", + "-show_entries", "packet=pts_time", "-of", "csv=p=0", str(seg)], + capture_output=True, text=True, timeout=10, + ) + for line in out.stdout.splitlines(): + tok = line.strip().strip(",") + if tok: + return float(tok) + except (subprocess.SubprocessError, ValueError, OSError): + pass + return None + + def _kill(s: HlsSession) -> None: try: s.proc.terminate() @@ -100,6 +145,7 @@ def start_session( item: PlexItem, start_s: float, audio_ord: int | None = None, + aoff: float = 0.0, ) -> HlsSession | None: """(Re)start the HLS remux for an item at the given offset, with an optional selected audio track (ordinal among audio streams). Subtitles are separate WebVTT tracks (not muxed here). Returns @@ -113,19 +159,32 @@ def start_session( old = _sessions.pop(key, None) if old is not None: _kill(old) - directory = _HLS_ROOT / f"{key}_{int(start_s)}_{audio_ord}" + directory = _HLS_ROOT / f"{key}_{int(start_s)}_{audio_ord}_{aoff:+.2f}" shutil.rmtree(directory, ignore_errors=True) directory.mkdir(parents=True, exist_ok=True) proc = subprocess.Popen( - _ffmpeg_cmd(src, start_s, directory, audio_ord), + _ffmpeg_cmd(src, start_s, directory, audio_ord, aoff), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) s = HlsSession(key, directory, proc, start_s, "index.m3u8") _sessions[key] = s _enforce_cap() - log.info("plex hls session start key=%s start=%.1f audio=%s", key, start_s, audio_ord) - return s + # Learn the REAL start offset K from the first segment (outside the lock — this blocks on ffmpeg + # producing seg_0, and we must not hold up other sessions). The client reads `media_start_s`, so + # its absolute clock + subtitle shift key off the true keyframe, not the requested offset. On + # timeout/probe failure we keep the provisional requested offset (old behaviour, small drift). + if start_s > 0: + seg0 = directory / "seg_0.ts" + if wait_for(seg0, timeout=25.0): + k = _probe_first_pts(seg0) + if k is not None and k >= 0: + s.media_start_s = k + log.info( + "plex hls session start key=%s req=%.1f real=%.3f audio=%s", + key, start_s, s.media_start_s, audio_ord, + ) + return s def current_session(key: str) -> HlsSession | None: diff --git a/backend/app/routes/plex.py b/backend/app/routes/plex.py index 9da6260..bf1fa55 100644 --- a/backend/app/routes/plex.py +++ b/backend/app/routes/plex.py @@ -943,6 +943,63 @@ def _srt_to_vtt(text: str) -> str: return "WEBVTT\n\n" + body.strip() + "\n" +def _vtt_ts_to_s(ts: str) -> float: + parts = ts.split(":") + h = "0" if len(parts) == 2 else parts[0] + m, rest = parts[-2], parts[-1] + sec, _, ms = rest.partition(".") + return int(h) * 3600 + int(m) * 60 + int(sec) + int(ms or 0) / 1000.0 + + +def _vtt_s_to_ts(x: float) -> str: + x = max(0.0, x) + h = int(x // 3600); x -= h * 3600 + m = int(x // 60); x -= m * 60 + s = int(x); ms = int(round((x - s) * 1000)) + if ms >= 1000: + s += 1; ms -= 1000 + return f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}" + + +def _shift_vtt(vtt: str, offset: float) -> str: + """Subtract `offset` seconds from every cue's start/end. WebVTT cues carry ABSOLUTE content times, + but an HLS remux session's clock is zero-based at the real keyframe (see stream.py media_start_s), + so a subtitle overlaid on that session must be shifted by that offset to line up with speech. + + Processes cue BLOCKS so a cue that shifts fully into the past (end <= 0) can be DROPPED entirely. + (Collapsing them to 00:00:00.000-->00:00:00.000 made EVERY past cue count as active at currentTime 0 + — where a resume/seek lands — piling all of them on screen until playback advanced past 0.) A cue + straddling 0 is clamped to start at 0.""" + if not offset: + return vtt + out_blocks: list[str] = [] + for block in re.split(r"\n[ \t]*\n", vtt): + if "-->" not in block: + out_blocks.append(block) # header / NOTE / STYLE — keep verbatim + continue + lines = block.split("\n") + keep = True + for i, ln in enumerate(lines): + if "-->" not in ln: + continue + left, _, right = ln.partition("-->") + rparts = right.strip().split(None, 1) + settings = (" " + rparts[1]) if len(rparts) > 1 else "" + try: + st = _vtt_ts_to_s(left.strip()) - offset + en = _vtt_ts_to_s(rparts[0]) - offset + except (ValueError, IndexError): + break # unparseable timing → leave the block as-is + if en <= 0: + keep = False # whole cue is before the session start → drop the block + else: + lines[i] = f"{_vtt_s_to_ts(st)} --> {_vtt_s_to_ts(en)}{settings}" # _vtt_s_to_ts clamps st>=0 + break + if keep: + out_blocks.append("\n".join(lines)) + return "\n\n".join(out_blocks) + + def _ffmpeg_to_vtt(src: str, sub_ord: int | None) -> str: """Convert one subtitle stream of a file to WebVTT via ffmpeg (for embedded subs, and non-SRT external subs written to a temp file). `sub_ord` = ordinal among the file's subtitle streams.""" @@ -1047,16 +1104,18 @@ def image(rating_key: str, request: Request, variant: str = "thumb") -> Response @router.get("/subtitle/{rating_key}/{sub_ord}") -def subtitle(rating_key: str, sub_ord: int, request: Request) -> Response: +def subtitle(rating_key: str, sub_ord: int, request: Request, offset: float = 0.0) -> Response: """Serve the ord-th subtitle track as WebVTT for the browser's — so choosing a subtitle overlays it directly (no HLS re-mux / session restart). Handles EXTERNAL sidecar subs (fetched from Plex via the stream `key` — the old `-map 0:s` broke on these since they're not in the file) AND embedded text subs (extracted from the local file). Image subs (PGS/VobSub) → 415. Disk-cached.""" _require_session(request) + # Cache the ABSOLUTE-time VTT once (per rk+ord); the per-session cue-shift is applied on the way + # out, so a resume/seek to any offset reuses the same cached track (no cache-per-offset explosion). cache_key = f"sub_{rating_key}_{sub_ord}" hit = _sub_cache_read(cache_key) if hit is not None: - return Response(hit, media_type="text/vtt") + return Response(_shift_vtt(hit, offset), media_type="text/vtt") try: with SessionLocal() as db: it = _item_or_404(db, rating_key) @@ -1094,7 +1153,7 @@ def subtitle(rating_key: str, sub_ord: int, request: Request) -> Response: except PlexError as e: raise HTTPException(status_code=502, detail=f"Subtitle fetch failed: {e}") _sub_cache_write(cache_key, vtt) - return Response(vtt, media_type="text/vtt") + return Response(_shift_vtt(vtt, offset), media_type="text/vtt") # --- Playback (local file → direct 206 or on-the-fly HLS remux; P2) ---------------------------- @@ -1439,6 +1498,7 @@ def stream_session( rating_key: str, start: float = 0.0, audio: int | None = None, + aoff: float = 0.0, user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: @@ -1450,14 +1510,16 @@ def stream_session( it = _item_or_404(db, rating_key) if it.playable == "transcode": raise HTTPException(status_code=501, detail="This format needs transcoding (a later phase).") - if it.playable == "direct" and audio is None: + if it.playable == "direct" and audio is None and not aoff: return {"mode": "direct", "url": f"/api/plex/stream/{it.rating_key}/file", "start_s": 0.0} if plex_paths.local_media_path(db, it.file_path) is None: raise HTTPException(status_code=404, detail="Media file not found on disk") - s = plex_stream.start_session(db, it, start, audio) + s = plex_stream.start_session(db, it, start, audio, aoff) if s is None: raise HTTPException(status_code=404, detail="Media file not found on disk") - return {"mode": "hls", "start_s": s.start_s, "url": f"/api/plex/stream/{it.rating_key}/hls/{s.entry}"} + # Report the REAL media start (the keyframe ffmpeg landed on, which hls.js zero-bases to), not the + # requested offset — the client's absolute clock and the subtitle cue-shift both key off this. + return {"mode": "hls", "start_s": s.media_start_s, "url": f"/api/plex/stream/{it.rating_key}/hls/{s.entry}"} @router.get("/stream/{rating_key}/file") diff --git a/frontend/src/components/PlexPlayer.tsx b/frontend/src/components/PlexPlayer.tsx index d1bdd48..3873449 100644 --- a/frontend/src/components/PlexPlayer.tsx +++ b/frontend/src/components/PlexPlayer.tsx @@ -1,4 +1,14 @@ -import { Fragment, lazy, Suspense, useCallback, useEffect, useRef, useState, type ReactNode } from "react"; +import { + Fragment, + lazy, + Suspense, + useCallback, + useEffect, + useRef, + useState, + type ReactNode, + type WheelEvent as ReactWheelEvent, +} from "react"; import { useTranslation } from "react-i18next"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import Hls from "hls.js"; @@ -8,6 +18,7 @@ import { Download, Info, Keyboard, + Languages, Maximize, Minimize, Pause, @@ -22,10 +33,71 @@ import { X, } from "lucide-react"; import { api, type PlexMarker } from "../lib/api"; +import { useAccountPersistedObject } from "../lib/storage"; +import { useDismiss } from "../lib/useDismiss"; // The rich info overlay (poster/cast/ratings) reuses the same component as the card's info page. const PlexInfo = lazy(() => import("./PlexInfo")); +// Audio/subtitle selections are persisted per-account by LANGUAGE, not by ordinal — the same track +// ordinal means different languages in different files, so we re-resolve the saved language to this +// item's track layout on load (and it carries across episodes/movies, like Plex's preferred-language). +type AudioStream = { ord: number; language?: string | null; default: boolean }; +type SubStream = { ord: number; language?: string | null; text: boolean }; +function audioOrdForLang(streams: AudioStream[], lang: string): number | null { + if (!lang) return null; // "" = use the file's default first track (keeps direct-play, no forced HLS) + const i = streams.findIndex((s) => (s.language ?? "") === lang); + // absent (-1) or already the FIRST track (0, which the backend plays by default) → no override. + // (Match by index, NOT the `default` flag — some files mark a non-first track default, which made + // the restored language silently fall back to track 0.) + return i <= 0 ? null : streams[i].ord; +} +function subOrdForLang(subs: SubStream[], lang: string): number | null { + if (!lang) return null; // "" = subtitles off + const m = subs.find((s) => s.text && (s.language ?? "") === lang); + return m ? m.ord : null; // preferred language not available here → off +} + +// Per-account player preferences, persisted so they survive F5 and carry across items. +type PlexPlayerPrefs = { + volume: number; // 0..1 + muted: boolean; + wasPlaying: boolean; // resume-playing intent across reloads (best-effort; browsers may block autoplay) + audioLang: string; // "" = default first audio + subLang: string; // "" = subtitles off + subOffset: number; // seconds, +later / -earlier (client-side cue shift) + audioOffset: number; // seconds, +later / -earlier (server-side -itsoffset) + barAutoHide: boolean; // auto-hide the control bar during playback + seekStep: number; // plain ←/→ jump (s) + fineSeekStep: number; // Ctrl+←/→ jump (s) + subSize: number; // subtitle font size (%) + subColor: string; // subtitle text color (hex) + subBg: string; // subtitle background ("transparent" or hex) + subPos: number; // subtitle vertical position (cue line %, higher = lower on screen) +}; +const DEFAULT_PREFS: PlexPlayerPrefs = { + volume: 1, + muted: false, + wasPlaying: true, + audioLang: "", + subLang: "", + subOffset: 0, + audioOffset: 0, + barAutoHide: true, + seekStep: 10, + fineSeekStep: 1, + subSize: 100, + subColor: "#ffffff", + subBg: "transparent", + subPos: 88, +}; +const clamp01 = (n: number) => Math.max(0, Math.min(1, n)); +// hls.js plays our non-zero-start `-copyts` HLS stream ~1.0s BEHIND the media clock (measured by +// frame-capture: the first frame at source K lands at currentTime≈1.0, not 0 — constant, framerate- and +// segment-type-independent). Native-time subtitles (shifted by K) would therefore run 1s early, so we +// bake this compensation into the shift. Only on the HLS path; direct-play uses the file's real clock. +const HLS_SUB_LAG = 1.0; + // The Plex module's rich, full-page player. Plays the LOCAL file: direct-playable files stream raw // (native