Merge feature/s4c-youtube-playlist-pull: YouTube playlist read-sync

S4c (read direction, YT -> local):
- feat(playlists): mirror users' YouTube playlists into source=youtube playlists,
  ingesting catalog-missing videos; manual /api/playlists/sync-youtube + scheduler job
- feat(playlists): YouTube-mirrored playlists shown read-only with a Sync button
- fix(notifications): success level for success toasts (green check, not accent)
This commit is contained in:
npeter83 2026-06-15 21:02:49 +02:00
commit 35831dc3ce
12 changed files with 321 additions and 25 deletions

View file

@ -89,6 +89,7 @@ class Settings(BaseSettings):
autotag_title_sample: int = 40 autotag_title_sample: int = 40
autotag_interval_minutes: int = 30 autotag_interval_minutes: int = 30
subscriptions_resync_minutes: int = 360 subscriptions_resync_minutes: int = 360
playlist_sync_minutes: int = 360
# live_status values hidden from the feed by default. Completed-stream VODs # live_status values hidden from the feed by default. Completed-stream VODs
# ("was_live") are real watchable content and stay visible. # ("was_live") are real watchable content and stay visible.
feed_default_hidden_live: str = "live,upcoming" feed_default_hidden_live: str = "live,upcoming"

View file

@ -5,10 +5,12 @@ from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import func, select, and_ from sqlalchemy import func, select, and_
from sqlalchemy.orm import Session, aliased from sqlalchemy.orm import Session, aliased
from app.auth import current_user from app import quota
from app.auth import current_user, has_read_scope
from app.db import get_db from app.db import get_db
from app.models import Channel, Playlist, PlaylistItem, User, Video, VideoState from app.models import Channel, Playlist, PlaylistItem, User, Video, VideoState
from app.routes.feed import _saved_expr, _serialize from app.routes.feed import _saved_expr, _serialize
from app.sync.playlists import sync_user_playlists
router = APIRouter(prefix="/api/playlists", tags=["playlists"]) router = APIRouter(prefix="/api/playlists", tags=["playlists"])
@ -187,6 +189,22 @@ def remove_watch_later(
return {"saved": False} return {"saved": False}
@router.post("/sync-youtube")
def sync_youtube(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
"""Pull the user's YouTube playlists into local mirrors now (read-only direction).
Also runs automatically on the scheduler; this is the manual 'sync now'."""
if not has_read_scope(user):
raise HTTPException(
status_code=403,
detail="Connect YouTube (read access) to sync your playlists.",
)
with quota.attribute(user.id, "playlist_sync"):
return sync_user_playlists(db, user)
@router.get("/{playlist_id}") @router.get("/{playlist_id}")
def get_playlist( def get_playlist(
playlist_id: int, playlist_id: int,

View file

@ -9,6 +9,7 @@ from app.config import settings
from app.db import SessionLocal from app.db import SessionLocal
from app.state import is_sync_paused from app.state import is_sync_paused
from app.sync.autotag import run_autotag_all from app.sync.autotag import run_autotag_all
from app.sync.playlists import sync_all_playlists
from app.sync.runner import ( from app.sync.runner import (
run_deep_backfill, run_deep_backfill,
run_enrich, run_enrich,
@ -79,6 +80,12 @@ def _subscriptions_job() -> None:
_job("subscriptions", work) _job("subscriptions", work)
def _playlist_sync_job() -> None:
# Mirror each read-scope user's YouTube playlists; per-user quota attribution is
# handled inside sync_all_playlists.
_job("playlist_sync", sync_all_playlists)
def start_scheduler() -> None: def start_scheduler() -> None:
global _scheduler global _scheduler
if not settings.scheduler_enabled or _scheduler is not None: if not settings.scheduler_enabled or _scheduler is not None:
@ -114,6 +121,12 @@ def start_scheduler() -> None:
minutes=settings.subscriptions_resync_minutes, minutes=settings.subscriptions_resync_minutes,
id="subscriptions", id="subscriptions",
) )
scheduler.add_job(
_playlist_sync_job,
"interval",
minutes=settings.playlist_sync_minutes,
id="playlist_sync",
)
scheduler.start() scheduler.start()
_scheduler = scheduler _scheduler = scheduler
logger.info("scheduler started") logger.info("scheduler started")

View file

@ -0,0 +1,152 @@
"""Read-direction YouTube playlist sync: mirror each user's own YouTube playlists into
local `source='youtube'` playlists (kept fresh by the scheduler). One-way (YT -> local);
the write-back direction (local edits -> YouTube) is a later phase.
YouTube's special Watch Later / History playlists are not exposed by the Data API and are
never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and
ingested (with a stub channel) so the mirror is faithful."""
import logging
from datetime import datetime, timezone
from sqlalchemy import delete, select
from sqlalchemy.orm import Session
from app import quota
from app.auth import has_read_scope
from app.models import Channel, OAuthToken, Playlist, PlaylistItem, User, Video
from app.sync.videos import apply_video_details, parse_dt
from app.youtube.client import YouTubeClient, YouTubeError
log = logging.getLogger("subfeed.sync")
def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
"""Make sure every given video id exists in the catalog, fetching + ingesting the
missing ones (and a stub channel for any unknown channel). Deleted/private videos that
the API won't return are simply left out."""
if not video_ids:
return
have = set(
db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all()
)
missing = [v for v in dict.fromkeys(video_ids) if v not in have]
if not missing:
return
items = yt.get_videos(missing)
chan_ids = {
it.get("snippet", {}).get("channelId")
for it in items
if it.get("snippet", {}).get("channelId")
}
existing_ch = set(
db.execute(select(Channel.id).where(Channel.id.in_(chan_ids))).scalars().all()
)
now = datetime.now(timezone.utc)
for it in items:
sn = it.get("snippet", {})
cid = sn.get("channelId")
if cid and cid not in existing_ch:
db.add(Channel(id=cid, title=sn.get("channelTitle")))
existing_ch.add(cid)
db.flush()
seen: set[str] = set()
for it in items:
vid = it.get("id")
sn = it.get("snippet", {})
cid = sn.get("channelId")
if not vid or not cid or vid in seen:
continue
seen.add(vid)
v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt")))
apply_video_details(v, it)
v.enriched_at = now
db.add(v)
db.commit()
def sync_user_playlists(db: Session, user: User) -> dict:
"""Mirror the user's YouTube playlists into local source='youtube' playlists. Leaves
the user's local playlists and the built-in Watch later untouched."""
if not has_read_scope(user):
return {"synced": 0, "reason": "no read scope"}
synced = 0
with YouTubeClient(db, user) as yt:
yt_playlists = list(yt.iter_my_playlists())
yt_ids = {p["id"] for p in yt_playlists if p.get("id")}
existing = {
pl.yt_playlist_id: pl
for pl in db.execute(
select(Playlist).where(
Playlist.user_id == user.id, Playlist.source == "youtube"
)
).scalars()
}
# Drop mirrors whose YouTube playlist no longer exists.
for ytid, pl in existing.items():
if ytid not in yt_ids:
db.delete(pl)
db.flush()
for idx, p in enumerate(yt_playlists):
ytid = p.get("id")
if not ytid:
continue
pl = existing.get(ytid)
if pl is None:
pl = Playlist(
user_id=user.id,
name=p.get("title") or "Untitled",
kind="user",
source="youtube",
yt_playlist_id=ytid,
position=1000 + idx, # sort mirrored lists after local ones
)
db.add(pl)
db.flush()
else:
pl.name = p.get("title") or pl.name
ids = list(yt.iter_my_playlist_video_ids(ytid))
_ensure_videos(db, yt, ids)
present = (
set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all())
if ids
else set()
)
ordered: list[str] = []
seen: set[str] = set()
for v in ids:
if v in present and v not in seen:
seen.add(v)
ordered.append(v)
# Mirror is authoritative: replace the items to match YouTube's order.
db.execute(delete(PlaylistItem).where(PlaylistItem.playlist_id == pl.id))
for pos, vid in enumerate(ordered):
db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=pos))
db.commit()
synced += 1
return {"synced": synced}
def sync_all_playlists(db: Session) -> dict:
"""Scheduler entry point: mirror playlists for every user with a read scope + token."""
users = (
db.execute(
select(User).join(OAuthToken).where(OAuthToken.refresh_token_enc.is_not(None))
)
.scalars()
.all()
)
total = 0
for user in users:
if not has_read_scope(user):
continue
try:
with quota.attribute(user.id, "playlist_sync"):
total += sync_user_playlists(db, user).get("synced", 0)
except YouTubeError:
db.rollback()
log.warning("Playlist sync failed for user %s", user.id)
except Exception:
db.rollback()
log.exception("Playlist sync crashed for user %s", user.id)
return {"users": len(users), "playlists_synced": total}

View file

@ -125,6 +125,49 @@ class YouTubeClient:
if not page_token: if not page_token:
break break
def iter_my_playlists(self) -> Iterator[dict]:
"""Yield the authenticated user's own playlists (OAuth, so private ones are
included). Note: YouTube's special Watch Later / History playlists are NOT
returned by the Data API and cannot be synced."""
page_token = None
while True:
params = {"part": "snippet,contentDetails", "mine": "true", "maxResults": 50}
if page_token:
params["pageToken"] = page_token
data = self._get("playlists", params, cost=1, allow_key=False)
for item in data.get("items", []):
sn = item.get("snippet", {})
yield {
"id": item.get("id"),
"title": sn.get("title"),
"item_count": item.get("contentDetails", {}).get("itemCount"),
"thumbnail_url": best_thumbnail(sn.get("thumbnails")),
}
page_token = data.get("nextPageToken")
if not page_token:
break
def iter_my_playlist_video_ids(self, playlist_id: str) -> Iterator[str]:
"""Yield the video ids of one of the user's own playlists, in playlist order
(OAuth, so private/unlisted playlists work)."""
page_token = None
while True:
params = {
"part": "contentDetails",
"playlistId": playlist_id,
"maxResults": 50,
}
if page_token:
params["pageToken"] = page_token
data = self._get("playlistItems", params, cost=1, allow_key=False)
for item in data.get("items", []):
vid = item.get("contentDetails", {}).get("videoId")
if vid:
yield vid
page_token = data.get("nextPageToken")
if not page_token:
break
def get_channels(self, channel_ids: list[str]) -> list[dict]: def get_channels(self, channel_ids: list[str]) -> list[dict]:
items: list[dict] = [] items: list[dict] = []
for batch in _chunks(channel_ids, 50): for batch in _chunks(channel_ids, 50):

View file

@ -118,7 +118,9 @@ export default function AddToPlaylist({
} }
} }
const lists = membership.data ?? []; // YouTube-mirrored playlists are read-only until the write-back phase, so they can't be
// added to here yet.
const lists = (membership.data ?? []).filter((pl) => pl.source !== "youtube");
return ( return (
<> <>

View file

@ -23,27 +23,32 @@ import {
Pencil, Pencil,
Play, Play,
Plus, Plus,
RefreshCw,
Trash2, Trash2,
X, X,
Youtube,
} from "lucide-react"; } from "lucide-react";
import { api, type Playlist, type Video } from "../lib/api"; import { api, type Playlist, type Video } from "../lib/api";
import { formatDuration } from "../lib/format"; import { formatDuration } from "../lib/format";
import { notify } from "../lib/notifications";
import PlayerModal from "./PlayerModal"; import PlayerModal from "./PlayerModal";
import { useConfirm } from "./ConfirmProvider"; import { useConfirm } from "./ConfirmProvider";
function Row({ function Row({
video, video,
index, index,
readOnly,
onPlay, onPlay,
onRemove, onRemove,
}: { }: {
video: Video; video: Video;
index: number; index: number;
readOnly?: boolean;
onPlay: () => void; onPlay: () => void;
onRemove: () => void; onRemove: () => void;
}) { }) {
const { attributes, listeners, setNodeRef, transform, transition, isDragging } = const { attributes, listeners, setNodeRef, transform, transition, isDragging } =
useSortable({ id: video.id }); useSortable({ id: video.id, disabled: readOnly });
const style = { const style = {
transform: CSS.Transform.toString(transform), transform: CSS.Transform.toString(transform),
transition, transition,
@ -55,14 +60,18 @@ function Row({
style={style} style={style}
className="flex items-center gap-2.5 p-2 rounded-lg border border-border bg-card" className="flex items-center gap-2.5 p-2 rounded-lg border border-border bg-card"
> >
<button {readOnly ? (
{...attributes} <span className="w-4" />
{...listeners} ) : (
className="text-muted hover:text-fg cursor-grab active:cursor-grabbing" <button
aria-label="reorder" {...attributes}
> {...listeners}
<GripVertical className="w-4 h-4" /> className="text-muted hover:text-fg cursor-grab active:cursor-grabbing"
</button> aria-label="reorder"
>
<GripVertical className="w-4 h-4" />
</button>
)}
<span className="text-xs text-muted w-4 text-center tabular-nums">{index + 1}</span> <span className="text-xs text-muted w-4 text-center tabular-nums">{index + 1}</span>
<button <button
onClick={onPlay} onClick={onPlay}
@ -84,14 +93,16 @@ function Row({
{formatDuration(video.duration_seconds)} {formatDuration(video.duration_seconds)}
</span> </span>
)} )}
<button {!readOnly && (
onClick={onRemove} <button
title="remove" onClick={onRemove}
className="text-muted hover:text-fg p-1" title="remove"
aria-label="remove from playlist" className="text-muted hover:text-fg p-1"
> aria-label="remove from playlist"
<X className="w-4 h-4" /> >
</button> <X className="w-4 h-4" />
</button>
)}
</div> </div>
); );
} }
@ -149,6 +160,25 @@ export default function Playlists() {
const plName = (p: { kind: string; name: string }) => const plName = (p: { kind: string; name: string }) =>
p.kind === "watch_later" ? t("playlists.watchLater") : p.name; p.kind === "watch_later" ? t("playlists.watchLater") : p.name;
const builtin = detail?.kind === "watch_later"; const builtin = detail?.kind === "watch_later";
// YouTube-mirrored playlists are read-only here until the write-back phase.
const mirrored = detail?.source === "youtube";
const editable = !builtin && !mirrored;
const [syncing, setSyncing] = useState(false);
async function syncYoutube() {
if (syncing) return;
setSyncing(true);
try {
const r = await api.syncYoutubePlaylists();
qc.invalidateQueries({ queryKey: ["playlists"] });
qc.invalidateQueries({ queryKey: ["playlist"] });
notify({ level: "success", message: t("playlists.syncedToast", { count: r.synced }) });
} catch {
notify({ level: "warning", message: t("playlists.syncFailed") });
} finally {
setSyncing(false);
}
}
const createMut = useMutation({ const createMut = useMutation({
mutationFn: (name: string) => api.createPlaylist(name), mutationFn: (name: string) => api.createPlaylist(name),
@ -166,7 +196,7 @@ export default function Playlists() {
async function onDragEnd(e: DragEndEvent) { async function onDragEnd(e: DragEndEvent) {
const { active: a, over } = e; const { active: a, over } = e;
if (!over || a.id === over.id || selectedId == null) return; if (!over || a.id === over.id || selectedId == null || mirrored) return;
const oldIndex = items.findIndex((v) => v.id === a.id); const oldIndex = items.findIndex((v) => v.id === a.id);
const newIndex = items.findIndex((v) => v.id === over.id); const newIndex = items.findIndex((v) => v.id === over.id);
if (oldIndex < 0 || newIndex < 0) return; if (oldIndex < 0 || newIndex < 0) return;
@ -222,7 +252,19 @@ export default function Playlists() {
return ( return (
<div className="flex h-full min-h-0"> <div className="flex h-full min-h-0">
<aside className="w-64 shrink-0 border-r border-border bg-surface/40 overflow-y-auto p-3"> <aside className="w-64 shrink-0 border-r border-border bg-surface/40 overflow-y-auto p-3">
<div className="text-sm font-semibold mb-2">{t("playlists.title")}</div> <div className="flex items-center justify-between mb-2">
<span className="text-sm font-semibold">{t("playlists.title")}</span>
<button
onClick={syncYoutube}
disabled={syncing}
title={t("playlists.syncYoutube")}
aria-label={t("playlists.syncYoutube")}
className="inline-flex items-center gap-1 text-[11px] text-muted hover:text-accent disabled:opacity-40 transition"
>
<RefreshCw className={`w-3.5 h-3.5 ${syncing ? "animate-spin" : ""}`} />
<Youtube className="w-3.5 h-3.5" />
</button>
</div>
{playlists.length === 0 && !listQuery.isLoading && ( {playlists.length === 0 && !listQuery.isLoading && (
<div className="text-xs text-muted px-1 py-2">{t("playlists.noneYet")}</div> <div className="text-xs text-muted px-1 py-2">{t("playlists.noneYet")}</div>
)} )}
@ -243,7 +285,12 @@ export default function Playlists() {
)} )}
</div> </div>
<div className="min-w-0"> <div className="min-w-0">
<div className="text-[13px] text-fg truncate">{plName(pl)}</div> <div className="flex items-center gap-1 text-[13px] text-fg">
<span className="truncate">{plName(pl)}</span>
{pl.source === "youtube" && (
<Youtube className="w-3 h-3 shrink-0 text-muted" />
)}
</div>
<div className="text-[11px] text-muted"> <div className="text-[11px] text-muted">
{t("playlists.itemCount", { count: pl.item_count })} {t("playlists.itemCount", { count: pl.item_count })}
</div> </div>
@ -301,7 +348,7 @@ export default function Playlists() {
) : ( ) : (
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<h2 className="text-lg font-semibold truncate">{plName(detail)}</h2> <h2 className="text-lg font-semibold truncate">{plName(detail)}</h2>
{!builtin && ( {editable && (
<button <button
onClick={() => { onClick={() => {
setRenameValue(detail.name); setRenameValue(detail.name);
@ -326,7 +373,7 @@ export default function Playlists() {
> >
<Play className="w-3.5 h-3.5 fill-current" /> {t("playlists.playAll")} <Play className="w-3.5 h-3.5 fill-current" /> {t("playlists.playAll")}
</button> </button>
{!builtin && ( {editable && (
<button <button
onClick={deletePlaylist} onClick={deletePlaylist}
className="inline-flex items-center gap-1.5 text-xs px-3 py-1.5 rounded-lg border border-border text-muted hover:text-fg hover:border-accent transition" className="inline-flex items-center gap-1.5 text-xs px-3 py-1.5 rounded-lg border border-border text-muted hover:text-fg hover:border-accent transition"
@ -334,6 +381,11 @@ export default function Playlists() {
<Trash2 className="w-3.5 h-3.5" /> {t("playlists.delete")} <Trash2 className="w-3.5 h-3.5" /> {t("playlists.delete")}
</button> </button>
)} )}
{mirrored && (
<span className="inline-flex items-center gap-1.5 text-[11px] px-2.5 py-1.5 rounded-lg text-muted">
<Youtube className="w-3.5 h-3.5" /> {t("playlists.ytReadonly")}
</span>
)}
</div> </div>
</div> </div>
</div> </div>
@ -361,6 +413,7 @@ export default function Playlists() {
key={v.id} key={v.id}
video={v} video={v}
index={i} index={i}
readOnly={mirrored}
onPlay={() => setPlayingIndex(i)} onPlay={() => setPlayingIndex(i)}
onRemove={() => removeItem(v.id)} onRemove={() => removeItem(v.id)}
/> />

View file

@ -202,7 +202,7 @@ export default function Sidebar({
async function shareView() { async function shareView() {
try { try {
await navigator.clipboard.writeText(shareUrl(filters)); await navigator.clipboard.writeText(shareUrl(filters));
notify({ message: t("sidebar.shareCopied") }); notify({ level: "success", message: t("sidebar.shareCopied") });
} catch { } catch {
notify({ level: "warning", message: t("sidebar.shareFailed") }); notify({ level: "warning", message: t("sidebar.shareFailed") });
} }

View file

@ -1,6 +1,10 @@
{ {
"title": "Wiedergabelisten", "title": "Wiedergabelisten",
"watchLater": "Später ansehen", "watchLater": "Später ansehen",
"syncYoutube": "Von YouTube synchronisieren",
"syncedToast": "{{count}} Wiedergabelisten von YouTube synchronisiert",
"syncFailed": "Synchronisierung von YouTube fehlgeschlagen",
"ytReadonly": "Von YouTube synchronisiert — vorerst schreibgeschützt",
"noneYet": "Noch keine Wiedergabelisten", "noneYet": "Noch keine Wiedergabelisten",
"newPlaylist": "Neue Liste…", "newPlaylist": "Neue Liste…",
"itemCount_one": "{{count}} Video", "itemCount_one": "{{count}} Video",

View file

@ -1,6 +1,10 @@
{ {
"title": "Playlists", "title": "Playlists",
"watchLater": "Watch later", "watchLater": "Watch later",
"syncYoutube": "Sync from YouTube",
"syncedToast": "Synced {{count}} playlists from YouTube",
"syncFailed": "Couldn't sync from YouTube",
"ytReadonly": "Synced from YouTube — read-only for now",
"noneYet": "No playlists yet", "noneYet": "No playlists yet",
"newPlaylist": "New playlist…", "newPlaylist": "New playlist…",
"itemCount_one": "{{count}} video", "itemCount_one": "{{count}} video",

View file

@ -1,6 +1,10 @@
{ {
"title": "Lejátszási listák", "title": "Lejátszási listák",
"watchLater": "Megnézem később", "watchLater": "Megnézem később",
"syncYoutube": "Szinkron YouTube-ról",
"syncedToast": "{{count}} lista szinkronizálva a YouTube-ról",
"syncFailed": "Nem sikerült a YouTube-szinkron",
"ytReadonly": "YouTube-ról szinkronizálva — egyelőre csak olvasható",
"noneYet": "Még nincs lejátszási lista", "noneYet": "Még nincs lejátszási lista",
"newPlaylist": "Új lista…", "newPlaylist": "Új lista…",
"itemCount_one": "{{count}} videó", "itemCount_one": "{{count}} videó",

View file

@ -344,6 +344,8 @@ export const api = {
}), }),
watchLaterRemove: (videoId: string) => watchLaterRemove: (videoId: string) =>
req(`/api/playlists/watch-later/${videoId}`, { method: "DELETE" }), req(`/api/playlists/watch-later/${videoId}`, { method: "DELETE" }),
syncYoutubePlaylists: (): Promise<{ synced: number; reason?: string }> =>
req("/api/playlists/sync-youtube", { method: "POST" }),
// --- quota usage --- // --- quota usage ---
myUsage: (): Promise<MyUsage> => req("/api/quota/my-usage"), myUsage: (): Promise<MyUsage> => req("/api/quota/my-usage"),