"""Read-direction YouTube playlist sync: mirror each user's own YouTube playlists into local `source='youtube'` playlists (kept fresh by the scheduler). One-way (YT -> local); the write-back direction (local edits -> YouTube) is a later phase. YouTube's special Watch Later / History playlists are not exposed by the Data API and are never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and ingested (with a stub channel) so the mirror is faithful.""" import logging from datetime import datetime, timezone from sqlalchemy import delete, select from sqlalchemy.orm import Session from app import quota from app.auth import has_read_scope, has_write_scope from app.models import Channel, OAuthToken, Playlist, PlaylistItem, User, Video from app.sync.videos import apply_video_details, parse_dt from app.youtube.client import YouTubeClient, YouTubeError log = logging.getLogger("subfeed.sync") # Cost of a single YouTube write op (playlists/playlistItems insert/update/delete). WRITE_UNIT_COST = 50 def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None: """Make sure every given video id exists in the catalog, fetching + ingesting the missing ones (and a stub channel for any unknown channel). Deleted/private videos that the API won't return are simply left out.""" if not video_ids: return have = set( db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all() ) missing = [v for v in dict.fromkeys(video_ids) if v not in have] if not missing: return items = yt.get_videos(missing) chan_ids = { it.get("snippet", {}).get("channelId") for it in items if it.get("snippet", {}).get("channelId") } existing_ch = set( db.execute(select(Channel.id).where(Channel.id.in_(chan_ids))).scalars().all() ) now = datetime.now(timezone.utc) for it in items: sn = it.get("snippet", {}) cid = sn.get("channelId") if cid and cid not in existing_ch: db.add(Channel(id=cid, title=sn.get("channelTitle"))) existing_ch.add(cid) db.flush() seen: set[str] = set() for it in items: vid = it.get("id") sn = it.get("snippet", {}) cid = sn.get("channelId") if not vid or not cid or vid in seen: continue seen.add(vid) v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt"))) apply_video_details(v, it) v.enriched_at = now db.add(v) db.commit() def sync_user_playlists(db: Session, user: User) -> dict: """Mirror the user's YouTube playlists into local source='youtube' playlists. Leaves the user's local playlists and the built-in Watch later untouched.""" if not has_read_scope(user): return {"synced": 0, "reason": "no read scope"} synced = 0 with YouTubeClient(db, user) as yt: yt_playlists = list(yt.iter_my_playlists()) yt_ids = {p["id"] for p in yt_playlists if p.get("id")} existing = { pl.yt_playlist_id: pl for pl in db.execute( select(Playlist).where( Playlist.user_id == user.id, Playlist.source == "youtube" ) ).scalars() } # YouTube playlists that we already own as local (exported) playlists must not be # re-mirrored, or every exported list would also show up as a read-only duplicate. owned_yt_ids = set( db.execute( select(Playlist.yt_playlist_id).where( Playlist.user_id == user.id, Playlist.source == "local", Playlist.yt_playlist_id.is_not(None), ) ) .scalars() .all() ) # Drop mirrors whose YouTube playlist no longer exists. for ytid, pl in existing.items(): if ytid not in yt_ids: db.delete(pl) db.flush() for idx, p in enumerate(yt_playlists): ytid = p.get("id") if not ytid or ytid in owned_yt_ids: continue pl = existing.get(ytid) if pl is None: pl = Playlist( user_id=user.id, name=p.get("title") or "Untitled", kind="user", source="youtube", yt_playlist_id=ytid, position=1000 + idx, # sort mirrored lists after local ones ) db.add(pl) db.flush() else: pl.name = p.get("title") or pl.name ids = list(yt.iter_my_playlist_video_ids(ytid)) _ensure_videos(db, yt, ids) present = ( set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all()) if ids else set() ) ordered: list[str] = [] seen: set[str] = set() for v in ids: if v in present and v not in seen: seen.add(v) ordered.append(v) # Mirror is authoritative: replace the items to match YouTube's order. db.execute(delete(PlaylistItem).where(PlaylistItem.playlist_id == pl.id)) for pos, vid in enumerate(ordered): db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=pos)) db.commit() synced += 1 return {"synced": synced} def _local_video_ids(db: Session, pl: Playlist) -> list[str]: return list( db.execute( select(PlaylistItem.video_id) .where(PlaylistItem.playlist_id == pl.id) .order_by(PlaylistItem.position) ) .scalars() .all() ) def _reorder_moves(current: list[str], desired: list[str]) -> int: """Count the playlistItems.update moves an insertion-sort would need to turn `current` order into `desired` order (both already the same set of video ids). Items already in place are skipped; this matches what apply_push actually executes.""" model = list(current) moves = 0 for i, want in enumerate(desired): if i < len(model) and model[i] == want: continue j = model.index(want, i) model.pop(j) model.insert(i, want) moves += 1 return moves def plan_push(db: Session, user: User, pl: Playlist) -> dict: """Compute (without writing) what pushing this local playlist to YouTube would do: create vs update, how many inserts/deletes/reorders, the quota estimate, and whether YouTube has diverged (items present there that local will remove). Costs read units (1/page) for a linked playlist; nothing for a fresh export.""" desired = _local_video_ids(db, pl) if not pl.yt_playlist_id: ops = 1 + len(desired) # create the playlist + one insert per video return { "action": "create", "to_insert": len(desired), "to_delete": 0, "to_reorder": 0, "yt_extra": 0, "units_estimate": ops * WRITE_UNIT_COST, } with YouTubeClient(db, user) as yt: current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)) cur_vids = [it["video_id"] for it in current] cur_set = set(cur_vids) desired_set = set(desired) to_insert = [v for v in desired if v not in cur_set] to_delete = [v for v in cur_vids if v not in desired_set] # After membership reconciliation, the YT order is current-minus-deletes plus inserts # appended; the reorder pass then moves items into the desired order. after_membership = [v for v in cur_vids if v in desired_set] + to_insert reorders = _reorder_moves(after_membership, desired) ops = len(to_insert) + len(to_delete) + reorders return { "action": "update", "to_insert": len(to_insert), "to_delete": len(to_delete), "to_reorder": reorders, "yt_extra": len(to_delete), "units_estimate": ops * WRITE_UNIT_COST, } def push_playlist(db: Session, user: User, pl: Playlist) -> dict: """Push a local playlist to YouTube: create it if needed, then reconcile membership and order so YouTube matches local (local wins). Marks the playlist clean on success. Caller must have checked the write scope and the quota budget.""" desired = _local_video_ids(db, pl) inserted = deleted = reordered = created = 0 failures: list[str] = [] with YouTubeClient(db, user) as yt: if not pl.yt_playlist_id: pl.yt_playlist_id = yt.create_playlist(pl.name) db.commit() created = 1 # Brand-new playlist: just append every video in order. for vid in desired: try: yt.add_playlist_item(pl.yt_playlist_id, vid) inserted += 1 except YouTubeError: failures.append(vid) pl.dirty = False db.commit() return { "created": created, "inserted": inserted, "deleted": deleted, "reordered": reordered, "failures": failures, "yt_playlist_id": pl.yt_playlist_id, } # Existing link: diff against the live YouTube state. current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)) item_id_by_vid = {it["video_id"]: it["item_id"] for it in current} cur_vids = [it["video_id"] for it in current] desired_set = set(desired) # Delete extras (local is authoritative). for it in current: if it["video_id"] not in desired_set: try: yt.delete_playlist_item(it["item_id"]) deleted += 1 except YouTubeError: failures.append(it["video_id"]) # Insert missing (append; the reorder pass below fixes positions). cur_set = set(cur_vids) for vid in desired: if vid not in cur_set: try: item_id_by_vid[vid] = yt.add_playlist_item(pl.yt_playlist_id, vid) inserted += 1 except YouTubeError: failures.append(vid) # Reorder to match `desired` via insertion-sort moves over a local model. model = [v for v in cur_vids if v in desired_set] + [ v for v in desired if v not in cur_set and v in item_id_by_vid ] for i, want in enumerate(desired): if want not in item_id_by_vid: continue # failed insert; skip if i < len(model) and model[i] == want: continue try: yt.move_playlist_item(item_id_by_vid[want], pl.yt_playlist_id, want, i) reordered += 1 except YouTubeError: failures.append(want) continue j = model.index(want) model.pop(j) model.insert(i, want) pl.dirty = False db.commit() return { "created": created, "inserted": inserted, "deleted": deleted, "reordered": reordered, "failures": failures, "yt_playlist_id": pl.yt_playlist_id, } def sync_all_playlists(db: Session) -> dict: """Scheduler entry point: mirror playlists for every user with a read scope + token.""" users = ( db.execute( select(User).join(OAuthToken).where(OAuthToken.refresh_token_enc.is_not(None)) ) .scalars() .all() ) total = 0 for user in users: if not has_read_scope(user): continue try: with quota.attribute(user.id, "playlist_sync"): total += sync_user_playlists(db, user).get("synced", 0) except YouTubeError: db.rollback() log.warning("Playlist sync failed for user %s", user.id) except Exception: db.rollback() log.exception("Playlist sync crashed for user %s", user.id) return {"users": len(users), "playlists_synced": total}