fix(deferred): close backend correctness/efficiency tails from the hygiene sweep

Verified each deferred per-subsystem item against current source, then fixed the
real ones (tradeoffs left as-is, see siftlode-ops/CODE-HYGIENE.md closeout):

- search BUG-3: don't finalise shorts_probed on un-enriched stubs (enrichment
  failure/omitted rows) — restores the scheduler's enriched_at guard so a real
  Short can't leak into the feed and never get reclassified.
- downloads GC: 4th pass reaps orphaned errored, fileless, unreferenced assets
  (they carry no TTL, so the expiry passes never cleared them).
- downloads B6: quota/edit errors now return a structured {code,reason,limit}
  the trilingual client localises, instead of hardcoded English + dot-decimal GB.
- channels BB1: reset-backfill re-arms deep sync on every subscriber (bulk update)
  instead of 404ing when the admin isn't personally subscribed.
- channels BB2: enrich the stub on BOTH the normal and "already exists" desync
  path (nest the insert try inside the client `with`).
- channels CB3: drop the redundant second _discovery_rows read (identity-mapped
  rows are already enriched; re-sort in Python for the title tie-break).
- playlists PC1: read the live playlist once in push() and share it with plan_push
  + push_playlist (halves read-quota, closes the second TOCTOU window).
- playlists PC2/PC5: batch the cover thumbnails in one window query (was N+1);
  rename combines count+duration into one aggregate + a single cover lookup.
- messages MB2: get_thread only resolves a messageable partner OR one we already
  share a thread with (closes the user-enumeration oracle).
- messages MB3: push a live unread-count to the user's other tabs after mark-read.
- messages MC4: list_conversations uses DISTINCT ON + grouped COUNT instead of
  loading the whole message history into memory.
- config AB3: per-spec allow_empty so smtp_user/youtube_api_proxy can be blanked
  to disable them rather than snapping back to the env default.
This commit is contained in:
npeter83 2026-07-12 05:59:03 +02:00
parent 182dddf5ed
commit 4e80e2b39b
8 changed files with 205 additions and 102 deletions

View file

@ -1,12 +1,14 @@
"""Download retention garbage collector (a scheduler job). """Download retention garbage collector (a scheduler job).
Three passes each run: Four passes each run:
1. Pre-expiry warning for ready assets expiring within the grace window, notify each holder 1. Pre-expiry warning for ready assets expiring within the grace window, notify each holder
once (gc_notified flag) so a download isn't silently deleted. once (gc_notified flag) so a download isn't silently deleted.
2. Expiry deletion delete assets past their TTL (files + row). The FK SET NULL nulls the 2. Expiry deletion delete assets past their TTL (files + row). The FK SET NULL nulls the
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI. holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
3. LRU eviction if a total cache cap is set and exceeded, delete least-recently-accessed 3. LRU eviction if a total cache cap is set and exceeded, delete least-recently-accessed
ready assets until back under it. ready assets until back under it.
4. Orphan reap delete errored, fileless, unreferenced asset rows (they carry no TTL, so
passes 1-3 skip them; benign row-count hygiene).
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work no YouTube quota. Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work no YouTube quota.
""" """
@ -58,7 +60,7 @@ def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
def run_download_gc(db: Session) -> dict: def run_download_gc(db: Session) -> dict:
now = datetime.now(timezone.utc) now = datetime.now(timezone.utc)
grace_days = sysconfig.get_int(db, "download_gc_grace_days") grace_days = sysconfig.get_int(db, "download_gc_grace_days")
warned = expired = evicted = 0 warned = expired = evicted = reaped = 0
# 1. Pre-expiry warnings. # 1. Pre-expiry warnings.
warn_before = now + timedelta(days=grace_days) warn_before = now + timedelta(days=grace_days)
@ -114,6 +116,22 @@ def run_download_gc(db: Session) -> dict:
evicted += 1 evicted += 1
db.commit() db.commit()
result = {"warned": warned, "expired": expired, "evicted": evicted} # 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
# referenced errored row to pending on re-enqueue, so these only accumulate once every
# holder is gone; they carry no expires_at, so passes 1-3 never touch them.
orphans = db.execute(
select(MediaAsset).where(
MediaAsset.status == "error",
MediaAsset.ref_count == 0,
MediaAsset.rel_path.is_(None),
)
).scalars().all()
for asset in orphans:
db.delete(asset) # no file to unlink; FK SET NULL clears any stray job.asset_id
reaped += 1
if orphans:
db.commit()
result = {"warned": warned, "expired": expired, "evicted": evicted, "reaped": reaped}
log.info("download_gc %s", result) log.info("download_gc %s", result)
return result return result

View file

@ -175,15 +175,17 @@ def discover_channels(
rows = _discovery_rows(db, user) rows = _discovery_rows(db, user)
# Enrich stub channels' metadata (uses the API key — no write scope needed). Re-read # Enrich stub channels' metadata (uses the API key — no write scope needed). The Channel
# the rows afterwards so the response carries the fresh subscriber counts/thumbnails. # objects are identity-mapped, so apply_channel_details mutates the very rows we already hold
# (fresh subscriber counts/thumbnails carry through without a re-query). Only the title can
# change, which affects the tie-break order → re-sort in Python instead of re-hitting the DB.
need = [ch.id for ch, _, _ in rows if ch.details_synced_at is None][:DISCOVERY_ENRICH_CAP] need = [ch.id for ch, _, _ in rows if ch.details_synced_at is None][:DISCOVERY_ENRICH_CAP]
if need: if need:
try: try:
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), YouTubeClient(db, user) as yt: with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), YouTubeClient(db, user) as yt:
apply_channel_details(db, yt.get_channels(need)) apply_channel_details(db, yt.get_channels(need))
db.commit() db.commit()
rows = _discovery_rows(db, user) rows = sorted(rows, key=lambda r: (-int(r[1]), (r[0].title or "").lower()))
except YouTubeError as exc: except YouTubeError as exc:
log.warning("discovery: channel enrich failed (user %s): %s", user.id, exc) log.warning("discovery: channel enrich failed (user %s): %s", user.id, exc)
db.rollback() db.rollback()
@ -418,11 +420,18 @@ def reset_backfill(
channel = db.get(Channel, channel_id) channel = db.get(Channel, channel_id)
if channel is None: if channel is None:
raise HTTPException(status_code=404, detail="Unknown channel") raise HTTPException(status_code=404, detail="Unknown channel")
sub = _user_subscription(db, user, channel_id)
channel.backfill_done = False channel.backfill_done = False
channel.backfill_cursor = None channel.backfill_cursor = None
channel.recent_synced_at = None channel.recent_synced_at = None
sub.deep_requested = True # Re-arm the demand-driven deep scheduler for this channel. run_deep_backfill gates on an
# EXISTS over subscriptions with deep_requested=True, so flip it on EVERY subscriber rather
# than requiring the admin to be personally subscribed (they reset from the manager, not
# their own feed). No-op row count if the channel has no subscribers.
db.execute(
update(Subscription)
.where(Subscription.channel_id == channel_id)
.values(deep_requested=True)
)
db.commit() db.commit()
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT): with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
run_recent_backfill(db, [channel], max_channels=1) run_recent_backfill(db, [channel], max_channels=1)
@ -455,13 +464,9 @@ def subscribe(
).scalar_one_or_none() ).scalar_one_or_none()
if existing is not None: if existing is not None:
raise HTTPException(status_code=409, detail="Already subscribed to this channel.") raise HTTPException(status_code=409, detail="Already subscribed to this channel.")
try:
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_SUBSCRIBE), YouTubeClient(db, user) as yt: with quota.attribute(user.id, quota.QuotaAction.CHANNELS_SUBSCRIBE), YouTubeClient(db, user) as yt:
try:
yt_sub_id = yt.insert_subscription(channel_id) yt_sub_id = yt.insert_subscription(channel_id)
# Enrich a stub channel (title/thumbnail/subscriber count) so it shows up
# properly right away; no video pull here.
if channel.details_synced_at is None:
apply_channel_details(db, yt.get_channels([channel_id]))
except YouTubeError as exc: except YouTubeError as exc:
# Already following on YouTube but not recorded here (a desync — e.g. a stale import # Already following on YouTube but not recorded here (a desync — e.g. a stale import
# dropped the local row, so it resurfaced in discovery). That's not a failure: the end # dropped the local row, so it resurfaced in discovery). That's not a failure: the end
@ -474,6 +479,15 @@ def subscribe(
yt_sub_id = "" yt_sub_id = ""
else: else:
raise HTTPException(status_code=422, detail=f"YouTube couldn't subscribe you: {exc}") raise HTTPException(status_code=422, detail=f"YouTube couldn't subscribe you: {exc}")
# Enrich a stub channel (title/thumbnail/subscriber count) so it shows up properly right
# away, on BOTH the normal and the desync path — a resurfaced stub still needs metadata to
# render. Keep the client open (we're still inside the `with`); a metadata failure here
# must not block recording the subscription the user asked for.
if channel.details_synced_at is None:
try:
apply_channel_details(db, yt.get_channels([channel_id]))
except YouTubeError as exc:
log.warning("subscribe: stub enrich failed (%s): %s", channel_id, exc)
db.add( db.add(
Subscription( Subscription(
user_id=user.id, user_id=user.id,

View file

@ -265,7 +265,7 @@ def enqueue_download(
db, user.id, source_kind, source_ref, spec, profile_id, display_name db, user.id, source_kind, source_ref, spec, profile_id, display_name
) )
except quota.QuotaExceeded as e: except quota.QuotaExceeded as e:
raise HTTPException(status_code=422, detail=_quota_message(e)) raise HTTPException(status_code=422, detail=_quota_detail(e))
return _serialize_job(db, job) return _serialize_job(db, job)
@ -288,27 +288,16 @@ def enqueue_edit(
try: try:
job = service.enqueue_edit(db, user.id, source_job, spec, display_name) job = service.enqueue_edit(db, user.id, source_job, spec, display_name)
except quota.QuotaExceeded as e: except quota.QuotaExceeded as e:
raise HTTPException(status_code=422, detail=_quota_message(e)) raise HTTPException(status_code=422, detail=_quota_detail(e))
except service.EditError as e: except service.EditError as e:
raise HTTPException(status_code=400, detail=_edit_message(e)) raise HTTPException(status_code=400, detail={"code": "edit", "reason": e.reason})
return _serialize_job(db, job) return _serialize_job(db, job)
def _edit_message(e: service.EditError) -> str: def _quota_detail(e: quota.QuotaExceeded) -> dict:
if e.reason == "empty_edit": """Structured error the trilingual client localizes (reason key + numbers), instead of a
return "Choose a trim range or crop area first." pre-baked English string with a dot-decimal GB value. See lib/api.ts localizeDetail()."""
if e.reason == "source_not_ready": return {"code": "quota", "reason": e.reason, "limit": e.limit, "current": e.current}
return "The source download isn't ready to edit."
return "That edit couldn't be created."
def _quota_message(e: quota.QuotaExceeded) -> str:
if e.reason == "max_jobs":
return f"You've reached your download limit ({e.limit} items). Remove some first."
if e.reason == "max_bytes":
gb = e.limit / 1_073_741_824
return f"You've used your download storage quota ({gb:.1f} GB). Free up space first."
return "Download quota exceeded."
@router.get("") @router.get("")

View file

@ -17,7 +17,7 @@ from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect
from pydantic import BaseModel from pydantic import BaseModel
from sqlalchemy import and_, func, or_, select from sqlalchemy import and_, case, func, or_, select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.auth import require_human, resolved_user_id from app.auth import require_human, resolved_user_id
@ -220,25 +220,40 @@ def list_conversations(
with the latest message + my unread count, newest first. Lazily creates the welcome on the with the latest message + my unread count, newest first. Lazily creates the welcome on the
first open. User-message previews are ciphertext the client decrypts them.""" first open. User-message previews are ciphertext the client decrypts them."""
ensure_welcome(db, user) ensure_welcome(db, user)
msgs = ( # The conversation partner for a message, from this user's perspective (system → 0).
partner_expr = case(
(Message.kind == "system", SYSTEM_PARTNER_ID),
(Message.sender_id == user.id, Message.recipient_id),
else_=Message.sender_id,
)
mine = or_(Message.sender_id == user.id, Message.recipient_id == user.id)
# Latest message per conversation in ONE query (DISTINCT ON, Postgres) instead of loading the
# user's whole message history into Python — bounded by the number of conversations.
latest = (
db.execute( db.execute(
select(Message) select(Message)
.where(or_(Message.sender_id == user.id, Message.recipient_id == user.id)) .where(mine)
.order_by(Message.created_at.desc()) .distinct(partner_expr)
.order_by(partner_expr, Message.created_at.desc())
) )
.scalars() .scalars()
.all() .all()
) )
convs: dict[int, dict] = {} # Unread incoming, grouped per conversation.
for m in msgs: unread = dict(
db.execute(
select(partner_expr, func.count())
.where(mine, Message.recipient_id == user.id, Message.read_at.is_(None))
.group_by(partner_expr)
).all()
)
def _pid(m: Message) -> int:
if m.kind == "system": if m.kind == "system":
pid = SYSTEM_PARTNER_ID return SYSTEM_PARTNER_ID
else: return m.recipient_id if m.sender_id == user.id else m.sender_id
pid = m.recipient_id if m.sender_id == user.id else m.sender_id
c = convs.setdefault(pid, {"last": m, "unread": 0}) user_ids = [_pid(m) for m in latest if m.kind != "system"]
if m.recipient_id == user.id and m.read_at is None:
c["unread"] += 1
user_ids = [pid for pid in convs if pid != SYSTEM_PARTNER_ID]
partners = {} partners = {}
if user_ids: if user_ids:
partners = { partners = {
@ -246,7 +261,8 @@ def list_conversations(
for u in db.execute(select(User).where(User.id.in_(user_ids))).scalars().all() for u in db.execute(select(User).where(User.id.in_(user_ids))).scalars().all()
} }
items = [] items = []
for pid, c in convs.items(): for m in latest:
pid = _pid(m)
if pid == SYSTEM_PARTNER_ID: if pid == SYSTEM_PARTNER_ID:
partner = _system_partner() partner = _system_partner()
elif pid in partners: elif pid in partners:
@ -254,7 +270,7 @@ def list_conversations(
else: else:
continue continue
items.append( items.append(
{"partner": partner, "last_message": _serialize_msg(c["last"]), "unread": c["unread"]} {"partner": partner, "last_message": _serialize_msg(m), "unread": int(unread.get(pid, 0))}
) )
items.sort(key=lambda x: x["last_message"]["created_at"] or "", reverse=True) items.sort(key=lambda x: x["last_message"]["created_at"] or "", reverse=True)
return {"items": items} return {"items": items}
@ -275,9 +291,6 @@ def get_thread(
cond = and_(Message.recipient_id == user.id, Message.kind == "system") cond = and_(Message.recipient_id == user.id, Message.kind == "system")
else: else:
p = db.get(User, partner_id) p = db.get(User, partner_id)
if p is None:
raise HTTPException(status_code=404, detail="User not found")
partner = _serialize_user(p)
cond = and_( cond = and_(
Message.kind == "user", Message.kind == "user",
or_( or_(
@ -285,6 +298,13 @@ def get_thread(
and_(Message.sender_id == partner_id, Message.recipient_id == user.id), and_(Message.sender_id == partner_id, Message.recipient_id == user.id),
), ),
) )
# MB2: don't leak arbitrary user identities. Resolve a partner only if they're messageable
# OR we already share a thread (so a conversation with a since-suspended/deactivated user
# still opens). Otherwise the same 404 as a non-existent id → no enumeration oracle.
has_history = db.scalar(select(Message.id).where(cond).limit(1))
if p is None or (not is_messageable_user(p) and not has_history):
raise HTTPException(status_code=404, detail="User not found")
partner = _serialize_user(p)
msgs = ( msgs = (
db.execute(select(Message).where(cond).order_by(Message.created_at.asc())) db.execute(select(Message).where(cond).order_by(Message.created_at.asc()))
.scalars() .scalars()
@ -299,6 +319,15 @@ def get_thread(
changed = True changed = True
if changed: if changed:
db.commit() db.commit()
# MB3: tell this user's OTHER tabs/devices their unread dropped so the badge updates
# live instead of waiting for the next 20s poll. Carries the fresh total so the client
# needs no extra fetch.
n = db.scalar(
select(func.count())
.select_from(Message)
.where(Message.recipient_id == user.id, Message.read_at.is_(None))
)
manager.push(user.id, {"type": "unread", "count": int(n or 0)})
return {"partner": partner, "items": [_serialize_msg(m) for m in msgs]} return {"partner": partner, "items": [_serialize_msg(m) for m in msgs]}

View file

@ -92,19 +92,12 @@ def _remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool)
def _summary( def _summary(
db: Session,
pl: Playlist, pl: Playlist,
count: int, count: int,
total_duration: int = 0, total_duration: int = 0,
has_video: bool | None = None, has_video: bool | None = None,
cover: str | None = None,
) -> dict: ) -> dict:
cover = db.scalar(
select(Video.thumbnail_url)
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
.where(PlaylistItem.playlist_id == pl.id)
.order_by(PlaylistItem.position)
.limit(1)
)
out = { out = {
"id": pl.id, "id": pl.id,
"name": pl.name, "name": pl.name,
@ -121,18 +114,6 @@ def _summary(
return out return out
def _total_duration(db: Session, playlist_id: int) -> int:
return (
db.scalar(
select(func.coalesce(func.sum(Video.duration_seconds), 0))
.select_from(PlaylistItem)
.join(Video, Video.id == PlaylistItem.video_id)
.where(PlaylistItem.playlist_id == playlist_id)
)
or 0
)
@router.get("") @router.get("")
def list_playlists( def list_playlists(
contains: str | None = None, contains: str | None = None,
@ -183,13 +164,34 @@ def list_playlists(
.scalars() .scalars()
.all() .all()
) )
# Batch the cover thumbnails (first item per playlist by position) in ONE window query
# instead of a per-playlist SELECT inside _summary (the old N+1).
rn = func.row_number().over(
partition_by=PlaylistItem.playlist_id,
order_by=PlaylistItem.position,
).label("rn")
first_items = (
select(
PlaylistItem.playlist_id.label("pid"),
Video.thumbnail_url.label("thumb"),
rn,
)
.join(Video, Video.id == PlaylistItem.video_id)
.where(PlaylistItem.playlist_id.in_(ids))
.subquery()
)
covers = dict(
db.execute(
select(first_items.c.pid, first_items.c.thumb).where(first_items.c.rn == 1)
).all()
)
return [ return [
_summary( _summary(
db,
p, p,
counts.get(p.id, 0), counts.get(p.id, 0),
durations.get(p.id, 0), durations.get(p.id, 0),
(p.id in member) if contains else None, (p.id in member) if contains else None,
covers.get(p.id),
) )
for p in pls for p in pls
] ]
@ -207,7 +209,7 @@ def create_playlist(
pl = Playlist(user_id=user.id, name=name, position=_next_playlist_position(db, user.id)) pl = Playlist(user_id=user.id, name=name, position=_next_playlist_position(db, user.id))
db.add(pl) db.add(pl)
db.commit() db.commit()
return _summary(db, pl, 0) return _summary(pl, 0)
def _watch_later(db: Session, user: User) -> Playlist: def _watch_later(db: Session, user: User) -> Playlist:
@ -350,13 +352,23 @@ def push(
) )
try: try:
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH): with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
plan = plan_push(db, user, pl) # Read the live playlist ONCE (linked playlists only) and share it with both the
# plan and the push, halving read-quota and closing the second read→write TOCTOU
# window. A fresh export (no yt_playlist_id) reads nothing.
live = None
if pl.yt_playlist_id:
with YouTubeClient(db, user) as yt:
live = (
list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)),
yt.get_playlist_snippet(pl.yt_playlist_id),
)
plan = plan_push(db, user, pl, live=live)
if plan["units_estimate"] > quota.remaining_today(db): if plan["units_estimate"] > quota.remaining_today(db):
raise HTTPException( raise HTTPException(
status_code=429, status_code=429,
detail="Not enough YouTube quota left today for this sync.", detail="Not enough YouTube quota left today for this sync.",
) )
return push_playlist(db, user, pl) return push_playlist(db, user, pl, live=live)
except YouTubeError: except YouTubeError:
db.rollback() db.rollback()
raise HTTPException(status_code=422, detail="YouTube playlist sync failed.") raise HTTPException(status_code=422, detail="YouTube playlist sync failed.")
@ -405,10 +417,25 @@ def rename_playlist(
pl.name = name pl.name = name
recompute_dirty(db, pl) recompute_dirty(db, pl)
db.commit() db.commit()
count = db.scalar( # One aggregate for count + duration (outer join so an empty playlist still returns 0),
select(func.count()).where(PlaylistItem.playlist_id == pl.id) # plus a single cover lookup — was 3 separate round-trips.
count, total = db.execute(
select(
func.count(PlaylistItem.id),
func.coalesce(func.sum(Video.duration_seconds), 0),
) )
return _summary(db, pl, count or 0, _total_duration(db, pl.id)) .select_from(PlaylistItem)
.outerjoin(Video, Video.id == PlaylistItem.video_id)
.where(PlaylistItem.playlist_id == pl.id)
).one()
cover = db.scalar(
select(Video.thumbnail_url)
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
.where(PlaylistItem.playlist_id == pl.id)
.order_by(PlaylistItem.position)
.limit(1)
)
return _summary(pl, count or 0, total or 0, cover=cover)
@router.delete("/{playlist_id}") @router.delete("/{playlist_id}")

View file

@ -60,6 +60,11 @@ def _classify_shorts(db: Session, videos: list[Video]) -> None:
for v in videos: for v in videos:
if v.shorts_probed: if v.shorts_probed:
continue continue
if v.enriched_at is None:
# Enrichment failed (proxy blip) or YouTube omitted the row — duration is unknown,
# so DON'T finalise shorts_probed off a null duration; leave it for the scheduler
# (which only classifies enriched rows) to retry. Otherwise a real Short would leak.
continue
if ( if (
v.live_status != "none" v.live_status != "none"
or v.duration_seconds is None or v.duration_seconds is None
@ -164,7 +169,13 @@ def _ingest_candidates(
# 4) Confirm/deny Shorts (no quota) so none enter via search. # 4) Confirm/deny Shorts (no quota) so none enter via search.
_classify_shorts(db, videos) _classify_shorts(db, videos)
keep = {v.id for v in videos if not v.is_short and v.live_status not in LIVE_OR_UPCOMING} keep = {
v.id
for v in videos
if v.enriched_at is not None
and not v.is_short
and v.live_status not in LIVE_OR_UPCOMING
}
return [vid for vid in ids if vid in keep] return [vid for vid in ids if vid in keep]

View file

@ -217,11 +217,12 @@ def _reorder_moves(current: list[str], desired: list[str]) -> int:
return moves return moves
def plan_push(db: Session, user: User, pl: Playlist) -> dict: def plan_push(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
"""Compute (without writing) what pushing this local playlist to YouTube would do: """Compute (without writing) what pushing this local playlist to YouTube would do:
create vs update, how many inserts/deletes/reorders, the quota estimate, and whether create vs update, how many inserts/deletes/reorders, the quota estimate, and whether
YouTube has diverged (items present there that local will remove). Costs read units YouTube has diverged (items present there that local will remove). Costs read units
(1/page) for a linked playlist; nothing for a fresh export.""" (1/page) for a linked playlist; nothing for a fresh export. `live` = a pre-fetched
(current_items, snippet) tuple so the caller can share one read with push_playlist."""
desired = _local_video_ids(db, pl) desired = _local_video_ids(db, pl)
if not pl.yt_playlist_id: if not pl.yt_playlist_id:
ops = 1 + len(desired) # create the playlist + one insert per video ops = 1 + len(desired) # create the playlist + one insert per video
@ -234,6 +235,9 @@ def plan_push(db: Session, user: User, pl: Playlist) -> dict:
"yt_extra": 0, "yt_extra": 0,
"units_estimate": ops * WRITE_UNIT_COST, "units_estimate": ops * WRITE_UNIT_COST,
} }
if live is not None:
current, snippet = live
else:
with YouTubeClient(db, user) as yt: with YouTubeClient(db, user) as yt:
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)) current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
snippet = yt.get_playlist_snippet(pl.yt_playlist_id) snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
@ -259,10 +263,11 @@ def plan_push(db: Session, user: User, pl: Playlist) -> dict:
} }
def push_playlist(db: Session, user: User, pl: Playlist) -> dict: def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
"""Push a local playlist to YouTube: create it if needed, then reconcile membership """Push a local playlist to YouTube: create it if needed, then reconcile membership
and order so YouTube matches local (local wins). Marks the playlist clean on success. and order so YouTube matches local (local wins). Marks the playlist clean on success.
Caller must have checked the write scope and the quota budget.""" Caller must have checked the write scope and the quota budget. `live` = a pre-fetched
(current_items, snippet) tuple (from plan_push's read) to avoid a second live page-through."""
desired = _local_video_ids(db, pl) desired = _local_video_ids(db, pl)
inserted = deleted = reordered = created = 0 inserted = deleted = reordered = created = 0
failures: list[str] = [] failures: list[str] = []
@ -291,14 +296,17 @@ def push_playlist(db: Session, user: User, pl: Playlist) -> dict:
"yt_playlist_id": pl.yt_playlist_id, "yt_playlist_id": pl.yt_playlist_id,
} }
# Existing link: diff against the live YouTube state. # Existing link: diff against the live YouTube state (reuse plan_push's read if given).
if live is not None:
current, snippet = live
else:
snippet = yt.get_playlist_snippet(pl.yt_playlist_id) snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
if snippet is not None and (snippet.get("title") or "") != pl.name: if snippet is not None and (snippet.get("title") or "") != pl.name:
try: try:
yt.update_playlist_title(pl.yt_playlist_id, pl.name) yt.update_playlist_title(pl.yt_playlist_id, pl.name)
except YouTubeError: except YouTubeError:
failures.append("title") failures.append("title")
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current} item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
cur_vids = [it["video_id"] for it in current] cur_vids = [it["video_id"] for it in current]
desired_set = set(desired) desired_set = set(desired)

View file

@ -25,6 +25,7 @@ class ConfigSpec:
secret: bool = False secret: bool = False
min: int | None = None min: int | None = None
max: int | None = None max: int | None = None
allow_empty: bool = False # str keys: a stored "" is an explicit empty value, not "unset"
# Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it # Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it
@ -33,7 +34,7 @@ SPECS: tuple[ConfigSpec, ...] = (
# --- Email / SMTP (one logical group; the password is the only secret) --- # --- Email / SMTP (one logical group; the password is the only secret) ---
ConfigSpec("smtp_host", "str", "email", "smtp_host"), ConfigSpec("smtp_host", "str", "email", "smtp_host"),
ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535), ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535),
ConfigSpec("smtp_user", "str", "email", "smtp_user"), ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True),
ConfigSpec("smtp_from", "str", "email", "smtp_from"), ConfigSpec("smtp_from", "str", "email", "smtp_from"),
ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True), ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True),
# --- Quota (shared YouTube Data API daily budget) --- # --- Quota (shared YouTube Data API daily budget) ---
@ -59,7 +60,7 @@ SPECS: tuple[ConfigSpec, ...] = (
# --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) --- # --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) ---
ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True), ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True),
# Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key). # Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key).
ConfigSpec("youtube_api_proxy", "str", "youtube", "youtube_api_proxy"), ConfigSpec("youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True),
# --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the # --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the
# install wizard / Configuration page so no restart is needed when the creds change. --- # install wizard / Configuration page so no restart is needed when the creds change. ---
ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True), ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True),
@ -124,7 +125,12 @@ def get(db: Session, key: str):
"""Effective value: the DB override (typed) if set, else the env/config default.""" """Effective value: the DB override (typed) if set, else the env/config default."""
s = _BY_KEY[key] s = _BY_KEY[key]
raw = _raw_override(db, key) raw = _raw_override(db, key)
if raw is None or raw == "": if raw is None:
return _default(s)
if raw == "" and not s.allow_empty:
# For most keys an empty override means "fall back to the env default". Keys that opt
# into allow_empty (e.g. smtp_user, youtube_api_proxy) treat "" as a real value — so an
# admin can blank one to DISABLE it rather than being stuck with the env default.
return _default(s) return _default(s)
try: try:
return _coerce(s, raw) return _coerce(s, raw)
@ -197,6 +203,7 @@ def describe(db: Session) -> dict:
"min": s.min, "min": s.min,
"max": s.max, "max": s.max,
"is_set": is_set, # an override row exists "is_set": is_set, # an override row exists
"allow_empty": s.allow_empty, # if true, the UI stores "" rather than deleting the row
} }
if s.secret: if s.secret:
item["default_is_set"] = bool(_default(s)) item["default_is_set"] = bool(_default(s))