fix(deferred): close backend correctness/efficiency tails from the hygiene sweep
Verified each deferred per-subsystem item against current source, then fixed the
real ones (tradeoffs left as-is, see siftlode-ops/CODE-HYGIENE.md closeout):
- search BUG-3: don't finalise shorts_probed on un-enriched stubs (enrichment
failure/omitted rows) — restores the scheduler's enriched_at guard so a real
Short can't leak into the feed and never get reclassified.
- downloads GC: 4th pass reaps orphaned errored, fileless, unreferenced assets
(they carry no TTL, so the expiry passes never cleared them).
- downloads B6: quota/edit errors now return a structured {code,reason,limit}
the trilingual client localises, instead of hardcoded English + dot-decimal GB.
- channels BB1: reset-backfill re-arms deep sync on every subscriber (bulk update)
instead of 404ing when the admin isn't personally subscribed.
- channels BB2: enrich the stub on BOTH the normal and "already exists" desync
path (nest the insert try inside the client `with`).
- channels CB3: drop the redundant second _discovery_rows read (identity-mapped
rows are already enriched; re-sort in Python for the title tie-break).
- playlists PC1: read the live playlist once in push() and share it with plan_push
+ push_playlist (halves read-quota, closes the second TOCTOU window).
- playlists PC2/PC5: batch the cover thumbnails in one window query (was N+1);
rename combines count+duration into one aggregate + a single cover lookup.
- messages MB2: get_thread only resolves a messageable partner OR one we already
share a thread with (closes the user-enumeration oracle).
- messages MB3: push a live unread-count to the user's other tabs after mark-read.
- messages MC4: list_conversations uses DISTINCT ON + grouped COUNT instead of
loading the whole message history into memory.
- config AB3: per-spec allow_empty so smtp_user/youtube_api_proxy can be blanked
to disable them rather than snapping back to the env default.
This commit is contained in:
parent
182dddf5ed
commit
4e80e2b39b
8 changed files with 205 additions and 102 deletions
|
|
@ -1,12 +1,14 @@
|
||||||
"""Download retention garbage collector (a scheduler job).
|
"""Download retention garbage collector (a scheduler job).
|
||||||
|
|
||||||
Three passes each run:
|
Four passes each run:
|
||||||
1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
|
1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
|
||||||
once (gc_notified flag) so a download isn't silently deleted.
|
once (gc_notified flag) so a download isn't silently deleted.
|
||||||
2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
|
2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
|
||||||
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
|
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
|
||||||
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
|
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
|
||||||
ready assets until back under it.
|
ready assets until back under it.
|
||||||
|
4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
|
||||||
|
passes 1-3 skip them; benign row-count hygiene).
|
||||||
|
|
||||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||||
"""
|
"""
|
||||||
|
|
@ -58,7 +60,7 @@ def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
|
||||||
def run_download_gc(db: Session) -> dict:
|
def run_download_gc(db: Session) -> dict:
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
|
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
|
||||||
warned = expired = evicted = 0
|
warned = expired = evicted = reaped = 0
|
||||||
|
|
||||||
# 1. Pre-expiry warnings.
|
# 1. Pre-expiry warnings.
|
||||||
warn_before = now + timedelta(days=grace_days)
|
warn_before = now + timedelta(days=grace_days)
|
||||||
|
|
@ -114,6 +116,22 @@ def run_download_gc(db: Session) -> dict:
|
||||||
evicted += 1
|
evicted += 1
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
result = {"warned": warned, "expired": expired, "evicted": evicted}
|
# 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
|
||||||
|
# referenced errored row to pending on re-enqueue, so these only accumulate once every
|
||||||
|
# holder is gone; they carry no expires_at, so passes 1-3 never touch them.
|
||||||
|
orphans = db.execute(
|
||||||
|
select(MediaAsset).where(
|
||||||
|
MediaAsset.status == "error",
|
||||||
|
MediaAsset.ref_count == 0,
|
||||||
|
MediaAsset.rel_path.is_(None),
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
for asset in orphans:
|
||||||
|
db.delete(asset) # no file to unlink; FK SET NULL clears any stray job.asset_id
|
||||||
|
reaped += 1
|
||||||
|
if orphans:
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
result = {"warned": warned, "expired": expired, "evicted": evicted, "reaped": reaped}
|
||||||
log.info("download_gc %s", result)
|
log.info("download_gc %s", result)
|
||||||
return result
|
return result
|
||||||
|
|
|
||||||
|
|
@ -175,15 +175,17 @@ def discover_channels(
|
||||||
|
|
||||||
rows = _discovery_rows(db, user)
|
rows = _discovery_rows(db, user)
|
||||||
|
|
||||||
# Enrich stub channels' metadata (uses the API key — no write scope needed). Re-read
|
# Enrich stub channels' metadata (uses the API key — no write scope needed). The Channel
|
||||||
# the rows afterwards so the response carries the fresh subscriber counts/thumbnails.
|
# objects are identity-mapped, so apply_channel_details mutates the very rows we already hold
|
||||||
|
# (fresh subscriber counts/thumbnails carry through without a re-query). Only the title can
|
||||||
|
# change, which affects the tie-break order → re-sort in Python instead of re-hitting the DB.
|
||||||
need = [ch.id for ch, _, _ in rows if ch.details_synced_at is None][:DISCOVERY_ENRICH_CAP]
|
need = [ch.id for ch, _, _ in rows if ch.details_synced_at is None][:DISCOVERY_ENRICH_CAP]
|
||||||
if need:
|
if need:
|
||||||
try:
|
try:
|
||||||
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), YouTubeClient(db, user) as yt:
|
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), YouTubeClient(db, user) as yt:
|
||||||
apply_channel_details(db, yt.get_channels(need))
|
apply_channel_details(db, yt.get_channels(need))
|
||||||
db.commit()
|
db.commit()
|
||||||
rows = _discovery_rows(db, user)
|
rows = sorted(rows, key=lambda r: (-int(r[1]), (r[0].title or "").lower()))
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
log.warning("discovery: channel enrich failed (user %s): %s", user.id, exc)
|
log.warning("discovery: channel enrich failed (user %s): %s", user.id, exc)
|
||||||
db.rollback()
|
db.rollback()
|
||||||
|
|
@ -418,11 +420,18 @@ def reset_backfill(
|
||||||
channel = db.get(Channel, channel_id)
|
channel = db.get(Channel, channel_id)
|
||||||
if channel is None:
|
if channel is None:
|
||||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||||
sub = _user_subscription(db, user, channel_id)
|
|
||||||
channel.backfill_done = False
|
channel.backfill_done = False
|
||||||
channel.backfill_cursor = None
|
channel.backfill_cursor = None
|
||||||
channel.recent_synced_at = None
|
channel.recent_synced_at = None
|
||||||
sub.deep_requested = True
|
# Re-arm the demand-driven deep scheduler for this channel. run_deep_backfill gates on an
|
||||||
|
# EXISTS over subscriptions with deep_requested=True, so flip it on EVERY subscriber rather
|
||||||
|
# than requiring the admin to be personally subscribed (they reset from the manager, not
|
||||||
|
# their own feed). No-op row count if the channel has no subscribers.
|
||||||
|
db.execute(
|
||||||
|
update(Subscription)
|
||||||
|
.where(Subscription.channel_id == channel_id)
|
||||||
|
.values(deep_requested=True)
|
||||||
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
|
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
|
||||||
run_recent_backfill(db, [channel], max_channels=1)
|
run_recent_backfill(db, [channel], max_channels=1)
|
||||||
|
|
@ -455,13 +464,9 @@ def subscribe(
|
||||||
).scalar_one_or_none()
|
).scalar_one_or_none()
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
raise HTTPException(status_code=409, detail="Already subscribed to this channel.")
|
raise HTTPException(status_code=409, detail="Already subscribed to this channel.")
|
||||||
try:
|
|
||||||
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_SUBSCRIBE), YouTubeClient(db, user) as yt:
|
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_SUBSCRIBE), YouTubeClient(db, user) as yt:
|
||||||
|
try:
|
||||||
yt_sub_id = yt.insert_subscription(channel_id)
|
yt_sub_id = yt.insert_subscription(channel_id)
|
||||||
# Enrich a stub channel (title/thumbnail/subscriber count) so it shows up
|
|
||||||
# properly right away; no video pull here.
|
|
||||||
if channel.details_synced_at is None:
|
|
||||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
# Already following on YouTube but not recorded here (a desync — e.g. a stale import
|
# Already following on YouTube but not recorded here (a desync — e.g. a stale import
|
||||||
# dropped the local row, so it resurfaced in discovery). That's not a failure: the end
|
# dropped the local row, so it resurfaced in discovery). That's not a failure: the end
|
||||||
|
|
@ -474,6 +479,15 @@ def subscribe(
|
||||||
yt_sub_id = ""
|
yt_sub_id = ""
|
||||||
else:
|
else:
|
||||||
raise HTTPException(status_code=422, detail=f"YouTube couldn't subscribe you: {exc}")
|
raise HTTPException(status_code=422, detail=f"YouTube couldn't subscribe you: {exc}")
|
||||||
|
# Enrich a stub channel (title/thumbnail/subscriber count) so it shows up properly right
|
||||||
|
# away, on BOTH the normal and the desync path — a resurfaced stub still needs metadata to
|
||||||
|
# render. Keep the client open (we're still inside the `with`); a metadata failure here
|
||||||
|
# must not block recording the subscription the user asked for.
|
||||||
|
if channel.details_synced_at is None:
|
||||||
|
try:
|
||||||
|
apply_channel_details(db, yt.get_channels([channel_id]))
|
||||||
|
except YouTubeError as exc:
|
||||||
|
log.warning("subscribe: stub enrich failed (%s): %s", channel_id, exc)
|
||||||
db.add(
|
db.add(
|
||||||
Subscription(
|
Subscription(
|
||||||
user_id=user.id,
|
user_id=user.id,
|
||||||
|
|
|
||||||
|
|
@ -265,7 +265,7 @@ def enqueue_download(
|
||||||
db, user.id, source_kind, source_ref, spec, profile_id, display_name
|
db, user.id, source_kind, source_ref, spec, profile_id, display_name
|
||||||
)
|
)
|
||||||
except quota.QuotaExceeded as e:
|
except quota.QuotaExceeded as e:
|
||||||
raise HTTPException(status_code=422, detail=_quota_message(e))
|
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
||||||
return _serialize_job(db, job)
|
return _serialize_job(db, job)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -288,27 +288,16 @@ def enqueue_edit(
|
||||||
try:
|
try:
|
||||||
job = service.enqueue_edit(db, user.id, source_job, spec, display_name)
|
job = service.enqueue_edit(db, user.id, source_job, spec, display_name)
|
||||||
except quota.QuotaExceeded as e:
|
except quota.QuotaExceeded as e:
|
||||||
raise HTTPException(status_code=422, detail=_quota_message(e))
|
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
||||||
except service.EditError as e:
|
except service.EditError as e:
|
||||||
raise HTTPException(status_code=400, detail=_edit_message(e))
|
raise HTTPException(status_code=400, detail={"code": "edit", "reason": e.reason})
|
||||||
return _serialize_job(db, job)
|
return _serialize_job(db, job)
|
||||||
|
|
||||||
|
|
||||||
def _edit_message(e: service.EditError) -> str:
|
def _quota_detail(e: quota.QuotaExceeded) -> dict:
|
||||||
if e.reason == "empty_edit":
|
"""Structured error the trilingual client localizes (reason key + numbers), instead of a
|
||||||
return "Choose a trim range or crop area first."
|
pre-baked English string with a dot-decimal GB value. See lib/api.ts localizeDetail()."""
|
||||||
if e.reason == "source_not_ready":
|
return {"code": "quota", "reason": e.reason, "limit": e.limit, "current": e.current}
|
||||||
return "The source download isn't ready to edit."
|
|
||||||
return "That edit couldn't be created."
|
|
||||||
|
|
||||||
|
|
||||||
def _quota_message(e: quota.QuotaExceeded) -> str:
|
|
||||||
if e.reason == "max_jobs":
|
|
||||||
return f"You've reached your download limit ({e.limit} items). Remove some first."
|
|
||||||
if e.reason == "max_bytes":
|
|
||||||
gb = e.limit / 1_073_741_824
|
|
||||||
return f"You've used your download storage quota ({gb:.1f} GB). Free up space first."
|
|
||||||
return "Download quota exceeded."
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@ from datetime import datetime, timezone
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect
|
from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from sqlalchemy import and_, func, or_, select
|
from sqlalchemy import and_, case, func, or_, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.auth import require_human, resolved_user_id
|
from app.auth import require_human, resolved_user_id
|
||||||
|
|
@ -220,25 +220,40 @@ def list_conversations(
|
||||||
with the latest message + my unread count, newest first. Lazily creates the welcome on the
|
with the latest message + my unread count, newest first. Lazily creates the welcome on the
|
||||||
first open. User-message previews are ciphertext — the client decrypts them."""
|
first open. User-message previews are ciphertext — the client decrypts them."""
|
||||||
ensure_welcome(db, user)
|
ensure_welcome(db, user)
|
||||||
msgs = (
|
# The conversation partner for a message, from this user's perspective (system → 0).
|
||||||
|
partner_expr = case(
|
||||||
|
(Message.kind == "system", SYSTEM_PARTNER_ID),
|
||||||
|
(Message.sender_id == user.id, Message.recipient_id),
|
||||||
|
else_=Message.sender_id,
|
||||||
|
)
|
||||||
|
mine = or_(Message.sender_id == user.id, Message.recipient_id == user.id)
|
||||||
|
# Latest message per conversation in ONE query (DISTINCT ON, Postgres) instead of loading the
|
||||||
|
# user's whole message history into Python — bounded by the number of conversations.
|
||||||
|
latest = (
|
||||||
db.execute(
|
db.execute(
|
||||||
select(Message)
|
select(Message)
|
||||||
.where(or_(Message.sender_id == user.id, Message.recipient_id == user.id))
|
.where(mine)
|
||||||
.order_by(Message.created_at.desc())
|
.distinct(partner_expr)
|
||||||
|
.order_by(partner_expr, Message.created_at.desc())
|
||||||
)
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
convs: dict[int, dict] = {}
|
# Unread incoming, grouped per conversation.
|
||||||
for m in msgs:
|
unread = dict(
|
||||||
|
db.execute(
|
||||||
|
select(partner_expr, func.count())
|
||||||
|
.where(mine, Message.recipient_id == user.id, Message.read_at.is_(None))
|
||||||
|
.group_by(partner_expr)
|
||||||
|
).all()
|
||||||
|
)
|
||||||
|
|
||||||
|
def _pid(m: Message) -> int:
|
||||||
if m.kind == "system":
|
if m.kind == "system":
|
||||||
pid = SYSTEM_PARTNER_ID
|
return SYSTEM_PARTNER_ID
|
||||||
else:
|
return m.recipient_id if m.sender_id == user.id else m.sender_id
|
||||||
pid = m.recipient_id if m.sender_id == user.id else m.sender_id
|
|
||||||
c = convs.setdefault(pid, {"last": m, "unread": 0})
|
user_ids = [_pid(m) for m in latest if m.kind != "system"]
|
||||||
if m.recipient_id == user.id and m.read_at is None:
|
|
||||||
c["unread"] += 1
|
|
||||||
user_ids = [pid for pid in convs if pid != SYSTEM_PARTNER_ID]
|
|
||||||
partners = {}
|
partners = {}
|
||||||
if user_ids:
|
if user_ids:
|
||||||
partners = {
|
partners = {
|
||||||
|
|
@ -246,7 +261,8 @@ def list_conversations(
|
||||||
for u in db.execute(select(User).where(User.id.in_(user_ids))).scalars().all()
|
for u in db.execute(select(User).where(User.id.in_(user_ids))).scalars().all()
|
||||||
}
|
}
|
||||||
items = []
|
items = []
|
||||||
for pid, c in convs.items():
|
for m in latest:
|
||||||
|
pid = _pid(m)
|
||||||
if pid == SYSTEM_PARTNER_ID:
|
if pid == SYSTEM_PARTNER_ID:
|
||||||
partner = _system_partner()
|
partner = _system_partner()
|
||||||
elif pid in partners:
|
elif pid in partners:
|
||||||
|
|
@ -254,7 +270,7 @@ def list_conversations(
|
||||||
else:
|
else:
|
||||||
continue
|
continue
|
||||||
items.append(
|
items.append(
|
||||||
{"partner": partner, "last_message": _serialize_msg(c["last"]), "unread": c["unread"]}
|
{"partner": partner, "last_message": _serialize_msg(m), "unread": int(unread.get(pid, 0))}
|
||||||
)
|
)
|
||||||
items.sort(key=lambda x: x["last_message"]["created_at"] or "", reverse=True)
|
items.sort(key=lambda x: x["last_message"]["created_at"] or "", reverse=True)
|
||||||
return {"items": items}
|
return {"items": items}
|
||||||
|
|
@ -275,9 +291,6 @@ def get_thread(
|
||||||
cond = and_(Message.recipient_id == user.id, Message.kind == "system")
|
cond = and_(Message.recipient_id == user.id, Message.kind == "system")
|
||||||
else:
|
else:
|
||||||
p = db.get(User, partner_id)
|
p = db.get(User, partner_id)
|
||||||
if p is None:
|
|
||||||
raise HTTPException(status_code=404, detail="User not found")
|
|
||||||
partner = _serialize_user(p)
|
|
||||||
cond = and_(
|
cond = and_(
|
||||||
Message.kind == "user",
|
Message.kind == "user",
|
||||||
or_(
|
or_(
|
||||||
|
|
@ -285,6 +298,13 @@ def get_thread(
|
||||||
and_(Message.sender_id == partner_id, Message.recipient_id == user.id),
|
and_(Message.sender_id == partner_id, Message.recipient_id == user.id),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
# MB2: don't leak arbitrary user identities. Resolve a partner only if they're messageable
|
||||||
|
# OR we already share a thread (so a conversation with a since-suspended/deactivated user
|
||||||
|
# still opens). Otherwise the same 404 as a non-existent id → no enumeration oracle.
|
||||||
|
has_history = db.scalar(select(Message.id).where(cond).limit(1))
|
||||||
|
if p is None or (not is_messageable_user(p) and not has_history):
|
||||||
|
raise HTTPException(status_code=404, detail="User not found")
|
||||||
|
partner = _serialize_user(p)
|
||||||
msgs = (
|
msgs = (
|
||||||
db.execute(select(Message).where(cond).order_by(Message.created_at.asc()))
|
db.execute(select(Message).where(cond).order_by(Message.created_at.asc()))
|
||||||
.scalars()
|
.scalars()
|
||||||
|
|
@ -299,6 +319,15 @@ def get_thread(
|
||||||
changed = True
|
changed = True
|
||||||
if changed:
|
if changed:
|
||||||
db.commit()
|
db.commit()
|
||||||
|
# MB3: tell this user's OTHER tabs/devices their unread dropped so the badge updates
|
||||||
|
# live instead of waiting for the next 20s poll. Carries the fresh total so the client
|
||||||
|
# needs no extra fetch.
|
||||||
|
n = db.scalar(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(Message)
|
||||||
|
.where(Message.recipient_id == user.id, Message.read_at.is_(None))
|
||||||
|
)
|
||||||
|
manager.push(user.id, {"type": "unread", "count": int(n or 0)})
|
||||||
return {"partner": partner, "items": [_serialize_msg(m) for m in msgs]}
|
return {"partner": partner, "items": [_serialize_msg(m) for m in msgs]}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -92,19 +92,12 @@ def _remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool)
|
||||||
|
|
||||||
|
|
||||||
def _summary(
|
def _summary(
|
||||||
db: Session,
|
|
||||||
pl: Playlist,
|
pl: Playlist,
|
||||||
count: int,
|
count: int,
|
||||||
total_duration: int = 0,
|
total_duration: int = 0,
|
||||||
has_video: bool | None = None,
|
has_video: bool | None = None,
|
||||||
|
cover: str | None = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
cover = db.scalar(
|
|
||||||
select(Video.thumbnail_url)
|
|
||||||
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
|
|
||||||
.where(PlaylistItem.playlist_id == pl.id)
|
|
||||||
.order_by(PlaylistItem.position)
|
|
||||||
.limit(1)
|
|
||||||
)
|
|
||||||
out = {
|
out = {
|
||||||
"id": pl.id,
|
"id": pl.id,
|
||||||
"name": pl.name,
|
"name": pl.name,
|
||||||
|
|
@ -121,18 +114,6 @@ def _summary(
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def _total_duration(db: Session, playlist_id: int) -> int:
|
|
||||||
return (
|
|
||||||
db.scalar(
|
|
||||||
select(func.coalesce(func.sum(Video.duration_seconds), 0))
|
|
||||||
.select_from(PlaylistItem)
|
|
||||||
.join(Video, Video.id == PlaylistItem.video_id)
|
|
||||||
.where(PlaylistItem.playlist_id == playlist_id)
|
|
||||||
)
|
|
||||||
or 0
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
def list_playlists(
|
def list_playlists(
|
||||||
contains: str | None = None,
|
contains: str | None = None,
|
||||||
|
|
@ -183,13 +164,34 @@ def list_playlists(
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
|
# Batch the cover thumbnails (first item per playlist by position) in ONE window query
|
||||||
|
# instead of a per-playlist SELECT inside _summary (the old N+1).
|
||||||
|
rn = func.row_number().over(
|
||||||
|
partition_by=PlaylistItem.playlist_id,
|
||||||
|
order_by=PlaylistItem.position,
|
||||||
|
).label("rn")
|
||||||
|
first_items = (
|
||||||
|
select(
|
||||||
|
PlaylistItem.playlist_id.label("pid"),
|
||||||
|
Video.thumbnail_url.label("thumb"),
|
||||||
|
rn,
|
||||||
|
)
|
||||||
|
.join(Video, Video.id == PlaylistItem.video_id)
|
||||||
|
.where(PlaylistItem.playlist_id.in_(ids))
|
||||||
|
.subquery()
|
||||||
|
)
|
||||||
|
covers = dict(
|
||||||
|
db.execute(
|
||||||
|
select(first_items.c.pid, first_items.c.thumb).where(first_items.c.rn == 1)
|
||||||
|
).all()
|
||||||
|
)
|
||||||
return [
|
return [
|
||||||
_summary(
|
_summary(
|
||||||
db,
|
|
||||||
p,
|
p,
|
||||||
counts.get(p.id, 0),
|
counts.get(p.id, 0),
|
||||||
durations.get(p.id, 0),
|
durations.get(p.id, 0),
|
||||||
(p.id in member) if contains else None,
|
(p.id in member) if contains else None,
|
||||||
|
covers.get(p.id),
|
||||||
)
|
)
|
||||||
for p in pls
|
for p in pls
|
||||||
]
|
]
|
||||||
|
|
@ -207,7 +209,7 @@ def create_playlist(
|
||||||
pl = Playlist(user_id=user.id, name=name, position=_next_playlist_position(db, user.id))
|
pl = Playlist(user_id=user.id, name=name, position=_next_playlist_position(db, user.id))
|
||||||
db.add(pl)
|
db.add(pl)
|
||||||
db.commit()
|
db.commit()
|
||||||
return _summary(db, pl, 0)
|
return _summary(pl, 0)
|
||||||
|
|
||||||
|
|
||||||
def _watch_later(db: Session, user: User) -> Playlist:
|
def _watch_later(db: Session, user: User) -> Playlist:
|
||||||
|
|
@ -350,13 +352,23 @@ def push(
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||||
plan = plan_push(db, user, pl)
|
# Read the live playlist ONCE (linked playlists only) and share it with both the
|
||||||
|
# plan and the push, halving read-quota and closing the second read→write TOCTOU
|
||||||
|
# window. A fresh export (no yt_playlist_id) reads nothing.
|
||||||
|
live = None
|
||||||
|
if pl.yt_playlist_id:
|
||||||
|
with YouTubeClient(db, user) as yt:
|
||||||
|
live = (
|
||||||
|
list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)),
|
||||||
|
yt.get_playlist_snippet(pl.yt_playlist_id),
|
||||||
|
)
|
||||||
|
plan = plan_push(db, user, pl, live=live)
|
||||||
if plan["units_estimate"] > quota.remaining_today(db):
|
if plan["units_estimate"] > quota.remaining_today(db):
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=429,
|
status_code=429,
|
||||||
detail="Not enough YouTube quota left today for this sync.",
|
detail="Not enough YouTube quota left today for this sync.",
|
||||||
)
|
)
|
||||||
return push_playlist(db, user, pl)
|
return push_playlist(db, user, pl, live=live)
|
||||||
except YouTubeError:
|
except YouTubeError:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise HTTPException(status_code=422, detail="YouTube playlist sync failed.")
|
raise HTTPException(status_code=422, detail="YouTube playlist sync failed.")
|
||||||
|
|
@ -405,10 +417,25 @@ def rename_playlist(
|
||||||
pl.name = name
|
pl.name = name
|
||||||
recompute_dirty(db, pl)
|
recompute_dirty(db, pl)
|
||||||
db.commit()
|
db.commit()
|
||||||
count = db.scalar(
|
# One aggregate for count + duration (outer join so an empty playlist still returns 0),
|
||||||
select(func.count()).where(PlaylistItem.playlist_id == pl.id)
|
# plus a single cover lookup — was 3 separate round-trips.
|
||||||
|
count, total = db.execute(
|
||||||
|
select(
|
||||||
|
func.count(PlaylistItem.id),
|
||||||
|
func.coalesce(func.sum(Video.duration_seconds), 0),
|
||||||
)
|
)
|
||||||
return _summary(db, pl, count or 0, _total_duration(db, pl.id))
|
.select_from(PlaylistItem)
|
||||||
|
.outerjoin(Video, Video.id == PlaylistItem.video_id)
|
||||||
|
.where(PlaylistItem.playlist_id == pl.id)
|
||||||
|
).one()
|
||||||
|
cover = db.scalar(
|
||||||
|
select(Video.thumbnail_url)
|
||||||
|
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
|
||||||
|
.where(PlaylistItem.playlist_id == pl.id)
|
||||||
|
.order_by(PlaylistItem.position)
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
return _summary(pl, count or 0, total or 0, cover=cover)
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/{playlist_id}")
|
@router.delete("/{playlist_id}")
|
||||||
|
|
|
||||||
|
|
@ -60,6 +60,11 @@ def _classify_shorts(db: Session, videos: list[Video]) -> None:
|
||||||
for v in videos:
|
for v in videos:
|
||||||
if v.shorts_probed:
|
if v.shorts_probed:
|
||||||
continue
|
continue
|
||||||
|
if v.enriched_at is None:
|
||||||
|
# Enrichment failed (proxy blip) or YouTube omitted the row — duration is unknown,
|
||||||
|
# so DON'T finalise shorts_probed off a null duration; leave it for the scheduler
|
||||||
|
# (which only classifies enriched rows) to retry. Otherwise a real Short would leak.
|
||||||
|
continue
|
||||||
if (
|
if (
|
||||||
v.live_status != "none"
|
v.live_status != "none"
|
||||||
or v.duration_seconds is None
|
or v.duration_seconds is None
|
||||||
|
|
@ -164,7 +169,13 @@ def _ingest_candidates(
|
||||||
# 4) Confirm/deny Shorts (no quota) so none enter via search.
|
# 4) Confirm/deny Shorts (no quota) so none enter via search.
|
||||||
_classify_shorts(db, videos)
|
_classify_shorts(db, videos)
|
||||||
|
|
||||||
keep = {v.id for v in videos if not v.is_short and v.live_status not in LIVE_OR_UPCOMING}
|
keep = {
|
||||||
|
v.id
|
||||||
|
for v in videos
|
||||||
|
if v.enriched_at is not None
|
||||||
|
and not v.is_short
|
||||||
|
and v.live_status not in LIVE_OR_UPCOMING
|
||||||
|
}
|
||||||
return [vid for vid in ids if vid in keep]
|
return [vid for vid in ids if vid in keep]
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -217,11 +217,12 @@ def _reorder_moves(current: list[str], desired: list[str]) -> int:
|
||||||
return moves
|
return moves
|
||||||
|
|
||||||
|
|
||||||
def plan_push(db: Session, user: User, pl: Playlist) -> dict:
|
def plan_push(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||||
"""Compute (without writing) what pushing this local playlist to YouTube would do:
|
"""Compute (without writing) what pushing this local playlist to YouTube would do:
|
||||||
create vs update, how many inserts/deletes/reorders, the quota estimate, and whether
|
create vs update, how many inserts/deletes/reorders, the quota estimate, and whether
|
||||||
YouTube has diverged (items present there that local will remove). Costs read units
|
YouTube has diverged (items present there that local will remove). Costs read units
|
||||||
(1/page) for a linked playlist; nothing for a fresh export."""
|
(1/page) for a linked playlist; nothing for a fresh export. `live` = a pre-fetched
|
||||||
|
(current_items, snippet) tuple so the caller can share one read with push_playlist."""
|
||||||
desired = _local_video_ids(db, pl)
|
desired = _local_video_ids(db, pl)
|
||||||
if not pl.yt_playlist_id:
|
if not pl.yt_playlist_id:
|
||||||
ops = 1 + len(desired) # create the playlist + one insert per video
|
ops = 1 + len(desired) # create the playlist + one insert per video
|
||||||
|
|
@ -234,6 +235,9 @@ def plan_push(db: Session, user: User, pl: Playlist) -> dict:
|
||||||
"yt_extra": 0,
|
"yt_extra": 0,
|
||||||
"units_estimate": ops * WRITE_UNIT_COST,
|
"units_estimate": ops * WRITE_UNIT_COST,
|
||||||
}
|
}
|
||||||
|
if live is not None:
|
||||||
|
current, snippet = live
|
||||||
|
else:
|
||||||
with YouTubeClient(db, user) as yt:
|
with YouTubeClient(db, user) as yt:
|
||||||
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
||||||
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
||||||
|
|
@ -259,10 +263,11 @@ def plan_push(db: Session, user: User, pl: Playlist) -> dict:
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def push_playlist(db: Session, user: User, pl: Playlist) -> dict:
|
def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||||
"""Push a local playlist to YouTube: create it if needed, then reconcile membership
|
"""Push a local playlist to YouTube: create it if needed, then reconcile membership
|
||||||
and order so YouTube matches local (local wins). Marks the playlist clean on success.
|
and order so YouTube matches local (local wins). Marks the playlist clean on success.
|
||||||
Caller must have checked the write scope and the quota budget."""
|
Caller must have checked the write scope and the quota budget. `live` = a pre-fetched
|
||||||
|
(current_items, snippet) tuple (from plan_push's read) to avoid a second live page-through."""
|
||||||
desired = _local_video_ids(db, pl)
|
desired = _local_video_ids(db, pl)
|
||||||
inserted = deleted = reordered = created = 0
|
inserted = deleted = reordered = created = 0
|
||||||
failures: list[str] = []
|
failures: list[str] = []
|
||||||
|
|
@ -291,14 +296,17 @@ def push_playlist(db: Session, user: User, pl: Playlist) -> dict:
|
||||||
"yt_playlist_id": pl.yt_playlist_id,
|
"yt_playlist_id": pl.yt_playlist_id,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Existing link: diff against the live YouTube state.
|
# Existing link: diff against the live YouTube state (reuse plan_push's read if given).
|
||||||
|
if live is not None:
|
||||||
|
current, snippet = live
|
||||||
|
else:
|
||||||
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
||||||
|
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
||||||
if snippet is not None and (snippet.get("title") or "") != pl.name:
|
if snippet is not None and (snippet.get("title") or "") != pl.name:
|
||||||
try:
|
try:
|
||||||
yt.update_playlist_title(pl.yt_playlist_id, pl.name)
|
yt.update_playlist_title(pl.yt_playlist_id, pl.name)
|
||||||
except YouTubeError:
|
except YouTubeError:
|
||||||
failures.append("title")
|
failures.append("title")
|
||||||
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
|
||||||
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
|
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
|
||||||
cur_vids = [it["video_id"] for it in current]
|
cur_vids = [it["video_id"] for it in current]
|
||||||
desired_set = set(desired)
|
desired_set = set(desired)
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ class ConfigSpec:
|
||||||
secret: bool = False
|
secret: bool = False
|
||||||
min: int | None = None
|
min: int | None = None
|
||||||
max: int | None = None
|
max: int | None = None
|
||||||
|
allow_empty: bool = False # str keys: a stored "" is an explicit empty value, not "unset"
|
||||||
|
|
||||||
|
|
||||||
# Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it
|
# Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it
|
||||||
|
|
@ -33,7 +34,7 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||||
# --- Email / SMTP (one logical group; the password is the only secret) ---
|
# --- Email / SMTP (one logical group; the password is the only secret) ---
|
||||||
ConfigSpec("smtp_host", "str", "email", "smtp_host"),
|
ConfigSpec("smtp_host", "str", "email", "smtp_host"),
|
||||||
ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535),
|
ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535),
|
||||||
ConfigSpec("smtp_user", "str", "email", "smtp_user"),
|
ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True),
|
||||||
ConfigSpec("smtp_from", "str", "email", "smtp_from"),
|
ConfigSpec("smtp_from", "str", "email", "smtp_from"),
|
||||||
ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True),
|
ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True),
|
||||||
# --- Quota (shared YouTube Data API daily budget) ---
|
# --- Quota (shared YouTube Data API daily budget) ---
|
||||||
|
|
@ -59,7 +60,7 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||||
# --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) ---
|
# --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) ---
|
||||||
ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True),
|
ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True),
|
||||||
# Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key).
|
# Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key).
|
||||||
ConfigSpec("youtube_api_proxy", "str", "youtube", "youtube_api_proxy"),
|
ConfigSpec("youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True),
|
||||||
# --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the
|
# --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the
|
||||||
# install wizard / Configuration page so no restart is needed when the creds change. ---
|
# install wizard / Configuration page so no restart is needed when the creds change. ---
|
||||||
ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True),
|
ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True),
|
||||||
|
|
@ -124,7 +125,12 @@ def get(db: Session, key: str):
|
||||||
"""Effective value: the DB override (typed) if set, else the env/config default."""
|
"""Effective value: the DB override (typed) if set, else the env/config default."""
|
||||||
s = _BY_KEY[key]
|
s = _BY_KEY[key]
|
||||||
raw = _raw_override(db, key)
|
raw = _raw_override(db, key)
|
||||||
if raw is None or raw == "":
|
if raw is None:
|
||||||
|
return _default(s)
|
||||||
|
if raw == "" and not s.allow_empty:
|
||||||
|
# For most keys an empty override means "fall back to the env default". Keys that opt
|
||||||
|
# into allow_empty (e.g. smtp_user, youtube_api_proxy) treat "" as a real value — so an
|
||||||
|
# admin can blank one to DISABLE it rather than being stuck with the env default.
|
||||||
return _default(s)
|
return _default(s)
|
||||||
try:
|
try:
|
||||||
return _coerce(s, raw)
|
return _coerce(s, raw)
|
||||||
|
|
@ -197,6 +203,7 @@ def describe(db: Session) -> dict:
|
||||||
"min": s.min,
|
"min": s.min,
|
||||||
"max": s.max,
|
"max": s.max,
|
||||||
"is_set": is_set, # an override row exists
|
"is_set": is_set, # an override row exists
|
||||||
|
"allow_empty": s.allow_empty, # if true, the UI stores "" rather than deleting the row
|
||||||
}
|
}
|
||||||
if s.secret:
|
if s.secret:
|
||||||
item["default_is_set"] = bool(_default(s))
|
item["default_is_set"] = bool(_default(s))
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue