feat(downloads): M3 — per-user quota + retention GC

- downloads/quota.py: per-user limits (DownloadQuota row or sysconfig defaults; unlimited
  bypass), footprint = distinct ready-asset bytes the user holds, check_enqueue (max_jobs +
  max_bytes hard at enqueue), at_concurrency_limit (worker-side max_concurrent), usage snapshot
- downloads/gc.py: run_download_gc — pre-expiry warning (once, gc_notified flag), TTL deletion
  (files + row; FK SET NULL orphans holders' jobs -> 'expired'), LRU eviction over a total-cache
  cap; notifies holders on each event
- migration 0038: media_assets.gc_notified
- config/sysconfig: download_total_max_bytes (0=unlimited) in the downloads group
- service.enqueue enforces quota; worker claim skips users at their concurrency limit
- scheduler: download_gc job registered (default 360 min, admin-tunable)

Verified on localdev: footprint accounting, max_jobs block, pre-expiry warning (notifies all
holders), TTL deletion (asset+files gone, dirs pruned, jobs orphaned, holders notified).
This commit is contained in:
npeter83 2026-07-03 00:26:40 +02:00
parent 7f9847c2c2
commit 7148335c09
9 changed files with 325 additions and 15 deletions

119
backend/app/downloads/gc.py Normal file
View file

@ -0,0 +1,119 @@
"""Download retention garbage collector (a scheduler job).
Three passes each run:
1. Pre-expiry warning for ready assets expiring within the grace window, notify each holder
once (gc_notified flag) so a download isn't silently deleted.
2. Expiry deletion delete assets past their TTL (files + row). The FK SET NULL nulls the
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
3. LRU eviction if a total cache cap is set and exceeded, delete least-recently-accessed
ready assets until back under it.
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work no YouTube quota.
"""
import logging
from datetime import datetime, timedelta, timezone
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app import sysconfig
from app.config import settings
from app.downloads import storage
from app.models import DownloadJob, MediaAsset
from app.notifications import create_notification
log = logging.getLogger("siftlode.downloads.gc")
def _holders(db: Session, asset_id: int) -> list[int]:
return list(
db.execute(
select(DownloadJob.user_id)
.where(DownloadJob.asset_id == asset_id)
.distinct()
).scalars()
)
def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None:
title = asset.title or "download"
for uid in _holders(db, asset.id):
create_notification(
db,
user_id=uid,
type=ntype,
title=title, # English fallback; the client renders a translated message from data
data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref},
commit=False,
)
def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
if asset.rel_path:
storage.delete_asset_files(settings.download_root, asset.rel_path)
_notify_holders(db, asset, ntype)
db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL
def run_download_gc(db: Session) -> dict:
now = datetime.now(timezone.utc)
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
warned = expired = evicted = 0
# 1. Pre-expiry warnings.
warn_before = now + timedelta(days=grace_days)
soon = db.execute(
select(MediaAsset).where(
MediaAsset.status == "ready",
MediaAsset.gc_notified.is_(False),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at > now,
MediaAsset.expires_at <= warn_before,
)
).scalars().all()
for asset in soon:
_notify_holders(db, asset, "download_expiring")
asset.gc_notified = True
warned += 1
if soon:
db.commit()
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
dead = db.execute(
select(MediaAsset).where(
MediaAsset.status.in_(("ready", "error")),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at <= now,
)
).scalars().all()
for asset in dead:
_delete_asset(db, asset, "download_expired")
expired += 1
if dead:
db.commit()
# 3. LRU eviction over the total cache cap.
cap = sysconfig.get_int(db, "download_total_max_bytes")
if cap > 0:
total = db.execute(
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
MediaAsset.status == "ready"
)
).scalar() or 0
if total > cap:
candidates = db.execute(
select(MediaAsset)
.where(MediaAsset.status == "ready")
.order_by(MediaAsset.last_access_at.asc().nulls_first())
).scalars().all()
for asset in candidates:
if total <= cap:
break
total -= asset.size_bytes or 0
_delete_asset(db, asset, "download_evicted")
evicted += 1
db.commit()
result = {"warned": warned, "expired": expired, "evicted": evicted}
log.info("download_gc %s", result)
return result

View file

@ -0,0 +1,129 @@
"""Per-user download limits + footprint accounting.
A user's *footprint* is the total size of the ready assets their library references (cache hits
count it's "how much disk you're responsible for", not "how much you personally downloaded").
Limits come from a per-user DownloadQuota row if the admin set one, else the sysconfig defaults.
`unlimited` (e.g. the admin's own account) bypasses the byte cap.
Enforcement split: max_jobs + max_bytes are checked at enqueue (they bound holdings); per-user
max_concurrent is enforced by the worker at claim time (it bounds simultaneous downloads).
"""
from dataclasses import dataclass
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app import sysconfig
from app.models import DownloadJob, DownloadQuota, MediaAsset
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
# (error, canceled) don't count.
_HOLDING = ("queued", "running", "paused", "done")
_RUNNING = ("queued", "running")
@dataclass
class QuotaLimits:
max_bytes: int
max_concurrent: int
max_jobs: int
unlimited: bool
class QuotaExceeded(Exception):
"""Raised by check_enqueue; `reason` is one of max_jobs|max_bytes (i18n key on the client)."""
def __init__(self, reason: str, limit: int, current: int):
self.reason = reason
self.limit = limit
self.current = current
super().__init__(f"download quota exceeded: {reason} ({current}/{limit})")
def resolve(db: Session, user_id: int) -> QuotaLimits:
row = db.get(DownloadQuota, user_id)
if row is not None:
return QuotaLimits(row.max_bytes, row.max_concurrent, row.max_jobs, row.unlimited)
return QuotaLimits(
max_bytes=sysconfig.get_int(db, "download_default_max_bytes"),
max_concurrent=sysconfig.get_int(db, "download_default_max_concurrent"),
max_jobs=sysconfig.get_int(db, "download_default_max_jobs"),
unlimited=False,
)
def footprint(db: Session, user_id: int) -> int:
"""Total bytes of the distinct ready assets this user's active jobs reference."""
held_assets = (
select(DownloadJob.asset_id)
.where(
DownloadJob.user_id == user_id,
DownloadJob.status.in_(_HOLDING),
DownloadJob.asset_id.is_not(None),
)
.distinct()
)
return (
db.execute(
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
MediaAsset.id.in_(held_assets), MediaAsset.status == "ready"
)
).scalar()
or 0
)
def _count(db: Session, user_id: int, states) -> int:
return (
db.execute(
select(func.count())
.select_from(DownloadJob)
.where(DownloadJob.user_id == user_id, DownloadJob.status.in_(states))
).scalar()
or 0
)
def active_jobs(db: Session, user_id: int) -> int:
return _count(db, user_id, _HOLDING)
def running_jobs(db: Session, user_id: int) -> int:
return _count(db, user_id, _RUNNING)
def at_concurrency_limit(db: Session, user_id: int) -> bool:
"""Worker-side gate: is this user already downloading their max? Unlimited users never are."""
lim = resolve(db, user_id)
if lim.unlimited:
return False
running = _count(db, user_id, ("running",))
return running >= lim.max_concurrent
def check_enqueue(db: Session, user_id: int) -> None:
"""Raise QuotaExceeded if the user can't take another job (holdings caps). Concurrency is
enforced later by the worker, so a big queue is allowed; it just drains max_concurrent-wide."""
lim = resolve(db, user_id)
if lim.unlimited:
return
n = active_jobs(db, user_id)
if n >= lim.max_jobs:
raise QuotaExceeded("max_jobs", lim.max_jobs, n)
used = footprint(db, user_id)
if used >= lim.max_bytes:
raise QuotaExceeded("max_bytes", lim.max_bytes, used)
def usage(db: Session, user_id: int) -> dict:
"""Compact usage snapshot for the UI (footprint + counts vs limits)."""
lim = resolve(db, user_id)
return {
"footprint_bytes": footprint(db, user_id),
"active_jobs": active_jobs(db, user_id),
"running_jobs": running_jobs(db, user_id),
"max_bytes": lim.max_bytes,
"max_concurrent": lim.max_concurrent,
"max_jobs": lim.max_jobs,
"unlimited": lim.unlimited,
}

View file

@ -12,7 +12,7 @@ from sqlalchemy import func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.downloads import formats
from app.downloads import formats, quota
from app.models import DownloadJob, MediaAsset
@ -67,7 +67,10 @@ def enqueue(
profile_id: int | None = None,
display_name: str | None = None,
) -> DownloadJob:
"""Queue a download for `user_id`. Returns the DownloadJob (already `done` on a cache hit)."""
"""Queue a download for `user_id`. Returns the DownloadJob (already `done` on a cache hit).
Raises quota.QuotaExceeded if the user is at their job-count or footprint cap."""
quota.check_enqueue(db, user_id)
spec = formats.normalize(spec)
sig = formats.format_sig(spec)
asset = get_or_create_asset(db, source_kind, source_ref, sig)