feat(downloads): M3 — per-user quota + retention GC
- downloads/quota.py: per-user limits (DownloadQuota row or sysconfig defaults; unlimited bypass), footprint = distinct ready-asset bytes the user holds, check_enqueue (max_jobs + max_bytes hard at enqueue), at_concurrency_limit (worker-side max_concurrent), usage snapshot - downloads/gc.py: run_download_gc — pre-expiry warning (once, gc_notified flag), TTL deletion (files + row; FK SET NULL orphans holders' jobs -> 'expired'), LRU eviction over a total-cache cap; notifies holders on each event - migration 0038: media_assets.gc_notified - config/sysconfig: download_total_max_bytes (0=unlimited) in the downloads group - service.enqueue enforces quota; worker claim skips users at their concurrency limit - scheduler: download_gc job registered (default 360 min, admin-tunable) Verified on localdev: footprint accounting, max_jobs block, pre-expiry warning (notifies all holders), TTL deletion (asset+files gone, dirs pruned, jobs orphaned, holders notified).
This commit is contained in:
parent
7f9847c2c2
commit
7148335c09
9 changed files with 325 additions and 15 deletions
119
backend/app/downloads/gc.py
Normal file
119
backend/app/downloads/gc.py
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
"""Download retention garbage collector (a scheduler job).
|
||||
|
||||
Three passes each run:
|
||||
1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
|
||||
once (gc_notified flag) so a download isn't silently deleted.
|
||||
2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
|
||||
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
|
||||
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
|
||||
ready assets until back under it.
|
||||
|
||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
|
||||
log = logging.getLogger("siftlode.downloads.gc")
|
||||
|
||||
|
||||
def _holders(db: Session, asset_id: int) -> list[int]:
|
||||
return list(
|
||||
db.execute(
|
||||
select(DownloadJob.user_id)
|
||||
.where(DownloadJob.asset_id == asset_id)
|
||||
.distinct()
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None:
|
||||
title = asset.title or "download"
|
||||
for uid in _holders(db, asset.id):
|
||||
create_notification(
|
||||
db,
|
||||
user_id=uid,
|
||||
type=ntype,
|
||||
title=title, # English fallback; the client renders a translated message from data
|
||||
data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref},
|
||||
commit=False,
|
||||
)
|
||||
|
||||
|
||||
def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||
_notify_holders(db, asset, ntype)
|
||||
db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL
|
||||
|
||||
|
||||
def run_download_gc(db: Session) -> dict:
|
||||
now = datetime.now(timezone.utc)
|
||||
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
|
||||
warned = expired = evicted = 0
|
||||
|
||||
# 1. Pre-expiry warnings.
|
||||
warn_before = now + timedelta(days=grace_days)
|
||||
soon = db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status == "ready",
|
||||
MediaAsset.gc_notified.is_(False),
|
||||
MediaAsset.expires_at.is_not(None),
|
||||
MediaAsset.expires_at > now,
|
||||
MediaAsset.expires_at <= warn_before,
|
||||
)
|
||||
).scalars().all()
|
||||
for asset in soon:
|
||||
_notify_holders(db, asset, "download_expiring")
|
||||
asset.gc_notified = True
|
||||
warned += 1
|
||||
if soon:
|
||||
db.commit()
|
||||
|
||||
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
|
||||
dead = db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status.in_(("ready", "error")),
|
||||
MediaAsset.expires_at.is_not(None),
|
||||
MediaAsset.expires_at <= now,
|
||||
)
|
||||
).scalars().all()
|
||||
for asset in dead:
|
||||
_delete_asset(db, asset, "download_expired")
|
||||
expired += 1
|
||||
if dead:
|
||||
db.commit()
|
||||
|
||||
# 3. LRU eviction over the total cache cap.
|
||||
cap = sysconfig.get_int(db, "download_total_max_bytes")
|
||||
if cap > 0:
|
||||
total = db.execute(
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
||||
MediaAsset.status == "ready"
|
||||
)
|
||||
).scalar() or 0
|
||||
if total > cap:
|
||||
candidates = db.execute(
|
||||
select(MediaAsset)
|
||||
.where(MediaAsset.status == "ready")
|
||||
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
||||
).scalars().all()
|
||||
for asset in candidates:
|
||||
if total <= cap:
|
||||
break
|
||||
total -= asset.size_bytes or 0
|
||||
_delete_asset(db, asset, "download_evicted")
|
||||
evicted += 1
|
||||
db.commit()
|
||||
|
||||
result = {"warned": warned, "expired": expired, "evicted": evicted}
|
||||
log.info("download_gc %s", result)
|
||||
return result
|
||||
129
backend/app/downloads/quota.py
Normal file
129
backend/app/downloads/quota.py
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
"""Per-user download limits + footprint accounting.
|
||||
|
||||
A user's *footprint* is the total size of the ready assets their library references (cache hits
|
||||
count — it's "how much disk you're responsible for", not "how much you personally downloaded").
|
||||
Limits come from a per-user DownloadQuota row if the admin set one, else the sysconfig defaults.
|
||||
`unlimited` (e.g. the admin's own account) bypasses the byte cap.
|
||||
|
||||
Enforcement split: max_jobs + max_bytes are checked at enqueue (they bound holdings); per-user
|
||||
max_concurrent is enforced by the worker at claim time (it bounds simultaneous downloads).
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
|
||||
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
|
||||
# (error, canceled) don't count.
|
||||
_HOLDING = ("queued", "running", "paused", "done")
|
||||
_RUNNING = ("queued", "running")
|
||||
|
||||
|
||||
@dataclass
|
||||
class QuotaLimits:
|
||||
max_bytes: int
|
||||
max_concurrent: int
|
||||
max_jobs: int
|
||||
unlimited: bool
|
||||
|
||||
|
||||
class QuotaExceeded(Exception):
|
||||
"""Raised by check_enqueue; `reason` is one of max_jobs|max_bytes (i18n key on the client)."""
|
||||
|
||||
def __init__(self, reason: str, limit: int, current: int):
|
||||
self.reason = reason
|
||||
self.limit = limit
|
||||
self.current = current
|
||||
super().__init__(f"download quota exceeded: {reason} ({current}/{limit})")
|
||||
|
||||
|
||||
def resolve(db: Session, user_id: int) -> QuotaLimits:
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
if row is not None:
|
||||
return QuotaLimits(row.max_bytes, row.max_concurrent, row.max_jobs, row.unlimited)
|
||||
return QuotaLimits(
|
||||
max_bytes=sysconfig.get_int(db, "download_default_max_bytes"),
|
||||
max_concurrent=sysconfig.get_int(db, "download_default_max_concurrent"),
|
||||
max_jobs=sysconfig.get_int(db, "download_default_max_jobs"),
|
||||
unlimited=False,
|
||||
)
|
||||
|
||||
|
||||
def footprint(db: Session, user_id: int) -> int:
|
||||
"""Total bytes of the distinct ready assets this user's active jobs reference."""
|
||||
held_assets = (
|
||||
select(DownloadJob.asset_id)
|
||||
.where(
|
||||
DownloadJob.user_id == user_id,
|
||||
DownloadJob.status.in_(_HOLDING),
|
||||
DownloadJob.asset_id.is_not(None),
|
||||
)
|
||||
.distinct()
|
||||
)
|
||||
return (
|
||||
db.execute(
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
||||
MediaAsset.id.in_(held_assets), MediaAsset.status == "ready"
|
||||
)
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
def _count(db: Session, user_id: int, states) -> int:
|
||||
return (
|
||||
db.execute(
|
||||
select(func.count())
|
||||
.select_from(DownloadJob)
|
||||
.where(DownloadJob.user_id == user_id, DownloadJob.status.in_(states))
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
def active_jobs(db: Session, user_id: int) -> int:
|
||||
return _count(db, user_id, _HOLDING)
|
||||
|
||||
|
||||
def running_jobs(db: Session, user_id: int) -> int:
|
||||
return _count(db, user_id, _RUNNING)
|
||||
|
||||
|
||||
def at_concurrency_limit(db: Session, user_id: int) -> bool:
|
||||
"""Worker-side gate: is this user already downloading their max? Unlimited users never are."""
|
||||
lim = resolve(db, user_id)
|
||||
if lim.unlimited:
|
||||
return False
|
||||
running = _count(db, user_id, ("running",))
|
||||
return running >= lim.max_concurrent
|
||||
|
||||
|
||||
def check_enqueue(db: Session, user_id: int) -> None:
|
||||
"""Raise QuotaExceeded if the user can't take another job (holdings caps). Concurrency is
|
||||
enforced later by the worker, so a big queue is allowed; it just drains max_concurrent-wide."""
|
||||
lim = resolve(db, user_id)
|
||||
if lim.unlimited:
|
||||
return
|
||||
n = active_jobs(db, user_id)
|
||||
if n >= lim.max_jobs:
|
||||
raise QuotaExceeded("max_jobs", lim.max_jobs, n)
|
||||
used = footprint(db, user_id)
|
||||
if used >= lim.max_bytes:
|
||||
raise QuotaExceeded("max_bytes", lim.max_bytes, used)
|
||||
|
||||
|
||||
def usage(db: Session, user_id: int) -> dict:
|
||||
"""Compact usage snapshot for the UI (footprint + counts vs limits)."""
|
||||
lim = resolve(db, user_id)
|
||||
return {
|
||||
"footprint_bytes": footprint(db, user_id),
|
||||
"active_jobs": active_jobs(db, user_id),
|
||||
"running_jobs": running_jobs(db, user_id),
|
||||
"max_bytes": lim.max_bytes,
|
||||
"max_concurrent": lim.max_concurrent,
|
||||
"max_jobs": lim.max_jobs,
|
||||
"unlimited": lim.unlimited,
|
||||
}
|
||||
|
|
@ -12,7 +12,7 @@ from sqlalchemy import func, select
|
|||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.downloads import formats
|
||||
from app.downloads import formats, quota
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
|
|
@ -67,7 +67,10 @@ def enqueue(
|
|||
profile_id: int | None = None,
|
||||
display_name: str | None = None,
|
||||
) -> DownloadJob:
|
||||
"""Queue a download for `user_id`. Returns the DownloadJob (already `done` on a cache hit)."""
|
||||
"""Queue a download for `user_id`. Returns the DownloadJob (already `done` on a cache hit).
|
||||
|
||||
Raises quota.QuotaExceeded if the user is at their job-count or footprint cap."""
|
||||
quota.check_enqueue(db, user_id)
|
||||
spec = formats.normalize(spec)
|
||||
sig = formats.format_sig(spec)
|
||||
asset = get_or_create_asset(db, source_kind, source_ref, sig)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue