feat(downloads): M3 — per-user quota + retention GC

- downloads/quota.py: per-user limits (DownloadQuota row or sysconfig defaults; unlimited
  bypass), footprint = distinct ready-asset bytes the user holds, check_enqueue (max_jobs +
  max_bytes hard at enqueue), at_concurrency_limit (worker-side max_concurrent), usage snapshot
- downloads/gc.py: run_download_gc — pre-expiry warning (once, gc_notified flag), TTL deletion
  (files + row; FK SET NULL orphans holders' jobs -> 'expired'), LRU eviction over a total-cache
  cap; notifies holders on each event
- migration 0038: media_assets.gc_notified
- config/sysconfig: download_total_max_bytes (0=unlimited) in the downloads group
- service.enqueue enforces quota; worker claim skips users at their concurrency limit
- scheduler: download_gc job registered (default 360 min, admin-tunable)

Verified on localdev: footprint accounting, max_jobs block, pre-expiry warning (notifies all
holders), TTL deletion (asset+files gone, dirs pruned, jobs orphaned, holders notified).
This commit is contained in:
npeter83 2026-07-03 00:26:40 +02:00
parent 7f9847c2c2
commit 7148335c09
9 changed files with 325 additions and 15 deletions

View file

@ -28,7 +28,7 @@ from sqlalchemy import text
from app.config import settings
from app.db import SessionLocal
from app.downloads import formats, service, storage
from app.downloads import formats, quota, service, storage
from app.models import DownloadJob, MediaAsset
log = logging.getLogger("siftlode.worker")
@ -83,24 +83,37 @@ def _recover_orphans() -> None:
def _claim_job() -> int | None:
"""Claim the oldest queued job whose user is under their per-user concurrency limit.
Fetches a window of locked candidates (FOR UPDATE SKIP LOCKED) and picks the first eligible
one, so one user flooding the queue can't starve others beyond their max_concurrent."""
with SessionLocal() as db:
row = db.execute(
rows = db.execute(
text(
"""
UPDATE download_jobs SET status='running', phase='starting', updated_at=now()
WHERE id = (
SELECT id FROM download_jobs
WHERE status='queued'
ORDER BY queue_pos, id
FOR UPDATE SKIP LOCKED
LIMIT 1
)
RETURNING id
SELECT id, user_id FROM download_jobs
WHERE status='queued'
ORDER BY queue_pos, id
FOR UPDATE SKIP LOCKED
LIMIT 50
"""
)
).fetchone()
).fetchall()
chosen = next(
(jid for jid, uid in rows if not quota.at_concurrency_limit(db, uid)), None
)
if chosen is None:
db.commit() # release the row locks; nothing eligible right now
return None
db.execute(
text(
"UPDATE download_jobs SET status='running', phase='starting', updated_at=now() "
"WHERE id = :id"
),
{"id": chosen},
)
db.commit()
return row[0] if row else None
return chosen
def _claim_asset(asset_id: int) -> bool: