feat(downloads): M3 — per-user quota + retention GC
- downloads/quota.py: per-user limits (DownloadQuota row or sysconfig defaults; unlimited bypass), footprint = distinct ready-asset bytes the user holds, check_enqueue (max_jobs + max_bytes hard at enqueue), at_concurrency_limit (worker-side max_concurrent), usage snapshot - downloads/gc.py: run_download_gc — pre-expiry warning (once, gc_notified flag), TTL deletion (files + row; FK SET NULL orphans holders' jobs -> 'expired'), LRU eviction over a total-cache cap; notifies holders on each event - migration 0038: media_assets.gc_notified - config/sysconfig: download_total_max_bytes (0=unlimited) in the downloads group - service.enqueue enforces quota; worker claim skips users at their concurrency limit - scheduler: download_gc job registered (default 360 min, admin-tunable) Verified on localdev: footprint accounting, max_jobs block, pre-expiry warning (notifies all holders), TTL deletion (asset+files gone, dirs pruned, jobs orphaned, holders notified).
This commit is contained in:
parent
7f9847c2c2
commit
7148335c09
9 changed files with 325 additions and 15 deletions
|
|
@ -28,7 +28,7 @@ from sqlalchemy import text
|
|||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import formats, service, storage
|
||||
from app.downloads import formats, quota, service, storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
|
||||
log = logging.getLogger("siftlode.worker")
|
||||
|
|
@ -83,24 +83,37 @@ def _recover_orphans() -> None:
|
|||
|
||||
|
||||
def _claim_job() -> int | None:
|
||||
"""Claim the oldest queued job whose user is under their per-user concurrency limit.
|
||||
|
||||
Fetches a window of locked candidates (FOR UPDATE SKIP LOCKED) and picks the first eligible
|
||||
one, so one user flooding the queue can't starve others beyond their max_concurrent."""
|
||||
with SessionLocal() as db:
|
||||
row = db.execute(
|
||||
rows = db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE download_jobs SET status='running', phase='starting', updated_at=now()
|
||||
WHERE id = (
|
||||
SELECT id FROM download_jobs
|
||||
WHERE status='queued'
|
||||
ORDER BY queue_pos, id
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id
|
||||
SELECT id, user_id FROM download_jobs
|
||||
WHERE status='queued'
|
||||
ORDER BY queue_pos, id
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 50
|
||||
"""
|
||||
)
|
||||
).fetchone()
|
||||
).fetchall()
|
||||
chosen = next(
|
||||
(jid for jid, uid in rows if not quota.at_concurrency_limit(db, uid)), None
|
||||
)
|
||||
if chosen is None:
|
||||
db.commit() # release the row locks; nothing eligible right now
|
||||
return None
|
||||
db.execute(
|
||||
text(
|
||||
"UPDATE download_jobs SET status='running', phase='starting', updated_at=now() "
|
||||
"WHERE id = :id"
|
||||
),
|
||||
{"id": chosen},
|
||||
)
|
||||
db.commit()
|
||||
return row[0] if row else None
|
||||
return chosen
|
||||
|
||||
|
||||
def _claim_asset(asset_id: int) -> bool:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue