7148335c09
- downloads/quota.py: per-user limits (DownloadQuota row or sysconfig defaults; unlimited bypass), footprint = distinct ready-asset bytes the user holds, check_enqueue (max_jobs + max_bytes hard at enqueue), at_concurrency_limit (worker-side max_concurrent), usage snapshot - downloads/gc.py: run_download_gc — pre-expiry warning (once, gc_notified flag), TTL deletion (files + row; FK SET NULL orphans holders' jobs -> 'expired'), LRU eviction over a total-cache cap; notifies holders on each event - migration 0038: media_assets.gc_notified - config/sysconfig: download_total_max_bytes (0=unlimited) in the downloads group - service.enqueue enforces quota; worker claim skips users at their concurrency limit - scheduler: download_gc job registered (default 360 min, admin-tunable) Verified on localdev: footprint accounting, max_jobs block, pre-expiry warning (notifies all holders), TTL deletion (asset+files gone, dirs pruned, jobs orphaned, holders notified).
119 lines
4.1 KiB
Python
119 lines
4.1 KiB
Python
"""Download retention garbage collector (a scheduler job).
|
|
|
|
Three passes each run:
|
|
1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
|
|
once (gc_notified flag) so a download isn't silently deleted.
|
|
2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
|
|
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
|
|
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
|
|
ready assets until back under it.
|
|
|
|
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
|
"""
|
|
import logging
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import sysconfig
|
|
from app.config import settings
|
|
from app.downloads import storage
|
|
from app.models import DownloadJob, MediaAsset
|
|
from app.notifications import create_notification
|
|
|
|
log = logging.getLogger("siftlode.downloads.gc")
|
|
|
|
|
|
def _holders(db: Session, asset_id: int) -> list[int]:
|
|
return list(
|
|
db.execute(
|
|
select(DownloadJob.user_id)
|
|
.where(DownloadJob.asset_id == asset_id)
|
|
.distinct()
|
|
).scalars()
|
|
)
|
|
|
|
|
|
def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None:
|
|
title = asset.title or "download"
|
|
for uid in _holders(db, asset.id):
|
|
create_notification(
|
|
db,
|
|
user_id=uid,
|
|
type=ntype,
|
|
title=title, # English fallback; the client renders a translated message from data
|
|
data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref},
|
|
commit=False,
|
|
)
|
|
|
|
|
|
def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
|
|
if asset.rel_path:
|
|
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
|
_notify_holders(db, asset, ntype)
|
|
db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL
|
|
|
|
|
|
def run_download_gc(db: Session) -> dict:
|
|
now = datetime.now(timezone.utc)
|
|
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
|
|
warned = expired = evicted = 0
|
|
|
|
# 1. Pre-expiry warnings.
|
|
warn_before = now + timedelta(days=grace_days)
|
|
soon = db.execute(
|
|
select(MediaAsset).where(
|
|
MediaAsset.status == "ready",
|
|
MediaAsset.gc_notified.is_(False),
|
|
MediaAsset.expires_at.is_not(None),
|
|
MediaAsset.expires_at > now,
|
|
MediaAsset.expires_at <= warn_before,
|
|
)
|
|
).scalars().all()
|
|
for asset in soon:
|
|
_notify_holders(db, asset, "download_expiring")
|
|
asset.gc_notified = True
|
|
warned += 1
|
|
if soon:
|
|
db.commit()
|
|
|
|
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
|
|
dead = db.execute(
|
|
select(MediaAsset).where(
|
|
MediaAsset.status.in_(("ready", "error")),
|
|
MediaAsset.expires_at.is_not(None),
|
|
MediaAsset.expires_at <= now,
|
|
)
|
|
).scalars().all()
|
|
for asset in dead:
|
|
_delete_asset(db, asset, "download_expired")
|
|
expired += 1
|
|
if dead:
|
|
db.commit()
|
|
|
|
# 3. LRU eviction over the total cache cap.
|
|
cap = sysconfig.get_int(db, "download_total_max_bytes")
|
|
if cap > 0:
|
|
total = db.execute(
|
|
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
|
MediaAsset.status == "ready"
|
|
)
|
|
).scalar() or 0
|
|
if total > cap:
|
|
candidates = db.execute(
|
|
select(MediaAsset)
|
|
.where(MediaAsset.status == "ready")
|
|
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
|
).scalars().all()
|
|
for asset in candidates:
|
|
if total <= cap:
|
|
break
|
|
total -= asset.size_bytes or 0
|
|
_delete_asset(db, asset, "download_evicted")
|
|
evicted += 1
|
|
db.commit()
|
|
|
|
result = {"warned": warned, "expired": expired, "evicted": evicted}
|
|
log.info("download_gc %s", result)
|
|
return result
|