7148335c09
- downloads/quota.py: per-user limits (DownloadQuota row or sysconfig defaults; unlimited bypass), footprint = distinct ready-asset bytes the user holds, check_enqueue (max_jobs + max_bytes hard at enqueue), at_concurrency_limit (worker-side max_concurrent), usage snapshot - downloads/gc.py: run_download_gc — pre-expiry warning (once, gc_notified flag), TTL deletion (files + row; FK SET NULL orphans holders' jobs -> 'expired'), LRU eviction over a total-cache cap; notifies holders on each event - migration 0038: media_assets.gc_notified - config/sysconfig: download_total_max_bytes (0=unlimited) in the downloads group - service.enqueue enforces quota; worker claim skips users at their concurrency limit - scheduler: download_gc job registered (default 360 min, admin-tunable) Verified on localdev: footprint accounting, max_jobs block, pre-expiry warning (notifies all holders), TTL deletion (asset+files gone, dirs pruned, jobs orphaned, holders notified).
100 lines
3.4 KiB
Python
100 lines
3.4 KiB
Python
"""Enqueue + cache resolution for the download center (the per-user layer over MediaAsset).
|
|
|
|
`enqueue` is the single entry point used by the API: it resolves (or creates) the shared
|
|
MediaAsset for the requested source+format, links a per-user DownloadJob to it, and — on a
|
|
cache hit (asset already ready) — marks the job done instantly with no re-download. The worker
|
|
handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and
|
|
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
|
"""
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.downloads import formats, quota
|
|
from app.models import DownloadJob, MediaAsset
|
|
|
|
|
|
def source_url(source_kind: str, source_ref: str) -> str:
|
|
"""The URL yt-dlp should fetch. YouTube stores the 11-char id; ad-hoc stores the URL."""
|
|
if source_kind == "youtube":
|
|
return f"https://www.youtube.com/watch?v={source_ref}"
|
|
return source_ref
|
|
|
|
|
|
def get_or_create_asset(
|
|
db: Session, source_kind: str, source_ref: str, format_sig: str
|
|
) -> MediaAsset:
|
|
"""Fetch the cache row for this (source, format) or create a fresh `pending` one.
|
|
|
|
Races on the unique cache key are resolved by re-selecting after an IntegrityError, so two
|
|
simultaneous enqueues of the same video+format converge on one shared asset."""
|
|
stmt = select(MediaAsset).where(
|
|
MediaAsset.source_kind == source_kind,
|
|
MediaAsset.source_ref == source_ref,
|
|
MediaAsset.format_sig == format_sig,
|
|
)
|
|
asset = db.execute(stmt).scalar_one_or_none()
|
|
if asset is not None:
|
|
return asset
|
|
asset = MediaAsset(
|
|
source_kind=source_kind,
|
|
source_ref=source_ref,
|
|
format_sig=format_sig,
|
|
status="pending",
|
|
ref_count=0,
|
|
)
|
|
db.add(asset)
|
|
try:
|
|
db.flush()
|
|
except IntegrityError:
|
|
db.rollback()
|
|
asset = db.execute(stmt).scalar_one()
|
|
return asset
|
|
|
|
|
|
def _next_queue_pos(db: Session) -> int:
|
|
return (db.execute(select(func.coalesce(func.max(DownloadJob.queue_pos), 0))).scalar() or 0) + 1
|
|
|
|
|
|
def enqueue(
|
|
db: Session,
|
|
user_id: int,
|
|
source_kind: str,
|
|
source_ref: str,
|
|
spec: dict,
|
|
profile_id: int | None = None,
|
|
display_name: str | None = None,
|
|
) -> DownloadJob:
|
|
"""Queue a download for `user_id`. Returns the DownloadJob (already `done` on a cache hit).
|
|
|
|
Raises quota.QuotaExceeded if the user is at their job-count or footprint cap."""
|
|
quota.check_enqueue(db, user_id)
|
|
spec = formats.normalize(spec)
|
|
sig = formats.format_sig(spec)
|
|
asset = get_or_create_asset(db, source_kind, source_ref, sig)
|
|
|
|
job = DownloadJob(
|
|
user_id=user_id,
|
|
asset_id=asset.id,
|
|
source_kind=source_kind,
|
|
source_ref=source_ref,
|
|
profile_id=profile_id,
|
|
profile_snapshot=spec,
|
|
display_name=display_name or (asset.title if asset.status == "ready" else None),
|
|
status="queued",
|
|
queue_pos=_next_queue_pos(db),
|
|
)
|
|
db.add(job)
|
|
asset.ref_count = (asset.ref_count or 0) + 1
|
|
|
|
# Cache hit: the file already exists — no worker round-trip needed.
|
|
if asset.status == "ready":
|
|
job.status = "done"
|
|
job.progress = 100
|
|
asset.last_access_at = datetime.now(timezone.utc)
|
|
|
|
db.commit()
|
|
db.refresh(job)
|
|
return job
|