siftlode/backend/app/downloads/gc.py

"""Download retention garbage collector (a scheduler job).

Four passes each run:
  1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
     once (gc_notified flag) so a download isn't silently deleted.
  2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
     holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
  3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
     ready assets until back under it.
  4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
     passes 1-3 skip them; benign row-count hygiene).

Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
"""
import logging
from datetime import datetime, timedelta, timezone

from sqlalchemy import func, select
from sqlalchemy.orm import Session

from app import sysconfig
from app.config import settings
from app.downloads import storage
from app.models import DownloadJob, MediaAsset
from app.notifications import create_notification

log = logging.getLogger("siftlode.downloads.gc")


def _holders(db: Session, asset_id: int) -> list[int]:
    return list(
        db.execute(
            select(DownloadJob.user_id)
            .where(DownloadJob.asset_id == asset_id)
            .distinct()
        ).scalars()
    )


def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None:
    title = asset.title or "download"
    for uid in _holders(db, asset.id):
        create_notification(
            db,
            user_id=uid,
            type=ntype,
            title=title,  # English fallback; the client renders a translated message from data
            data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref},
            commit=False,
        )


def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
    if asset.rel_path:
        storage.delete_asset_files(settings.download_root, asset.rel_path)
    _notify_holders(db, asset, ntype)
    db.delete(asset)  # jobs.asset_id -> NULL via FK ON DELETE SET NULL


def run_download_gc(db: Session) -> dict:
    now = datetime.now(timezone.utc)
    grace_days = sysconfig.get_int(db, "download_gc_grace_days")
    warned = expired = evicted = reaped = 0

    # 1. Pre-expiry warnings.
    warn_before = now + timedelta(days=grace_days)
    soon = db.execute(
        select(MediaAsset).where(
            MediaAsset.status == "ready",
            MediaAsset.gc_notified.is_(False),
            MediaAsset.expires_at.is_not(None),
            MediaAsset.expires_at > now,
            MediaAsset.expires_at <= warn_before,
        )
    ).scalars().all()
    for asset in soon:
        _notify_holders(db, asset, "download_expiring")
        asset.gc_notified = True
        warned += 1
    if soon:
        db.commit()

    # 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
    dead = db.execute(
        select(MediaAsset).where(
            MediaAsset.status.in_(("ready", "error")),
            MediaAsset.expires_at.is_not(None),
            MediaAsset.expires_at <= now,
        )
    ).scalars().all()
    for asset in dead:
        _delete_asset(db, asset, "download_expired")
        expired += 1
    if dead:
        db.commit()

    # 3. LRU eviction over the total cache cap.
    cap = sysconfig.get_int(db, "download_total_max_bytes")
    if cap > 0:
        total = db.execute(
            select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
                MediaAsset.status == "ready"
            )
        ).scalar() or 0
        if total > cap:
            candidates = db.execute(
                select(MediaAsset)
                .where(MediaAsset.status == "ready")
                .order_by(MediaAsset.last_access_at.asc().nulls_first())
            ).scalars().all()
            for asset in candidates:
                if total <= cap:
                    break
                total -= asset.size_bytes or 0
                _delete_asset(db, asset, "download_evicted")
                evicted += 1
            db.commit()

    # 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
    #    referenced errored row to pending on re-enqueue, so these only accumulate once every
    #    holder is gone; they carry no expires_at, so passes 1-3 never touch them.
    orphans = db.execute(
        select(MediaAsset).where(
            MediaAsset.status == "error",
            MediaAsset.ref_count == 0,
            MediaAsset.rel_path.is_(None),
        )
    ).scalars().all()
    for asset in orphans:
        db.delete(asset)  # no file to unlink; FK SET NULL clears any stray job.asset_id
        reaped += 1
    if orphans:
        db.commit()

    result = {"warned": warned, "expired": expired, "evicted": evicted, "reaped": reaped}
    log.info("download_gc %s", result)
    return result