feat(m5d): demand-driven deep backfill + per-user ETA

Per-user opt-in to full-history (deep) backfill so a new user's unique
channels no longer trigger a big shared-quota burst.

- migration 0007: Subscription.deep_requested (default false); seed admins'
  existing subscriptions to preserve today's global-backfill behaviour
- run_deep_backfill is now demand-driven: only channels at least one user has
  requested are deep-backfilled; recent backfill stays unconditional (cheap)
- estimate_deep_backfill ETA helper (quota-bound) surfaced in /api/sync/my-status
- POST /api/sync/deep-all to opt all my channels in; PATCH channels accepts
  deep_requested
- UI: per-channel Full history toggle, Backfill everything action, deep
  progress + ETA in Channels header and Settings - Sync
This commit is contained in:
npeter83 2026-06-11 23:07:09 +02:00
parent bc8db45323
commit 5f60b3e9fe
9 changed files with 256 additions and 18 deletions

View file

@ -1,12 +1,17 @@
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import and_, case, func, select
from sqlalchemy import and_, case, func, select, update
from sqlalchemy.orm import Session
from app import quota, state
from app.auth import current_user
from app.db import get_db
from app.models import Channel, Subscription, User, Video
from app.sync.runner import run_enrich, run_recent_backfill, run_rss_poll
from app.sync.runner import (
estimate_deep_backfill,
run_enrich,
run_recent_backfill,
run_rss_poll,
)
from app.sync.subscriptions import import_subscriptions
router = APIRouter(prefix="/api/sync", tags=["sync"])
@ -113,6 +118,21 @@ def my_status(
).one()
total = total or 0
deep_done = int(deep_done or 0)
# Per-user full-history (deep) backfill: channels this user has opted in, and an ETA
# for the still-pending ones (quota-bound estimate).
deep_requested_channels = (
db.execute(
select(Channel).join(
Subscription,
and_(sub_join, Subscription.deep_requested.is_(True)),
)
)
.scalars()
.all()
)
deep_eta = estimate_deep_backfill(db, deep_requested_channels)
my_videos = db.scalar(
select(func.count(Video.id)).join(
Subscription,
@ -130,6 +150,9 @@ def my_status(
"channels_deep_done": deep_done,
"channels_recent_pending": total - recent_synced,
"channels_deep_pending": total - deep_done,
"channels_deep_requested": len(deep_requested_channels),
"deep_pending_count": deep_eta["channels_pending"],
"deep_eta_seconds": deep_eta["eta_seconds"],
"my_videos": my_videos or 0,
"quota_used_today": quota.units_used_today(db),
"quota_remaining_today": quota.remaining_today(db),
@ -137,6 +160,22 @@ def my_status(
}
@router.post("/deep-all")
def deep_all(
user: User = Depends(current_user),
db: Session = Depends(get_db),
on: bool = True,
) -> dict:
"""Opt every one of the user's channels into (or out of) full-history backfill."""
result = db.execute(
update(Subscription)
.where(Subscription.user_id == user.id)
.values(deep_requested=on)
)
db.commit()
return {"updated": result.rowcount, "deep_requested": on}
@router.post("/pause")
def pause_sync(
user: User = Depends(current_user), db: Session = Depends(get_db)