feat(m5d): demand-driven deep backfill + per-user ETA

Per-user opt-in to full-history (deep) backfill so a new user's unique
channels no longer trigger a big shared-quota burst.

- migration 0007: Subscription.deep_requested (default false); seed admins'
  existing subscriptions to preserve today's global-backfill behaviour
- run_deep_backfill is now demand-driven: only channels at least one user has
  requested are deep-backfilled; recent backfill stays unconditional (cheap)
- estimate_deep_backfill ETA helper (quota-bound) surfaced in /api/sync/my-status
- POST /api/sync/deep-all to opt all my channels in; PATCH channels accepts
  deep_requested
- UI: per-channel Full history toggle, Backfill everything action, deep
  progress + ETA in Channels header and Settings - Sync
This commit is contained in:
npeter83 2026-06-11 23:07:09 +02:00
parent bc8db45323
commit 5f60b3e9fe
9 changed files with 256 additions and 18 deletions

View file

@ -13,7 +13,7 @@ from app import quota
from app.config import settings
log = logging.getLogger("subfeed.sync")
from app.models import Channel, OAuthToken, User
from app.models import Channel, OAuthToken, Subscription, User
from app.sync.subscriptions import import_subscriptions
from app.sync.videos import (
backfill_channel_deep,
@ -125,11 +125,22 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
user = get_service_user(db)
if user is None:
return {"channels_processed": 0, "videos_added": 0, "reason": "no service user"}
# Demand-driven: only deep-backfill channels at least one user has opted into
# (Subscription.deep_requested). Recent backfill stays unconditional (cheap).
deep_requested = (
select(Subscription.id)
.where(
Subscription.channel_id == Channel.id,
Subscription.deep_requested.is_(True),
)
.exists()
)
channels = (
db.execute(
select(Channel).where(
Channel.backfill_done.is_(False),
Channel.recent_synced_at.is_not(None),
deep_requested,
)
)
.scalars()
@ -148,3 +159,26 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
db.rollback()
log.exception("Deep backfill failed for channel %s", channel.id)
return {"channels_processed": processed, "videos_added": videos_added}
def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
"""Rough ETA to finish full-history backfill of `channels`. Quota-bound: total
estimated API units / the daily budget left for backfill. Deliberately approximate
a channel's cost ~= playlistItems pages (1 unit / 50 videos) + enrichment (likewise)."""
pending = [c for c in channels if not c.backfill_done]
def units(c: Channel) -> int:
videos = c.video_count or 0
pages = max(1, (videos + 49) // 50)
return 2 * pages
total_units = sum(units(c) for c in pending)
daily_budget = max(1, settings.quota_daily_budget - settings.backfill_quota_reserve)
eta_seconds = int(total_units / daily_budget * 86_400) if pending else 0
return {
"channels_pending": len(pending),
"videos_pending_est": sum(c.video_count or 0 for c in pending),
"units_est": total_units,
"daily_budget": daily_budget,
"eta_seconds": eta_seconds,
}