feat(m5d): demand-driven deep backfill + per-user ETA
Per-user opt-in to full-history (deep) backfill so a new user's unique channels no longer trigger a big shared-quota burst. - migration 0007: Subscription.deep_requested (default false); seed admins' existing subscriptions to preserve today's global-backfill behaviour - run_deep_backfill is now demand-driven: only channels at least one user has requested are deep-backfilled; recent backfill stays unconditional (cheap) - estimate_deep_backfill ETA helper (quota-bound) surfaced in /api/sync/my-status - POST /api/sync/deep-all to opt all my channels in; PATCH channels accepts deep_requested - UI: per-channel Full history toggle, Backfill everything action, deep progress + ETA in Channels header and Settings - Sync
This commit is contained in:
parent
bbcfd46ea9
commit
beb961c021
9 changed files with 256 additions and 18 deletions
|
|
@ -13,7 +13,7 @@ from app import quota
|
|||
from app.config import settings
|
||||
|
||||
log = logging.getLogger("subfeed.sync")
|
||||
from app.models import Channel, OAuthToken, User
|
||||
from app.models import Channel, OAuthToken, Subscription, User
|
||||
from app.sync.subscriptions import import_subscriptions
|
||||
from app.sync.videos import (
|
||||
backfill_channel_deep,
|
||||
|
|
@ -125,11 +125,22 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
|||
user = get_service_user(db)
|
||||
if user is None:
|
||||
return {"channels_processed": 0, "videos_added": 0, "reason": "no service user"}
|
||||
# Demand-driven: only deep-backfill channels at least one user has opted into
|
||||
# (Subscription.deep_requested). Recent backfill stays unconditional (cheap).
|
||||
deep_requested = (
|
||||
select(Subscription.id)
|
||||
.where(
|
||||
Subscription.channel_id == Channel.id,
|
||||
Subscription.deep_requested.is_(True),
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
channels = (
|
||||
db.execute(
|
||||
select(Channel).where(
|
||||
Channel.backfill_done.is_(False),
|
||||
Channel.recent_synced_at.is_not(None),
|
||||
deep_requested,
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
|
|
@ -148,3 +159,26 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
|||
db.rollback()
|
||||
log.exception("Deep backfill failed for channel %s", channel.id)
|
||||
return {"channels_processed": processed, "videos_added": videos_added}
|
||||
|
||||
|
||||
def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
|
||||
"""Rough ETA to finish full-history backfill of `channels`. Quota-bound: total
|
||||
estimated API units / the daily budget left for backfill. Deliberately approximate —
|
||||
a channel's cost ~= playlistItems pages (1 unit / 50 videos) + enrichment (likewise)."""
|
||||
pending = [c for c in channels if not c.backfill_done]
|
||||
|
||||
def units(c: Channel) -> int:
|
||||
videos = c.video_count or 0
|
||||
pages = max(1, (videos + 49) // 50)
|
||||
return 2 * pages
|
||||
|
||||
total_units = sum(units(c) for c in pending)
|
||||
daily_budget = max(1, settings.quota_daily_budget - settings.backfill_quota_reserve)
|
||||
eta_seconds = int(total_units / daily_budget * 86_400) if pending else 0
|
||||
return {
|
||||
"channels_pending": len(pending),
|
||||
"videos_pending_est": sum(c.video_count or 0 for c in pending),
|
||||
"units_est": total_units,
|
||||
"daily_budget": daily_budget,
|
||||
"eta_seconds": eta_seconds,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue