From c06631cc9eab4cd017a5c44ef832775b28f8105f Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 12 Jun 2026 00:15:20 +0200 Subject: [PATCH] fix(backfill): don't skip the tail of the recent/deep boundary page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When recent backfill stopped partway through a page (age cutoff or count cap), it stored the *next* page token as the deep-backfill cursor, so the older items remaining on that same page were collected by neither pass — a channel could report backfill_done while silently missing a band of videos right around the 365-day cutoff. Now resume deep from the page we stopped on (re-fetched; inserts are idempotent). Verified: Pánczél went 200 -> 209 stored (= full uploads playlist) after re-backfill. --- backend/app/sync/videos.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/backend/app/sync/videos.py b/backend/app/sync/videos.py index 2ac9c56..38c6cc3 100644 --- a/backend/app/sync/videos.py +++ b/backend/app/sync/videos.py @@ -107,25 +107,34 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> next_cursor: str | None = None done = False - while len(collected) < settings.backfill_recent_max_videos: + while True: + page_start = page_token # token that fetched THIS page (None = first page) data = yt.get_playlist_items(channel.uploads_playlist_id, page_token) - reached_cutoff = False + stopped_mid_page = False for item in data.get("items", []): stub = _stub_from_playlist_item(item, channel.id) if stub is None: continue if stub["published_at"] and stub["published_at"] < cutoff: - reached_cutoff = True + stopped_mid_page = True break collected.append(stub) if len(collected) >= settings.backfill_recent_max_videos: + stopped_mid_page = True break page_token = data.get("nextPageToken") - if reached_cutoff or not page_token: - next_cursor = page_token - done = not page_token + if stopped_mid_page: + # We stopped partway through this page (age cutoff or count cap), so the older + # items still on it weren't collected. Resume the deep backfill FROM THIS page + # (re-fetch it; inserts are idempotent) so that tail isn't silently skipped. + next_cursor = page_start + done = False break - next_cursor = page_token + if not page_token: + next_cursor = None + done = True + break + next_cursor = page_token # whole page consumed, more pages remain channel.recent_synced_at = _now() channel.backfill_cursor = next_cursor