fix(backfill): don't skip the tail of the recent/deep boundary page

When recent backfill stopped partway through a page (age cutoff or count cap),
it stored the *next* page token as the deep-backfill cursor, so the older items
remaining on that same page were collected by neither pass — a channel could
report backfill_done while silently missing a band of videos right around the
365-day cutoff. Now resume deep from the page we stopped on (re-fetched; inserts
are idempotent). Verified: Pánczél went 200 -> 209 stored (= full uploads
playlist) after re-backfill.
This commit is contained in:
npeter83 2026-06-12 00:15:20 +02:00
parent d022037286
commit a5fc1e6d0b

View file

@ -107,25 +107,34 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
next_cursor: str | None = None
done = False
while len(collected) < settings.backfill_recent_max_videos:
while True:
page_start = page_token # token that fetched THIS page (None = first page)
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
reached_cutoff = False
stopped_mid_page = False
for item in data.get("items", []):
stub = _stub_from_playlist_item(item, channel.id)
if stub is None:
continue
if stub["published_at"] and stub["published_at"] < cutoff:
reached_cutoff = True
stopped_mid_page = True
break
collected.append(stub)
if len(collected) >= settings.backfill_recent_max_videos:
stopped_mid_page = True
break
page_token = data.get("nextPageToken")
if reached_cutoff or not page_token:
next_cursor = page_token
done = not page_token
if stopped_mid_page:
# We stopped partway through this page (age cutoff or count cap), so the older
# items still on it weren't collected. Resume the deep backfill FROM THIS page
# (re-fetch it; inserts are idempotent) so that tail isn't silently skipped.
next_cursor = page_start
done = False
break
next_cursor = page_token
if not page_token:
next_cursor = None
done = True
break
next_cursor = page_token # whole page consumed, more pages remain
channel.recent_synced_at = _now()
channel.backfill_cursor = next_cursor