peter/siftlode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(backfill): don't skip the tail of the recent/deep boundary page

Browse source 
When recent backfill stopped partway through a page (age cutoff or count cap),
it stored the *next* page token as the deep-backfill cursor, so the older items
remaining on that same page were collected by neither pass — a channel could
report backfill_done while silently missing a band of videos right around the
365-day cutoff. Now resume deep from the page we stopped on (re-fetched; inserts
are idempotent). Verified: Pánczél went 200 -> 209 stored (= full uploads
playlist) after re-backfill.
This commit is contained in:
npeter83 2026-06-12 00:15:20 +02:00
parent 70d4a9eb69
commit c06631cc9e
1 changed files with 16 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

23
backend/app/sync/videos.py
View file

@ -107,25 +107,34 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
next_cursor: str | None = None next_cursor: str | None = None
done = False done = False
while len(collected) < settings.backfill_recent_max_videos: while True:
page_start = page_token # token that fetched THIS page (None = first page)
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token) data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
reached_cutoff = False stopped_mid_page = False
for item in data.get("items", []): for item in data.get("items", []):
stub = _stub_from_playlist_item(item, channel.id) stub = _stub_from_playlist_item(item, channel.id)
if stub is None: if stub is None:
continue continue
if stub["published_at"] and stub["published_at"] < cutoff: if stub["published_at"] and stub["published_at"] < cutoff:
reached_cutoff = True stopped_mid_page = True
break break
collected.append(stub) collected.append(stub)
if len(collected) >= settings.backfill_recent_max_videos: if len(collected) >= settings.backfill_recent_max_videos:
stopped_mid_page = True
break break
page_token = data.get("nextPageToken") page_token = data.get("nextPageToken")
if reached_cutoff or not page_token: if stopped_mid_page:
next_cursor = page_token # We stopped partway through this page (age cutoff or count cap), so the older
done = not page_token # items still on it weren't collected. Resume the deep backfill FROM THIS page
# (re-fetch it; inserts are idempotent) so that tail isn't silently skipped.
next_cursor = page_start
done = False
break break
next_cursor = page_token if not page_token:
next_cursor = None
done = True
break
next_cursor = page_token # whole page consumed, more pages remain
channel.recent_synced_at = _now() channel.recent_synced_at = _now()
channel.backfill_cursor = next_cursor channel.backfill_cursor = next_cursor