fix(backfill): don't skip the tail of the recent/deep boundary page
When recent backfill stopped partway through a page (age cutoff or count cap), it stored the *next* page token as the deep-backfill cursor, so the older items remaining on that same page were collected by neither pass — a channel could report backfill_done while silently missing a band of videos right around the 365-day cutoff. Now resume deep from the page we stopped on (re-fetched; inserts are idempotent). Verified: Pánczél went 200 -> 209 stored (= full uploads playlist) after re-backfill.
This commit is contained in:
parent
69e9c60ca1
commit
18aa3726fb
1 changed files with 16 additions and 7 deletions
|
|
@ -107,25 +107,34 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
|
|||
next_cursor: str | None = None
|
||||
done = False
|
||||
|
||||
while len(collected) < settings.backfill_recent_max_videos:
|
||||
while True:
|
||||
page_start = page_token # token that fetched THIS page (None = first page)
|
||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
||||
reached_cutoff = False
|
||||
stopped_mid_page = False
|
||||
for item in data.get("items", []):
|
||||
stub = _stub_from_playlist_item(item, channel.id)
|
||||
if stub is None:
|
||||
continue
|
||||
if stub["published_at"] and stub["published_at"] < cutoff:
|
||||
reached_cutoff = True
|
||||
stopped_mid_page = True
|
||||
break
|
||||
collected.append(stub)
|
||||
if len(collected) >= settings.backfill_recent_max_videos:
|
||||
stopped_mid_page = True
|
||||
break
|
||||
page_token = data.get("nextPageToken")
|
||||
if reached_cutoff or not page_token:
|
||||
next_cursor = page_token
|
||||
done = not page_token
|
||||
if stopped_mid_page:
|
||||
# We stopped partway through this page (age cutoff or count cap), so the older
|
||||
# items still on it weren't collected. Resume the deep backfill FROM THIS page
|
||||
# (re-fetch it; inserts are idempotent) so that tail isn't silently skipped.
|
||||
next_cursor = page_start
|
||||
done = False
|
||||
break
|
||||
next_cursor = page_token
|
||||
if not page_token:
|
||||
next_cursor = None
|
||||
done = True
|
||||
break
|
||||
next_cursor = page_token # whole page consumed, more pages remain
|
||||
|
||||
channel.recent_synced_at = _now()
|
||||
channel.backfill_cursor = next_cursor
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue