fix(backfill): auto-complete full history when stored >= video_count
A channel whose stored uploads already meet or exceed YouTube's advertised video_count holds its whole history, but backfill_done could stay false forever when the deep cursor never reached the end (e.g. a small channel that was never deep-requested, so the demand-driven deep job never ran). Such channels nagged as 'needs full history' despite having every video. Add reconcile_full_history() (idempotent, no quota) and run it at the end of each recent/deep backfill cycle so backfill_done self-heals.
This commit is contained in:
parent
30a45eea30
commit
64c6e44686
2 changed files with 40 additions and 3 deletions
|
|
@ -5,7 +5,7 @@ import re
|
|||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import or_, select, update
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
|
@ -173,6 +173,32 @@ def backfill_channel_deep(
|
|||
return total
|
||||
|
||||
|
||||
def reconcile_full_history(db: Session) -> int:
|
||||
"""Mark channels fully backfilled once their stored uploads meet or exceed YouTube's
|
||||
advertised video_count. The uploads playlist length is normally <= the advertised
|
||||
count, so 'stored >= video_count' means we already hold the whole history — even if
|
||||
the deep cursor never reached the end (e.g. a small channel that was never
|
||||
deep-requested, so the demand-driven deep job never ran). Without this such a channel
|
||||
sits in 'needs full history' forever despite having every video. Idempotent, no API
|
||||
quota; returns the number of channels flipped."""
|
||||
stored = (
|
||||
select(func.count(Video.id))
|
||||
.where(Video.channel_id == Channel.id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
result = db.execute(
|
||||
update(Channel)
|
||||
.where(
|
||||
Channel.backfill_done.is_(False),
|
||||
Channel.video_count.is_not(None),
|
||||
stored >= Channel.video_count,
|
||||
)
|
||||
.values(backfill_done=True, backfill_cursor=None)
|
||||
)
|
||||
db.commit()
|
||||
return result.rowcount
|
||||
|
||||
|
||||
# --- Enrichment (videos.list) ---
|
||||
def _live_status(snippet: dict, live_details: dict | None) -> str:
|
||||
broadcast = snippet.get("liveBroadcastContent")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue