peter/siftlode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(backfill): auto-complete full history when stored >= video_count

Browse source 
A channel whose stored uploads already meet or exceed YouTube's advertised
video_count holds its whole history, but backfill_done could stay false forever
when the deep cursor never reached the end (e.g. a small channel that was never
deep-requested, so the demand-driven deep job never ran). Such channels nagged
as 'needs full history' despite having every video. Add reconcile_full_history()
(idempotent, no quota) and run it at the end of each recent/deep backfill cycle
so backfill_done self-heals.
This commit is contained in:
npeter83 2026-06-14 19:01:04 +02:00
parent 30a45eea30
commit 64c6e44686
2 changed files with 40 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

15
backend/app/sync/runner.py
View file

@ -20,6 +20,7 @@ from app.sync.videos import (
backfill_channel_recent,
enrich_pending,
poll_rss_channel,
reconcile_full_history,
run_shorts_classification,
)
from app.youtube.client import YouTubeClient
@ -118,7 +119,12 @@ def run_recent_backfill(
except Exception:
db.rollback()
log.exception("Recent backfill failed for channel %s", channel.id)
return {"channels_processed": processed, "videos_added": videos_added}
reconciled = reconcile_full_history(db)
return {
"channels_processed": processed,
"videos_added": videos_added,
"reconciled_full": reconciled,
}
def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -> dict:
@ -158,7 +164,12 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
except Exception:
db.rollback()
log.exception("Deep backfill failed for channel %s", channel.id)
return {"channels_processed": processed, "videos_added": videos_added}
reconciled = reconcile_full_history(db)
return {
"channels_processed": processed,
"videos_added": videos_added,
"reconciled_full": reconciled,
}
def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict: