feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler

- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
  backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
  language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
  quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
This commit is contained in:
npeter83 2026-06-11 01:36:41 +02:00
parent 24b6e0026d
commit cff1d23071
10 changed files with 569 additions and 4 deletions

77
backend/app/scheduler.py Normal file
View file

@ -0,0 +1,77 @@
"""Background scheduler: free RSS detection, enrichment of new videos, and quota-aware
recent-first / deep backfill. Runs inside the API process (single worker)."""
import logging
from apscheduler.schedulers.background import BackgroundScheduler
from app.config import settings
from app.db import SessionLocal
from app.sync.runner import (
run_deep_backfill,
run_enrich,
run_recent_backfill,
run_rss_poll,
)
logger = logging.getLogger("subfeed.scheduler")
_scheduler: BackgroundScheduler | None = None
def _job(name: str, fn) -> None:
db = SessionLocal()
try:
result = fn(db)
logger.info("job %s -> %s", name, result)
except Exception:
db.rollback()
logger.exception("job %s failed", name)
finally:
db.close()
def _rss_job() -> None:
_job("rss_poll", lambda db: run_rss_poll(db))
def _enrich_job() -> None:
_job("enrich", lambda db: run_enrich(db))
def _backfill_job() -> None:
# Recent-first for not-yet-synced channels, then deep backfill for the rest.
def work(db):
recent = run_recent_backfill(db, max_channels=25)
deep = run_deep_backfill(db, max_channels=10)
return {"recent": recent, "deep": deep}
_job("backfill", work)
def start_scheduler() -> None:
global _scheduler
if not settings.scheduler_enabled or _scheduler is not None:
return
scheduler = BackgroundScheduler(timezone="UTC")
scheduler.add_job(
_rss_job, "interval", minutes=settings.rss_poll_minutes, id="rss_poll"
)
scheduler.add_job(
_enrich_job, "interval", minutes=settings.enrich_interval_minutes, id="enrich"
)
scheduler.add_job(
_backfill_job,
"interval",
minutes=settings.backfill_interval_minutes,
id="backfill",
)
scheduler.start()
_scheduler = scheduler
logger.info("scheduler started")
def shutdown_scheduler() -> None:
global _scheduler
if _scheduler is not None:
_scheduler.shutdown(wait=False)
_scheduler = None