feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler
- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
This commit is contained in:
parent
24b6e0026d
commit
cff1d23071
10 changed files with 569 additions and 4 deletions
77
backend/app/scheduler.py
Normal file
77
backend/app/scheduler.py
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
"""Background scheduler: free RSS detection, enrichment of new videos, and quota-aware
|
||||
recent-first / deep backfill. Runs inside the API process (single worker)."""
|
||||
import logging
|
||||
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.sync.runner import (
|
||||
run_deep_backfill,
|
||||
run_enrich,
|
||||
run_recent_backfill,
|
||||
run_rss_poll,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("subfeed.scheduler")
|
||||
|
||||
_scheduler: BackgroundScheduler | None = None
|
||||
|
||||
|
||||
def _job(name: str, fn) -> None:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
result = fn(db)
|
||||
logger.info("job %s -> %s", name, result)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("job %s failed", name)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _rss_job() -> None:
|
||||
_job("rss_poll", lambda db: run_rss_poll(db))
|
||||
|
||||
|
||||
def _enrich_job() -> None:
|
||||
_job("enrich", lambda db: run_enrich(db))
|
||||
|
||||
|
||||
def _backfill_job() -> None:
|
||||
# Recent-first for not-yet-synced channels, then deep backfill for the rest.
|
||||
def work(db):
|
||||
recent = run_recent_backfill(db, max_channels=25)
|
||||
deep = run_deep_backfill(db, max_channels=10)
|
||||
return {"recent": recent, "deep": deep}
|
||||
|
||||
_job("backfill", work)
|
||||
|
||||
|
||||
def start_scheduler() -> None:
|
||||
global _scheduler
|
||||
if not settings.scheduler_enabled or _scheduler is not None:
|
||||
return
|
||||
scheduler = BackgroundScheduler(timezone="UTC")
|
||||
scheduler.add_job(
|
||||
_rss_job, "interval", minutes=settings.rss_poll_minutes, id="rss_poll"
|
||||
)
|
||||
scheduler.add_job(
|
||||
_enrich_job, "interval", minutes=settings.enrich_interval_minutes, id="enrich"
|
||||
)
|
||||
scheduler.add_job(
|
||||
_backfill_job,
|
||||
"interval",
|
||||
minutes=settings.backfill_interval_minutes,
|
||||
id="backfill",
|
||||
)
|
||||
scheduler.start()
|
||||
_scheduler = scheduler
|
||||
logger.info("scheduler started")
|
||||
|
||||
|
||||
def shutdown_scheduler() -> None:
|
||||
global _scheduler
|
||||
if _scheduler is not None:
|
||||
_scheduler.shutdown(wait=False)
|
||||
_scheduler = None
|
||||
Loading…
Add table
Add a link
Reference in a new issue