feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler

- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
  backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
  language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
  quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
This commit is contained in:
npeter83 2026-06-11 01:36:41 +02:00
parent 24b6e0026d
commit cff1d23071
10 changed files with 569 additions and 4 deletions

View file

@ -6,11 +6,24 @@ from app import quota
from app.auth import current_user
from app.db import get_db
from app.models import Channel, Subscription, User, Video
from app.sync.runner import run_enrich, run_recent_backfill, run_rss_poll
from app.sync.subscriptions import import_subscriptions
router = APIRouter(prefix="/api/sync", tags=["sync"])
def _user_channels(db: Session, user: User) -> list[Channel]:
return (
db.execute(
select(Channel)
.join(Subscription, Subscription.channel_id == Channel.id)
.where(Subscription.user_id == user.id)
)
.scalars()
.all()
)
@router.post("/subscriptions")
def sync_subscriptions(
user: User = Depends(current_user), db: Session = Depends(get_db)
@ -22,6 +35,38 @@ def sync_subscriptions(
return result
@router.post("/rss")
def sync_rss(
user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
new = run_rss_poll(db, _user_channels(db, user))
return {"new_videos": new}
@router.post("/backfill")
def sync_backfill(
user: User = Depends(current_user),
db: Session = Depends(get_db),
max_channels: int = 25,
) -> dict:
before = quota.units_used_today(db)
result = run_recent_backfill(db, _user_channels(db, user), max_channels=max_channels)
result["quota_used_estimate"] = quota.units_used_today(db) - before
return result
@router.post("/enrich")
def sync_enrich(
user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
before = quota.units_used_today(db)
enriched = run_enrich(db)
return {
"enriched": enriched,
"quota_used_estimate": quota.units_used_today(db) - before,
}
@router.get("/status")
def sync_status(
user: User = Depends(current_user), db: Session = Depends(get_db)