feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler
- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
This commit is contained in:
parent
24b6e0026d
commit
cff1d23071
10 changed files with 569 additions and 4 deletions
|
|
@ -6,11 +6,24 @@ from app import quota
|
|||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import Channel, Subscription, User, Video
|
||||
from app.sync.runner import run_enrich, run_recent_backfill, run_rss_poll
|
||||
from app.sync.subscriptions import import_subscriptions
|
||||
|
||||
router = APIRouter(prefix="/api/sync", tags=["sync"])
|
||||
|
||||
|
||||
def _user_channels(db: Session, user: User) -> list[Channel]:
|
||||
return (
|
||||
db.execute(
|
||||
select(Channel)
|
||||
.join(Subscription, Subscription.channel_id == Channel.id)
|
||||
.where(Subscription.user_id == user.id)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
@router.post("/subscriptions")
|
||||
def sync_subscriptions(
|
||||
user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
|
|
@ -22,6 +35,38 @@ def sync_subscriptions(
|
|||
return result
|
||||
|
||||
|
||||
@router.post("/rss")
|
||||
def sync_rss(
|
||||
user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
new = run_rss_poll(db, _user_channels(db, user))
|
||||
return {"new_videos": new}
|
||||
|
||||
|
||||
@router.post("/backfill")
|
||||
def sync_backfill(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
max_channels: int = 25,
|
||||
) -> dict:
|
||||
before = quota.units_used_today(db)
|
||||
result = run_recent_backfill(db, _user_channels(db, user), max_channels=max_channels)
|
||||
result["quota_used_estimate"] = quota.units_used_today(db) - before
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/enrich")
|
||||
def sync_enrich(
|
||||
user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
before = quota.units_used_today(db)
|
||||
enriched = run_enrich(db)
|
||||
return {
|
||||
"enriched": enriched,
|
||||
"quota_used_estimate": quota.units_used_today(db) - before,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
def sync_status(
|
||||
user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue