feat(scheduler): report progress from every job + parallelize RSS polling
Every background job now reports progress, so the dashboard can show a live bar for any run (not only enrich/backfill/shorts/maintenance): rss_poll and subscriptions (runner), autotag, and playlist_sync gained progress.report calls over their loops. RSS polling now fetches the channel feeds concurrently (16 workers) instead of one-at-a-time — a slow/unreachable feed no longer blocks the rest, cutting a full poll of the catalogue from minutes to seconds. Feed fetches are network-bound and run in the pool; DB inserts stay on the session's thread (SQLAlchemy sessions aren't thread-safe), applied as each fetch completes. Split the DB-write half of poll_rss_channel into apply_rss_feed so the fetch and apply phases compose cleanly.
This commit is contained in:
parent
5ae12a1752
commit
6eb552fa04
4 changed files with 46 additions and 13 deletions
|
|
@ -12,7 +12,7 @@ from datetime import datetime, timezone
|
|||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import quota
|
||||
from app import progress, quota
|
||||
from app.auth import has_read_scope, has_write_scope
|
||||
from app.models import Channel, OAuthToken, Playlist, PlaylistItem, User, Video
|
||||
from app.sync.videos import apply_video_details, parse_dt
|
||||
|
|
@ -382,8 +382,9 @@ def sync_all_playlists(db: Session) -> dict:
|
|||
.all()
|
||||
)
|
||||
total = 0
|
||||
for user in users:
|
||||
for i, user in enumerate(users, 1):
|
||||
if not has_read_scope(user):
|
||||
progress.report(i, len(users), "playlist_sync")
|
||||
continue
|
||||
try:
|
||||
with quota.attribute(user.id, "playlist_sync"):
|
||||
|
|
@ -394,4 +395,5 @@ def sync_all_playlists(db: Session) -> dict:
|
|||
except Exception:
|
||||
db.rollback()
|
||||
log.exception("Playlist sync crashed for user %s", user.id)
|
||||
progress.report(i, len(users), "playlist_sync")
|
||||
return {"users": len(users), "playlists_synced": total}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue