feat(scheduler): report progress from every job + parallelize RSS polling

Every background job now reports progress, so the dashboard can show a live
bar for any run (not only enrich/backfill/shorts/maintenance): rss_poll and
subscriptions (runner), autotag, and playlist_sync gained progress.report
calls over their loops.

RSS polling now fetches the channel feeds concurrently (16 workers) instead
of one-at-a-time — a slow/unreachable feed no longer blocks the rest, cutting
a full poll of the catalogue from minutes to seconds. Feed fetches are
network-bound and run in the pool; DB inserts stay on the session's thread
(SQLAlchemy sessions aren't thread-safe), applied as each fetch completes.
Split the DB-write half of poll_rss_channel into apply_rss_feed so the fetch
and apply phases compose cleanly.
This commit is contained in:
npeter83 2026-06-19 02:43:37 +02:00
parent b5141ab112
commit e0971a23ec
4 changed files with 46 additions and 13 deletions

View file

@ -5,6 +5,7 @@ user" (any user with a stored refresh token) unless a YOUTUBE_API_KEY is configu
public reads.
"""
import logging
from concurrent.futures import ThreadPoolExecutor, as_completed
from sqlalchemy import select
from sqlalchemy.orm import Session
@ -16,15 +17,19 @@ log = logging.getLogger("subfeed.sync")
from app.models import Channel, OAuthToken, Subscription, User
from app.sync.subscriptions import import_subscriptions
from app.sync.videos import (
apply_rss_feed,
backfill_channel_deep,
backfill_channel_recent,
enrich_pending,
poll_rss_channel,
reconcile_full_history,
refresh_live,
run_shorts_classification,
)
from app.youtube.client import YouTubeClient
from app.youtube.rss import fetch_channel_feed
# RSS feeds are free and network-bound, so fetch many at once; DB writes stay single-threaded.
RSS_POLL_WORKERS = 16
def get_service_user(db: Session) -> User | None:
@ -43,13 +48,30 @@ def get_service_user(db: Session) -> User | None:
def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> int:
if channels is None:
channels = db.execute(select(Channel)).scalars().all()
total = len(channels)
new = 0
for channel in channels:
try:
new += poll_rss_channel(db, channel)
except Exception:
db.rollback()
log.exception("RSS poll failed for channel %s", channel.id)
done = 0
# Fetch feeds concurrently (network-bound, no DB), then apply inserts on this thread as
# each fetch returns — a SQLAlchemy session isn't thread-safe. Pass the channel id (a
# plain str) to the workers; the ORM instance is touched only here on the main thread.
futures = {}
with ThreadPoolExecutor(max_workers=RSS_POLL_WORKERS) as pool:
for channel in channels:
futures[pool.submit(fetch_channel_feed, channel.id)] = channel
for fut in as_completed(futures):
channel = futures[fut]
try:
entries = fut.result()
except Exception:
log.exception("RSS fetch failed for channel %s", channel.id)
entries = []
try:
new += apply_rss_feed(db, channel, entries)
except Exception:
db.rollback()
log.exception("RSS apply failed for channel %s", channel.id)
done += 1
progress.report(done, total, "rss_poll")
return new
@ -90,12 +112,13 @@ def run_subscription_resync(db: Session) -> dict:
.scalars()
.all()
)
for user in users:
for i, user in enumerate(users, 1):
try:
import_subscriptions(db, user)
except Exception:
db.rollback()
log.exception("Subscription resync failed for user %s", user.id)
progress.report(i, len(users), "subscriptions")
return {"users": len(users)}