peter/siftlode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(config): move operational params to DB (quota/backfill/shorts/batch)

Browse source 
Register 8 more keys in the sysconfig registry (quota daily budget + backfill
reserve, recent-backfill window, shorts-probe params, enrich/autotag batch sizes)
and route their reads through the DB-override-or-env resolver at every call site
(quota.py, sync/runner.py, sync/videos.py, sync/autotag.py, sync/maintenance.py,
routes/scheduler.py). All read sites already had a db session in scope. Reads are
read-through (no cache), matching app.state; admin can tune them live, no redeploy.
This commit is contained in:
npeter83 2026-06-19 13:07:45 +02:00
parent ff03b51352
commit e6bcf5ba1e
7 changed files with 35 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

15
backend/app/sync/videos.py
View file

@ -9,8 +9,7 @@ from sqlalchemy import and_, func, or_, select, update
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from app import progress
from app.config import settings
from app import progress, sysconfig
from app.models import Channel, Video
from app.youtube.client import YouTubeClient, best_thumbnail
from app.youtube.rss import fetch_channel_feed
@ -108,7 +107,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
db.commit()
return 0
cutoff = _now() - timedelta(days=settings.backfill_recent_max_days)
cutoff = _now() - timedelta(days=sysconfig.get_int(db, "backfill_recent_max_days"))
collected: list[dict] = []
page_token: str | None = None
next_cursor: str | None = None
@ -126,7 +125,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
stopped_mid_page = True
break
collected.append(stub)
if len(collected) >= settings.backfill_recent_max_videos:
if len(collected) >= sysconfig.get_int(db, "backfill_recent_max_videos"):
stopped_mid_page = True
break
page_token = data.get("nextPageToken")
@ -249,7 +248,7 @@ def apply_video_details(video: Video, item: dict) -> None:
def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int:
limit = limit or settings.enrich_batch_size
limit = limit or sysconfig.get_int(db, "enrich_batch_size")
videos = (
db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit))
.scalars()
@ -282,7 +281,7 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in
ends. So we revisit anything still live/upcoming, plus a just-ended stream that hasn't got
its duration yet (YouTube can lag a bit after the broadcast), until it settles. Cheap:
videos.list is 1 unit per 50 ids, and the live/upcoming set is normally tiny."""
limit = limit or settings.enrich_batch_size
limit = limit or sysconfig.get_int(db, "enrich_batch_size")
videos = (
db.execute(
select(Video)
@ -322,8 +321,8 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in
def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
"""Finalize Shorts classification: cheaply mark non-candidates, then probe the
youtube.com/shorts URL for short-enough, non-live, enriched videos."""
limit = limit or settings.shorts_probe_batch
probe_max = settings.shorts_probe_max_seconds
limit = limit or sysconfig.get_int(db, "shorts_probe_batch")
probe_max = sysconfig.get_int(db, "shorts_probe_max_seconds")
# 1) Anything enriched that can't be a Short -> finalize without a probe.
db.execute(