feat(config): move operational params to DB (quota/backfill/shorts/batch)

Register 8 more keys in the sysconfig registry (quota daily budget + backfill
reserve, recent-backfill window, shorts-probe params, enrich/autotag batch sizes)
and route their reads through the DB-override-or-env resolver at every call site
(quota.py, sync/runner.py, sync/videos.py, sync/autotag.py, sync/maintenance.py,
routes/scheduler.py). All read sites already had a db session in scope. Reads are
read-through (no cache), matching app.state; admin can tune them live, no redeploy.
This commit is contained in:
npeter83 2026-06-19 13:07:45 +02:00
parent e127ecf490
commit 7081b57150
7 changed files with 35 additions and 24 deletions

View file

@ -12,7 +12,7 @@ from zoneinfo import ZoneInfo
from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.config import settings from app import sysconfig
from app.models import ApiQuotaUsage, QuotaEvent from app.models import ApiQuotaUsage, QuotaEvent
_PACIFIC = ZoneInfo("America/Los_Angeles") _PACIFIC = ZoneInfo("America/Los_Angeles")
@ -82,7 +82,7 @@ def units_used_today(db: Session) -> int:
def remaining_today(db: Session) -> int: def remaining_today(db: Session) -> int:
return max(0, settings.quota_daily_budget - units_used_today(db)) return max(0, sysconfig.get_int(db, "quota_daily_budget") - units_used_today(db))
def can_spend(db: Session, units: int) -> bool: def can_spend(db: Session, units: int) -> bool:

View file

@ -5,7 +5,7 @@ from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import and_, func, or_, select from sqlalchemy import and_, func, or_, select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import quota, state from app import quota, state, sysconfig
from app.config import settings from app.config import settings
from app.db import get_db from app.db import get_db
from app.models import Channel, Subscription, User, Video from app.models import Channel, Subscription, User, Video
@ -152,8 +152,8 @@ def get_scheduler(
quota_info = { quota_info = {
"used_today": used, "used_today": used,
"remaining_today": quota.remaining_today(db), "remaining_today": quota.remaining_today(db),
"daily_budget": settings.quota_daily_budget, "daily_budget": sysconfig.get_int(db, "quota_daily_budget"),
"backfill_reserve": settings.backfill_quota_reserve, "backfill_reserve": sysconfig.get_int(db, "backfill_quota_reserve"),
} }
return { return {

View file

@ -13,8 +13,7 @@ from sqlalchemy.orm import Session
log = logging.getLogger("subfeed.autotag") log = logging.getLogger("subfeed.autotag")
from app import progress from app import progress, sysconfig
from app.config import settings
from app.models import Channel, ChannelTag, Tag, Video from app.models import Channel, ChannelTag, Tag, Video
# --- language id (offline, deterministic, small) --- # --- language id (offline, deterministic, small) ---
@ -166,7 +165,7 @@ def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None,
select(Video.title) select(Video.title)
.where(Video.channel_id == channel.id, Video.title.is_not(None)) .where(Video.channel_id == channel.id, Video.title.is_not(None))
.order_by(Video.published_at.desc()) .order_by(Video.published_at.desc())
.limit(settings.autotag_title_sample) .limit(sysconfig.get_int(db, "autotag_title_sample"))
) )
.scalars() .scalars()
.all() .all()

View file

@ -25,7 +25,7 @@ from datetime import datetime, timedelta, timezone
from sqlalchemy import or_, select from sqlalchemy import or_, select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import progress, quota from app import progress, quota, sysconfig
from app.config import settings from app.config import settings
from app.models import Playlist, PlaylistItem, Video, VideoState from app.models import Playlist, PlaylistItem, Video, VideoState
from app.notifications import create_notification from app.notifications import create_notification
@ -196,7 +196,7 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
now = _now() now = _now()
# Process in videos.list-sized pages so we commit incrementally and stop on low quota. # Process in videos.list-sized pages so we commit incrementally and stop on low quota.
while checked < batch: while checked < batch:
if quota.remaining_today(db) <= settings.backfill_quota_reserve: if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
break break
page = ( page = (
db.execute( db.execute(
@ -236,7 +236,7 @@ def run_maintenance(db: Session) -> dict:
phase1 = _recheck_flagged(db, yt) phase1 = _recheck_flagged(db, yt)
phase2 = ( phase2 = (
_revalidate_rolling(db, yt) _revalidate_rolling(db, yt)
if quota.remaining_today(db) > settings.backfill_quota_reserve if quota.remaining_today(db) > sysconfig.get_int(db, "backfill_quota_reserve")
else {"checked": 0, "flagged": 0, "skipped": "low quota"} else {"checked": 0, "flagged": 0, "skipped": "low quota"}
) )
return { return {

View file

@ -10,8 +10,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
from sqlalchemy import select from sqlalchemy import select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import progress, quota from app import progress, quota, sysconfig
from app.config import settings
log = logging.getLogger("subfeed.sync") log = logging.getLogger("subfeed.sync")
from app.models import Channel, OAuthToken, Subscription, User from app.models import Channel, OAuthToken, Subscription, User
@ -140,7 +139,7 @@ def run_recent_backfill(
videos_added = 0 videos_added = 0
with YouTubeClient(db, user) as yt: with YouTubeClient(db, user) as yt:
for channel in channels: for channel in channels:
if quota.remaining_today(db) <= settings.backfill_quota_reserve: if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
break break
try: try:
videos_added += backfill_channel_recent(db, yt, channel) videos_added += backfill_channel_recent(db, yt, channel)
@ -186,7 +185,7 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
videos_added = 0 videos_added = 0
with YouTubeClient(db, user) as yt: with YouTubeClient(db, user) as yt:
for channel in channels: for channel in channels:
if quota.remaining_today(db) <= settings.backfill_quota_reserve: if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
break break
try: try:
videos_added += backfill_channel_deep(db, yt, channel, max_pages) videos_added += backfill_channel_deep(db, yt, channel, max_pages)
@ -215,7 +214,9 @@ def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
return 2 * pages return 2 * pages
total_units = sum(units(c) for c in pending) total_units = sum(units(c) for c in pending)
daily_budget = max(1, settings.quota_daily_budget - settings.backfill_quota_reserve) daily_budget = max(
1, sysconfig.get_int(db, "quota_daily_budget") - sysconfig.get_int(db, "backfill_quota_reserve")
)
eta_seconds = int(total_units / daily_budget * 86_400) if pending else 0 eta_seconds = int(total_units / daily_budget * 86_400) if pending else 0
return { return {
"channels_pending": len(pending), "channels_pending": len(pending),

View file

@ -9,8 +9,7 @@ from sqlalchemy import and_, func, or_, select, update
from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import progress from app import progress, sysconfig
from app.config import settings
from app.models import Channel, Video from app.models import Channel, Video
from app.youtube.client import YouTubeClient, best_thumbnail from app.youtube.client import YouTubeClient, best_thumbnail
from app.youtube.rss import fetch_channel_feed from app.youtube.rss import fetch_channel_feed
@ -108,7 +107,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
db.commit() db.commit()
return 0 return 0
cutoff = _now() - timedelta(days=settings.backfill_recent_max_days) cutoff = _now() - timedelta(days=sysconfig.get_int(db, "backfill_recent_max_days"))
collected: list[dict] = [] collected: list[dict] = []
page_token: str | None = None page_token: str | None = None
next_cursor: str | None = None next_cursor: str | None = None
@ -126,7 +125,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
stopped_mid_page = True stopped_mid_page = True
break break
collected.append(stub) collected.append(stub)
if len(collected) >= settings.backfill_recent_max_videos: if len(collected) >= sysconfig.get_int(db, "backfill_recent_max_videos"):
stopped_mid_page = True stopped_mid_page = True
break break
page_token = data.get("nextPageToken") page_token = data.get("nextPageToken")
@ -249,7 +248,7 @@ def apply_video_details(video: Video, item: dict) -> None:
def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int: def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int:
limit = limit or settings.enrich_batch_size limit = limit or sysconfig.get_int(db, "enrich_batch_size")
videos = ( videos = (
db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit)) db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit))
.scalars() .scalars()
@ -282,7 +281,7 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in
ends. So we revisit anything still live/upcoming, plus a just-ended stream that hasn't got ends. So we revisit anything still live/upcoming, plus a just-ended stream that hasn't got
its duration yet (YouTube can lag a bit after the broadcast), until it settles. Cheap: its duration yet (YouTube can lag a bit after the broadcast), until it settles. Cheap:
videos.list is 1 unit per 50 ids, and the live/upcoming set is normally tiny.""" videos.list is 1 unit per 50 ids, and the live/upcoming set is normally tiny."""
limit = limit or settings.enrich_batch_size limit = limit or sysconfig.get_int(db, "enrich_batch_size")
videos = ( videos = (
db.execute( db.execute(
select(Video) select(Video)
@ -322,8 +321,8 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in
def run_shorts_classification(db: Session, limit: int | None = None) -> dict: def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
"""Finalize Shorts classification: cheaply mark non-candidates, then probe the """Finalize Shorts classification: cheaply mark non-candidates, then probe the
youtube.com/shorts URL for short-enough, non-live, enriched videos.""" youtube.com/shorts URL for short-enough, non-live, enriched videos."""
limit = limit or settings.shorts_probe_batch limit = limit or sysconfig.get_int(db, "shorts_probe_batch")
probe_max = settings.shorts_probe_max_seconds probe_max = sysconfig.get_int(db, "shorts_probe_max_seconds")
# 1) Anything enriched that can't be a Short -> finalize without a probe. # 1) Anything enriched that can't be a Short -> finalize without a probe.
db.execute( db.execute(

View file

@ -36,6 +36,18 @@ SPECS: tuple[ConfigSpec, ...] = (
ConfigSpec("smtp_user", "str", "email", "smtp_user"), ConfigSpec("smtp_user", "str", "email", "smtp_user"),
ConfigSpec("smtp_from", "str", "email", "smtp_from"), ConfigSpec("smtp_from", "str", "email", "smtp_from"),
ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True), ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True),
# --- Quota (shared YouTube Data API daily budget) ---
ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000),
ConfigSpec("backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000),
# --- Backfill (recent-first first pass per channel) ---
ConfigSpec("backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000),
ConfigSpec("backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500),
# --- Shorts probe ---
ConfigSpec("shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600),
ConfigSpec("shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000),
# --- Batch sizes ---
ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50),
ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000),
) )
_BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS} _BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS}