Merge feature/config-db-4b (epic 4b)

Move the operational params (quota budget/reserve, backfill window, shorts-probe,
enrich/autotag batch) off env into the DB-backed system_config + admin Configuration
page, wiring each read through the resolver.
This commit is contained in:
npeter83 2026-06-19 13:07:55 +02:00
commit 93d11ef299
11 changed files with 78 additions and 31 deletions

View file

@ -12,7 +12,7 @@ from zoneinfo import ZoneInfo
from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.config import settings from app import sysconfig
from app.models import ApiQuotaUsage, QuotaEvent from app.models import ApiQuotaUsage, QuotaEvent
_PACIFIC = ZoneInfo("America/Los_Angeles") _PACIFIC = ZoneInfo("America/Los_Angeles")
@ -82,7 +82,7 @@ def units_used_today(db: Session) -> int:
def remaining_today(db: Session) -> int: def remaining_today(db: Session) -> int:
return max(0, settings.quota_daily_budget - units_used_today(db)) return max(0, sysconfig.get_int(db, "quota_daily_budget") - units_used_today(db))
def can_spend(db: Session, units: int) -> bool: def can_spend(db: Session, units: int) -> bool:

View file

@ -5,7 +5,7 @@ from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import and_, func, or_, select from sqlalchemy import and_, func, or_, select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import quota, state from app import quota, state, sysconfig
from app.config import settings from app.config import settings
from app.db import get_db from app.db import get_db
from app.models import Channel, Subscription, User, Video from app.models import Channel, Subscription, User, Video
@ -152,8 +152,8 @@ def get_scheduler(
quota_info = { quota_info = {
"used_today": used, "used_today": used,
"remaining_today": quota.remaining_today(db), "remaining_today": quota.remaining_today(db),
"daily_budget": settings.quota_daily_budget, "daily_budget": sysconfig.get_int(db, "quota_daily_budget"),
"backfill_reserve": settings.backfill_quota_reserve, "backfill_reserve": sysconfig.get_int(db, "backfill_quota_reserve"),
} }
return { return {

View file

@ -13,8 +13,7 @@ from sqlalchemy.orm import Session
log = logging.getLogger("subfeed.autotag") log = logging.getLogger("subfeed.autotag")
from app import progress from app import progress, sysconfig
from app.config import settings
from app.models import Channel, ChannelTag, Tag, Video from app.models import Channel, ChannelTag, Tag, Video
# --- language id (offline, deterministic, small) --- # --- language id (offline, deterministic, small) ---
@ -166,7 +165,7 @@ def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None,
select(Video.title) select(Video.title)
.where(Video.channel_id == channel.id, Video.title.is_not(None)) .where(Video.channel_id == channel.id, Video.title.is_not(None))
.order_by(Video.published_at.desc()) .order_by(Video.published_at.desc())
.limit(settings.autotag_title_sample) .limit(sysconfig.get_int(db, "autotag_title_sample"))
) )
.scalars() .scalars()
.all() .all()

View file

@ -25,7 +25,7 @@ from datetime import datetime, timedelta, timezone
from sqlalchemy import or_, select from sqlalchemy import or_, select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import progress, quota from app import progress, quota, sysconfig
from app.config import settings from app.config import settings
from app.models import Playlist, PlaylistItem, Video, VideoState from app.models import Playlist, PlaylistItem, Video, VideoState
from app.notifications import create_notification from app.notifications import create_notification
@ -196,7 +196,7 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
now = _now() now = _now()
# Process in videos.list-sized pages so we commit incrementally and stop on low quota. # Process in videos.list-sized pages so we commit incrementally and stop on low quota.
while checked < batch: while checked < batch:
if quota.remaining_today(db) <= settings.backfill_quota_reserve: if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
break break
page = ( page = (
db.execute( db.execute(
@ -236,7 +236,7 @@ def run_maintenance(db: Session) -> dict:
phase1 = _recheck_flagged(db, yt) phase1 = _recheck_flagged(db, yt)
phase2 = ( phase2 = (
_revalidate_rolling(db, yt) _revalidate_rolling(db, yt)
if quota.remaining_today(db) > settings.backfill_quota_reserve if quota.remaining_today(db) > sysconfig.get_int(db, "backfill_quota_reserve")
else {"checked": 0, "flagged": 0, "skipped": "low quota"} else {"checked": 0, "flagged": 0, "skipped": "low quota"}
) )
return { return {

View file

@ -10,8 +10,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
from sqlalchemy import select from sqlalchemy import select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import progress, quota from app import progress, quota, sysconfig
from app.config import settings
log = logging.getLogger("subfeed.sync") log = logging.getLogger("subfeed.sync")
from app.models import Channel, OAuthToken, Subscription, User from app.models import Channel, OAuthToken, Subscription, User
@ -140,7 +139,7 @@ def run_recent_backfill(
videos_added = 0 videos_added = 0
with YouTubeClient(db, user) as yt: with YouTubeClient(db, user) as yt:
for channel in channels: for channel in channels:
if quota.remaining_today(db) <= settings.backfill_quota_reserve: if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
break break
try: try:
videos_added += backfill_channel_recent(db, yt, channel) videos_added += backfill_channel_recent(db, yt, channel)
@ -186,7 +185,7 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
videos_added = 0 videos_added = 0
with YouTubeClient(db, user) as yt: with YouTubeClient(db, user) as yt:
for channel in channels: for channel in channels:
if quota.remaining_today(db) <= settings.backfill_quota_reserve: if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
break break
try: try:
videos_added += backfill_channel_deep(db, yt, channel, max_pages) videos_added += backfill_channel_deep(db, yt, channel, max_pages)
@ -215,7 +214,9 @@ def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
return 2 * pages return 2 * pages
total_units = sum(units(c) for c in pending) total_units = sum(units(c) for c in pending)
daily_budget = max(1, settings.quota_daily_budget - settings.backfill_quota_reserve) daily_budget = max(
1, sysconfig.get_int(db, "quota_daily_budget") - sysconfig.get_int(db, "backfill_quota_reserve")
)
eta_seconds = int(total_units / daily_budget * 86_400) if pending else 0 eta_seconds = int(total_units / daily_budget * 86_400) if pending else 0
return { return {
"channels_pending": len(pending), "channels_pending": len(pending),

View file

@ -9,8 +9,7 @@ from sqlalchemy import and_, func, or_, select, update
from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import progress from app import progress, sysconfig
from app.config import settings
from app.models import Channel, Video from app.models import Channel, Video
from app.youtube.client import YouTubeClient, best_thumbnail from app.youtube.client import YouTubeClient, best_thumbnail
from app.youtube.rss import fetch_channel_feed from app.youtube.rss import fetch_channel_feed
@ -108,7 +107,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
db.commit() db.commit()
return 0 return 0
cutoff = _now() - timedelta(days=settings.backfill_recent_max_days) cutoff = _now() - timedelta(days=sysconfig.get_int(db, "backfill_recent_max_days"))
collected: list[dict] = [] collected: list[dict] = []
page_token: str | None = None page_token: str | None = None
next_cursor: str | None = None next_cursor: str | None = None
@ -126,7 +125,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
stopped_mid_page = True stopped_mid_page = True
break break
collected.append(stub) collected.append(stub)
if len(collected) >= settings.backfill_recent_max_videos: if len(collected) >= sysconfig.get_int(db, "backfill_recent_max_videos"):
stopped_mid_page = True stopped_mid_page = True
break break
page_token = data.get("nextPageToken") page_token = data.get("nextPageToken")
@ -249,7 +248,7 @@ def apply_video_details(video: Video, item: dict) -> None:
def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int: def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int:
limit = limit or settings.enrich_batch_size limit = limit or sysconfig.get_int(db, "enrich_batch_size")
videos = ( videos = (
db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit)) db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit))
.scalars() .scalars()
@ -282,7 +281,7 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in
ends. So we revisit anything still live/upcoming, plus a just-ended stream that hasn't got ends. So we revisit anything still live/upcoming, plus a just-ended stream that hasn't got
its duration yet (YouTube can lag a bit after the broadcast), until it settles. Cheap: its duration yet (YouTube can lag a bit after the broadcast), until it settles. Cheap:
videos.list is 1 unit per 50 ids, and the live/upcoming set is normally tiny.""" videos.list is 1 unit per 50 ids, and the live/upcoming set is normally tiny."""
limit = limit or settings.enrich_batch_size limit = limit or sysconfig.get_int(db, "enrich_batch_size")
videos = ( videos = (
db.execute( db.execute(
select(Video) select(Video)
@ -322,8 +321,8 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in
def run_shorts_classification(db: Session, limit: int | None = None) -> dict: def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
"""Finalize Shorts classification: cheaply mark non-candidates, then probe the """Finalize Shorts classification: cheaply mark non-candidates, then probe the
youtube.com/shorts URL for short-enough, non-live, enriched videos.""" youtube.com/shorts URL for short-enough, non-live, enriched videos."""
limit = limit or settings.shorts_probe_batch limit = limit or sysconfig.get_int(db, "shorts_probe_batch")
probe_max = settings.shorts_probe_max_seconds probe_max = sysconfig.get_int(db, "shorts_probe_max_seconds")
# 1) Anything enriched that can't be a Short -> finalize without a probe. # 1) Anything enriched that can't be a Short -> finalize without a probe.
db.execute( db.execute(

View file

@ -36,6 +36,18 @@ SPECS: tuple[ConfigSpec, ...] = (
ConfigSpec("smtp_user", "str", "email", "smtp_user"), ConfigSpec("smtp_user", "str", "email", "smtp_user"),
ConfigSpec("smtp_from", "str", "email", "smtp_from"), ConfigSpec("smtp_from", "str", "email", "smtp_from"),
ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True), ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True),
# --- Quota (shared YouTube Data API daily budget) ---
ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000),
ConfigSpec("backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000),
# --- Backfill (recent-first first pass per channel) ---
ConfigSpec("backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000),
ConfigSpec("backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500),
# --- Shorts probe ---
ConfigSpec("shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600),
ConfigSpec("shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000),
# --- Batch sizes ---
ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50),
ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000),
) )
_BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS} _BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS}

View file

@ -11,7 +11,7 @@ import Tooltip from "./Tooltip";
// applied together via one Save/Discard bar (consistent with the Settings page); nothing hits // applied together via one Save/Discard bar (consistent with the Settings page); nothing hits
// the server until Save. Group order is fixed where known so logically related settings (e.g. // the server until Save. Group order is fixed where known so logically related settings (e.g.
// Email/SMTP) stay together. // Email/SMTP) stay together.
const GROUP_ORDER = ["email"]; const GROUP_ORDER = ["email", "quota", "backfill", "shorts", "batch"];
type SaveState = "idle" | "saving" | "saved" | "error"; type SaveState = "idle" | "saving" | "saved" | "error";
export default function ConfigPanel() { export default function ConfigPanel() {

View file

@ -2,14 +2,26 @@
"intro": "In der Datenbank gespeicherte Betriebseinstellungen — sie überschreiben die Datei-/Env-Standardwerte und wirken ohne erneutes Deployment. Lass ein Feld leer (oder setze es zurück), um auf den Standard zurückzufallen.", "intro": "In der Datenbank gespeicherte Betriebseinstellungen — sie überschreiben die Datei-/Env-Standardwerte und wirken ohne erneutes Deployment. Lass ein Feld leer (oder setze es zurück), um auf den Standard zurückzufallen.",
"loading": "Konfiguration wird geladen…", "loading": "Konfiguration wird geladen…",
"groups": { "groups": {
"email": "E-Mail / SMTP" "email": "E-Mail / SMTP",
"quota": "Kontingent",
"backfill": "Nachladen (Backfill)",
"shorts": "Shorts-Prüfung",
"batch": "Stapelgrößen"
}, },
"fields": { "fields": {
"smtp_host": { "label": "SMTP-Host", "hint": "z. B. smtp.gmail.com" }, "smtp_host": { "label": "SMTP-Host", "hint": "z. B. smtp.gmail.com" },
"smtp_port": { "label": "SMTP-Port", "hint": "Üblicherweise 587 (STARTTLS)." }, "smtp_port": { "label": "SMTP-Port", "hint": "Üblicherweise 587 (STARTTLS)." },
"smtp_user": { "label": "SMTP-Benutzername", "hint": "Das sendende Konto / der Login." }, "smtp_user": { "label": "SMTP-Benutzername", "hint": "Das sendende Konto / der Login." },
"smtp_from": { "label": "Absenderadresse", "hint": "z. B. Siftlode <addr@gmail.com>. Standard ist der Benutzername." }, "smtp_from": { "label": "Absenderadresse", "hint": "z. B. Siftlode <addr@gmail.com>. Standard ist der Benutzername." },
"smtp_password": { "label": "SMTP-Passwort", "hint": "App-Passwort. Verschlüsselt gespeichert; nur schreibbar — wird nie wieder angezeigt." } "smtp_password": { "label": "SMTP-Passwort", "hint": "App-Passwort. Verschlüsselt gespeichert; nur schreibbar — wird nie wieder angezeigt." },
"quota_daily_budget": { "label": "Tägliches Kontingentbudget", "hint": "YouTube-Data-API-Einheiten pro Tag (kostenloses Limit 10.000). Standard 9000 lässt Puffer." },
"backfill_quota_reserve": { "label": "Nachlade-Kontingentreserve", "hint": "Reservierte Einheiten, damit geplantes Nachladen interaktive Syncs / Anreicherung nie aushungert." },
"backfill_recent_max_videos": { "label": "Aktuelles Nachladen — max. Videos", "hint": "Neueste Videos pro Kanal beim ersten Durchlauf." },
"backfill_recent_max_days": { "label": "Aktuelles Nachladen — max. Alter (Tage)", "hint": "Wie weit der erste Durchlauf pro Kanal zurückreicht." },
"shorts_probe_max_seconds": { "label": "Shorts-Prüfung — max. Dauer (s)", "hint": "Nur Videos bis zu dieser Länge werden als mögliche Shorts geprüft." },
"shorts_probe_batch": { "label": "Shorts-Prüfung — Stapelgröße", "hint": "Wie viele Videos pro Lauf klassifiziert werden." },
"enrich_batch_size": { "label": "Anreicherungs-Stapelgröße", "hint": "videos.list-IDs pro Aufruf (YouTube begrenzt auf 50)." },
"autotag_title_sample": { "label": "Auto-Tag-Titelstichprobe", "hint": "Pro Kanal abgetastete aktuelle Videotitel zur Spracherkennung." }
}, },
"save": "Speichern", "save": "Speichern",
"saving": "Speichern…", "saving": "Speichern…",

View file

@ -2,14 +2,26 @@
"intro": "Operational settings stored in the database — these override the file/env defaults and take effect without a redeploy. Leave a field empty (or reset it) to fall back to the default.", "intro": "Operational settings stored in the database — these override the file/env defaults and take effect without a redeploy. Leave a field empty (or reset it) to fall back to the default.",
"loading": "Loading configuration…", "loading": "Loading configuration…",
"groups": { "groups": {
"email": "Email / SMTP" "email": "Email / SMTP",
"quota": "Quota",
"backfill": "Backfill",
"shorts": "Shorts probe",
"batch": "Batch sizes"
}, },
"fields": { "fields": {
"smtp_host": { "label": "SMTP host", "hint": "e.g. smtp.gmail.com" }, "smtp_host": { "label": "SMTP host", "hint": "e.g. smtp.gmail.com" },
"smtp_port": { "label": "SMTP port", "hint": "Usually 587 (STARTTLS)." }, "smtp_port": { "label": "SMTP port", "hint": "Usually 587 (STARTTLS)." },
"smtp_user": { "label": "SMTP username", "hint": "The sending account / login." }, "smtp_user": { "label": "SMTP username", "hint": "The sending account / login." },
"smtp_from": { "label": "From address", "hint": "e.g. Siftlode <addr@gmail.com>. Defaults to the username." }, "smtp_from": { "label": "From address", "hint": "e.g. Siftlode <addr@gmail.com>. Defaults to the username." },
"smtp_password": { "label": "SMTP password", "hint": "App password. Stored encrypted; write-only — it's never shown back." } "smtp_password": { "label": "SMTP password", "hint": "App password. Stored encrypted; write-only — it's never shown back." },
"quota_daily_budget": { "label": "Daily quota budget", "hint": "Total YouTube Data API units to spend per day (free tier is 10,000). Default 9000 leaves headroom." },
"backfill_quota_reserve": { "label": "Backfill quota reserve", "hint": "Units kept in reserve so scheduled backfill never starves interactive syncs / enrichment." },
"backfill_recent_max_videos": { "label": "Recent backfill — max videos", "hint": "Newest videos fetched per channel on the first pass." },
"backfill_recent_max_days": { "label": "Recent backfill — max age (days)", "hint": "How far back the first pass reaches per channel." },
"shorts_probe_max_seconds": { "label": "Shorts probe — max duration (s)", "hint": "Only videos at or below this length are probed as possible Shorts." },
"shorts_probe_batch": { "label": "Shorts probe — batch size", "hint": "How many videos to classify per run." },
"enrich_batch_size": { "label": "Enrichment batch size", "hint": "videos.list ids per call (YouTube caps this at 50)." },
"autotag_title_sample": { "label": "Auto-tag title sample", "hint": "Recent video titles sampled per channel for language detection." }
}, },
"save": "Save", "save": "Save",
"saving": "Saving…", "saving": "Saving…",

View file

@ -2,14 +2,26 @@
"intro": "Az adatbázisban tárolt működési beállítások — felülírják a fájl/env alapértékeket, és újratelepítés nélkül lépnek életbe. Hagyd üresen (vagy állítsd vissza) a mezőt az alapértékhez való visszatéréshez.", "intro": "Az adatbázisban tárolt működési beállítások — felülírják a fájl/env alapértékeket, és újratelepítés nélkül lépnek életbe. Hagyd üresen (vagy állítsd vissza) a mezőt az alapértékhez való visszatéréshez.",
"loading": "Konfiguráció betöltése…", "loading": "Konfiguráció betöltése…",
"groups": { "groups": {
"email": "E-mail / SMTP" "email": "E-mail / SMTP",
"quota": "Kvóta",
"backfill": "Letöltés (backfill)",
"shorts": "Shorts-vizsgálat",
"batch": "Kötegméretek"
}, },
"fields": { "fields": {
"smtp_host": { "label": "SMTP-kiszolgáló", "hint": "pl. smtp.gmail.com" }, "smtp_host": { "label": "SMTP-kiszolgáló", "hint": "pl. smtp.gmail.com" },
"smtp_port": { "label": "SMTP-port", "hint": "Általában 587 (STARTTLS)." }, "smtp_port": { "label": "SMTP-port", "hint": "Általában 587 (STARTTLS)." },
"smtp_user": { "label": "SMTP-felhasználónév", "hint": "A küldő fiók / bejelentkezés." }, "smtp_user": { "label": "SMTP-felhasználónév", "hint": "A küldő fiók / bejelentkezés." },
"smtp_from": { "label": "Feladó cím", "hint": "pl. Siftlode <addr@gmail.com>. Alapból a felhasználónév." }, "smtp_from": { "label": "Feladó cím", "hint": "pl. Siftlode <addr@gmail.com>. Alapból a felhasználónév." },
"smtp_password": { "label": "SMTP-jelszó", "hint": "Alkalmazásjelszó. Titkosítva tárolva; csak írható — soha nem jelenik meg újra." } "smtp_password": { "label": "SMTP-jelszó", "hint": "Alkalmazásjelszó. Titkosítva tárolva; csak írható — soha nem jelenik meg újra." },
"quota_daily_budget": { "label": "Napi kvótakeret", "hint": "Naponta elkölthető YouTube Data API-egységek (az ingyenes keret 10 000). Az alap 9000 tartalékot hagy." },
"backfill_quota_reserve": { "label": "Letöltési kvótatartalék", "hint": "Tartalékban tartott egységek, hogy az ütemezett letöltés ne éheztesse ki az interaktív szinkront / gazdagítást." },
"backfill_recent_max_videos": { "label": "Friss letöltés — max videó", "hint": "Csatornánként az első körben letöltött legújabb videók száma." },
"backfill_recent_max_days": { "label": "Friss letöltés — max kor (nap)", "hint": "Az első kör csatornánként meddig nyúl vissza." },
"shorts_probe_max_seconds": { "label": "Shorts-vizsgálat — max hossz (mp)", "hint": "Csak az ennél nem hosszabb videókat vizsgáljuk lehetséges Shortsként." },
"shorts_probe_batch": { "label": "Shorts-vizsgálat — kötegméret", "hint": "Futásonként ennyi videót osztályozunk." },
"enrich_batch_size": { "label": "Gazdagítási kötegméret", "hint": "videos.list azonosítók hívásonként (a YouTube 50-ben maximálja)." },
"autotag_title_sample": { "label": "Auto-címke címmintavétel", "hint": "Csatornánként ennyi friss videócímet mintázunk a nyelvfelismeréshez." }
}, },
"save": "Mentés", "save": "Mentés",
"saving": "Mentés…", "saving": "Mentés…",