Merge feature/config-db-4b (epic 4b)

Move the operational params (quota budget/reserve, backfill window, shorts-probe, enrich/autotag batch) off env into the DB-backed system_config + admin Configuration page, wiring each read through the resolver.
2026-06-19 13:07:55 +02:00 · 2026-06-19 13:07:55 +02:00 · 93d11ef299
commit 93d11ef299
parent e127ecf490 50d776cdb7
11 changed files with 78 additions and 31 deletions
--- a/backend/app/quota.py
+++ b/backend/app/quota.py
@ -12,7 +12,7 @@ from zoneinfo import ZoneInfo
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session

-from app.config import settings
+from app import sysconfig
 from app.models import ApiQuotaUsage, QuotaEvent

 _PACIFIC = ZoneInfo("America/Los_Angeles")
@ -82,7 +82,7 @@ def units_used_today(db: Session) -> int:


 def remaining_today(db: Session) -> int:
-    return max(0, settings.quota_daily_budget - units_used_today(db))
+    return max(0, sysconfig.get_int(db, "quota_daily_budget") - units_used_today(db))


 def can_spend(db: Session, units: int) -> bool:
--- a/backend/app/routes/scheduler.py
+++ b/backend/app/routes/scheduler.py
@ -5,7 +5,7 @@ from fastapi import APIRouter, Depends, HTTPException
 from sqlalchemy import and_, func, or_, select
 from sqlalchemy.orm import Session

-from app import quota, state
+from app import quota, state, sysconfig
 from app.config import settings
 from app.db import get_db
 from app.models import Channel, Subscription, User, Video
@ -152,8 +152,8 @@ def get_scheduler(
    quota_info = {
        "used_today": used,
        "remaining_today": quota.remaining_today(db),
-        "daily_budget": settings.quota_daily_budget,
-        "backfill_reserve": settings.backfill_quota_reserve,
+        "daily_budget": sysconfig.get_int(db, "quota_daily_budget"),
+        "backfill_reserve": sysconfig.get_int(db, "backfill_quota_reserve"),
    }

    return {
--- a/backend/app/sync/autotag.py
+++ b/backend/app/sync/autotag.py
@ -13,8 +13,7 @@ from sqlalchemy.orm import Session

 log = logging.getLogger("subfeed.autotag")

-from app import progress
-from app.config import settings
+from app import progress, sysconfig
 from app.models import Channel, ChannelTag, Tag, Video

 # --- language id (offline, deterministic, small) ---
@ -166,7 +165,7 @@ def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None,
            select(Video.title)
            .where(Video.channel_id == channel.id, Video.title.is_not(None))
            .order_by(Video.published_at.desc())
-            .limit(settings.autotag_title_sample)
+            .limit(sysconfig.get_int(db, "autotag_title_sample"))
        )
        .scalars()
        .all()
--- a/backend/app/sync/maintenance.py
+++ b/backend/app/sync/maintenance.py
@ -25,7 +25,7 @@ from datetime import datetime, timedelta, timezone
 from sqlalchemy import or_, select
 from sqlalchemy.orm import Session

-from app import progress, quota
+from app import progress, quota, sysconfig
 from app.config import settings
 from app.models import Playlist, PlaylistItem, Video, VideoState
 from app.notifications import create_notification
@ -196,7 +196,7 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
    now = _now()
    # Process in videos.list-sized pages so we commit incrementally and stop on low quota.
    while checked < batch:
-        if quota.remaining_today(db) <= settings.backfill_quota_reserve:
+        if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
            break
        page = (
            db.execute(
@ -236,7 +236,7 @@ def run_maintenance(db: Session) -> dict:
        phase1 = _recheck_flagged(db, yt)
        phase2 = (
            _revalidate_rolling(db, yt)
-            if quota.remaining_today(db) > settings.backfill_quota_reserve
+            if quota.remaining_today(db) > sysconfig.get_int(db, "backfill_quota_reserve")
            else {"checked": 0, "flagged": 0, "skipped": "low quota"}
        )
    return {
--- a/backend/app/sync/runner.py
+++ b/backend/app/sync/runner.py
@ -10,8 +10,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from sqlalchemy import select
 from sqlalchemy.orm import Session

-from app import progress, quota
-from app.config import settings
+from app import progress, quota, sysconfig

 log = logging.getLogger("subfeed.sync")
 from app.models import Channel, OAuthToken, Subscription, User
@ -140,7 +139,7 @@ def run_recent_backfill(
    videos_added = 0
    with YouTubeClient(db, user) as yt:
        for channel in channels:
-            if quota.remaining_today(db) <= settings.backfill_quota_reserve:
+            if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
                break
            try:
                videos_added += backfill_channel_recent(db, yt, channel)
@ -186,7 +185,7 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
    videos_added = 0
    with YouTubeClient(db, user) as yt:
        for channel in channels:
-            if quota.remaining_today(db) <= settings.backfill_quota_reserve:
+            if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
                break
            try:
                videos_added += backfill_channel_deep(db, yt, channel, max_pages)
@ -215,7 +214,9 @@ def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
        return 2 * pages

    total_units = sum(units(c) for c in pending)
-    daily_budget = max(1, settings.quota_daily_budget - settings.backfill_quota_reserve)
+    daily_budget = max(
+        1, sysconfig.get_int(db, "quota_daily_budget") - sysconfig.get_int(db, "backfill_quota_reserve")
+    )
    eta_seconds = int(total_units / daily_budget * 86_400) if pending else 0
    return {
        "channels_pending": len(pending),
--- a/backend/app/sync/videos.py
+++ b/backend/app/sync/videos.py
@ -9,8 +9,7 @@ from sqlalchemy import and_, func, or_, select, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session

-from app import progress
-from app.config import settings
+from app import progress, sysconfig
 from app.models import Channel, Video
 from app.youtube.client import YouTubeClient, best_thumbnail
 from app.youtube.rss import fetch_channel_feed
@ -108,7 +107,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
        db.commit()
        return 0

-    cutoff = _now() - timedelta(days=settings.backfill_recent_max_days)
+    cutoff = _now() - timedelta(days=sysconfig.get_int(db, "backfill_recent_max_days"))
    collected: list[dict] = []
    page_token: str | None = None
    next_cursor: str | None = None
@ -126,7 +125,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
                stopped_mid_page = True
                break
            collected.append(stub)
-            if len(collected) >= settings.backfill_recent_max_videos:
+            if len(collected) >= sysconfig.get_int(db, "backfill_recent_max_videos"):
                stopped_mid_page = True
                break
        page_token = data.get("nextPageToken")
@ -249,7 +248,7 @@ def apply_video_details(video: Video, item: dict) -> None:


 def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int:
-    limit = limit or settings.enrich_batch_size
+    limit = limit or sysconfig.get_int(db, "enrich_batch_size")
    videos = (
        db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit))
        .scalars()
@ -282,7 +281,7 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in
    ends. So we revisit anything still live/upcoming, plus a just-ended stream that hasn't got
    its duration yet (YouTube can lag a bit after the broadcast), until it settles. Cheap:
    videos.list is 1 unit per 50 ids, and the live/upcoming set is normally tiny."""
-    limit = limit or settings.enrich_batch_size
+    limit = limit or sysconfig.get_int(db, "enrich_batch_size")
    videos = (
        db.execute(
            select(Video)
@ -322,8 +321,8 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in
 def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
    """Finalize Shorts classification: cheaply mark non-candidates, then probe the
    youtube.com/shorts URL for short-enough, non-live, enriched videos."""
-    limit = limit or settings.shorts_probe_batch
-    probe_max = settings.shorts_probe_max_seconds
+    limit = limit or sysconfig.get_int(db, "shorts_probe_batch")
+    probe_max = sysconfig.get_int(db, "shorts_probe_max_seconds")

    # 1) Anything enriched that can't be a Short -> finalize without a probe.
    db.execute(
--- a/backend/app/sysconfig.py
+++ b/backend/app/sysconfig.py
@ -36,6 +36,18 @@ SPECS: tuple[ConfigSpec, ...] = (
    ConfigSpec("smtp_user", "str", "email", "smtp_user"),
    ConfigSpec("smtp_from", "str", "email", "smtp_from"),
    ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True),
+    # --- Quota (shared YouTube Data API daily budget) ---
+    ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000),
+    ConfigSpec("backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000),
+    # --- Backfill (recent-first first pass per channel) ---
+    ConfigSpec("backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000),
+    ConfigSpec("backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500),
+    # --- Shorts probe ---
+    ConfigSpec("shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600),
+    ConfigSpec("shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000),
+    # --- Batch sizes ---
+    ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50),
+    ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000),
 )

 _BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS}
--- a/frontend/src/components/ConfigPanel.tsx
+++ b/frontend/src/components/ConfigPanel.tsx
@ -11,7 +11,7 @@ import Tooltip from "./Tooltip";
 // applied together via one Save/Discard bar (consistent with the Settings page); nothing hits
 // the server until Save. Group order is fixed where known so logically related settings (e.g.
 // Email/SMTP) stay together.
-const GROUP_ORDER = ["email"];
+const GROUP_ORDER = ["email", "quota", "backfill", "shorts", "batch"];
 type SaveState = "idle" | "saving" | "saved" | "error";

 export default function ConfigPanel() {
--- a/frontend/src/i18n/locales/de/config.json
+++ b/frontend/src/i18n/locales/de/config.json
@ -2,14 +2,26 @@
  "intro": "In der Datenbank gespeicherte Betriebseinstellungen — sie überschreiben die Datei-/Env-Standardwerte und wirken ohne erneutes Deployment. Lass ein Feld leer (oder setze es zurück), um auf den Standard zurückzufallen.",
  "loading": "Konfiguration wird geladen…",
  "groups": {
-    "email": "E-Mail / SMTP"
+    "email": "E-Mail / SMTP",
+    "quota": "Kontingent",
+    "backfill": "Nachladen (Backfill)",
+    "shorts": "Shorts-Prüfung",
+    "batch": "Stapelgrößen"
  },
  "fields": {
    "smtp_host": { "label": "SMTP-Host", "hint": "z. B. smtp.gmail.com" },
    "smtp_port": { "label": "SMTP-Port", "hint": "Üblicherweise 587 (STARTTLS)." },
    "smtp_user": { "label": "SMTP-Benutzername", "hint": "Das sendende Konto / der Login." },
    "smtp_from": { "label": "Absenderadresse", "hint": "z. B. Siftlode <addr@gmail.com>. Standard ist der Benutzername." },
-    "smtp_password": { "label": "SMTP-Passwort", "hint": "App-Passwort. Verschlüsselt gespeichert; nur schreibbar — wird nie wieder angezeigt." }
+    "smtp_password": { "label": "SMTP-Passwort", "hint": "App-Passwort. Verschlüsselt gespeichert; nur schreibbar — wird nie wieder angezeigt." },
+    "quota_daily_budget": { "label": "Tägliches Kontingentbudget", "hint": "YouTube-Data-API-Einheiten pro Tag (kostenloses Limit 10.000). Standard 9000 lässt Puffer." },
+    "backfill_quota_reserve": { "label": "Nachlade-Kontingentreserve", "hint": "Reservierte Einheiten, damit geplantes Nachladen interaktive Syncs / Anreicherung nie aushungert." },
+    "backfill_recent_max_videos": { "label": "Aktuelles Nachladen — max. Videos", "hint": "Neueste Videos pro Kanal beim ersten Durchlauf." },
+    "backfill_recent_max_days": { "label": "Aktuelles Nachladen — max. Alter (Tage)", "hint": "Wie weit der erste Durchlauf pro Kanal zurückreicht." },
+    "shorts_probe_max_seconds": { "label": "Shorts-Prüfung — max. Dauer (s)", "hint": "Nur Videos bis zu dieser Länge werden als mögliche Shorts geprüft." },
+    "shorts_probe_batch": { "label": "Shorts-Prüfung — Stapelgröße", "hint": "Wie viele Videos pro Lauf klassifiziert werden." },
+    "enrich_batch_size": { "label": "Anreicherungs-Stapelgröße", "hint": "videos.list-IDs pro Aufruf (YouTube begrenzt auf 50)." },
+    "autotag_title_sample": { "label": "Auto-Tag-Titelstichprobe", "hint": "Pro Kanal abgetastete aktuelle Videotitel zur Spracherkennung." }
  },
  "save": "Speichern",
  "saving": "Speichern…",
--- a/frontend/src/i18n/locales/en/config.json
+++ b/frontend/src/i18n/locales/en/config.json
@ -2,14 +2,26 @@
  "intro": "Operational settings stored in the database — these override the file/env defaults and take effect without a redeploy. Leave a field empty (or reset it) to fall back to the default.",
  "loading": "Loading configuration…",
  "groups": {
-    "email": "Email / SMTP"
+    "email": "Email / SMTP",
+    "quota": "Quota",
+    "backfill": "Backfill",
+    "shorts": "Shorts probe",
+    "batch": "Batch sizes"
  },
  "fields": {
    "smtp_host": { "label": "SMTP host", "hint": "e.g. smtp.gmail.com" },
    "smtp_port": { "label": "SMTP port", "hint": "Usually 587 (STARTTLS)." },
    "smtp_user": { "label": "SMTP username", "hint": "The sending account / login." },
    "smtp_from": { "label": "From address", "hint": "e.g. Siftlode <addr@gmail.com>. Defaults to the username." },
-    "smtp_password": { "label": "SMTP password", "hint": "App password. Stored encrypted; write-only — it's never shown back." }
+    "smtp_password": { "label": "SMTP password", "hint": "App password. Stored encrypted; write-only — it's never shown back." },
+    "quota_daily_budget": { "label": "Daily quota budget", "hint": "Total YouTube Data API units to spend per day (free tier is 10,000). Default 9000 leaves headroom." },
+    "backfill_quota_reserve": { "label": "Backfill quota reserve", "hint": "Units kept in reserve so scheduled backfill never starves interactive syncs / enrichment." },
+    "backfill_recent_max_videos": { "label": "Recent backfill — max videos", "hint": "Newest videos fetched per channel on the first pass." },
+    "backfill_recent_max_days": { "label": "Recent backfill — max age (days)", "hint": "How far back the first pass reaches per channel." },
+    "shorts_probe_max_seconds": { "label": "Shorts probe — max duration (s)", "hint": "Only videos at or below this length are probed as possible Shorts." },
+    "shorts_probe_batch": { "label": "Shorts probe — batch size", "hint": "How many videos to classify per run." },
+    "enrich_batch_size": { "label": "Enrichment batch size", "hint": "videos.list ids per call (YouTube caps this at 50)." },
+    "autotag_title_sample": { "label": "Auto-tag title sample", "hint": "Recent video titles sampled per channel for language detection." }
  },
  "save": "Save",
  "saving": "Saving…",
--- a/frontend/src/i18n/locales/hu/config.json
+++ b/frontend/src/i18n/locales/hu/config.json
@ -2,14 +2,26 @@
  "intro": "Az adatbázisban tárolt működési beállítások — felülírják a fájl/env alapértékeket, és újratelepítés nélkül lépnek életbe. Hagyd üresen (vagy állítsd vissza) a mezőt az alapértékhez való visszatéréshez.",
  "loading": "Konfiguráció betöltése…",
  "groups": {
-    "email": "E-mail / SMTP"
+    "email": "E-mail / SMTP",
+    "quota": "Kvóta",
+    "backfill": "Letöltés (backfill)",
+    "shorts": "Shorts-vizsgálat",
+    "batch": "Kötegméretek"
  },
  "fields": {
    "smtp_host": { "label": "SMTP-kiszolgáló", "hint": "pl. smtp.gmail.com" },
    "smtp_port": { "label": "SMTP-port", "hint": "Általában 587 (STARTTLS)." },
    "smtp_user": { "label": "SMTP-felhasználónév", "hint": "A küldő fiók / bejelentkezés." },
    "smtp_from": { "label": "Feladó cím", "hint": "pl. Siftlode <addr@gmail.com>. Alapból a felhasználónév." },
-    "smtp_password": { "label": "SMTP-jelszó", "hint": "Alkalmazásjelszó. Titkosítva tárolva; csak írható — soha nem jelenik meg újra." }
+    "smtp_password": { "label": "SMTP-jelszó", "hint": "Alkalmazásjelszó. Titkosítva tárolva; csak írható — soha nem jelenik meg újra." },
+    "quota_daily_budget": { "label": "Napi kvótakeret", "hint": "Naponta elkölthető YouTube Data API-egységek (az ingyenes keret 10 000). Az alap 9000 tartalékot hagy." },
+    "backfill_quota_reserve": { "label": "Letöltési kvótatartalék", "hint": "Tartalékban tartott egységek, hogy az ütemezett letöltés ne éheztesse ki az interaktív szinkront / gazdagítást." },
+    "backfill_recent_max_videos": { "label": "Friss letöltés — max videó", "hint": "Csatornánként az első körben letöltött legújabb videók száma." },
+    "backfill_recent_max_days": { "label": "Friss letöltés — max kor (nap)", "hint": "Az első kör csatornánként meddig nyúl vissza." },
+    "shorts_probe_max_seconds": { "label": "Shorts-vizsgálat — max hossz (mp)", "hint": "Csak az ennél nem hosszabb videókat vizsgáljuk lehetséges Shortsként." },
+    "shorts_probe_batch": { "label": "Shorts-vizsgálat — kötegméret", "hint": "Futásonként ennyi videót osztályozunk." },
+    "enrich_batch_size": { "label": "Gazdagítási kötegméret", "hint": "videos.list azonosítók hívásonként (a YouTube 50-ben maximálja)." },
+    "autotag_title_sample": { "label": "Auto-címke címmintavétel", "hint": "Csatornánként ennyi friss videócímet mintázunk a nyelvfelismeréshez." }
  },
  "save": "Mentés",
  "saving": "Mentés…",