From e6bcf5ba1eddf58eb2681b612bfbc4ccd0aa3d30 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 19 Jun 2026 13:07:45 +0200 Subject: [PATCH 1/2] feat(config): move operational params to DB (quota/backfill/shorts/batch) Register 8 more keys in the sysconfig registry (quota daily budget + backfill reserve, recent-backfill window, shorts-probe params, enrich/autotag batch sizes) and route their reads through the DB-override-or-env resolver at every call site (quota.py, sync/runner.py, sync/videos.py, sync/autotag.py, sync/maintenance.py, routes/scheduler.py). All read sites already had a db session in scope. Reads are read-through (no cache), matching app.state; admin can tune them live, no redeploy. --- backend/app/quota.py | 4 ++-- backend/app/routes/scheduler.py | 6 +++--- backend/app/sync/autotag.py | 5 ++--- backend/app/sync/maintenance.py | 6 +++--- backend/app/sync/runner.py | 11 ++++++----- backend/app/sync/videos.py | 15 +++++++-------- backend/app/sysconfig.py | 12 ++++++++++++ 7 files changed, 35 insertions(+), 24 deletions(-) diff --git a/backend/app/quota.py b/backend/app/quota.py index e2c7696..be699a1 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -12,7 +12,7 @@ from zoneinfo import ZoneInfo from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session -from app.config import settings +from app import sysconfig from app.models import ApiQuotaUsage, QuotaEvent _PACIFIC = ZoneInfo("America/Los_Angeles") @@ -82,7 +82,7 @@ def units_used_today(db: Session) -> int: def remaining_today(db: Session) -> int: - return max(0, settings.quota_daily_budget - units_used_today(db)) + return max(0, sysconfig.get_int(db, "quota_daily_budget") - units_used_today(db)) def can_spend(db: Session, units: int) -> bool: diff --git a/backend/app/routes/scheduler.py b/backend/app/routes/scheduler.py index 66e4f8f..e88b1db 100644 --- a/backend/app/routes/scheduler.py +++ b/backend/app/routes/scheduler.py @@ -5,7 +5,7 @@ from fastapi import APIRouter, Depends, HTTPException from sqlalchemy import and_, func, or_, select from sqlalchemy.orm import Session -from app import quota, state +from app import quota, state, sysconfig from app.config import settings from app.db import get_db from app.models import Channel, Subscription, User, Video @@ -152,8 +152,8 @@ def get_scheduler( quota_info = { "used_today": used, "remaining_today": quota.remaining_today(db), - "daily_budget": settings.quota_daily_budget, - "backfill_reserve": settings.backfill_quota_reserve, + "daily_budget": sysconfig.get_int(db, "quota_daily_budget"), + "backfill_reserve": sysconfig.get_int(db, "backfill_quota_reserve"), } return { diff --git a/backend/app/sync/autotag.py b/backend/app/sync/autotag.py index 09cce68..0a1359e 100644 --- a/backend/app/sync/autotag.py +++ b/backend/app/sync/autotag.py @@ -13,8 +13,7 @@ from sqlalchemy.orm import Session log = logging.getLogger("subfeed.autotag") -from app import progress -from app.config import settings +from app import progress, sysconfig from app.models import Channel, ChannelTag, Tag, Video # --- language id (offline, deterministic, small) --- @@ -166,7 +165,7 @@ def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None, select(Video.title) .where(Video.channel_id == channel.id, Video.title.is_not(None)) .order_by(Video.published_at.desc()) - .limit(settings.autotag_title_sample) + .limit(sysconfig.get_int(db, "autotag_title_sample")) ) .scalars() .all() diff --git a/backend/app/sync/maintenance.py b/backend/app/sync/maintenance.py index 47e3569..06262e0 100644 --- a/backend/app/sync/maintenance.py +++ b/backend/app/sync/maintenance.py @@ -25,7 +25,7 @@ from datetime import datetime, timedelta, timezone from sqlalchemy import or_, select from sqlalchemy.orm import Session -from app import progress, quota +from app import progress, quota, sysconfig from app.config import settings from app.models import Playlist, PlaylistItem, Video, VideoState from app.notifications import create_notification @@ -196,7 +196,7 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict: now = _now() # Process in videos.list-sized pages so we commit incrementally and stop on low quota. while checked < batch: - if quota.remaining_today(db) <= settings.backfill_quota_reserve: + if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): break page = ( db.execute( @@ -236,7 +236,7 @@ def run_maintenance(db: Session) -> dict: phase1 = _recheck_flagged(db, yt) phase2 = ( _revalidate_rolling(db, yt) - if quota.remaining_today(db) > settings.backfill_quota_reserve + if quota.remaining_today(db) > sysconfig.get_int(db, "backfill_quota_reserve") else {"checked": 0, "flagged": 0, "skipped": "low quota"} ) return { diff --git a/backend/app/sync/runner.py b/backend/app/sync/runner.py index ed76ebc..6f5faf4 100644 --- a/backend/app/sync/runner.py +++ b/backend/app/sync/runner.py @@ -10,8 +10,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from sqlalchemy import select from sqlalchemy.orm import Session -from app import progress, quota -from app.config import settings +from app import progress, quota, sysconfig log = logging.getLogger("subfeed.sync") from app.models import Channel, OAuthToken, Subscription, User @@ -140,7 +139,7 @@ def run_recent_backfill( videos_added = 0 with YouTubeClient(db, user) as yt: for channel in channels: - if quota.remaining_today(db) <= settings.backfill_quota_reserve: + if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): break try: videos_added += backfill_channel_recent(db, yt, channel) @@ -186,7 +185,7 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) - videos_added = 0 with YouTubeClient(db, user) as yt: for channel in channels: - if quota.remaining_today(db) <= settings.backfill_quota_reserve: + if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): break try: videos_added += backfill_channel_deep(db, yt, channel, max_pages) @@ -215,7 +214,9 @@ def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict: return 2 * pages total_units = sum(units(c) for c in pending) - daily_budget = max(1, settings.quota_daily_budget - settings.backfill_quota_reserve) + daily_budget = max( + 1, sysconfig.get_int(db, "quota_daily_budget") - sysconfig.get_int(db, "backfill_quota_reserve") + ) eta_seconds = int(total_units / daily_budget * 86_400) if pending else 0 return { "channels_pending": len(pending), diff --git a/backend/app/sync/videos.py b/backend/app/sync/videos.py index 4b64c51..de0df77 100644 --- a/backend/app/sync/videos.py +++ b/backend/app/sync/videos.py @@ -9,8 +9,7 @@ from sqlalchemy import and_, func, or_, select, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session -from app import progress -from app.config import settings +from app import progress, sysconfig from app.models import Channel, Video from app.youtube.client import YouTubeClient, best_thumbnail from app.youtube.rss import fetch_channel_feed @@ -108,7 +107,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> db.commit() return 0 - cutoff = _now() - timedelta(days=settings.backfill_recent_max_days) + cutoff = _now() - timedelta(days=sysconfig.get_int(db, "backfill_recent_max_days")) collected: list[dict] = [] page_token: str | None = None next_cursor: str | None = None @@ -126,7 +125,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> stopped_mid_page = True break collected.append(stub) - if len(collected) >= settings.backfill_recent_max_videos: + if len(collected) >= sysconfig.get_int(db, "backfill_recent_max_videos"): stopped_mid_page = True break page_token = data.get("nextPageToken") @@ -249,7 +248,7 @@ def apply_video_details(video: Video, item: dict) -> None: def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int: - limit = limit or settings.enrich_batch_size + limit = limit or sysconfig.get_int(db, "enrich_batch_size") videos = ( db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit)) .scalars() @@ -282,7 +281,7 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in ends. So we revisit anything still live/upcoming, plus a just-ended stream that hasn't got its duration yet (YouTube can lag a bit after the broadcast), until it settles. Cheap: videos.list is 1 unit per 50 ids, and the live/upcoming set is normally tiny.""" - limit = limit or settings.enrich_batch_size + limit = limit or sysconfig.get_int(db, "enrich_batch_size") videos = ( db.execute( select(Video) @@ -322,8 +321,8 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in def run_shorts_classification(db: Session, limit: int | None = None) -> dict: """Finalize Shorts classification: cheaply mark non-candidates, then probe the youtube.com/shorts URL for short-enough, non-live, enriched videos.""" - limit = limit or settings.shorts_probe_batch - probe_max = settings.shorts_probe_max_seconds + limit = limit or sysconfig.get_int(db, "shorts_probe_batch") + probe_max = sysconfig.get_int(db, "shorts_probe_max_seconds") # 1) Anything enriched that can't be a Short -> finalize without a probe. db.execute( diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index eabba05..439de1a 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -36,6 +36,18 @@ SPECS: tuple[ConfigSpec, ...] = ( ConfigSpec("smtp_user", "str", "email", "smtp_user"), ConfigSpec("smtp_from", "str", "email", "smtp_from"), ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True), + # --- Quota (shared YouTube Data API daily budget) --- + ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000), + ConfigSpec("backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000), + # --- Backfill (recent-first first pass per channel) --- + ConfigSpec("backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000), + ConfigSpec("backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500), + # --- Shorts probe --- + ConfigSpec("shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600), + ConfigSpec("shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000), + # --- Batch sizes --- + ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50), + ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000), ) _BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS} From 060dbcffb05818c2760002288408512ceb262d21 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 19 Jun 2026 13:07:55 +0200 Subject: [PATCH 2/2] feat(config): render quota/backfill/shorts/batch groups on the Configuration page Add the four new groups to the page's group order and EN/HU/DE labels + field hints for the 8 newly DB-overridable settings. The page is registry-driven, so this is purely presentational (group order + i18n). --- frontend/src/components/ConfigPanel.tsx | 2 +- frontend/src/i18n/locales/de/config.json | 16 ++++++++++++++-- frontend/src/i18n/locales/en/config.json | 16 ++++++++++++++-- frontend/src/i18n/locales/hu/config.json | 16 ++++++++++++++-- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/ConfigPanel.tsx b/frontend/src/components/ConfigPanel.tsx index 49d69c9..ae2686a 100644 --- a/frontend/src/components/ConfigPanel.tsx +++ b/frontend/src/components/ConfigPanel.tsx @@ -11,7 +11,7 @@ import Tooltip from "./Tooltip"; // applied together via one Save/Discard bar (consistent with the Settings page); nothing hits // the server until Save. Group order is fixed where known so logically related settings (e.g. // Email/SMTP) stay together. -const GROUP_ORDER = ["email"]; +const GROUP_ORDER = ["email", "quota", "backfill", "shorts", "batch"]; type SaveState = "idle" | "saving" | "saved" | "error"; export default function ConfigPanel() { diff --git a/frontend/src/i18n/locales/de/config.json b/frontend/src/i18n/locales/de/config.json index 772bcfb..30dbd67 100644 --- a/frontend/src/i18n/locales/de/config.json +++ b/frontend/src/i18n/locales/de/config.json @@ -2,14 +2,26 @@ "intro": "In der Datenbank gespeicherte Betriebseinstellungen — sie überschreiben die Datei-/Env-Standardwerte und wirken ohne erneutes Deployment. Lass ein Feld leer (oder setze es zurück), um auf den Standard zurückzufallen.", "loading": "Konfiguration wird geladen…", "groups": { - "email": "E-Mail / SMTP" + "email": "E-Mail / SMTP", + "quota": "Kontingent", + "backfill": "Nachladen (Backfill)", + "shorts": "Shorts-Prüfung", + "batch": "Stapelgrößen" }, "fields": { "smtp_host": { "label": "SMTP-Host", "hint": "z. B. smtp.gmail.com" }, "smtp_port": { "label": "SMTP-Port", "hint": "Üblicherweise 587 (STARTTLS)." }, "smtp_user": { "label": "SMTP-Benutzername", "hint": "Das sendende Konto / der Login." }, "smtp_from": { "label": "Absenderadresse", "hint": "z. B. Siftlode . Standard ist der Benutzername." }, - "smtp_password": { "label": "SMTP-Passwort", "hint": "App-Passwort. Verschlüsselt gespeichert; nur schreibbar — wird nie wieder angezeigt." } + "smtp_password": { "label": "SMTP-Passwort", "hint": "App-Passwort. Verschlüsselt gespeichert; nur schreibbar — wird nie wieder angezeigt." }, + "quota_daily_budget": { "label": "Tägliches Kontingentbudget", "hint": "YouTube-Data-API-Einheiten pro Tag (kostenloses Limit 10.000). Standard 9000 lässt Puffer." }, + "backfill_quota_reserve": { "label": "Nachlade-Kontingentreserve", "hint": "Reservierte Einheiten, damit geplantes Nachladen interaktive Syncs / Anreicherung nie aushungert." }, + "backfill_recent_max_videos": { "label": "Aktuelles Nachladen — max. Videos", "hint": "Neueste Videos pro Kanal beim ersten Durchlauf." }, + "backfill_recent_max_days": { "label": "Aktuelles Nachladen — max. Alter (Tage)", "hint": "Wie weit der erste Durchlauf pro Kanal zurückreicht." }, + "shorts_probe_max_seconds": { "label": "Shorts-Prüfung — max. Dauer (s)", "hint": "Nur Videos bis zu dieser Länge werden als mögliche Shorts geprüft." }, + "shorts_probe_batch": { "label": "Shorts-Prüfung — Stapelgröße", "hint": "Wie viele Videos pro Lauf klassifiziert werden." }, + "enrich_batch_size": { "label": "Anreicherungs-Stapelgröße", "hint": "videos.list-IDs pro Aufruf (YouTube begrenzt auf 50)." }, + "autotag_title_sample": { "label": "Auto-Tag-Titelstichprobe", "hint": "Pro Kanal abgetastete aktuelle Videotitel zur Spracherkennung." } }, "save": "Speichern", "saving": "Speichern…", diff --git a/frontend/src/i18n/locales/en/config.json b/frontend/src/i18n/locales/en/config.json index cd5c4b1..8d0c903 100644 --- a/frontend/src/i18n/locales/en/config.json +++ b/frontend/src/i18n/locales/en/config.json @@ -2,14 +2,26 @@ "intro": "Operational settings stored in the database — these override the file/env defaults and take effect without a redeploy. Leave a field empty (or reset it) to fall back to the default.", "loading": "Loading configuration…", "groups": { - "email": "Email / SMTP" + "email": "Email / SMTP", + "quota": "Quota", + "backfill": "Backfill", + "shorts": "Shorts probe", + "batch": "Batch sizes" }, "fields": { "smtp_host": { "label": "SMTP host", "hint": "e.g. smtp.gmail.com" }, "smtp_port": { "label": "SMTP port", "hint": "Usually 587 (STARTTLS)." }, "smtp_user": { "label": "SMTP username", "hint": "The sending account / login." }, "smtp_from": { "label": "From address", "hint": "e.g. Siftlode . Defaults to the username." }, - "smtp_password": { "label": "SMTP password", "hint": "App password. Stored encrypted; write-only — it's never shown back." } + "smtp_password": { "label": "SMTP password", "hint": "App password. Stored encrypted; write-only — it's never shown back." }, + "quota_daily_budget": { "label": "Daily quota budget", "hint": "Total YouTube Data API units to spend per day (free tier is 10,000). Default 9000 leaves headroom." }, + "backfill_quota_reserve": { "label": "Backfill quota reserve", "hint": "Units kept in reserve so scheduled backfill never starves interactive syncs / enrichment." }, + "backfill_recent_max_videos": { "label": "Recent backfill — max videos", "hint": "Newest videos fetched per channel on the first pass." }, + "backfill_recent_max_days": { "label": "Recent backfill — max age (days)", "hint": "How far back the first pass reaches per channel." }, + "shorts_probe_max_seconds": { "label": "Shorts probe — max duration (s)", "hint": "Only videos at or below this length are probed as possible Shorts." }, + "shorts_probe_batch": { "label": "Shorts probe — batch size", "hint": "How many videos to classify per run." }, + "enrich_batch_size": { "label": "Enrichment batch size", "hint": "videos.list ids per call (YouTube caps this at 50)." }, + "autotag_title_sample": { "label": "Auto-tag title sample", "hint": "Recent video titles sampled per channel for language detection." } }, "save": "Save", "saving": "Saving…", diff --git a/frontend/src/i18n/locales/hu/config.json b/frontend/src/i18n/locales/hu/config.json index 2cdcb8b..555d698 100644 --- a/frontend/src/i18n/locales/hu/config.json +++ b/frontend/src/i18n/locales/hu/config.json @@ -2,14 +2,26 @@ "intro": "Az adatbázisban tárolt működési beállítások — felülírják a fájl/env alapértékeket, és újratelepítés nélkül lépnek életbe. Hagyd üresen (vagy állítsd vissza) a mezőt az alapértékhez való visszatéréshez.", "loading": "Konfiguráció betöltése…", "groups": { - "email": "E-mail / SMTP" + "email": "E-mail / SMTP", + "quota": "Kvóta", + "backfill": "Letöltés (backfill)", + "shorts": "Shorts-vizsgálat", + "batch": "Kötegméretek" }, "fields": { "smtp_host": { "label": "SMTP-kiszolgáló", "hint": "pl. smtp.gmail.com" }, "smtp_port": { "label": "SMTP-port", "hint": "Általában 587 (STARTTLS)." }, "smtp_user": { "label": "SMTP-felhasználónév", "hint": "A küldő fiók / bejelentkezés." }, "smtp_from": { "label": "Feladó cím", "hint": "pl. Siftlode . Alapból a felhasználónév." }, - "smtp_password": { "label": "SMTP-jelszó", "hint": "Alkalmazásjelszó. Titkosítva tárolva; csak írható — soha nem jelenik meg újra." } + "smtp_password": { "label": "SMTP-jelszó", "hint": "Alkalmazásjelszó. Titkosítva tárolva; csak írható — soha nem jelenik meg újra." }, + "quota_daily_budget": { "label": "Napi kvótakeret", "hint": "Naponta elkölthető YouTube Data API-egységek (az ingyenes keret 10 000). Az alap 9000 tartalékot hagy." }, + "backfill_quota_reserve": { "label": "Letöltési kvótatartalék", "hint": "Tartalékban tartott egységek, hogy az ütemezett letöltés ne éheztesse ki az interaktív szinkront / gazdagítást." }, + "backfill_recent_max_videos": { "label": "Friss letöltés — max videó", "hint": "Csatornánként az első körben letöltött legújabb videók száma." }, + "backfill_recent_max_days": { "label": "Friss letöltés — max kor (nap)", "hint": "Az első kör csatornánként meddig nyúl vissza." }, + "shorts_probe_max_seconds": { "label": "Shorts-vizsgálat — max hossz (mp)", "hint": "Csak az ennél nem hosszabb videókat vizsgáljuk lehetséges Shortsként." }, + "shorts_probe_batch": { "label": "Shorts-vizsgálat — kötegméret", "hint": "Futásonként ennyi videót osztályozunk." }, + "enrich_batch_size": { "label": "Gazdagítási kötegméret", "hint": "videos.list azonosítók hívásonként (a YouTube 50-ben maximálja)." }, + "autotag_title_sample": { "label": "Auto-címke címmintavétel", "hint": "Csatornánként ennyi friss videócímet mintázunk a nyelvfelismeréshez." } }, "save": "Mentés", "saving": "Mentés…",