diff --git a/backend/app/quota.py b/backend/app/quota.py index e2c7696..be699a1 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -12,7 +12,7 @@ from zoneinfo import ZoneInfo from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session -from app.config import settings +from app import sysconfig from app.models import ApiQuotaUsage, QuotaEvent _PACIFIC = ZoneInfo("America/Los_Angeles") @@ -82,7 +82,7 @@ def units_used_today(db: Session) -> int: def remaining_today(db: Session) -> int: - return max(0, settings.quota_daily_budget - units_used_today(db)) + return max(0, sysconfig.get_int(db, "quota_daily_budget") - units_used_today(db)) def can_spend(db: Session, units: int) -> bool: diff --git a/backend/app/routes/scheduler.py b/backend/app/routes/scheduler.py index 66e4f8f..e88b1db 100644 --- a/backend/app/routes/scheduler.py +++ b/backend/app/routes/scheduler.py @@ -5,7 +5,7 @@ from fastapi import APIRouter, Depends, HTTPException from sqlalchemy import and_, func, or_, select from sqlalchemy.orm import Session -from app import quota, state +from app import quota, state, sysconfig from app.config import settings from app.db import get_db from app.models import Channel, Subscription, User, Video @@ -152,8 +152,8 @@ def get_scheduler( quota_info = { "used_today": used, "remaining_today": quota.remaining_today(db), - "daily_budget": settings.quota_daily_budget, - "backfill_reserve": settings.backfill_quota_reserve, + "daily_budget": sysconfig.get_int(db, "quota_daily_budget"), + "backfill_reserve": sysconfig.get_int(db, "backfill_quota_reserve"), } return { diff --git a/backend/app/sync/autotag.py b/backend/app/sync/autotag.py index 09cce68..0a1359e 100644 --- a/backend/app/sync/autotag.py +++ b/backend/app/sync/autotag.py @@ -13,8 +13,7 @@ from sqlalchemy.orm import Session log = logging.getLogger("subfeed.autotag") -from app import progress -from app.config import settings +from app import progress, sysconfig from app.models import Channel, ChannelTag, Tag, Video # --- language id (offline, deterministic, small) --- @@ -166,7 +165,7 @@ def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None, select(Video.title) .where(Video.channel_id == channel.id, Video.title.is_not(None)) .order_by(Video.published_at.desc()) - .limit(settings.autotag_title_sample) + .limit(sysconfig.get_int(db, "autotag_title_sample")) ) .scalars() .all() diff --git a/backend/app/sync/maintenance.py b/backend/app/sync/maintenance.py index 47e3569..06262e0 100644 --- a/backend/app/sync/maintenance.py +++ b/backend/app/sync/maintenance.py @@ -25,7 +25,7 @@ from datetime import datetime, timedelta, timezone from sqlalchemy import or_, select from sqlalchemy.orm import Session -from app import progress, quota +from app import progress, quota, sysconfig from app.config import settings from app.models import Playlist, PlaylistItem, Video, VideoState from app.notifications import create_notification @@ -196,7 +196,7 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict: now = _now() # Process in videos.list-sized pages so we commit incrementally and stop on low quota. while checked < batch: - if quota.remaining_today(db) <= settings.backfill_quota_reserve: + if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): break page = ( db.execute( @@ -236,7 +236,7 @@ def run_maintenance(db: Session) -> dict: phase1 = _recheck_flagged(db, yt) phase2 = ( _revalidate_rolling(db, yt) - if quota.remaining_today(db) > settings.backfill_quota_reserve + if quota.remaining_today(db) > sysconfig.get_int(db, "backfill_quota_reserve") else {"checked": 0, "flagged": 0, "skipped": "low quota"} ) return { diff --git a/backend/app/sync/runner.py b/backend/app/sync/runner.py index ed76ebc..6f5faf4 100644 --- a/backend/app/sync/runner.py +++ b/backend/app/sync/runner.py @@ -10,8 +10,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from sqlalchemy import select from sqlalchemy.orm import Session -from app import progress, quota -from app.config import settings +from app import progress, quota, sysconfig log = logging.getLogger("subfeed.sync") from app.models import Channel, OAuthToken, Subscription, User @@ -140,7 +139,7 @@ def run_recent_backfill( videos_added = 0 with YouTubeClient(db, user) as yt: for channel in channels: - if quota.remaining_today(db) <= settings.backfill_quota_reserve: + if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): break try: videos_added += backfill_channel_recent(db, yt, channel) @@ -186,7 +185,7 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) - videos_added = 0 with YouTubeClient(db, user) as yt: for channel in channels: - if quota.remaining_today(db) <= settings.backfill_quota_reserve: + if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): break try: videos_added += backfill_channel_deep(db, yt, channel, max_pages) @@ -215,7 +214,9 @@ def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict: return 2 * pages total_units = sum(units(c) for c in pending) - daily_budget = max(1, settings.quota_daily_budget - settings.backfill_quota_reserve) + daily_budget = max( + 1, sysconfig.get_int(db, "quota_daily_budget") - sysconfig.get_int(db, "backfill_quota_reserve") + ) eta_seconds = int(total_units / daily_budget * 86_400) if pending else 0 return { "channels_pending": len(pending), diff --git a/backend/app/sync/videos.py b/backend/app/sync/videos.py index 4b64c51..de0df77 100644 --- a/backend/app/sync/videos.py +++ b/backend/app/sync/videos.py @@ -9,8 +9,7 @@ from sqlalchemy import and_, func, or_, select, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session -from app import progress -from app.config import settings +from app import progress, sysconfig from app.models import Channel, Video from app.youtube.client import YouTubeClient, best_thumbnail from app.youtube.rss import fetch_channel_feed @@ -108,7 +107,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> db.commit() return 0 - cutoff = _now() - timedelta(days=settings.backfill_recent_max_days) + cutoff = _now() - timedelta(days=sysconfig.get_int(db, "backfill_recent_max_days")) collected: list[dict] = [] page_token: str | None = None next_cursor: str | None = None @@ -126,7 +125,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> stopped_mid_page = True break collected.append(stub) - if len(collected) >= settings.backfill_recent_max_videos: + if len(collected) >= sysconfig.get_int(db, "backfill_recent_max_videos"): stopped_mid_page = True break page_token = data.get("nextPageToken") @@ -249,7 +248,7 @@ def apply_video_details(video: Video, item: dict) -> None: def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int: - limit = limit or settings.enrich_batch_size + limit = limit or sysconfig.get_int(db, "enrich_batch_size") videos = ( db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit)) .scalars() @@ -282,7 +281,7 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in ends. So we revisit anything still live/upcoming, plus a just-ended stream that hasn't got its duration yet (YouTube can lag a bit after the broadcast), until it settles. Cheap: videos.list is 1 unit per 50 ids, and the live/upcoming set is normally tiny.""" - limit = limit or settings.enrich_batch_size + limit = limit or sysconfig.get_int(db, "enrich_batch_size") videos = ( db.execute( select(Video) @@ -322,8 +321,8 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in def run_shorts_classification(db: Session, limit: int | None = None) -> dict: """Finalize Shorts classification: cheaply mark non-candidates, then probe the youtube.com/shorts URL for short-enough, non-live, enriched videos.""" - limit = limit or settings.shorts_probe_batch - probe_max = settings.shorts_probe_max_seconds + limit = limit or sysconfig.get_int(db, "shorts_probe_batch") + probe_max = sysconfig.get_int(db, "shorts_probe_max_seconds") # 1) Anything enriched that can't be a Short -> finalize without a probe. db.execute( diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index eabba05..439de1a 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -36,6 +36,18 @@ SPECS: tuple[ConfigSpec, ...] = ( ConfigSpec("smtp_user", "str", "email", "smtp_user"), ConfigSpec("smtp_from", "str", "email", "smtp_from"), ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True), + # --- Quota (shared YouTube Data API daily budget) --- + ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000), + ConfigSpec("backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000), + # --- Backfill (recent-first first pass per channel) --- + ConfigSpec("backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000), + ConfigSpec("backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500), + # --- Shorts probe --- + ConfigSpec("shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600), + ConfigSpec("shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000), + # --- Batch sizes --- + ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50), + ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000), ) _BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS}