feat(config): move operational params to DB (quota/backfill/shorts/batch)
Register 8 more keys in the sysconfig registry (quota daily budget + backfill reserve, recent-backfill window, shorts-probe params, enrich/autotag batch sizes) and route their reads through the DB-override-or-env resolver at every call site (quota.py, sync/runner.py, sync/videos.py, sync/autotag.py, sync/maintenance.py, routes/scheduler.py). All read sites already had a db session in scope. Reads are read-through (no cache), matching app.state; admin can tune them live, no redeploy.
This commit is contained in:
npeter83
parent
ff03b51352
commit
e6bcf5ba1e
7 changed files with 35 additions and 24 deletions
|
|
@ -13,8 +13,7 @@ from sqlalchemy.orm import Session
|
|||
|
||||
log = logging.getLogger("subfeed.autotag")
|
||||
|
||||
from app import progress
|
||||
from app.config import settings
|
||||
from app import progress, sysconfig
|
||||
from app.models import Channel, ChannelTag, Tag, Video
|
||||
|
||||
# --- language id (offline, deterministic, small) ---
|
||||
|
|
@ -166,7 +165,7 @@ def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None,
|
|||
select(Video.title)
|
||||
.where(Video.channel_id == channel.id, Video.title.is_not(None))
|
||||
.order_by(Video.published_at.desc())
|
||||
.limit(settings.autotag_title_sample)
|
||||
.limit(sysconfig.get_int(db, "autotag_title_sample"))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ from datetime import datetime, timedelta, timezone
|
|||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import progress, quota
|
||||
from app import progress, quota, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import Playlist, PlaylistItem, Video, VideoState
|
||||
from app.notifications import create_notification
|
||||
|
|
@ -196,7 +196,7 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
|
|||
now = _now()
|
||||
# Process in videos.list-sized pages so we commit incrementally and stop on low quota.
|
||||
while checked < batch:
|
||||
if quota.remaining_today(db) <= settings.backfill_quota_reserve:
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
break
|
||||
page = (
|
||||
db.execute(
|
||||
|
|
@ -236,7 +236,7 @@ def run_maintenance(db: Session) -> dict:
|
|||
phase1 = _recheck_flagged(db, yt)
|
||||
phase2 = (
|
||||
_revalidate_rolling(db, yt)
|
||||
if quota.remaining_today(db) > settings.backfill_quota_reserve
|
||||
if quota.remaining_today(db) > sysconfig.get_int(db, "backfill_quota_reserve")
|
||||
else {"checked": 0, "flagged": 0, "skipped": "low quota"}
|
||||
)
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -10,8 +10,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
|
|||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import progress, quota
|
||||
from app.config import settings
|
||||
from app import progress, quota, sysconfig
|
||||
|
||||
log = logging.getLogger("subfeed.sync")
|
||||
from app.models import Channel, OAuthToken, Subscription, User
|
||||
|
|
@ -140,7 +139,7 @@ def run_recent_backfill(
|
|||
videos_added = 0
|
||||
with YouTubeClient(db, user) as yt:
|
||||
for channel in channels:
|
||||
if quota.remaining_today(db) <= settings.backfill_quota_reserve:
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
break
|
||||
try:
|
||||
videos_added += backfill_channel_recent(db, yt, channel)
|
||||
|
|
@ -186,7 +185,7 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
|||
videos_added = 0
|
||||
with YouTubeClient(db, user) as yt:
|
||||
for channel in channels:
|
||||
if quota.remaining_today(db) <= settings.backfill_quota_reserve:
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
break
|
||||
try:
|
||||
videos_added += backfill_channel_deep(db, yt, channel, max_pages)
|
||||
|
|
@ -215,7 +214,9 @@ def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
|
|||
return 2 * pages
|
||||
|
||||
total_units = sum(units(c) for c in pending)
|
||||
daily_budget = max(1, settings.quota_daily_budget - settings.backfill_quota_reserve)
|
||||
daily_budget = max(
|
||||
1, sysconfig.get_int(db, "quota_daily_budget") - sysconfig.get_int(db, "backfill_quota_reserve")
|
||||
)
|
||||
eta_seconds = int(total_units / daily_budget * 86_400) if pending else 0
|
||||
return {
|
||||
"channels_pending": len(pending),
|
||||
|
|
|
|||
|
|
@ -9,8 +9,7 @@ from sqlalchemy import and_, func, or_, select, update
|
|||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import progress
|
||||
from app.config import settings
|
||||
from app import progress, sysconfig
|
||||
from app.models import Channel, Video
|
||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
||||
from app.youtube.rss import fetch_channel_feed
|
||||
|
|
@ -108,7 +107,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
|
|||
db.commit()
|
||||
return 0
|
||||
|
||||
cutoff = _now() - timedelta(days=settings.backfill_recent_max_days)
|
||||
cutoff = _now() - timedelta(days=sysconfig.get_int(db, "backfill_recent_max_days"))
|
||||
collected: list[dict] = []
|
||||
page_token: str | None = None
|
||||
next_cursor: str | None = None
|
||||
|
|
@ -126,7 +125,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
|
|||
stopped_mid_page = True
|
||||
break
|
||||
collected.append(stub)
|
||||
if len(collected) >= settings.backfill_recent_max_videos:
|
||||
if len(collected) >= sysconfig.get_int(db, "backfill_recent_max_videos"):
|
||||
stopped_mid_page = True
|
||||
break
|
||||
page_token = data.get("nextPageToken")
|
||||
|
|
@ -249,7 +248,7 @@ def apply_video_details(video: Video, item: dict) -> None:
|
|||
|
||||
|
||||
def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int:
|
||||
limit = limit or settings.enrich_batch_size
|
||||
limit = limit or sysconfig.get_int(db, "enrich_batch_size")
|
||||
videos = (
|
||||
db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit))
|
||||
.scalars()
|
||||
|
|
@ -282,7 +281,7 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in
|
|||
ends. So we revisit anything still live/upcoming, plus a just-ended stream that hasn't got
|
||||
its duration yet (YouTube can lag a bit after the broadcast), until it settles. Cheap:
|
||||
videos.list is 1 unit per 50 ids, and the live/upcoming set is normally tiny."""
|
||||
limit = limit or settings.enrich_batch_size
|
||||
limit = limit or sysconfig.get_int(db, "enrich_batch_size")
|
||||
videos = (
|
||||
db.execute(
|
||||
select(Video)
|
||||
|
|
@ -322,8 +321,8 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in
|
|||
def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
|
||||
"""Finalize Shorts classification: cheaply mark non-candidates, then probe the
|
||||
youtube.com/shorts URL for short-enough, non-live, enriched videos."""
|
||||
limit = limit or settings.shorts_probe_batch
|
||||
probe_max = settings.shorts_probe_max_seconds
|
||||
limit = limit or sysconfig.get_int(db, "shorts_probe_batch")
|
||||
probe_max = sysconfig.get_int(db, "shorts_probe_max_seconds")
|
||||
|
||||
# 1) Anything enriched that can't be a Short -> finalize without a probe.
|
||||
db.execute(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue