siftlode/backend/app/config.py
npeter83 9b1bdb6b42 feat(search): live YouTube search backend
Add a live YouTube search that materialises results into the shared catalog so
they render with the normal feed cards + in-app player and gain per-user state.

- YouTubeClient.search_videos(): search.list (100 units), embeddable-only, returns
  flat stubs + nextPageToken; surfaces liveBroadcastContent for live filtering.
- routes/search.py GET /api/search/youtube: require_human + per-user daily cap
  (search_daily_limit_per_user, default 70) + can_spend pre-check (429 on either);
  drops live/upcoming, upserts channel stubs (channels.list) + video stubs, enriches
  (videos.list), runs the youtube.com/shorts probe, then excludes Shorts/live and
  returns feed cards in relevance order with the YouTube pageToken as the cursor.
- Provenance: videos.via_search / channels.from_search (migration 0028) flag
  search-discovered rows; the feed hides them from the Library (scope=all) by default
  via exclude_search_discovered, leaving the Mine feed untouched.
- quota.actions_today() counts a user's per-action events today for the cap; only the
  search.list call is attributed VIDEOS_SEARCH so the counter is exactly 1 per search.
2026-06-29 02:01:31 +02:00

165 lines
7.7 KiB
Python

from pathlib import Path
from pydantic import model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
# Shipped placeholder; production must override it. Used by the startup guard below.
_DEFAULT_SECRET_KEY = "change-me-session-key"
def _version_from_file() -> str:
"""The committed VERSION file (copied into the image at /app/VERSION) is the single
source of truth for the app version; falls back to 'dev' for un-built/local runs."""
for p in (Path(__file__).resolve().parents[1] / "VERSION", Path("VERSION")):
try:
v = p.read_text(encoding="utf-8").strip()
if v:
return v
except OSError:
continue
return "dev"
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
app_name: str = "Siftlode"
# Version from the committed VERSION file (an explicit APP_VERSION env still overrides);
# git_sha/build_date are injected at image build time. Surfaced by /api/version.
app_version: str = _version_from_file()
git_sha: str = "unknown"
build_date: str = ""
database_url: str = "postgresql+psycopg://siftlode:siftlode@db:5432/siftlode"
# Session cookie signing key.
secret_key: str = _DEFAULT_SECRET_KEY
# Fernet key (urlsafe base64, 32 bytes) for encrypting stored refresh tokens.
token_encryption_key: str = ""
google_client_id: str = ""
google_client_secret: str = ""
oauth_redirect_url: str = "http://localhost:8080/auth/callback"
# Comma-separated invite list / admin list of Google account emails.
allowed_emails: str = ""
admin_emails: str = ""
# Whether anyone may submit an email+password registration (still gated by email
# verification + admin approval before they can sign in). Admin-tunable via the DB.
allow_registration: bool = True
# Origin of a separately served frontend dev server (enables CORS). Empty in production.
frontend_origin: str = ""
# --- Outbound email (onboarding: access-request + approval notices) ---
# Gmail SMTP + App Password by default. All optional: if unset, email is skipped
# (fail-soft) and onboarding still works via in-app notifications.
smtp_host: str = ""
smtp_port: int = 587
smtp_user: str = ""
smtp_password: str = ""
smtp_from: str = "" # e.g. "Siftlode <addr@gmail.com>"; falls back to smtp_user
# --- Sync / YouTube Data API ---
# Optional API key for public reads (channels/videos/playlistItems). When set it is
# preferred for shared backfill/enrichment so it doesn't depend on a specific user.
youtube_api_key: str = ""
# Optional HTTP(S) proxy for outbound YouTube Data API calls. Set this to send the
# scheduler's googleapis traffic through a fixed-IP host (e.g. the VPS over WireGuard) so an
# IP-restricted API key keeps working even when this host's public IP is dynamic.
youtube_api_proxy: str = ""
# Daily quota budget (units). Default leaves headroom under the 10,000/day free limit.
quota_daily_budget: int = 9000
# Per-user cap on live YouTube searches per day. search.list costs 100 units each, so the
# shared budget only affords ~80-90/day total — this stops one user draining it. Admin-tunable.
search_daily_limit_per_user: int = 70
# Recent-first backfill: how far back to fetch on the first pass per channel.
backfill_recent_max_videos: int = 100
backfill_recent_max_days: int = 365
# Shorts are confirmed by probing youtube.com/shorts/<id>. Only videos at or below
# this duration (and not livestreams) are probed; longer videos are never Shorts.
shorts_probe_max_seconds: int = 180
shorts_probe_batch: int = 150
shorts_probe_interval_minutes: int = 2
# videos.list accepts up to 50 ids per call.
enrich_batch_size: int = 50
# --- Background scheduler ---
scheduler_enabled: bool = True
rss_poll_minutes: int = 20
enrich_interval_minutes: int = 3
backfill_interval_minutes: int = 10
# Keep this many quota units in reserve so scheduled backfill never starves
# interactive syncs / enrichment of fresh videos.
backfill_quota_reserve: int = 2000
# --- Auto-tagging / feed defaults ---
# Number of recent video titles sampled per channel for language detection.
autotag_title_sample: int = 40
autotag_interval_minutes: int = 30
subscriptions_resync_minutes: int = 360
playlist_sync_minutes: int = 360
# How often the shared demo account is auto-reset to a clean baseline (communal sandbox).
demo_reset_minutes: int = 720
# --- Maintenance / validation job ---
# Runs once a day by default (admin-tunable via the Scheduler dashboard). Grace periods
# are in days because that's the granularity that matters; the rolling re-validation
# re-checks the least-recently-checked N videos per run (1 unit / 50 videos), so on a
# ~233k catalog ~15000/day ≈ a 15-day full cycle for ~300 quota units.
maintenance_interval_minutes: int = 1440
maintenance_revalidate_batch: int = 15000
# Grace before a flagged-unavailable video is hard-deleted (cascades to states/playlist
# items). Deleted/private/paywalled get a recovery window; abandoned upcoming and
# confirmed-dead stuck-live have no age grace (deleted on the sweep that confirms them).
maintenance_grace_days: int = 7
# live_status values hidden from the feed by default. Completed-stream VODs
# ("was_live") are real watchable content and stay visible.
feed_default_hidden_live: str = "live,upcoming"
@property
def allowed_email_set(self) -> set[str]:
return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()}
@property
def admin_email_set(self) -> set[str]:
return {e.strip().lower() for e in self.admin_emails.split(",") if e.strip()}
@property
def app_base(self) -> str:
"""Public origin of the deployed app, derived from the OAuth redirect URL
(.../auth/callback). The single source for user-facing links (verification, reset,
approval emails); dev and prod differ, so links must always come from config."""
u = self.oauth_redirect_url
return u.split("/auth/")[0] if "/auth/" in u else u.rstrip("/")
@property
def session_https_only(self) -> bool:
"""Mark the session cookie Secure when we're served over HTTPS. We treat an
https OAUTH_REDIRECT_URL as the signal for 'real deployment' vs local dev."""
return self.oauth_redirect_url.lower().startswith("https://")
@model_validator(mode="after")
def _enforce_production_secrets(self) -> "Settings":
"""Fail fast rather than boot a public instance with the shipped placeholder
signing key (which would let anyone forge an admin session) or without token
encryption. Local dev (http redirect URL) keeps the convenient defaults."""
if not self.session_https_only:
return self # local/dev: allow the placeholder defaults
if self.secret_key == _DEFAULT_SECRET_KEY or len(self.secret_key) < 32:
raise ValueError(
"SECRET_KEY must be a unique random value of at least 32 chars in "
"production (an https OAUTH_REDIRECT_URL was detected). Generate one "
'with: python -c "import secrets; print(secrets.token_urlsafe(48))"'
)
if not self.token_encryption_key:
raise ValueError(
"TOKEN_ENCRYPTION_KEY must be set in production so refresh tokens are "
"encrypted at rest. Generate one with: "
'python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"'
)
return self
settings = Settings()