21c5508889
The web player clients need two things beyond the PO token: (1) a JavaScript runtime for YouTube's 'n' signature challenge — install Deno (v2.9.1) + yt-dlp[default] (bundles yt-dlp-ejs solver scripts); auto-detected on PATH. (2) an audio source — the web clients only expose video-only DASH, so add the 'android' client (also PO-token-backed) for audio/muxed formats. Player clients: web_safari,web,android (dropped tv — it falsely reports normal videos as DRM). End-to-end verified in the worker: extraction (POT + Deno n-challenge) + download completes, no bot/DRM/format errors. Full YouTube stack now: force-ipv4 + bgutil POT sidecar + Deno.
218 lines
11 KiB
Python
218 lines
11 KiB
Python
from pathlib import Path
|
|
|
|
from pydantic import model_validator
|
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
|
|
# Shipped placeholder; production must override it. Used by the startup guard below.
|
|
_DEFAULT_SECRET_KEY = "change-me-session-key"
|
|
|
|
|
|
def _version_from_file() -> str:
|
|
"""The committed VERSION file (copied into the image at /app/VERSION) is the single
|
|
source of truth for the app version; falls back to 'dev' for un-built/local runs."""
|
|
for p in (Path(__file__).resolve().parents[1] / "VERSION", Path("VERSION")):
|
|
try:
|
|
v = p.read_text(encoding="utf-8").strip()
|
|
if v:
|
|
return v
|
|
except OSError:
|
|
continue
|
|
return "dev"
|
|
|
|
|
|
class Settings(BaseSettings):
|
|
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
|
|
|
app_name: str = "Siftlode"
|
|
# Version from the committed VERSION file (an explicit APP_VERSION env still overrides);
|
|
# git_sha/build_date are injected at image build time. Surfaced by /api/version.
|
|
app_version: str = _version_from_file()
|
|
git_sha: str = "unknown"
|
|
build_date: str = ""
|
|
|
|
database_url: str = "postgresql+psycopg://siftlode:siftlode@db:5432/siftlode"
|
|
|
|
# Session cookie signing key.
|
|
secret_key: str = _DEFAULT_SECRET_KEY
|
|
# Fernet key (urlsafe base64, 32 bytes) for encrypting stored refresh tokens.
|
|
token_encryption_key: str = ""
|
|
|
|
google_client_id: str = ""
|
|
google_client_secret: str = ""
|
|
oauth_redirect_url: str = "http://localhost:8080/auth/callback"
|
|
|
|
# Comma-separated invite list / admin list of Google account emails.
|
|
allowed_emails: str = ""
|
|
admin_emails: str = ""
|
|
|
|
# Whether anyone may submit an email+password registration (still gated by email
|
|
# verification + admin approval before they can sign in). Admin-tunable via the DB.
|
|
allow_registration: bool = True
|
|
|
|
# Origin of a separately served frontend dev server (enables CORS). Empty in production.
|
|
frontend_origin: str = ""
|
|
|
|
# --- Outbound email (onboarding: access-request + approval notices) ---
|
|
# Gmail SMTP + App Password by default. All optional: if unset, email is skipped
|
|
# (fail-soft) and onboarding still works via in-app notifications.
|
|
smtp_host: str = ""
|
|
smtp_port: int = 587
|
|
smtp_user: str = ""
|
|
smtp_password: str = ""
|
|
smtp_from: str = "" # e.g. "Siftlode <addr@gmail.com>"; falls back to smtp_user
|
|
|
|
# --- Sync / YouTube Data API ---
|
|
# Optional API key for public reads (channels/videos/playlistItems). When set it is
|
|
# preferred for shared backfill/enrichment so it doesn't depend on a specific user.
|
|
youtube_api_key: str = ""
|
|
# Optional HTTP(S) proxy for outbound YouTube Data API calls. Set this to send the
|
|
# scheduler's googleapis traffic through a fixed-IP host so an IP-restricted API key
|
|
# keeps working even when this host's public IP is dynamic.
|
|
youtube_api_proxy: str = ""
|
|
# Daily quota budget (units). Default leaves headroom under the 10,000/day free limit.
|
|
quota_daily_budget: int = 9000
|
|
# Per-user cap on live YouTube searches per day. With the official API (search_source="api")
|
|
# each search.list costs 100 units, so the shared budget only affords ~80-90/day total; with
|
|
# the zero-quota scrape source it's a pure anti-abuse rate limit. Admin-tunable.
|
|
search_daily_limit_per_user: int = 70
|
|
# Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list,
|
|
# 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks.
|
|
search_source: str = "scrape"
|
|
# Recent-first backfill: how far back to fetch on the first pass per channel.
|
|
backfill_recent_max_videos: int = 100
|
|
backfill_recent_max_days: int = 365
|
|
|
|
# Shorts are confirmed by probing youtube.com/shorts/<id>. Only videos at or below
|
|
# this duration (and not livestreams) are probed; longer videos are never Shorts.
|
|
shorts_probe_max_seconds: int = 180
|
|
shorts_probe_batch: int = 150
|
|
shorts_probe_interval_minutes: int = 2
|
|
# videos.list accepts up to 50 ids per call.
|
|
enrich_batch_size: int = 50
|
|
|
|
# --- Background scheduler ---
|
|
scheduler_enabled: bool = True
|
|
rss_poll_minutes: int = 20
|
|
enrich_interval_minutes: int = 3
|
|
backfill_interval_minutes: int = 10
|
|
# Keep this many quota units in reserve so scheduled backfill never starves
|
|
# interactive syncs / enrichment of fresh videos.
|
|
backfill_quota_reserve: int = 2000
|
|
|
|
# --- Auto-tagging / feed defaults ---
|
|
# Number of recent video titles sampled per channel for language detection.
|
|
autotag_title_sample: int = 40
|
|
autotag_interval_minutes: int = 30
|
|
subscriptions_resync_minutes: int = 360
|
|
playlist_sync_minutes: int = 360
|
|
# How often the shared demo account is auto-reset to a clean baseline (communal sandbox).
|
|
demo_reset_minutes: int = 720
|
|
# --- Maintenance / validation job ---
|
|
# Runs once a day by default (admin-tunable via the Scheduler dashboard). Grace periods
|
|
# are in days because that's the granularity that matters; the rolling re-validation
|
|
# re-checks the least-recently-checked N videos per run (1 unit / 50 videos), so on a
|
|
# ~233k catalog ~15000/day ≈ a 15-day full cycle for ~300 quota units.
|
|
maintenance_interval_minutes: int = 1440
|
|
maintenance_revalidate_batch: int = 15000
|
|
# Grace before a flagged-unavailable video is hard-deleted (cascades to states/playlist
|
|
# items). Deleted/private/paywalled get a recovery window; abandoned upcoming and
|
|
# confirmed-dead stuck-live have no age grace (deleted on the sweep that confirms them).
|
|
maintenance_grace_days: int = 7
|
|
# --- Channel-explore cleanup job ---
|
|
# Runs daily by default (admin-tunable via the Scheduler dashboard). An explored-but-never-
|
|
# kept channel (no subscription, no per-user state on its videos) is hard-deleted this many
|
|
# days after it was last explored — the grace clock is ExploredChannel.created_at.
|
|
explore_cleanup_minutes: int = 1440
|
|
explore_grace_days: int = 7
|
|
# How long an un-kept live-search result (a via_search video nobody watched / saved /
|
|
# subscribed to) is kept before the same cleanup job removes it — "as if never added".
|
|
search_grace_days: int = 7
|
|
# live_status values hidden from the feed by default. Completed-stream VODs
|
|
# ("was_live") are real watchable content and stay visible.
|
|
feed_default_hidden_live: str = "live,upcoming"
|
|
|
|
# --- Download center (yt-dlp) ---
|
|
# Root directory (inside the container) for downloaded media — a Docker volume/bind mount.
|
|
# Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it.
|
|
download_root: str = "/downloads"
|
|
# Whether THIS process runs the download worker loop. On for the dedicated worker
|
|
# container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled.
|
|
worker_enabled: bool = False
|
|
# Base URL of the bgutil PO-token provider sidecar. When set, the download worker asks it
|
|
# for YouTube Proof-of-Origin tokens (bypasses bot-detection + unlocks high-quality
|
|
# formats). Empty disables it (yt-dlp still works, but may hit "confirm you're not a bot").
|
|
download_pot_base_url: str = "http://bgutil-pot:4416"
|
|
# yt-dlp player clients to use (comma-separated). POT-capable clients (web_safari/web) must
|
|
# be used for the PO token to actually apply; the default clients (android_vr) don't request
|
|
# one. Empty = yt-dlp's own default. Admin-tunable when YouTube shifts what works.
|
|
download_player_clients: str = "web_safari,web,android"
|
|
|
|
@property
|
|
def player_client_list(self) -> list[str]:
|
|
return [c.strip() for c in self.download_player_clients.split(",") if c.strip()]
|
|
# How many downloads the worker runs concurrently (claimed via FOR UPDATE SKIP LOCKED).
|
|
download_worker_concurrency: int = 2
|
|
# How often (empty pending queue) the worker polls for a new job, in seconds.
|
|
download_poll_seconds: int = 5
|
|
# Total cache size cap across ALL users (bytes). When >0 and exceeded, the GC evicts the
|
|
# least-recently-accessed ready assets until under it. 0 = unlimited (disk is the limit).
|
|
download_total_max_bytes: int = 0
|
|
# Default per-user quota (admin-tunable; a per-user download_quotas row overrides).
|
|
download_default_max_bytes: int = 5_368_709_120 # 5 GiB
|
|
download_default_max_concurrent: int = 2
|
|
download_default_max_jobs: int = 50
|
|
# Retention: an asset expires this many days after its last access; the GC job then
|
|
# deletes it (once no job references it) after an additional grace window.
|
|
download_retention_days: int = 30
|
|
download_gc_grace_days: int = 2
|
|
download_gc_minutes: int = 360
|
|
# On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat".
|
|
download_layout: str = "plex"
|
|
# Default SponsorBlock (skip sponsor segments) for new custom profiles.
|
|
sponsorblock_default: bool = False
|
|
|
|
@property
|
|
def allowed_email_set(self) -> set[str]:
|
|
return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()}
|
|
|
|
@property
|
|
def admin_email_set(self) -> set[str]:
|
|
return {e.strip().lower() for e in self.admin_emails.split(",") if e.strip()}
|
|
|
|
@property
|
|
def app_base(self) -> str:
|
|
"""Public origin of the deployed app, derived from the OAuth redirect URL
|
|
(.../auth/callback). The single source for user-facing links (verification, reset,
|
|
approval emails); dev and prod differ, so links must always come from config."""
|
|
u = self.oauth_redirect_url
|
|
return u.split("/auth/")[0] if "/auth/" in u else u.rstrip("/")
|
|
|
|
@property
|
|
def session_https_only(self) -> bool:
|
|
"""Mark the session cookie Secure when we're served over HTTPS. We treat an
|
|
https OAUTH_REDIRECT_URL as the signal for 'real deployment' vs local dev."""
|
|
return self.oauth_redirect_url.lower().startswith("https://")
|
|
|
|
@model_validator(mode="after")
|
|
def _enforce_production_secrets(self) -> "Settings":
|
|
"""Fail fast rather than boot a public instance with the shipped placeholder
|
|
signing key (which would let anyone forge an admin session) or without token
|
|
encryption. Local dev (http redirect URL) keeps the convenient defaults."""
|
|
if not self.session_https_only:
|
|
return self # local/dev: allow the placeholder defaults
|
|
if self.secret_key == _DEFAULT_SECRET_KEY or len(self.secret_key) < 32:
|
|
raise ValueError(
|
|
"SECRET_KEY must be a unique random value of at least 32 chars in "
|
|
"production (an https OAUTH_REDIRECT_URL was detected). Generate one "
|
|
'with: python -c "import secrets; print(secrets.token_urlsafe(48))"'
|
|
)
|
|
if not self.token_encryption_key:
|
|
raise ValueError(
|
|
"TOKEN_ENCRYPTION_KEY must be set in production so refresh tokens are "
|
|
"encrypted at rest. Generate one with: "
|
|
'python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"'
|
|
)
|
|
return self
|
|
|
|
|
|
settings = Settings()
|