05df599f7a
Add a daily maintenance/validation job that detects videos which can't be played anywhere and retires them safely. Two phases: re-check already-flagged videos (recover if available again, else hard-delete once the grace period elapses, cascading to states/playlist items), and a rolling re-validation of the least-recently-checked currently-available videos that flags newly-unplayable ones (hidden from the feed immediately via unavailable_since). Detection is ~free: a video missing from the videos.list response is deleted-or-private; an `upcoming` premiere >2 days past its scheduled start that never went live is abandoned. A still-live broadcast is kept (legit 24/7 stream). Enrichment now also fetches part=status to populate the status columns. Grace is 7 days for removed videos, none for abandoned. Before deleting, affected users get one batched notification (never per-video). Interval is admin-tunable via the Scheduler dashboard; batch size and grace are config. Quota-attributed to the system and bounded by the same backfill reserve as the other jobs.
144 lines
6.4 KiB
Python
144 lines
6.4 KiB
Python
from pathlib import Path
|
|
|
|
from pydantic import model_validator
|
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
|
|
# Shipped placeholder; production must override it. Used by the startup guard below.
|
|
_DEFAULT_SECRET_KEY = "change-me-session-key"
|
|
|
|
|
|
def _version_from_file() -> str:
|
|
"""The committed VERSION file (copied into the image at /app/VERSION) is the single
|
|
source of truth for the app version; falls back to 'dev' for un-built/local runs."""
|
|
for p in (Path(__file__).resolve().parents[1] / "VERSION", Path("VERSION")):
|
|
try:
|
|
v = p.read_text(encoding="utf-8").strip()
|
|
if v:
|
|
return v
|
|
except OSError:
|
|
continue
|
|
return "dev"
|
|
|
|
|
|
class Settings(BaseSettings):
|
|
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
|
|
|
app_name: str = "Siftlode"
|
|
# Version from the committed VERSION file (an explicit APP_VERSION env still overrides);
|
|
# git_sha/build_date are injected at image build time. Surfaced by /api/version.
|
|
app_version: str = _version_from_file()
|
|
git_sha: str = "unknown"
|
|
build_date: str = ""
|
|
|
|
database_url: str = "postgresql+psycopg://subfeed:subfeed@db:5432/subfeed"
|
|
|
|
# Session cookie signing key.
|
|
secret_key: str = _DEFAULT_SECRET_KEY
|
|
# Fernet key (urlsafe base64, 32 bytes) for encrypting stored refresh tokens.
|
|
token_encryption_key: str = ""
|
|
|
|
google_client_id: str = ""
|
|
google_client_secret: str = ""
|
|
oauth_redirect_url: str = "http://localhost:8080/auth/callback"
|
|
|
|
# Comma-separated invite list / admin list of Google account emails.
|
|
allowed_emails: str = ""
|
|
admin_emails: str = ""
|
|
|
|
# Origin of a separately served frontend dev server (enables CORS). Empty in production.
|
|
frontend_origin: str = ""
|
|
|
|
# --- Outbound email (onboarding: access-request + approval notices) ---
|
|
# Gmail SMTP + App Password by default. All optional: if unset, email is skipped
|
|
# (fail-soft) and onboarding still works via in-app notifications.
|
|
smtp_host: str = ""
|
|
smtp_port: int = 587
|
|
smtp_user: str = ""
|
|
smtp_password: str = ""
|
|
smtp_from: str = "" # e.g. "Siftlode <addr@gmail.com>"; falls back to smtp_user
|
|
|
|
# --- Sync / YouTube Data API ---
|
|
# Optional API key for public reads (channels/videos/playlistItems). When set it is
|
|
# preferred for shared backfill/enrichment so it doesn't depend on a specific user.
|
|
youtube_api_key: str = ""
|
|
# Daily quota budget (units). Default leaves headroom under the 10,000/day free limit.
|
|
quota_daily_budget: int = 9000
|
|
# Recent-first backfill: how far back to fetch on the first pass per channel.
|
|
backfill_recent_max_videos: int = 100
|
|
backfill_recent_max_days: int = 365
|
|
|
|
# Shorts are confirmed by probing youtube.com/shorts/<id>. Only videos at or below
|
|
# this duration (and not livestreams) are probed; longer videos are never Shorts.
|
|
shorts_probe_max_seconds: int = 180
|
|
shorts_probe_batch: int = 150
|
|
shorts_probe_interval_minutes: int = 2
|
|
# videos.list accepts up to 50 ids per call.
|
|
enrich_batch_size: int = 50
|
|
|
|
# --- Background scheduler ---
|
|
scheduler_enabled: bool = True
|
|
rss_poll_minutes: int = 20
|
|
enrich_interval_minutes: int = 3
|
|
backfill_interval_minutes: int = 10
|
|
# Keep this many quota units in reserve so scheduled backfill never starves
|
|
# interactive syncs / enrichment of fresh videos.
|
|
backfill_quota_reserve: int = 2000
|
|
|
|
# --- Auto-tagging / feed defaults ---
|
|
# Number of recent video titles sampled per channel for language detection.
|
|
autotag_title_sample: int = 40
|
|
autotag_interval_minutes: int = 30
|
|
subscriptions_resync_minutes: int = 360
|
|
playlist_sync_minutes: int = 360
|
|
# --- Maintenance / validation job ---
|
|
# Runs once a day by default (admin-tunable via the Scheduler dashboard). Grace periods
|
|
# are in days because that's the granularity that matters; the rolling re-validation
|
|
# re-checks the least-recently-checked N videos per run (1 unit / 50 videos), so on a
|
|
# ~233k catalog ~15000/day ≈ a 15-day full cycle for ~300 quota units.
|
|
maintenance_interval_minutes: int = 1440
|
|
maintenance_revalidate_batch: int = 15000
|
|
# Grace before a flagged-unavailable video is hard-deleted (cascades to states/playlist
|
|
# items). Deleted/private/paywalled get a recovery window; abandoned upcoming and
|
|
# confirmed-dead stuck-live have no age grace (deleted on the sweep that confirms them).
|
|
maintenance_grace_days: int = 7
|
|
# live_status values hidden from the feed by default. Completed-stream VODs
|
|
# ("was_live") are real watchable content and stay visible.
|
|
feed_default_hidden_live: str = "live,upcoming"
|
|
|
|
@property
|
|
def allowed_email_set(self) -> set[str]:
|
|
return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()}
|
|
|
|
@property
|
|
def admin_email_set(self) -> set[str]:
|
|
return {e.strip().lower() for e in self.admin_emails.split(",") if e.strip()}
|
|
|
|
@property
|
|
def session_https_only(self) -> bool:
|
|
"""Mark the session cookie Secure when we're served over HTTPS. We treat an
|
|
https OAUTH_REDIRECT_URL as the signal for 'real deployment' vs local dev."""
|
|
return self.oauth_redirect_url.lower().startswith("https://")
|
|
|
|
@model_validator(mode="after")
|
|
def _enforce_production_secrets(self) -> "Settings":
|
|
"""Fail fast rather than boot a public instance with the shipped placeholder
|
|
signing key (which would let anyone forge an admin session) or without token
|
|
encryption. Local dev (http redirect URL) keeps the convenient defaults."""
|
|
if not self.session_https_only:
|
|
return self # local/dev: allow the placeholder defaults
|
|
if self.secret_key == _DEFAULT_SECRET_KEY or len(self.secret_key) < 32:
|
|
raise ValueError(
|
|
"SECRET_KEY must be a unique random value of at least 32 chars in "
|
|
"production (an https OAUTH_REDIRECT_URL was detected). Generate one "
|
|
'with: python -c "import secrets; print(secrets.token_urlsafe(48))"'
|
|
)
|
|
if not self.token_encryption_key:
|
|
raise ValueError(
|
|
"TOKEN_ENCRYPTION_KEY must be set in production so refresh tokens are "
|
|
"encrypted at rest. Generate one with: "
|
|
'python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"'
|
|
)
|
|
return self
|
|
|
|
|
|
settings = Settings()
|