2026-06-15 02:22:27 +02:00
|
|
|
from pathlib import Path
|
|
|
|
|
|
2026-06-13 23:56:34 +02:00
|
|
|
from pydantic import model_validator
|
2026-06-11 01:01:37 +02:00
|
|
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
|
|
|
|
2026-06-13 23:56:34 +02:00
|
|
|
# Shipped placeholder; production must override it. Used by the startup guard below.
|
|
|
|
|
_DEFAULT_SECRET_KEY = "change-me-session-key"
|
|
|
|
|
|
2026-06-11 01:01:37 +02:00
|
|
|
|
2026-06-15 02:22:27 +02:00
|
|
|
def _version_from_file() -> str:
|
|
|
|
|
"""The committed VERSION file (copied into the image at /app/VERSION) is the single
|
|
|
|
|
source of truth for the app version; falls back to 'dev' for un-built/local runs."""
|
|
|
|
|
for p in (Path(__file__).resolve().parents[1] / "VERSION", Path("VERSION")):
|
|
|
|
|
try:
|
|
|
|
|
v = p.read_text(encoding="utf-8").strip()
|
|
|
|
|
if v:
|
|
|
|
|
return v
|
|
|
|
|
except OSError:
|
|
|
|
|
continue
|
|
|
|
|
return "dev"
|
|
|
|
|
|
|
|
|
|
|
2026-06-11 01:01:37 +02:00
|
|
|
class Settings(BaseSettings):
|
|
|
|
|
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
|
|
|
|
|
chore: rebrand Subfeed -> Siftlode
Rename all user-facing references (UI wordmark Sift+lode, titles, app name,
legal pages, onboarding wizard, emails, README/docs) and infra paths
(/srv/subfeed -> /srv/siftlode, image tag, deploy script, backup filenames).
Internal identifiers kept on purpose: Postgres user/db "subfeed", logger
namespace, localStorage keys, and the subfeed_pgdata volume (renaming would
orphan the migrated production data).
2026-06-14 04:40:22 +02:00
|
|
|
app_name: str = "Siftlode"
|
2026-06-15 02:22:27 +02:00
|
|
|
# Version from the committed VERSION file (an explicit APP_VERSION env still overrides);
|
|
|
|
|
# git_sha/build_date are injected at image build time. Surfaced by /api/version.
|
|
|
|
|
app_version: str = _version_from_file()
|
2026-06-15 00:06:57 +02:00
|
|
|
git_sha: str = "unknown"
|
|
|
|
|
build_date: str = ""
|
2026-06-11 01:01:37 +02:00
|
|
|
|
2026-06-21 06:53:12 +02:00
|
|
|
database_url: str = "postgresql+psycopg://siftlode:siftlode@db:5432/siftlode"
|
2026-06-11 01:01:37 +02:00
|
|
|
|
|
|
|
|
# Session cookie signing key.
|
2026-06-13 23:56:34 +02:00
|
|
|
secret_key: str = _DEFAULT_SECRET_KEY
|
2026-06-11 01:01:37 +02:00
|
|
|
# Fernet key (urlsafe base64, 32 bytes) for encrypting stored refresh tokens.
|
|
|
|
|
token_encryption_key: str = ""
|
|
|
|
|
|
|
|
|
|
google_client_id: str = ""
|
|
|
|
|
google_client_secret: str = ""
|
|
|
|
|
oauth_redirect_url: str = "http://localhost:8080/auth/callback"
|
|
|
|
|
|
|
|
|
|
# Comma-separated invite list / admin list of Google account emails.
|
|
|
|
|
allowed_emails: str = ""
|
|
|
|
|
admin_emails: str = ""
|
|
|
|
|
|
2026-06-19 14:03:11 +02:00
|
|
|
# Whether anyone may submit an email+password registration (still gated by email
|
|
|
|
|
# verification + admin approval before they can sign in). Admin-tunable via the DB.
|
|
|
|
|
allow_registration: bool = True
|
|
|
|
|
|
2026-06-11 01:01:37 +02:00
|
|
|
# Origin of a separately served frontend dev server (enables CORS). Empty in production.
|
|
|
|
|
frontend_origin: str = ""
|
|
|
|
|
|
2026-06-12 01:43:07 +02:00
|
|
|
# --- Outbound email (onboarding: access-request + approval notices) ---
|
|
|
|
|
# Gmail SMTP + App Password by default. All optional: if unset, email is skipped
|
|
|
|
|
# (fail-soft) and onboarding still works via in-app notifications.
|
|
|
|
|
smtp_host: str = ""
|
|
|
|
|
smtp_port: int = 587
|
|
|
|
|
smtp_user: str = ""
|
|
|
|
|
smtp_password: str = ""
|
chore: rebrand Subfeed -> Siftlode
Rename all user-facing references (UI wordmark Sift+lode, titles, app name,
legal pages, onboarding wizard, emails, README/docs) and infra paths
(/srv/subfeed -> /srv/siftlode, image tag, deploy script, backup filenames).
Internal identifiers kept on purpose: Postgres user/db "subfeed", logger
namespace, localStorage keys, and the subfeed_pgdata volume (renaming would
orphan the migrated production data).
2026-06-14 04:40:22 +02:00
|
|
|
smtp_from: str = "" # e.g. "Siftlode <addr@gmail.com>"; falls back to smtp_user
|
2026-06-12 01:43:07 +02:00
|
|
|
|
2026-06-11 01:22:07 +02:00
|
|
|
# --- Sync / YouTube Data API ---
|
|
|
|
|
# Optional API key for public reads (channels/videos/playlistItems). When set it is
|
|
|
|
|
# preferred for shared backfill/enrichment so it doesn't depend on a specific user.
|
|
|
|
|
youtube_api_key: str = ""
|
2026-06-26 00:32:17 +02:00
|
|
|
# Optional HTTP(S) proxy for outbound YouTube Data API calls. Set this to send the
|
|
|
|
|
# scheduler's googleapis traffic through a fixed-IP host (e.g. the VPS over WireGuard) so an
|
|
|
|
|
# IP-restricted API key keeps working even when this host's public IP is dynamic.
|
|
|
|
|
youtube_api_proxy: str = ""
|
2026-06-11 01:22:07 +02:00
|
|
|
# Daily quota budget (units). Default leaves headroom under the 10,000/day free limit.
|
|
|
|
|
quota_daily_budget: int = 9000
|
2026-06-29 22:29:54 +02:00
|
|
|
# Per-user cap on live YouTube searches per day. With the official API (search_source="api")
|
|
|
|
|
# each search.list costs 100 units, so the shared budget only affords ~80-90/day total; with
|
|
|
|
|
# the zero-quota scrape source it's a pure anti-abuse rate limit. Admin-tunable.
|
2026-06-29 02:01:31 +02:00
|
|
|
search_daily_limit_per_user: int = 70
|
2026-06-29 22:29:54 +02:00
|
|
|
# Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list,
|
|
|
|
|
# 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks.
|
|
|
|
|
search_source: str = "scrape"
|
2026-06-11 01:22:07 +02:00
|
|
|
# Recent-first backfill: how far back to fetch on the first pass per channel.
|
feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler
- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
2026-06-11 01:36:41 +02:00
|
|
|
backfill_recent_max_videos: int = 100
|
2026-06-11 01:22:07 +02:00
|
|
|
backfill_recent_max_days: int = 365
|
|
|
|
|
|
fix: address reader-UI feedback (shorts, search, tags, login, UX)
- Shorts: confirm via youtube.com/shorts/<id> probe (SOCS cookie bypasses the
consent redirect) instead of a 60s heuristic; concurrent probing, shorts_probed
flag, scheduled refinement (migration 0005)
- Search: match title + channel name only (descriptions caused noisy results)
- Faceted tag filtering: AND across categories (language AND topic narrows),
OR within a category; any/all toggle applies to topics
- Language detection: majority vote over individual titles (fixes misdetections
like multipoleguy -> English; drops bogus Polish/Romanian)
- Login: drop forced consent so returning sign-in is quick (select_account)
- Feed cards: clickable channel name (opens channel), persistent saved badge,
undo toast on hide, Hidden view to restore; tag chips show counts in tooltip
2026-06-11 03:07:49 +02:00
|
|
|
# Shorts are confirmed by probing youtube.com/shorts/<id>. Only videos at or below
|
|
|
|
|
# this duration (and not livestreams) are probed; longer videos are never Shorts.
|
|
|
|
|
shorts_probe_max_seconds: int = 180
|
|
|
|
|
shorts_probe_batch: int = 150
|
|
|
|
|
shorts_probe_interval_minutes: int = 2
|
feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler
- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
2026-06-11 01:36:41 +02:00
|
|
|
# videos.list accepts up to 50 ids per call.
|
|
|
|
|
enrich_batch_size: int = 50
|
|
|
|
|
|
|
|
|
|
# --- Background scheduler ---
|
|
|
|
|
scheduler_enabled: bool = True
|
|
|
|
|
rss_poll_minutes: int = 20
|
|
|
|
|
enrich_interval_minutes: int = 3
|
|
|
|
|
backfill_interval_minutes: int = 10
|
|
|
|
|
# Keep this many quota units in reserve so scheduled backfill never starves
|
|
|
|
|
# interactive syncs / enrichment of fresh videos.
|
|
|
|
|
backfill_quota_reserve: int = 2000
|
|
|
|
|
|
2026-06-11 01:57:19 +02:00
|
|
|
# --- Auto-tagging / feed defaults ---
|
|
|
|
|
# Number of recent video titles sampled per channel for language detection.
|
|
|
|
|
autotag_title_sample: int = 40
|
|
|
|
|
autotag_interval_minutes: int = 30
|
2026-06-11 03:28:45 +02:00
|
|
|
subscriptions_resync_minutes: int = 360
|
2026-06-15 19:37:03 +02:00
|
|
|
playlist_sync_minutes: int = 360
|
2026-06-19 15:25:13 +02:00
|
|
|
# How often the shared demo account is auto-reset to a clean baseline (communal sandbox).
|
|
|
|
|
demo_reset_minutes: int = 720
|
2026-06-18 03:20:28 +02:00
|
|
|
# --- Maintenance / validation job ---
|
|
|
|
|
# Runs once a day by default (admin-tunable via the Scheduler dashboard). Grace periods
|
|
|
|
|
# are in days because that's the granularity that matters; the rolling re-validation
|
|
|
|
|
# re-checks the least-recently-checked N videos per run (1 unit / 50 videos), so on a
|
|
|
|
|
# ~233k catalog ~15000/day ≈ a 15-day full cycle for ~300 quota units.
|
|
|
|
|
maintenance_interval_minutes: int = 1440
|
|
|
|
|
maintenance_revalidate_batch: int = 15000
|
|
|
|
|
# Grace before a flagged-unavailable video is hard-deleted (cascades to states/playlist
|
|
|
|
|
# items). Deleted/private/paywalled get a recovery window; abandoned upcoming and
|
|
|
|
|
# confirmed-dead stuck-live have no age grace (deleted on the sweep that confirms them).
|
|
|
|
|
maintenance_grace_days: int = 7
|
feat(channels): channel-explore backend — about metadata, ephemeral provenance, cleanup
Adds the backend for a dedicated channel page + ephemeral browsing of un-subscribed
channels:
- migration 0033: channels.{total_view_count,published_at,banner_url,external_links,
from_explore}, videos.via_explore, explored_channels (per-user, grace-clocked).
- enrich channels with part=brandingSettings (banner/links) + statistics.viewCount +
snippet.publishedAt — no extra quota.
- GET /api/channels/{id}: About detail + this user's relationship; lazy-enriches the new
About fields (published_at sentinel).
- POST /api/channels/{id}/explore (require_human, quota-reserve guarded): records the
exploration, flags the channel ephemeral unless followed, ingests one page of uploads
(via_explore) + enriches; returns next_page_token for load-more.
- feed: per-explorer gate — a via_explore video is visible only to users who explored or
subscribe to its channel, so exploration never leaks into everyone's catalog.
- subscribe = keep: clears from_explore + via_explore on the channel's videos.
- scheduler explore_cleanup job + explore_grace_days config: hard-delete explored-but-
unkept channels and their untouched ephemeral videos after a grace period.
2026-06-30 02:52:44 +02:00
|
|
|
# --- Channel-explore cleanup job ---
|
|
|
|
|
# Runs daily by default (admin-tunable via the Scheduler dashboard). An explored-but-never-
|
|
|
|
|
# kept channel (no subscription, no per-user state on its videos) is hard-deleted this many
|
|
|
|
|
# days after it was last explored — the grace clock is ExploredChannel.created_at.
|
|
|
|
|
explore_cleanup_minutes: int = 1440
|
|
|
|
|
explore_grace_days: int = 7
|
2026-06-11 01:57:19 +02:00
|
|
|
# live_status values hidden from the feed by default. Completed-stream VODs
|
|
|
|
|
# ("was_live") are real watchable content and stay visible.
|
|
|
|
|
feed_default_hidden_live: str = "live,upcoming"
|
|
|
|
|
|
2026-06-11 01:01:37 +02:00
|
|
|
@property
|
|
|
|
|
def allowed_email_set(self) -> set[str]:
|
|
|
|
|
return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()}
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def admin_email_set(self) -> set[str]:
|
|
|
|
|
return {e.strip().lower() for e in self.admin_emails.split(",") if e.strip()}
|
|
|
|
|
|
2026-06-19 19:52:02 +02:00
|
|
|
@property
|
|
|
|
|
def app_base(self) -> str:
|
|
|
|
|
"""Public origin of the deployed app, derived from the OAuth redirect URL
|
|
|
|
|
(.../auth/callback). The single source for user-facing links (verification, reset,
|
|
|
|
|
approval emails); dev and prod differ, so links must always come from config."""
|
|
|
|
|
u = self.oauth_redirect_url
|
|
|
|
|
return u.split("/auth/")[0] if "/auth/" in u else u.rstrip("/")
|
|
|
|
|
|
2026-06-13 23:56:34 +02:00
|
|
|
@property
|
|
|
|
|
def session_https_only(self) -> bool:
|
|
|
|
|
"""Mark the session cookie Secure when we're served over HTTPS. We treat an
|
|
|
|
|
https OAUTH_REDIRECT_URL as the signal for 'real deployment' vs local dev."""
|
|
|
|
|
return self.oauth_redirect_url.lower().startswith("https://")
|
|
|
|
|
|
|
|
|
|
@model_validator(mode="after")
|
|
|
|
|
def _enforce_production_secrets(self) -> "Settings":
|
|
|
|
|
"""Fail fast rather than boot a public instance with the shipped placeholder
|
|
|
|
|
signing key (which would let anyone forge an admin session) or without token
|
|
|
|
|
encryption. Local dev (http redirect URL) keeps the convenient defaults."""
|
|
|
|
|
if not self.session_https_only:
|
|
|
|
|
return self # local/dev: allow the placeholder defaults
|
|
|
|
|
if self.secret_key == _DEFAULT_SECRET_KEY or len(self.secret_key) < 32:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
"SECRET_KEY must be a unique random value of at least 32 chars in "
|
|
|
|
|
"production (an https OAUTH_REDIRECT_URL was detected). Generate one "
|
|
|
|
|
'with: python -c "import secrets; print(secrets.token_urlsafe(48))"'
|
|
|
|
|
)
|
|
|
|
|
if not self.token_encryption_key:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
"TOKEN_ENCRYPTION_KEY must be set in production so refresh tokens are "
|
|
|
|
|
"encrypted at rest. Generate one with: "
|
|
|
|
|
'python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"'
|
|
|
|
|
)
|
|
|
|
|
return self
|
|
|
|
|
|
2026-06-11 01:01:37 +02:00
|
|
|
|
|
|
|
|
settings = Settings()
|