diff --git a/.env.example b/.env.example index 3b2e0c6..18c3325 100644 --- a/.env.example +++ b/.env.example @@ -48,15 +48,9 @@ SMTP_USER= SMTP_PASSWORD= SMTP_FROM= -# --- Deployment role --- -# DATABASE_URL is set automatically by docker-compose.yml / docker-compose.server.yml to the -# bundled `db` service. Override it only for local dev against the central server DB, e.g.: -# DATABASE_URL=postgresql+psycopg://siftlode:@your-db-host:5432/siftlode -# Exactly one instance may run the background scheduler. The central server keeps it on; every -# other instance (local dev, etc.) must set SCHEDULER_ENABLED=false. The compose files set this -# for you (server=true via docker-compose.server.yml, local=false via docker-compose.localdev.yml). +# --- Scheduler --- +# The background scheduler (subscription sync, backfill, enrichment) runs inside the app. +# DATABASE_URL is set for you by docker-compose to the bundled `db` service. If you ever run +# more than one instance against the same database, keep SCHEDULER_ENABLED=true on exactly one +# of them and false on the rest, to avoid double quota use and write races. SCHEDULER_ENABLED=true - -# On the central server, set this so backup.sh / restore.sh and plain `docker compose` commands -# target the server stack automatically: -# COMPOSE_FILE=docker-compose.server.yml diff --git a/README.md b/README.md index 9c9c218..c9df4e6 100644 --- a/README.md +++ b/README.md @@ -1,106 +1,109 @@ # Siftlode -Self-hosted, multi-user web app for browsing **your own YouTube subscriptions** the way you -actually want: precise filtering and sorting (by language, topic, length, age, watch state…), -a fast local-first feed, and one-click playback that opens the real youtube.com — so your -browser's ad blocking and SponsorBlock keep working exactly as before. +**Your YouTube subscriptions, the way a feed should work.** Siftlode pulls every upload from the +channels you follow into one clean, filterable feed — no algorithm deciding what you see, and no +Shorts or livestream noise unless you want it. Self-hosted, multi-user, and private: your data +stays on your own server. -Each user signs in with their own Google account (invite-only) and sees only their own -subscriptions. All the expensive data (channels, videos, metadata) is fetched from YouTube -once and stored locally, so filtering/searching/sorting are instant and don't burn API quota. +![Siftlode feed](frontend/public/welcome/feed.png) -> Status: early development. -> - **M1** (foundation): docker-compose stack, FastAPI backend, Google OAuth login with an -> email invite-list, encrypted token storage. -> - **M2** (ingest core): subscription import, free RSS detection, recent-first + deep backfill -> from the uploads playlist, enrichment (duration/stats/category, Shorts & livestream -> classification), a shared daily quota guard, and a background scheduler. -> - **M3** (auto-tagging): system tags for channel language (offline detection) and topic -> (from YouTube topics + dominant category), regenerated automatically; user tags are -> never overwritten. -> - **M4** (reader UI): React + Vite SPA with four color schemes (dark/light) and adjustable -> text size; grid/list feed scoped to your subscriptions, with faceted tag filters, -> content-type toggles (Normal/Shorts/Live), date range, search, sort, watch/save/hide -> state and per-channel filtering; clicking a video opens youtube.com. Accurate Shorts -> detection via the /shorts probe, a sync-status indicator with admin pause/resume, a -> filtered video count, and structured timestamped logging. +Everything expensive (channels, videos, metadata) is fetched from YouTube once and stored locally, +so filtering, searching and sorting are instant and don't burn API quota. Click a video to watch it +in an in-app player that resumes where you left off — or open it on youtube.com so your own ad +blocker and SponsorBlock keep working. -## Requirements +## Features -- Docker + Docker Compose -- A Google Cloud project with an OAuth client (see below) +- **A readable subscription feed** — sort and filter by channel, tag, language, topic, length, + upload date or watch state; hide channels without unsubscribing; save filter setups as named views. +- **Search all of YouTube** from the feed — results play, save and add to playlists like any other + video, and are materialised into your catalog. +- **Channel pages & a channel manager** — per-channel stats and uploads, priorities, and your own + tags to slice the feed by. +- **Playlists with two-way YouTube sync** — build them locally, keep them in sync in both directions. +- **In-app player** with resume, plus keyboard/scroll controls. +- **Multi-user** with per-user private state, a shared catalog, and a fair daily API-quota guard. +- **Self-hosted & private**, with a first-run web setup wizard and the interface in **English, + Hungarian and German**. -## Quick start +## Quick start (self-hosting) -1. Copy the env template and generate secrets: +You don't need to build anything — Siftlode runs from a prebuilt public image, and all +configuration (your admin account, Google sign-in, email) happens in a **first-run web wizard**. +You need [Docker](https://docs.docker.com/get-docker/) with the Compose plugin. - ```sh - cp .env.example .env - python -c "import secrets;print('SECRET_KEY=',secrets.token_urlsafe(48))" - python -c "import base64,os;print('TOKEN_ENCRYPTION_KEY=',base64.urlsafe_b64encode(os.urandom(32)).decode())" - ``` - - Paste the generated values into `.env`. - -2. Create a Google OAuth client (Google Cloud Console → APIs & Services): - - Enable the **YouTube Data API v3**. - - OAuth consent screen: **External**, publishing status **Testing**, and add every invited - Google account as a **Test user** (up to 100 — no app verification needed at this scale). - - Create credentials → **OAuth client ID** → type **Web application**. - - Authorized redirect URI: `http://localhost:8080/auth/callback` - (must match `OAUTH_REDIRECT_URL` in `.env`). - - Put the client ID/secret into `.env`, and list invited emails in `ALLOWED_EMAILS`. - -3. Start it: - - ```sh - docker compose up --build - ``` - - Open http://localhost:8080 and sign in. Database migrations run automatically on startup. - -## Backup & moving to another machine - -All data lives in the Postgres database — moving to another host (e.g. a Proxmox Linux -server) does **not** require re-fetching from YouTube. Copy your `.env` (keep the same -`TOKEN_ENCRYPTION_KEY` and Google client so stored tokens stay valid), then: +**1. Get the files and run the installer:** ```sh -./scripts/backup.sh # -> backups/siftlode-.dump (Windows: scripts\backup.ps1) -./scripts/restore.sh backups/ # on the new host after `docker compose up` +git clone https://forge.b1fr0st.eu/peter/siftlode.git +cd siftlode +./install.sh # Windows (PowerShell): ./install.ps1 ``` -## Central server + local dev (single shared database) +The installer generates a private `.env` (secrets), pulls the image, starts the app + database, and +prints a one-time setup URL like `http://localhost:8080/setup?token=…`. -For always-on operation the recommended topology is a single **central Postgres** running on a -24/7 host (e.g. a Proxmox LXC) that also runs the background scheduler, with every other instance -(your local dev machine, a future VPS) pointing at that same database. One source of truth, no -sync. +**2. Finish in your browser.** Open that URL and follow the wizard: -- **Server** (`docker-compose.server.yml`): full stack, Postgres published on the LAN, scheduler - on, DB files in a host-visible `./pgdata` bind mount. In the LXC, from the project root: +1. **Admin account** — the email + password you'll sign in with. +2. **Google sign-in** *(optional)* — paste a Google OAuth client to enable "Sign in with Google" and + pulling your YouTube subscriptions. Skip it to use email + password only. +3. **Email / SMTP** *(optional)* — for verification/notification emails. Skip it and you (the admin) + simply approve new accounts yourself. - ```sh - echo 'COMPOSE_FILE=docker-compose.server.yml' >> .env # so backup/restore target this stack - docker compose up -d --build - ./scripts/restore.sh backups/ # migrate existing data in - ``` +Then sign in with your admin account. That's it. See **[docs/self-hosting.md](docs/self-hosting.md)** +for the full walkthrough. - Set `YOUTUBE_API_KEY` in `.env` so unattended backfill/enrichment use the API key and don't - depend on a refresh token (which expires after 7 days while the OAuth screen is in *Testing*). +> Just trying it out? Press Enter at the installer's URL prompt to run on `http://localhost:8080`. -- **Local dev** (`docker-compose.localdev.yml`): webapp only, no local DB, no scheduler. In `.env` - set `DATABASE_URL` to the central DB (e.g. `…@your-db-host:5432/siftlode`), then - `docker compose -f docker-compose.localdev.yml up --build` and browse http://localhost:8080. +## Build from source (alternative) -**Exactly one instance may run the scheduler.** The server keeps `SCHEDULER_ENABLED=true`; every -other instance must be `false` (the compose files enforce this) to avoid double quota burn and -write races on the shared DB. +Prefer to build the image yourself instead of pulling it: + +```sh +git clone https://forge.b1fr0st.eu/peter/siftlode.git +cd siftlode +cp .env.example .env +# generate the two secrets and paste them into .env: +python -c "import secrets;print('SECRET_KEY='+secrets.token_urlsafe(48))" +python -c "import base64,os;print('TOKEN_ENCRYPTION_KEY='+base64.urlsafe_b64encode(os.urandom(32)).decode())" +docker compose up --build -d # builds from the included Dockerfile +``` + +Open `http://localhost:8080` and finish in the setup wizard as above. (Set a `POSTGRES_PASSWORD` in +`.env` too.) + +## HTTPS / public access + +Port `8080` over plain HTTP is fine for a LAN or a quick trial. For public access put a reverse +proxy (Caddy, Nginx, Traefik…) in front to terminate TLS, and set the **public URL** (the installer +prompt, or `OAUTH_REDIRECT_URL` in `.env`) to your `https://…` address — this also marks the session +cookie secure. Add that same `…/auth/callback` URL to your Google OAuth client's authorized redirect +URIs. + +## Updating & backups + +```sh +docker compose -f docker-compose.selfhost.yml pull # or: docker compose pull +docker compose -f docker-compose.selfhost.yml up -d +``` + +Database migrations run automatically on startup. Your data (accounts, subscriptions, playlists, the +video catalog) lives in a Postgres volume — back it up with `scripts/backup.sh` (or `backup.ps1` on +Windows) and restore with `scripts/restore.sh`. + +## How it works + +- **Shared catalog, private state.** Channels and videos are stored once and shared; each user's + subscriptions, tags, playlists and watch/save/hide state are private. +- **Cheap by design.** Public reads are cached locally; a shared daily quota budget and a background + scheduler keep unattended syncing within YouTube's free API limits. An optional API key lets + backfill run without depending on a user's OAuth token. ## Tech -FastAPI + PostgreSQL backend, React + Vite frontend (added in a later milestone), packaged with -Docker Compose. +FastAPI + PostgreSQL (SQLAlchemy, Alembic) backend; React + Vite + Tailwind + TanStack Query +frontend; packaged as a single Docker image with Docker Compose. ## Note diff --git a/backend/app/auth.py b/backend/app/auth.py index f71b55a..68b235f 100644 --- a/backend/app/auth.py +++ b/backend/app/auth.py @@ -805,4 +805,6 @@ def auth_config(db: Session = Depends(get_db)) -> dict: return { "google_enabled": google_enabled(db), "allow_registration": sysconfig.get_bool(db, "allow_registration"), + # Contact shown on the public legal pages (the operator's admin email), if configured. + "operator_contact": operator_contact(), } diff --git a/backend/app/config.py b/backend/app/config.py index 9b13b62..938cfcb 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -66,8 +66,8 @@ class Settings(BaseSettings): # preferred for shared backfill/enrichment so it doesn't depend on a specific user. youtube_api_key: str = "" # Optional HTTP(S) proxy for outbound YouTube Data API calls. Set this to send the - # scheduler's googleapis traffic through a fixed-IP host (e.g. the VPS over WireGuard) so an - # IP-restricted API key keeps working even when this host's public IP is dynamic. + # scheduler's googleapis traffic through a fixed-IP host so an IP-restricted API key + # keeps working even when this host's public IP is dynamic. youtube_api_proxy: str = "" # Daily quota budget (units). Default leaves headroom under the 10,000/day free limit. quota_daily_budget: int = 9000 diff --git a/backend/app/youtube/client.py b/backend/app/youtube/client.py index 0fc0ebf..bf2293c 100644 --- a/backend/app/youtube/client.py +++ b/backend/app/youtube/client.py @@ -44,8 +44,8 @@ class YouTubeClient: def __init__(self, db, user: User): self.db = db self.user = user - # Optionally route all YouTube traffic through a fixed-IP egress proxy (e.g. the VPS - # over WireGuard) so an IP-restricted API key keeps working from a dynamic-IP host. + # Optionally route all YouTube traffic through a fixed-IP egress proxy so an + # IP-restricted API key keeps working from a host with a dynamic public IP. proxy = sysconfig.get_str(db, "youtube_api_proxy") or None self._http = httpx.Client(timeout=30.0, proxy=proxy) diff --git a/docker-compose.localdev.yml b/docker-compose.localdev.yml index d7974af..37260aa 100644 --- a/docker-compose.localdev.yml +++ b/docker-compose.localdev.yml @@ -46,7 +46,7 @@ services: depends_on: db: condition: service_healthy - # Harmless on Docker Desktop; needed if ever run under Docker-in-LXC. + # Harmless on Docker Desktop; needed on some nested/hardened Docker hosts. security_opt: - apparmor:unconfined ports: diff --git a/frontend/src/components/legal/LegalLayout.tsx b/frontend/src/components/legal/LegalLayout.tsx index ef32ed7..bcd1dd5 100644 --- a/frontend/src/components/legal/LegalLayout.tsx +++ b/frontend/src/components/legal/LegalLayout.tsx @@ -1,10 +1,47 @@ // Shared shell for the public, login-free legal pages (/privacy, /terms). These must be // reachable unauthenticated — Google's OAuth consent screen links to them and reviewers -// fetch them directly — so they render outside the authenticated App tree (see main.tsx). +// fetch them directly — so they render outside the authenticated App tree (see main.tsx), +// with no react-query provider: the operator contact is fetched with a plain fetch below. +import { useEffect, useState } from "react"; -export const CONTACT_EMAIL = "b1fr0stmailops@gmail.com"; export const LAST_UPDATED = "June 14, 2026"; +// Contact shown on the legal pages: the instance operator's configured admin email, from the +// public /auth/config. One shared fetch across the page; null when the operator set none. +let contactPromise: Promise | null = null; +function fetchOperatorContact(): Promise { + if (!contactPromise) { + contactPromise = fetch("/auth/config") + .then((r) => (r.ok ? r.json() : null)) + .then((c) => (c?.operator_contact as string | undefined) ?? null) + .catch(() => null); + } + return contactPromise; +} + +export function useOperatorContact(): string | null { + const [contact, setContact] = useState(null); + useEffect(() => { + let alive = true; + fetchOperatorContact().then((c) => alive && setContact(c)); + return () => { + alive = false; + }; + }, []); + return contact; +} + +// The operator's contact rendered as a mailto link, or a neutral fallback phrase when unset. +export function ContactEmail() { + const email = useOperatorContact(); + if (!email) return <>your instance's administrator; + return ( + + {email} + + ); +} + export function LegalLink({ href, children }: { href: string; children: React.ReactNode }) { return ( @@ -20,6 +57,7 @@ export default function LegalLayout({ title: string; children: React.ReactNode; }) { + const contact = useOperatorContact(); return ( diff --git a/frontend/src/components/legal/PrivacyPolicy.tsx b/frontend/src/components/legal/PrivacyPolicy.tsx index 0da6618..9ac63a6 100644 --- a/frontend/src/components/legal/PrivacyPolicy.tsx +++ b/frontend/src/components/legal/PrivacyPolicy.tsx @@ -1,4 +1,4 @@ -import LegalLayout, { CONTACT_EMAIL, H2, LegalLink } from "./LegalLayout"; +import LegalLayout, { ContactEmail, H2, LegalLink } from "./LegalLayout"; export default function PrivacyPolicy() { return ( @@ -71,11 +71,7 @@ export default function PrivacyPolicy() { myaccount.google.com/permissions ; once revoked, the stored tokens can no longer be used. To have your account and - associated data deleted, email{" "} - - {CONTACT_EMAIL} - - . + associated data deleted, contact .

Cookies

@@ -100,11 +96,7 @@ export default function PrivacyPolicy() {

Contact

- Questions about this policy or your data:{" "} - - {CONTACT_EMAIL} - - . + Questions about this policy or your data: contact .

); diff --git a/frontend/src/components/legal/Terms.tsx b/frontend/src/components/legal/Terms.tsx index e616329..5d8d37c 100644 --- a/frontend/src/components/legal/Terms.tsx +++ b/frontend/src/components/legal/Terms.tsx @@ -1,4 +1,4 @@ -import LegalLayout, { CONTACT_EMAIL, H2, LegalLink } from "./LegalLayout"; +import LegalLayout, { ContactEmail, H2, LegalLink } from "./LegalLayout"; export default function Terms() { return ( @@ -56,11 +56,7 @@ export default function Terms() {

Contact

- Questions:{" "} - - {CONTACT_EMAIL} - - . + Questions: contact .

); diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 852150a..cc2a873 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -773,8 +773,11 @@ export const api = { }), // Public: which sign-in options this instance offers (e.g. hide Google when not configured). - authConfig: (): Promise<{ google_enabled: boolean; allow_registration: boolean }> => - req("/auth/config"), + authConfig: (): Promise<{ + google_enabled: boolean; + allow_registration: boolean; + operator_contact: string | null; + }> => req("/auth/config"), // --- first-run install wizard (token from the setup URL printed to the container logs) --- setupStatus: (): Promise<{ configured: boolean }> => req("/api/setup/status"),