diff --git a/.env.example b/.env.example index f428ca2..6d00b52 100644 --- a/.env.example +++ b/.env.example @@ -28,3 +28,23 @@ ADMIN_EMAILS= # Optional: origin of a separately-served frontend dev server (enables CORS). Leave empty in production. FRONTEND_ORIGIN= + +# Optional YouTube Data API key (Google Cloud Console -> Credentials -> Create API key). +# When set, all public reads (channels/videos/playlist backfill + enrichment) use the key +# instead of a user's OAuth token, so 24/7 backfill never depends on a refresh token that +# would otherwise expire (Google expires refresh tokens after 7 days while the OAuth consent +# screen is in "Testing"). Strongly recommended for the always-on server instance. +YOUTUBE_API_KEY= + +# --- Deployment role --- +# DATABASE_URL is set automatically by docker-compose.yml / docker-compose.server.yml to the +# bundled `db` service. Override it only for local dev against the central server DB, e.g.: +# DATABASE_URL=postgresql+psycopg://subfeed:@your-db-host:5432/subfeed +# Exactly one instance may run the background scheduler. The central server keeps it on; every +# other instance (local dev, etc.) must set SCHEDULER_ENABLED=false. The compose files set this +# for you (server=true via docker-compose.server.yml, local=false via docker-compose.localdev.yml). +SCHEDULER_ENABLED=true + +# On the central server, set this so backup.sh / restore.sh and plain `docker compose` commands +# target the server stack automatically: +# COMPOSE_FILE=docker-compose.server.yml diff --git a/README.md b/README.md index 95af6e8..7812ba0 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ once and stored locally, so filtering/searching/sorting are instant and don't bu ## Backup & moving to another machine -All data lives in the `pgdata` Postgres volume — moving to another host (e.g. a Proxmox Linux +All data lives in the Postgres database — moving to another host (e.g. a Proxmox Linux server) does **not** require re-fetching from YouTube. Copy your `.env` (keep the same `TOKEN_ENCRYPTION_KEY` and Google client so stored tokens stay valid), then: @@ -70,6 +70,33 @@ server) does **not** require re-fetching from YouTube. Copy your `.env` (keep th ./scripts/restore.sh backups/ # on the new host after `docker compose up` ``` +## Central server + local dev (single shared database) + +For always-on operation the recommended topology is a single **central Postgres** running on a +24/7 host (e.g. a Proxmox LXC) that also runs the background scheduler, with every other instance +(your local dev machine, a future VPS) pointing at that same database. One source of truth, no +sync. + +- **Server** (`docker-compose.server.yml`): full stack, Postgres published on the LAN, scheduler + on, DB files in a host-visible `./pgdata` bind mount. In the LXC, from the project root: + + ```sh + echo 'COMPOSE_FILE=docker-compose.server.yml' >> .env # so backup/restore target this stack + docker compose up -d --build + ./scripts/restore.sh backups/ # migrate existing data in + ``` + + Set `YOUTUBE_API_KEY` in `.env` so unattended backfill/enrichment use the API key and don't + depend on a refresh token (which expires after 7 days while the OAuth screen is in *Testing*). + +- **Local dev** (`docker-compose.localdev.yml`): webapp only, no local DB, no scheduler. In `.env` + set `DATABASE_URL` to the central DB (e.g. `…@your-db-host:5432/subfeed`), then + `docker compose -f docker-compose.localdev.yml up --build` and browse http://localhost:8080. + +**Exactly one instance may run the scheduler.** The server keeps `SCHEDULER_ENABLED=true`; every +other instance must be `false` (the compose files enforce this) to avoid double quota burn and +write races on the shared DB. + ## Tech FastAPI + PostgreSQL backend, React + Vite frontend (added in a later milestone), packaged with diff --git a/docker-compose.localdev.yml b/docker-compose.localdev.yml new file mode 100644 index 0000000..918b21e --- /dev/null +++ b/docker-compose.localdev.yml @@ -0,0 +1,30 @@ +# Local development against the central (Proxmox) Postgres. +# +# Runs only the webapp/API — no local database, no scheduler — pointed at the shared +# DB on the server. You see the same live data the 24/7 backfill is filling, and you +# never run a second scheduler against it. +# +# Setup: in .env set +# DATABASE_URL=postgresql+psycopg://subfeed:@your-db-host:5432/subfeed +# (use the central DB's host/port and the real POSTGRES_PASSWORD), then: +# docker compose -f docker-compose.localdev.yml up --build +# +# Browse at http://localhost:8080 — Google login works here because the OAuth redirect +# is http://localhost:8080/auth/callback. +# +# Note: the API runs `alembic upgrade head` on startup, so launching this with newer +# local migrations will apply them to the shared DB. For risky schema work, point +# DATABASE_URL at a throwaway local Postgres instead. + +services: + api: + build: + context: . + dockerfile: Dockerfile + env_file: .env + environment: + # Exactly one scheduler may write to the shared DB; the server owns it. + SCHEDULER_ENABLED: "false" + ports: + - "${APP_PORT:-8080}:8000" + restart: unless-stopped diff --git a/docker-compose.server.yml b/docker-compose.server.yml new file mode 100644 index 0000000..279468b --- /dev/null +++ b/docker-compose.server.yml @@ -0,0 +1,58 @@ +# Server (Proxmox LXC) deployment of the full Subfeed stack. +# +# This is the "central" instance: it owns the single shared Postgres database and +# runs the background scheduler (RSS poll, enrichment, recent/deep backfill) 24/7. +# Local dev machines and any future VPS point their DATABASE_URL at this Postgres +# and run with SCHEDULER_ENABLED=false (see docker-compose.localdev.yml) so exactly +# one scheduler ever writes to the DB. +# +# Usage (inside the LXC, from the project root which is the bind-mounted /docker): +# export COMPOSE_FILE=docker-compose.server.yml # or put it in .env +# docker compose up -d --build +# +# Postgres data lives in ./pgdata (a host-visible bind mount under /docker/subfeed) +# so it is covered by the host's /docker backups; logical pg_dump backups via +# scripts/backup.sh remain the portable mechanism for moving between hosts. + +services: + db: + image: postgres:16-alpine + environment: + POSTGRES_USER: ${POSTGRES_USER:-subfeed} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-subfeed} + POSTGRES_DB: ${POSTGRES_DB:-subfeed} + volumes: + - ./pgdata:/var/lib/postgresql/data + ports: + # Published on the LAN so the local dev webapp uses this central DB. + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-subfeed} -d ${POSTGRES_DB:-subfeed}"] + interval: 5s + timeout: 5s + retries: 12 + restart: unless-stopped + + api: + build: + context: . + dockerfile: Dockerfile + env_file: .env + environment: + DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-subfeed}:${POSTGRES_PASSWORD:-subfeed}@db:5432/${POSTGRES_DB:-subfeed} + # This instance owns the background sync. Keep it true here and false everywhere else. + SCHEDULER_ENABLED: "true" + depends_on: + db: + condition: service_healthy + ports: + # Exposed on the LAN for status/health checks only. Interactive Google login + # needs a public HTTPS redirect URL (set up later); backfill runs regardless. + - "${APP_PORT:-8080}:8000" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; exit(0 if urllib.request.urlopen('http://localhost:8000/healthz').status == 200 else 1)"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 25s + restart: unless-stopped