From 253108a032a97c838611d1e34447ba9ad216266d Mon Sep 17 00:00:00 2001 From: npeter83 Date: Thu, 11 Jun 2026 16:35:00 +0200 Subject: [PATCH 1/2] feat(deploy): central Postgres + scheduler split for Proxmox 24/7 backfill Add a server compose (full stack, Postgres on the LAN, scheduler on, host-visible pgdata bind mount) and a localdev compose (webapp only, no DB, scheduler off) that points at the central DB. Document the single-shared-database topology, the exactly-one-scheduler rule, and YOUTUBE_API_KEY for unattended backfill that does not depend on a 7-day OAuth refresh token. --- .env.example | 20 +++++++++++++ README.md | 29 ++++++++++++++++++- docker-compose.localdev.yml | 30 +++++++++++++++++++ docker-compose.server.yml | 58 +++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 docker-compose.localdev.yml create mode 100644 docker-compose.server.yml diff --git a/.env.example b/.env.example index f428ca2..eba2c14 100644 --- a/.env.example +++ b/.env.example @@ -28,3 +28,23 @@ ADMIN_EMAILS= # Optional: origin of a separately-served frontend dev server (enables CORS). Leave empty in production. FRONTEND_ORIGIN= + +# Optional YouTube Data API key (Google Cloud Console -> Credentials -> Create API key). +# When set, all public reads (channels/videos/playlist backfill + enrichment) use the key +# instead of a user's OAuth token, so 24/7 backfill never depends on a refresh token that +# would otherwise expire (Google expires refresh tokens after 7 days while the OAuth consent +# screen is in "Testing"). Strongly recommended for the always-on server instance. +YOUTUBE_API_KEY= + +# --- Deployment role --- +# DATABASE_URL is set automatically by docker-compose.yml / docker-compose.server.yml to the +# bundled `db` service. Override it only for local dev against the central server DB, e.g.: +# DATABASE_URL=postgresql+psycopg://subfeed:@192.168.1.118:5432/subfeed +# Exactly one instance may run the background scheduler. The central server keeps it on; every +# other instance (local dev, etc.) must set SCHEDULER_ENABLED=false. The compose files set this +# for you (server=true via docker-compose.server.yml, local=false via docker-compose.localdev.yml). +SCHEDULER_ENABLED=true + +# On the central server, set this so backup.sh / restore.sh and plain `docker compose` commands +# target the server stack automatically: +# COMPOSE_FILE=docker-compose.server.yml diff --git a/README.md b/README.md index 95af6e8..1b67ad9 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ once and stored locally, so filtering/searching/sorting are instant and don't bu ## Backup & moving to another machine -All data lives in the `pgdata` Postgres volume — moving to another host (e.g. a Proxmox Linux +All data lives in the Postgres database — moving to another host (e.g. a Proxmox Linux server) does **not** require re-fetching from YouTube. Copy your `.env` (keep the same `TOKEN_ENCRYPTION_KEY` and Google client so stored tokens stay valid), then: @@ -70,6 +70,33 @@ server) does **not** require re-fetching from YouTube. Copy your `.env` (keep th ./scripts/restore.sh backups/ # on the new host after `docker compose up` ``` +## Central server + local dev (single shared database) + +For always-on operation the recommended topology is a single **central Postgres** running on a +24/7 host (e.g. a Proxmox LXC) that also runs the background scheduler, with every other instance +(your local dev machine, a future VPS) pointing at that same database. One source of truth, no +sync. + +- **Server** (`docker-compose.server.yml`): full stack, Postgres published on the LAN, scheduler + on, DB files in a host-visible `./pgdata` bind mount. In the LXC, from the project root: + + ```sh + echo 'COMPOSE_FILE=docker-compose.server.yml' >> .env # so backup/restore target this stack + docker compose up -d --build + ./scripts/restore.sh backups/ # migrate existing data in + ``` + + Set `YOUTUBE_API_KEY` in `.env` so unattended backfill/enrichment use the API key and don't + depend on a refresh token (which expires after 7 days while the OAuth screen is in *Testing*). + +- **Local dev** (`docker-compose.localdev.yml`): webapp only, no local DB, no scheduler. In `.env` + set `DATABASE_URL` to the central DB (e.g. `…@192.168.1.118:5432/subfeed`), then + `docker compose -f docker-compose.localdev.yml up --build` and browse http://localhost:8080. + +**Exactly one instance may run the scheduler.** The server keeps `SCHEDULER_ENABLED=true`; every +other instance must be `false` (the compose files enforce this) to avoid double quota burn and +write races on the shared DB. + ## Tech FastAPI + PostgreSQL backend, React + Vite frontend (added in a later milestone), packaged with diff --git a/docker-compose.localdev.yml b/docker-compose.localdev.yml new file mode 100644 index 0000000..5f1017d --- /dev/null +++ b/docker-compose.localdev.yml @@ -0,0 +1,30 @@ +# Local development against the central (Proxmox) Postgres. +# +# Runs only the webapp/API — no local database, no scheduler — pointed at the shared +# DB on the server. You see the same live data the 24/7 backfill is filling, and you +# never run a second scheduler against it. +# +# Setup: in .env set +# DATABASE_URL=postgresql+psycopg://subfeed:@192.168.1.118:5432/subfeed +# (use the central DB's host/port and the real POSTGRES_PASSWORD), then: +# docker compose -f docker-compose.localdev.yml up --build +# +# Browse at http://localhost:8080 — Google login works here because the OAuth redirect +# is http://localhost:8080/auth/callback. +# +# Note: the API runs `alembic upgrade head` on startup, so launching this with newer +# local migrations will apply them to the shared DB. For risky schema work, point +# DATABASE_URL at a throwaway local Postgres instead. + +services: + api: + build: + context: . + dockerfile: Dockerfile + env_file: .env + environment: + # Exactly one scheduler may write to the shared DB; the server owns it. + SCHEDULER_ENABLED: "false" + ports: + - "${APP_PORT:-8080}:8000" + restart: unless-stopped diff --git a/docker-compose.server.yml b/docker-compose.server.yml new file mode 100644 index 0000000..279468b --- /dev/null +++ b/docker-compose.server.yml @@ -0,0 +1,58 @@ +# Server (Proxmox LXC) deployment of the full Subfeed stack. +# +# This is the "central" instance: it owns the single shared Postgres database and +# runs the background scheduler (RSS poll, enrichment, recent/deep backfill) 24/7. +# Local dev machines and any future VPS point their DATABASE_URL at this Postgres +# and run with SCHEDULER_ENABLED=false (see docker-compose.localdev.yml) so exactly +# one scheduler ever writes to the DB. +# +# Usage (inside the LXC, from the project root which is the bind-mounted /docker): +# export COMPOSE_FILE=docker-compose.server.yml # or put it in .env +# docker compose up -d --build +# +# Postgres data lives in ./pgdata (a host-visible bind mount under /docker/subfeed) +# so it is covered by the host's /docker backups; logical pg_dump backups via +# scripts/backup.sh remain the portable mechanism for moving between hosts. + +services: + db: + image: postgres:16-alpine + environment: + POSTGRES_USER: ${POSTGRES_USER:-subfeed} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-subfeed} + POSTGRES_DB: ${POSTGRES_DB:-subfeed} + volumes: + - ./pgdata:/var/lib/postgresql/data + ports: + # Published on the LAN so the local dev webapp uses this central DB. + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-subfeed} -d ${POSTGRES_DB:-subfeed}"] + interval: 5s + timeout: 5s + retries: 12 + restart: unless-stopped + + api: + build: + context: . + dockerfile: Dockerfile + env_file: .env + environment: + DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-subfeed}:${POSTGRES_PASSWORD:-subfeed}@db:5432/${POSTGRES_DB:-subfeed} + # This instance owns the background sync. Keep it true here and false everywhere else. + SCHEDULER_ENABLED: "true" + depends_on: + db: + condition: service_healthy + ports: + # Exposed on the LAN for status/health checks only. Interactive Google login + # needs a public HTTPS redirect URL (set up later); backfill runs regardless. + - "${APP_PORT:-8080}:8000" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; exit(0 if urllib.request.urlopen('http://localhost:8000/healthz').status == 200 else 1)"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 25s + restart: unless-stopped From ac0d0da07170a584d0ec79d23364368f79c39b14 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Thu, 11 Jun 2026 16:42:44 +0200 Subject: [PATCH 2/2] fix(deploy): apparmor:unconfined for Docker-in-LXC The docker-default AppArmor profile can't be loaded from inside the LXC; match the lab convention (finance/arr stacks) of running services unconfined. --- docker-compose.localdev.yml | 3 +++ docker-compose.server.yml | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/docker-compose.localdev.yml b/docker-compose.localdev.yml index 5f1017d..fbd0bf3 100644 --- a/docker-compose.localdev.yml +++ b/docker-compose.localdev.yml @@ -25,6 +25,9 @@ services: environment: # Exactly one scheduler may write to the shared DB; the server owns it. SCHEDULER_ENABLED: "false" + # Harmless on Docker Desktop; needed if ever run under Docker-in-LXC. + security_opt: + - apparmor:unconfined ports: - "${APP_PORT:-8080}:8000" restart: unless-stopped diff --git a/docker-compose.server.yml b/docker-compose.server.yml index 279468b..49ee062 100644 --- a/docker-compose.server.yml +++ b/docker-compose.server.yml @@ -26,6 +26,10 @@ services: ports: # Published on the LAN so the local dev webapp uses this central DB. - "5432:5432" + # Required for Docker-in-LXC: the container can't load the docker-default + # AppArmor profile inside an unprivileged-from-AppArmor's-view namespace. + security_opt: + - apparmor:unconfined healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-subfeed} -d ${POSTGRES_DB:-subfeed}"] interval: 5s @@ -45,6 +49,8 @@ services: depends_on: db: condition: service_healthy + security_opt: + - apparmor:unconfined ports: # Exposed on the LAN for status/health checks only. Interactive Google login # needs a public HTTPS redirect URL (set up later); backfill runs regardless.