2026-06-11 04:19:54 +02:00
|
|
|
import logging
|
|
|
|
|
import sys
|
feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler
- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
2026-06-11 01:36:41 +02:00
|
|
|
from contextlib import asynccontextmanager
|
2026-06-11 01:01:37 +02:00
|
|
|
from pathlib import Path
|
|
|
|
|
|
2026-06-11 02:19:47 +02:00
|
|
|
from fastapi import FastAPI, HTTPException
|
2026-06-11 04:19:54 +02:00
|
|
|
|
2026-06-21 06:53:12 +02:00
|
|
|
# When started via uvicorn --log-config the "siftlode" logger is already configured
|
2026-06-11 04:26:18 +02:00
|
|
|
# (see log_config.json). This block is a timestamped fallback for other entrypoints
|
|
|
|
|
# (tests, scripts) so our logs are never silently dropped.
|
2026-06-21 06:53:12 +02:00
|
|
|
_siftlode_logger = logging.getLogger("siftlode")
|
|
|
|
|
if not _siftlode_logger.handlers:
|
2026-06-11 04:19:54 +02:00
|
|
|
_handler = logging.StreamHandler(sys.stdout)
|
2026-06-11 04:26:18 +02:00
|
|
|
_handler.setFormatter(
|
|
|
|
|
logging.Formatter("%(asctime)s %(levelname)-5s [%(name)s] %(message)s")
|
|
|
|
|
)
|
2026-06-21 06:53:12 +02:00
|
|
|
_siftlode_logger.addHandler(_handler)
|
|
|
|
|
_siftlode_logger.setLevel(logging.INFO)
|
|
|
|
|
_siftlode_logger.propagate = False
|
2026-06-11 04:26:18 +02:00
|
|
|
|
2026-06-21 06:53:12 +02:00
|
|
|
log = logging.getLogger("siftlode.app")
|
2026-06-11 01:01:37 +02:00
|
|
|
from fastapi.middleware.cors import CORSMiddleware
|
2026-06-21 00:38:47 +02:00
|
|
|
from fastapi.responses import FileResponse, JSONResponse
|
2026-06-11 01:01:37 +02:00
|
|
|
from fastapi.staticfiles import StaticFiles
|
|
|
|
|
from starlette.middleware.sessions import SessionMiddleware
|
|
|
|
|
|
2026-06-21 00:38:47 +02:00
|
|
|
from app import auth, state
|
2026-06-11 01:01:37 +02:00
|
|
|
from app.config import settings
|
2026-06-21 00:38:47 +02:00
|
|
|
from app.db import SessionLocal
|
2026-06-16 14:38:51 +02:00
|
|
|
from app.routes import (
|
|
|
|
|
admin,
|
|
|
|
|
channels,
|
2026-06-19 12:22:36 +02:00
|
|
|
config as config_routes,
|
feat(downloads): M4 — REST API (enqueue, manage, file-serve, sharing, admin)
routes/downloads.py (require_human; admin_router adds admin_user):
- profiles: list (builtins + own), create/update/delete custom
- enqueue: resolve_source (bare id / watch / youtu.be / shorts URLs -> youtube id; else
raw URL for the generic extractor), profile_id or inline spec or builtin fallback; maps
quota.QuotaExceeded -> 422 speaking message
- list / usage / shared-with-me; rename (display name only); pause/resume/cancel/delete
(ref_count bookkeeping)
- GET /{id}/file: ownership-or-share check, path-traversal guard, range-aware FileResponse
with the user's custom display name (Content-Disposition), bumps last_access
- share/unshare by email + a download_shared notification
- admin: all-jobs, storage dashboard (totals + per-user footprint), per-user quota GET/PUT/DELETE
- cast SUM(bigint) footprints to int for clean numeric JSON
- wired both routers in main.py
Verified via TestClient: full enqueue->download->206 range fetch->share->access-control
(cross-user delete 404); unauth = 401 on user + admin routes; all 15 paths registered.
2026-07-03 00:34:08 +02:00
|
|
|
downloads,
|
2026-06-16 14:38:51 +02:00
|
|
|
feed,
|
|
|
|
|
health,
|
|
|
|
|
me,
|
feat(messages): end-to-end encrypted real-time direct messaging backend
Private user-to-user messages are end-to-end encrypted: the server only ever
stores ciphertext + iv and acts as a key directory (public keys) plus an opaque
store for each user's private key, wrapped client-side with a passphrase the
server never sees — so not even an admin can read a conversation. A separate
kind=system message (plaintext, no sender) powers a server-authored Siftlode
welcome shown on first open, reusable later for announcements.
- models: rework Message (kind, nullable sender/body, ciphertext+iv) + MessageKey;
migrations 0026 (table) + 0027 (E2EE rework).
- routes/messages.py: key directory/blob endpoints, ciphertext send, conversations
+ threads (system + user), lazy welcome, all gated by require_human.
- realtime.py: in-process WebSocket connection registry; /ws delivers sent
messages to a user's open tabs instantly (sync-callable push, single-process).
2026-06-25 22:05:35 +02:00
|
|
|
messages,
|
2026-06-18 03:20:17 +02:00
|
|
|
notifications,
|
2026-06-16 14:38:51 +02:00
|
|
|
playlists,
|
2026-07-04 04:19:29 +02:00
|
|
|
public as public_routes,
|
2026-06-16 14:38:51 +02:00
|
|
|
quota,
|
2026-07-01 03:17:36 +02:00
|
|
|
saved_views,
|
2026-06-16 14:38:51 +02:00
|
|
|
scheduler as scheduler_routes,
|
2026-06-29 02:01:31 +02:00
|
|
|
search as search_routes,
|
2026-06-21 00:38:47 +02:00
|
|
|
setup as setup_routes,
|
2026-06-16 14:38:51 +02:00
|
|
|
sync,
|
|
|
|
|
tags,
|
|
|
|
|
version,
|
|
|
|
|
)
|
feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler
- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
2026-06-11 01:36:41 +02:00
|
|
|
from app.scheduler import shutdown_scheduler, start_scheduler
|
2026-06-11 01:01:37 +02:00
|
|
|
|
feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler
- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
2026-06-11 01:36:41 +02:00
|
|
|
|
|
|
|
|
@asynccontextmanager
|
|
|
|
|
async def lifespan(app: FastAPI):
|
chore: rebrand Subfeed -> Siftlode
Rename all user-facing references (UI wordmark Sift+lode, titles, app name,
legal pages, onboarding wizard, emails, README/docs) and infra paths
(/srv/subfeed -> /srv/siftlode, image tag, deploy script, backup filenames).
Internal identifiers kept on purpose: Postgres user/db "subfeed", logger
namespace, localStorage keys, and the subfeed_pgdata volume (renaming would
orphan the migrated production data).
2026-06-14 04:40:22 +02:00
|
|
|
log.info("Siftlode starting up")
|
2026-06-21 00:38:47 +02:00
|
|
|
# First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print
|
|
|
|
|
# the wizard URL so the operator can open it (the app runs in setup mode until they finish).
|
|
|
|
|
with SessionLocal() as db:
|
|
|
|
|
if not state.is_configured(db):
|
|
|
|
|
token = state.rotate_setup_token(db)
|
|
|
|
|
url = f"{settings.app_base}/setup?token={token}"
|
|
|
|
|
log.warning(
|
|
|
|
|
"FIRST-RUN SETUP REQUIRED — this instance isn't configured yet.\n"
|
|
|
|
|
" Open the install wizard (internal access only):\n %s",
|
|
|
|
|
url,
|
|
|
|
|
)
|
feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler
- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
2026-06-11 01:36:41 +02:00
|
|
|
start_scheduler()
|
|
|
|
|
try:
|
|
|
|
|
yield
|
|
|
|
|
finally:
|
chore: rebrand Subfeed -> Siftlode
Rename all user-facing references (UI wordmark Sift+lode, titles, app name,
legal pages, onboarding wizard, emails, README/docs) and infra paths
(/srv/subfeed -> /srv/siftlode, image tag, deploy script, backup filenames).
Internal identifiers kept on purpose: Postgres user/db "subfeed", logger
namespace, localStorage keys, and the subfeed_pgdata volume (renaming would
orphan the migrated production data).
2026-06-14 04:40:22 +02:00
|
|
|
log.info("Siftlode shutting down")
|
feat: M2 (part 2) — RSS poller, backfill, enrichment, scheduler
- Free per-channel RSS reader for quota-less fresh-video detection
- Recent-first backfill (configurable: 100 videos / 1 year) plus resumable deep
backfill from the uploads playlist
- Enrichment via videos.list: duration, view/like counts, category, topics,
language, Shorts heuristic and livestream/premiere classification
- Reusable sync runners + APScheduler jobs (rss / enrich / backfill), all
quota-aware with a reserve so backfill never starves fresh enrichment
- Manual triggers: POST /api/sync/{rss,backfill,enrich}
- Exact insert counting via RETURNING with in-batch de-duplication
2026-06-11 01:36:41 +02:00
|
|
|
shutdown_scheduler()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app = FastAPI(title=settings.app_name, lifespan=lifespan)
|
2026-06-11 01:01:37 +02:00
|
|
|
|
|
|
|
|
app.add_middleware(
|
|
|
|
|
SessionMiddleware,
|
|
|
|
|
secret_key=settings.secret_key,
|
2026-06-13 23:56:34 +02:00
|
|
|
same_site="lax", # required so the cookie rides the OAuth redirect back from Google
|
|
|
|
|
https_only=settings.session_https_only, # Secure flag when served over HTTPS (prod)
|
2026-06-11 01:01:37 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
if settings.frontend_origin:
|
|
|
|
|
app.add_middleware(
|
|
|
|
|
CORSMiddleware,
|
|
|
|
|
allow_origins=[settings.frontend_origin],
|
|
|
|
|
allow_credentials=True,
|
|
|
|
|
allow_methods=["*"],
|
|
|
|
|
allow_headers=["*"],
|
|
|
|
|
)
|
|
|
|
|
|
2026-06-21 00:38:47 +02:00
|
|
|
# Paths reachable while the instance is unconfigured (setup mode). Everything else under /api or
|
|
|
|
|
# /auth is locked until the wizard finishes; static/SPA always loads so the wizard page can render.
|
|
|
|
|
_SETUP_OPEN_PREFIXES = ("/api/setup", "/api/version")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@app.middleware("http")
|
|
|
|
|
async def setup_gate(request, call_next):
|
|
|
|
|
"""Lock the app to the install wizard until it's configured. Skips the DB entirely once setup
|
|
|
|
|
completed in this process (the cached fast-path), so there's no steady-state overhead."""
|
|
|
|
|
if not state.setup_complete():
|
|
|
|
|
path = request.url.path
|
|
|
|
|
if path.startswith(("/api/", "/auth/")) and not path.startswith(_SETUP_OPEN_PREFIXES):
|
|
|
|
|
with SessionLocal() as db:
|
|
|
|
|
configured = state.is_configured_cached(db)
|
|
|
|
|
if not configured:
|
|
|
|
|
return JSONResponse(
|
|
|
|
|
{"detail": "This instance isn't set up yet."}, status_code=503
|
|
|
|
|
)
|
|
|
|
|
return await call_next(request)
|
|
|
|
|
|
|
|
|
|
|
2026-06-11 01:01:37 +02:00
|
|
|
app.include_router(health.router)
|
|
|
|
|
app.include_router(auth.router)
|
2026-06-11 01:22:07 +02:00
|
|
|
app.include_router(sync.router)
|
2026-06-11 01:57:19 +02:00
|
|
|
app.include_router(tags.router)
|
2026-06-11 02:11:02 +02:00
|
|
|
app.include_router(feed.router)
|
2026-06-29 02:01:31 +02:00
|
|
|
app.include_router(search_routes.router)
|
2026-06-11 02:11:02 +02:00
|
|
|
app.include_router(me.router)
|
2026-06-18 03:20:17 +02:00
|
|
|
app.include_router(notifications.router)
|
feat(messages): end-to-end encrypted real-time direct messaging backend
Private user-to-user messages are end-to-end encrypted: the server only ever
stores ciphertext + iv and acts as a key directory (public keys) plus an opaque
store for each user's private key, wrapped client-side with a passphrase the
server never sees — so not even an admin can read a conversation. A separate
kind=system message (plaintext, no sender) powers a server-authored Siftlode
welcome shown on first open, reusable later for announcements.
- models: rework Message (kind, nullable sender/body, ciphertext+iv) + MessageKey;
migrations 0026 (table) + 0027 (E2EE rework).
- routes/messages.py: key directory/blob endpoints, ciphertext send, conversations
+ threads (system + user), lazy welcome, all gated by require_human.
- realtime.py: in-process WebSocket connection registry; /ws delivers sent
messages to a user's open tabs instantly (sync-callable push, single-process).
2026-06-25 22:05:35 +02:00
|
|
|
app.include_router(messages.router)
|
feat(m5a): channel manager, tabbed settings panel, per-user sync status
Backend: /api/channels (list + PATCH priority/hidden + attach/detach user tags),
user-tag CRUD on /api/tags, /api/sync/my-status (per-user channel sync counts).
Frontend: feed|channels page nav (URL-synced) from the account menu; a slide-in
tabbed Settings panel (Appearance, Notifications=6b sound+duration, Sync status +
actions + admin pause/resume, Account); a channel manager with priority, hide,
per-channel user tags, sync badges and 'view in feed'. Notifications now honor the
configurable sound + auto-dismiss settings.
2026-06-11 20:45:48 +02:00
|
|
|
app.include_router(channels.router)
|
2026-06-15 14:37:09 +02:00
|
|
|
app.include_router(playlists.router)
|
2026-07-01 03:17:36 +02:00
|
|
|
app.include_router(saved_views.router)
|
feat(downloads): M4 — REST API (enqueue, manage, file-serve, sharing, admin)
routes/downloads.py (require_human; admin_router adds admin_user):
- profiles: list (builtins + own), create/update/delete custom
- enqueue: resolve_source (bare id / watch / youtu.be / shorts URLs -> youtube id; else
raw URL for the generic extractor), profile_id or inline spec or builtin fallback; maps
quota.QuotaExceeded -> 422 speaking message
- list / usage / shared-with-me; rename (display name only); pause/resume/cancel/delete
(ref_count bookkeeping)
- GET /{id}/file: ownership-or-share check, path-traversal guard, range-aware FileResponse
with the user's custom display name (Content-Disposition), bumps last_access
- share/unshare by email + a download_shared notification
- admin: all-jobs, storage dashboard (totals + per-user footprint), per-user quota GET/PUT/DELETE
- cast SUM(bigint) footprints to int for clean numeric JSON
- wired both routers in main.py
Verified via TestClient: full enqueue->download->206 range fetch->share->access-control
(cross-user delete 404); unauth = 401 on user + admin routes; all 15 paths registered.
2026-07-03 00:34:08 +02:00
|
|
|
app.include_router(downloads.router)
|
|
|
|
|
app.include_router(downloads.admin_router)
|
2026-07-04 04:19:29 +02:00
|
|
|
app.include_router(public_routes.router)
|
2026-06-12 01:43:07 +02:00
|
|
|
app.include_router(admin.router)
|
2026-06-19 12:22:36 +02:00
|
|
|
app.include_router(config_routes.router)
|
2026-06-16 14:38:51 +02:00
|
|
|
app.include_router(scheduler_routes.router)
|
2026-06-12 02:47:55 +02:00
|
|
|
app.include_router(quota.router)
|
2026-06-15 00:06:57 +02:00
|
|
|
app.include_router(version.router)
|
2026-06-21 00:38:47 +02:00
|
|
|
app.include_router(setup_routes.router)
|
2026-06-11 01:01:37 +02:00
|
|
|
|
2026-06-11 02:19:47 +02:00
|
|
|
# The built SPA (populated by the Docker frontend build stage).
|
|
|
|
|
STATIC_DIR = Path(__file__).parent / "static_spa"
|
2026-07-01 23:43:35 +02:00
|
|
|
# index.html is unhashed and references the content-hashed /assets bundles, so it MUST NOT be
|
|
|
|
|
# heuristically cached — otherwise, after a deploy, a browser keeps serving the old index.html
|
|
|
|
|
# (pointing at the previous bundle) and runs stale code until a hard refresh. `no-cache` lets it
|
|
|
|
|
# stay cached but forces revalidation (cheap 304 via the FileResponse ETag) on every load. The
|
|
|
|
|
# hashed bundles under /assets can cache forever (a new build changes their filename).
|
|
|
|
|
INDEX_HTML = STATIC_DIR / "index.html"
|
|
|
|
|
INDEX_HEADERS = {"Cache-Control": "no-cache"}
|
2026-06-11 02:19:47 +02:00
|
|
|
app.mount(
|
|
|
|
|
"/assets",
|
|
|
|
|
StaticFiles(directory=STATIC_DIR / "assets", check_dir=False),
|
|
|
|
|
name="assets",
|
|
|
|
|
)
|
2026-06-11 01:01:37 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
@app.get("/")
|
|
|
|
|
async def index() -> FileResponse:
|
2026-07-01 23:43:35 +02:00
|
|
|
return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)
|
2026-06-11 02:19:47 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
@app.get("/{full_path:path}")
|
|
|
|
|
async def spa_fallback(full_path: str) -> FileResponse:
|
|
|
|
|
# Client-side routes fall back to index.html; real API/asset paths are matched above.
|
|
|
|
|
if full_path.startswith(("api/", "auth/", "healthz", "assets/")):
|
|
|
|
|
raise HTTPException(status_code=404)
|
2026-06-19 19:52:29 +02:00
|
|
|
# Serve real files that live at the SPA root (Vite copies public/ there — e.g. the landing
|
|
|
|
|
# screenshots under /welcome/, favicon). /assets is already mounted above; everything else
|
|
|
|
|
# that isn't a real file is a client-side route → index.html. Guard against path traversal.
|
|
|
|
|
if full_path:
|
|
|
|
|
candidate = (STATIC_DIR / full_path).resolve()
|
|
|
|
|
if candidate.is_file() and STATIC_DIR.resolve() in candidate.parents:
|
|
|
|
|
return FileResponse(candidate)
|
2026-07-01 23:43:35 +02:00
|
|
|
return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)
|