fix(worker): wait for the DB schema before starting (no crash-loop on fresh deploy)

The worker and API start in parallel; the API applies migrations on its own startup, so on a fresh
deploy the worker hit a missing download_jobs table and crash-looped until migrations landed. The
worker now polls for the schema before recovering orphans, and (belt-and-suspenders) the compose
files gate it on the API being healthy. Fixes the observed prod restart-loop on the v0.22.0 deploy.
This commit is contained in:
npeter83 2026-07-04 06:43:04 +02:00
parent fe822598fa
commit 524507ce9b
3 changed files with 30 additions and 0 deletions

View file

@ -76,6 +76,26 @@ def _parse_upload_date(raw) -> date | None:
# --- claim + recovery -----------------------------------------------------------------------
def _wait_for_schema(timeout: float = 300.0) -> None:
"""Block until the download tables exist before doing any DB work.
The API applies migrations on its OWN startup, and the worker is a separate container that may
boot first (they start in parallel). Without this, the worker would hit a missing `download_jobs`
table and crash-loop until the API's migrations land. Poll patiently instead."""
deadline = time.monotonic() + timeout
while not _stop.is_set():
try:
with SessionLocal() as db:
db.execute(text("SELECT 1 FROM download_jobs LIMIT 1"))
return
except Exception as exc: # noqa: BLE001 — table not migrated yet / DB not up yet
if time.monotonic() > deadline:
log.warning("download schema still absent after %ss; continuing: %s", timeout, str(exc)[:120])
return
log.info("waiting for the database schema (API migrations)…")
_stop.wait(3.0)
def _recover_orphans() -> None:
"""Reset state left behind by a previous crash so nothing is stuck."""
with SessionLocal() as db:
@ -573,6 +593,9 @@ def main() -> None:
log.info("WORKER_ENABLED is off; worker exiting (nothing to do).")
return
_wait_for_schema() # the API may still be applying migrations when we boot
if _stop.is_set():
return
_recover_orphans()
n = max(1, settings.download_worker_concurrency)
log.info("Download worker started (concurrency=%d, root=%s).", n, settings.download_root)