fix(worker): wait for the DB schema before starting (no crash-loop on fresh deploy)
The worker and API start in parallel; the API applies migrations on its own startup, so on a fresh deploy the worker hit a missing download_jobs table and crash-looped until migrations landed. The worker now polls for the schema before recovering orphans, and (belt-and-suspenders) the compose files gate it on the API being healthy. Fixes the observed prod restart-loop on the v0.22.0 deploy.
This commit is contained in:
parent
fe822598fa
commit
524507ce9b
3 changed files with 30 additions and 0 deletions
|
|
@ -76,6 +76,26 @@ def _parse_upload_date(raw) -> date | None:
|
|||
|
||||
# --- claim + recovery -----------------------------------------------------------------------
|
||||
|
||||
def _wait_for_schema(timeout: float = 300.0) -> None:
|
||||
"""Block until the download tables exist before doing any DB work.
|
||||
|
||||
The API applies migrations on its OWN startup, and the worker is a separate container that may
|
||||
boot first (they start in parallel). Without this, the worker would hit a missing `download_jobs`
|
||||
table and crash-loop until the API's migrations land. Poll patiently instead."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while not _stop.is_set():
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
db.execute(text("SELECT 1 FROM download_jobs LIMIT 1"))
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001 — table not migrated yet / DB not up yet
|
||||
if time.monotonic() > deadline:
|
||||
log.warning("download schema still absent after %ss; continuing: %s", timeout, str(exc)[:120])
|
||||
return
|
||||
log.info("waiting for the database schema (API migrations)…")
|
||||
_stop.wait(3.0)
|
||||
|
||||
|
||||
def _recover_orphans() -> None:
|
||||
"""Reset state left behind by a previous crash so nothing is stuck."""
|
||||
with SessionLocal() as db:
|
||||
|
|
@ -573,6 +593,9 @@ def main() -> None:
|
|||
log.info("WORKER_ENABLED is off; worker exiting (nothing to do).")
|
||||
return
|
||||
|
||||
_wait_for_schema() # the API may still be applying migrations when we boot
|
||||
if _stop.is_set():
|
||||
return
|
||||
_recover_orphans()
|
||||
n = max(1, settings.download_worker_concurrency)
|
||||
log.info("Download worker started (concurrency=%d, root=%s).", n, settings.download_root)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue