From 8214e07abd531af01c4335005398c0b52d6acbea Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 28 Oct 2025 05:52:58 +0800 Subject: [PATCH] misc: properly handled multi-container setups --- backend/e2e-test/vitest-environment-knex.ts | 4 ++- backend/src/auto-start-migrations.ts | 9 +++++- backend/src/main.ts | 25 ++++++++++----- backend/src/server/app.ts | 34 +++++++++++++++------ 4 files changed, 54 insertions(+), 18 deletions(-) diff --git a/backend/e2e-test/vitest-environment-knex.ts b/backend/e2e-test/vitest-environment-knex.ts index 0f84dbee2..f33d32f6a 100644 --- a/backend/e2e-test/vitest-environment-knex.ts +++ b/backend/e2e-test/vitest-environment-knex.ts @@ -8,7 +8,7 @@ import path from "path"; import { seedData1 } from "@app/db/seed-data"; import { getDatabaseCredentials, getHsmConfig, initEnvConfig } from "@app/lib/config/env"; import { initLogger } from "@app/lib/logger"; -import { main } from "@app/server/app"; +import { main, markServerReady } from "@app/server/app"; import { AuthMethod, AuthTokenType } from "@app/services/auth/auth-type"; import { mockSmtpServer } from "./mocks/smtp"; @@ -96,6 +96,8 @@ export default { envConfig: envCfg }); + markServerReady(); + await bootstrapCheck({ db }); // @ts-expect-error type diff --git a/backend/src/auto-start-migrations.ts b/backend/src/auto-start-migrations.ts index 88f6dea69..38c30366d 100644 --- a/backend/src/auto-start-migrations.ts +++ b/backend/src/auto-start-migrations.ts @@ -12,6 +12,7 @@ type TArgs = { auditLogDb?: Knex; applicationDb: Knex; logger: Logger; + onMigrationLockAcquired?: () => void; }; const isProduction = process.env.NODE_ENV === "production"; @@ -30,7 +31,7 @@ const migrationStatusCheckErrorHandler = (err: Error) => { throw err; }; -export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs) => { +export const runMigrations = async ({ applicationDb, auditLogDb, logger, onMigrationLockAcquired }: TArgs) => { try { // akhilmhdh(Feb 10 2025): 2 years from now remove this if (isProduction) { @@ -85,6 +86,12 @@ export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs await applicationDb.transaction(async (tx) => { await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]); + + // Signal that this container is running migrations + if (onMigrationLockAcquired) { + onMigrationLockAcquired(); + } + logger.info("Running application migrations."); const didPreviousInstanceRunMigration = !(await applicationDb.migrate diff --git a/backend/src/main.ts b/backend/src/main.ts index 3936d1c47..d680fc006 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -16,7 +16,7 @@ import { buildRedisFromConfig } from "./lib/config/redis"; import { removeTemporaryBaseDirectory } from "./lib/files"; import { initLogger } from "./lib/logger"; import { queueServiceFactory } from "./queue"; -import { main, markServerReady } from "./server/app"; +import { main, markRunningMigrations, markServerReady } from "./server/app"; import { bootstrapCheck } from "./server/boot-strap-check"; import { kmsRootConfigDALFactory } from "./services/kms/kms-root-config-dal"; import { smtpServiceFactory } from "./services/smtp/smtp-service"; @@ -118,25 +118,36 @@ const run = async () => { } // Start listening BEFORE migrations - // At this point: /api/health returns 200, /api/ready returns 503 await server.listen({ port: envConfig.PORT, host: envConfig.HOST }); logger.info(`Server listening on ${envConfig.HOST}:${envConfig.PORT}`); - logger.info("Running migrations - health check available, other endpoints blocked..."); + logger.info("Running migrations..."); // Run migrations while server is up - await runMigrations({ applicationDb: db, auditLogDb, logger }); + // All containers start as NOT HEALTHY (waiting for migrations) + // Container that acquires lock: becomes HEALTHY (running migrations) + NOT READY (no traffic) + // Other containers waiting: stay NOT HEALTHY (waiting) + NOT READY (no traffic) + await runMigrations({ + applicationDb: db, + auditLogDb, + logger, + onMigrationLockAcquired: () => { + // Called after successfully acquiring the lock + // This container is now the migration runner + markRunningMigrations(); + logger.info("Migration lock acquired! This container is running migrations."); + } + }); - logger.info("Migrations complete. Marking server as ready..."); + logger.info("Migrations complete. Marking server as READY..."); - // Mark server as ready - now all endpoints work + // Now mark server as ready - it can accept traffic markServerReady(); logger.info("Server is ready to accept traffic"); - const bootstrap = await bootstrapCheck({ db }); void bootstrap(); }; diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index eadda27cd..2de20ab46 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -42,9 +42,11 @@ import { registerRoutes } from "./routes"; const histogram = monitorEventLoopDelay({ resolution: 20 }); histogram.enable(); -// Readiness state -const readinessState = { - isReady: false +// Server state tracking +const serverState = { + isReady: false, + isRunningMigrations: false, + isWaitingForMigrations: true // Start as true - containers are unhealthy until they acquire migration lock or complete }; type TMain = { @@ -156,8 +158,15 @@ export const main = async ({ }) }); - // Health check - always returns 200 when server is running - server.get("/api/health", async () => { + // Health check - returns 200 only if doing useful work (running migrations or ready) + // Returns 503 if waiting for another container to finish migrations + server.get("/api/health", async (_, reply) => { + if (serverState.isWaitingForMigrations) { + return reply.code(503).send({ + status: "waiting", + message: "Waiting for migrations to complete in another container" + }); + } return { status: "ok", message: "Server is alive" }; }); @@ -167,7 +176,7 @@ export const main = async ({ if (request.url === "/api/health" || request.url === "/api/ready") { return; } - if (!readinessState.isReady) { + if (!serverState.isReady) { return reply.code(503).send({ status: "unavailable", message: "Server is starting up, migrations in progress. Please try again in a moment." @@ -190,7 +199,7 @@ export const main = async ({ request.log.info(`Raw event loop stats: ${JSON.stringify(histogram, null, 2)}`); - if (!readinessState.isReady) { + if (!serverState.isReady) { return reply.code(503).send({ date: new Date(), message: "Server is starting up, migrations in progress", @@ -243,7 +252,14 @@ export const main = async ({ } }; -// Function to mark server as ready after migrations +// Functions to manage server state export const markServerReady = () => { - readinessState.isReady = true; + serverState.isReady = true; + serverState.isRunningMigrations = false; + serverState.isWaitingForMigrations = false; +}; + +export const markRunningMigrations = () => { + serverState.isRunningMigrations = true; + serverState.isWaitingForMigrations = false; };