misc: properly handled multi-container setups

backend/e2e-test/vitest-environment-knex.ts

@@ -8,7 +8,7 @@ import path from "path";
 import { seedData1 } from "@app/db/seed-data";
 import { getDatabaseCredentials, getHsmConfig, initEnvConfig } from "@app/lib/config/env";
 import { initLogger } from "@app/lib/logger";
-import { main } from "@app/server/app";
+import { main, markServerReady } from "@app/server/app";
 import { AuthMethod, AuthTokenType } from "@app/services/auth/auth-type";
 
 import { mockSmtpServer } from "./mocks/smtp";
@@ -96,6 +96,8 @@ export default {
         envConfig: envCfg
       });
 
+      markServerReady();
+
       await bootstrapCheck({ db });
 
       // @ts-expect-error type

backend/src/auto-start-migrations.ts

@@ -12,6 +12,7 @@ type TArgs = {
   auditLogDb?: Knex;
   applicationDb: Knex;
   logger: Logger;
+  onMigrationLockAcquired?: () => void;
 };
 
 const isProduction = process.env.NODE_ENV === "production";
@@ -30,7 +31,7 @@ const migrationStatusCheckErrorHandler = (err: Error) => {
   throw err;
 };
 
-export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs) => {
+export const runMigrations = async ({ applicationDb, auditLogDb, logger, onMigrationLockAcquired }: TArgs) => {
   try {
     // akhilmhdh(Feb 10 2025): 2 years  from now remove this
     if (isProduction) {
@@ -85,6 +86,12 @@ export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs
 
     await applicationDb.transaction(async (tx) => {
       await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]);
+
+      // Signal that this container is running migrations
+      if (onMigrationLockAcquired) {
+        onMigrationLockAcquired();
+      }
+
       logger.info("Running application migrations.");
 
       const didPreviousInstanceRunMigration = !(await applicationDb.migrate

backend/src/main.ts

@@ -16,7 +16,7 @@ import { buildRedisFromConfig } from "./lib/config/redis";
 import { removeTemporaryBaseDirectory } from "./lib/files";
 import { initLogger } from "./lib/logger";
 import { queueServiceFactory } from "./queue";
-import { main, markServerReady } from "./server/app";
+import { main, markRunningMigrations, markServerReady } from "./server/app";
 import { bootstrapCheck } from "./server/boot-strap-check";
 import { kmsRootConfigDALFactory } from "./services/kms/kms-root-config-dal";
 import { smtpServiceFactory } from "./services/smtp/smtp-service";
@@ -118,25 +118,36 @@ const run = async () => {
   }
 
   // Start listening BEFORE migrations
-  // At this point: /api/health returns 200, /api/ready returns 503
   await server.listen({
     port: envConfig.PORT,
     host: envConfig.HOST
   });
 
   logger.info(`Server listening on ${envConfig.HOST}:${envConfig.PORT}`);
-  logger.info("Running migrations - health check available, other endpoints blocked...");
+  logger.info("Running migrations...");
 
   // Run migrations while server is up
-  await runMigrations({ applicationDb: db, auditLogDb, logger });
+  // All containers start as NOT HEALTHY (waiting for migrations)
+  // Container that acquires lock: becomes HEALTHY (running migrations) + NOT READY (no traffic)
+  // Other containers waiting: stay NOT HEALTHY (waiting) + NOT READY (no traffic)
+  await runMigrations({
+    applicationDb: db,
+    auditLogDb,
+    logger,
+    onMigrationLockAcquired: () => {
+      // Called after successfully acquiring the lock
+      // This container is now the migration runner
+      markRunningMigrations();
+      logger.info("Migration lock acquired! This container is running migrations.");
+    }
+  });
 
-  logger.info("Migrations complete. Marking server as ready...");
+  logger.info("Migrations complete. Marking server as READY...");
 
-  // Mark server as ready - now all endpoints work
+  // Now mark server as ready - it can accept traffic
   markServerReady();
 
   logger.info("Server is ready to accept traffic");
-
   const bootstrap = await bootstrapCheck({ db });
   void bootstrap();
 };

backend/src/server/app.ts

@@ -42,9 +42,11 @@ import { registerRoutes } from "./routes";
 const histogram = monitorEventLoopDelay({ resolution: 20 });
 histogram.enable();
 
-// Readiness state
+// Server state tracking
-const readinessState = {
+const serverState = {
-  isReady: false
+  isReady: false,
+  isRunningMigrations: false,
+  isWaitingForMigrations: true // Start as true - containers are unhealthy until they acquire migration lock or complete
 };
 
 type TMain = {
@@ -156,8 +158,15 @@ export const main = async ({
       })
     });
 
-    // Health check - always returns 200 when server is running
+    // Health check - returns 200 only if doing useful work (running migrations or ready)
-    server.get("/api/health", async () => {
+    // Returns 503 if waiting for another container to finish migrations
+    server.get("/api/health", async (_, reply) => {
+      if (serverState.isWaitingForMigrations) {
+        return reply.code(503).send({
+          status: "waiting",
+          message: "Waiting for migrations to complete in another container"
+        });
+      }
       return { status: "ok", message: "Server is alive" };
     });
 
@@ -167,7 +176,7 @@ export const main = async ({
       if (request.url === "/api/health" || request.url === "/api/ready") {
         return;
       }
-      if (!readinessState.isReady) {
+      if (!serverState.isReady) {
         return reply.code(503).send({
           status: "unavailable",
           message: "Server is starting up, migrations in progress. Please try again in a moment."
@@ -190,7 +199,7 @@ export const main = async ({
 
       request.log.info(`Raw event loop stats: ${JSON.stringify(histogram, null, 2)}`);
 
-      if (!readinessState.isReady) {
+      if (!serverState.isReady) {
         return reply.code(503).send({
           date: new Date(),
           message: "Server is starting up, migrations in progress",
@@ -243,7 +252,14 @@ export const main = async ({
   }
 };
 
-// Function to mark server as ready after migrations
+// Functions to manage server state
 export const markServerReady = () => {
-  readinessState.isReady = true;
+  serverState.isReady = true;
+  serverState.isRunningMigrations = false;
+  serverState.isWaitingForMigrations = false;
+};
+
+export const markRunningMigrations = () => {
+  serverState.isRunningMigrations = true;
+  serverState.isWaitingForMigrations = false;
 };