misc: added health and ready probe endpoints

backend/src/main.ts

@@ -16,7 +16,7 @@ import { buildRedisFromConfig } from "./lib/config/redis";
 import { removeTemporaryBaseDirectory } from "./lib/files";
 import { initLogger } from "./lib/logger";
 import { queueServiceFactory } from "./queue";
-import { main } from "./server/app";
+import { main, markServerReady } from "./server/app";
 import { bootstrapCheck } from "./server/boot-strap-check";
 import { kmsRootConfigDALFactory } from "./services/kms/kms-root-config-dal";
 import { smtpServiceFactory } from "./services/smtp/smtp-service";
@@ -59,8 +59,6 @@ const run = async () => {
       })
     : undefined;
 
-  await runMigrations({ applicationDb: db, auditLogDb, logger });
-
   const smtp = smtpServiceFactory(formatSmtpConfig());
 
   const queue = queueServiceFactory(envConfig, {
@@ -87,8 +85,8 @@ const run = async () => {
     redis,
     envConfig
   });
-  const bootstrap = await bootstrapCheck({ db });
 
+  // Setup signal handlers
   // eslint-disable-next-line
   process.on("SIGINT", async () => {
     await server.close();
@@ -119,14 +117,28 @@ const run = async () => {
     });
   }
 
+  // Start listening BEFORE migrations
+  // At this point: /api/health returns 200, /api/ready returns 503
   await server.listen({
     port: envConfig.PORT,
-    host: envConfig.HOST,
-    listenTextResolver: (address) => {
-      void bootstrap();
-      return address;
-    }
+    host: envConfig.HOST
   });
+
+  logger.info(`Server listening on ${envConfig.HOST}:${envConfig.PORT}`);
+  logger.info("Running migrations - health check available, other endpoints blocked...");
+
+  // Run migrations while server is up
+  await runMigrations({ applicationDb: db, auditLogDb, logger });
+
+  logger.info("Migrations complete. Marking server as ready...");
+
+  // Mark server as ready - now all endpoints work
+  markServerReady();
+
+  logger.info("Server is ready to accept traffic");
+
+  const bootstrap = await bootstrapCheck({ db });
+  void bootstrap();
 };
 
 void run();

backend/src/server/app.ts

@@ -1,5 +1,6 @@
 /* eslint-disable import/extensions */
 import path from "node:path";
+import { monitorEventLoopDelay } from "perf_hooks";
 
 import type { FastifyCookieOptions } from "@fastify/cookie";
 import cookie from "@fastify/cookie";
@@ -24,6 +25,7 @@ import { TQueueServiceFactory } from "@app/queue";
 import { TKmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal";
 import { TSmtpService } from "@app/services/smtp/smtp-service";
 import { TSuperAdminDALFactory } from "@app/services/super-admin/super-admin-dal";
+import { getServerCfg } from "@app/services/super-admin/super-admin-service";
 
 import { globalRateLimiterCfg } from "./config/rateLimiter";
 import { addErrorsToResponseSchemas } from "./plugins/add-errors-to-response-schemas";
@@ -36,6 +38,15 @@ import { registerServeUI } from "./plugins/serve-ui";
 import { fastifySwagger } from "./plugins/swagger";
 import { registerRoutes } from "./routes";
 
+// Monitor event loop for readiness checks
+const histogram = monitorEventLoopDelay({ resolution: 20 });
+histogram.enable();
+
+// Readiness state
+const readinessState = {
+  isReady: false
+};
+
 type TMain = {
   auditLogDb?: Knex;
   db: Knex;
@@ -145,6 +156,66 @@ export const main = async ({
       })
     });
 
+    // Health check - always returns 200 when server is running
+    server.get("/api/health", async () => {
+      return { status: "ok", message: "Server is alive" };
+    });
+
+    // Global preHandler to block requests during migrations
+    // Excludes /api/health and /api/ready endpoints
+    server.addHook("preHandler", async (request, reply) => {
+      if (request.url === "/api/health" || request.url === "/api/ready") {
+        return;
+      }
+      if (!readinessState.isReady) {
+        return reply.code(503).send({
+          status: "unavailable",
+          message: "Server is starting up, migrations in progress. Please try again in a moment."
+        });
+      }
+    });
+
+    // Readiness check - returns 503 until migrations are complete
+    server.get("/api/ready", async (request, reply) => {
+      const cfg = getConfig();
+
+      // Calculate event loop statistics
+      const meanLagMs = histogram.mean / 1e6;
+      const maxLagMs = histogram.max / 1e6;
+      const p99LagMs = histogram.percentile(99) / 1e6;
+
+      request.log.info(
+        `Event loop stats - Mean: ${meanLagMs.toFixed(2)}ms, Max: ${maxLagMs.toFixed(2)}ms, p99: ${p99LagMs.toFixed(2)}ms`
+      );
+
+      request.log.info(`Raw event loop stats: ${JSON.stringify(histogram, null, 2)}`);
+
+      if (!readinessState.isReady) {
+        return reply.code(503).send({
+          date: new Date(),
+          message: "Server is starting up, migrations in progress",
+          emailConfigured: cfg.isSmtpConfigured,
+          redisConfigured: cfg.isRedisConfigured,
+          secretScanningConfigured: cfg.isSecretScanningConfigured,
+          samlDefaultOrgSlug: cfg.samlDefaultOrgSlug,
+          auditLogStorageDisabled: Boolean(cfg.DISABLE_AUDIT_LOG_STORAGE)
+        });
+      }
+
+      const serverCfg = await getServerCfg();
+
+      return {
+        date: new Date(),
+        message: "Ok",
+        emailConfigured: cfg.isSmtpConfigured,
+        inviteOnlySignup: Boolean(serverCfg.allowSignUp),
+        redisConfigured: cfg.isRedisConfigured,
+        secretScanningConfigured: cfg.isSecretScanningConfigured,
+        samlDefaultOrgSlug: cfg.samlDefaultOrgSlug,
+        auditLogStorageDisabled: Boolean(cfg.DISABLE_AUDIT_LOG_STORAGE)
+      };
+    });
+
     await server.register(registerRoutes, {
       smtp,
       queue,
@@ -171,3 +242,8 @@ export const main = async ({
     process.exit(1);
   }
 };
+
+// Function to mark server as ready after migrations
+export const markServerReady = () => {
+  readinessState.isReady = true;
+};