From b9c824559c03e66900d4410469afa23537376120 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 28 Oct 2025 04:57:25 +0800 Subject: [PATCH] misc: added health and ready probe endpoints --- backend/src/main.ts | 30 +++++++++++----- backend/src/server/app.ts | 76 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 9 deletions(-) diff --git a/backend/src/main.ts b/backend/src/main.ts index 400804804..3936d1c47 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -16,7 +16,7 @@ import { buildRedisFromConfig } from "./lib/config/redis"; import { removeTemporaryBaseDirectory } from "./lib/files"; import { initLogger } from "./lib/logger"; import { queueServiceFactory } from "./queue"; -import { main } from "./server/app"; +import { main, markServerReady } from "./server/app"; import { bootstrapCheck } from "./server/boot-strap-check"; import { kmsRootConfigDALFactory } from "./services/kms/kms-root-config-dal"; import { smtpServiceFactory } from "./services/smtp/smtp-service"; @@ -59,8 +59,6 @@ const run = async () => { }) : undefined; - await runMigrations({ applicationDb: db, auditLogDb, logger }); - const smtp = smtpServiceFactory(formatSmtpConfig()); const queue = queueServiceFactory(envConfig, { @@ -87,8 +85,8 @@ const run = async () => { redis, envConfig }); - const bootstrap = await bootstrapCheck({ db }); + // Setup signal handlers // eslint-disable-next-line process.on("SIGINT", async () => { await server.close(); @@ -119,14 +117,28 @@ const run = async () => { }); } + // Start listening BEFORE migrations + // At this point: /api/health returns 200, /api/ready returns 503 await server.listen({ port: envConfig.PORT, - host: envConfig.HOST, - listenTextResolver: (address) => { - void bootstrap(); - return address; - } + host: envConfig.HOST }); + + logger.info(`Server listening on ${envConfig.HOST}:${envConfig.PORT}`); + logger.info("Running migrations - health check available, other endpoints blocked..."); + + // Run migrations while server is up + await runMigrations({ applicationDb: db, auditLogDb, logger }); + + logger.info("Migrations complete. Marking server as ready..."); + + // Mark server as ready - now all endpoints work + markServerReady(); + + logger.info("Server is ready to accept traffic"); + + const bootstrap = await bootstrapCheck({ db }); + void bootstrap(); }; void run(); diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index f1176b932..eadda27cd 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -1,5 +1,6 @@ /* eslint-disable import/extensions */ import path from "node:path"; +import { monitorEventLoopDelay } from "perf_hooks"; import type { FastifyCookieOptions } from "@fastify/cookie"; import cookie from "@fastify/cookie"; @@ -24,6 +25,7 @@ import { TQueueServiceFactory } from "@app/queue"; import { TKmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal"; import { TSmtpService } from "@app/services/smtp/smtp-service"; import { TSuperAdminDALFactory } from "@app/services/super-admin/super-admin-dal"; +import { getServerCfg } from "@app/services/super-admin/super-admin-service"; import { globalRateLimiterCfg } from "./config/rateLimiter"; import { addErrorsToResponseSchemas } from "./plugins/add-errors-to-response-schemas"; @@ -36,6 +38,15 @@ import { registerServeUI } from "./plugins/serve-ui"; import { fastifySwagger } from "./plugins/swagger"; import { registerRoutes } from "./routes"; +// Monitor event loop for readiness checks +const histogram = monitorEventLoopDelay({ resolution: 20 }); +histogram.enable(); + +// Readiness state +const readinessState = { + isReady: false +}; + type TMain = { auditLogDb?: Knex; db: Knex; @@ -145,6 +156,66 @@ export const main = async ({ }) }); + // Health check - always returns 200 when server is running + server.get("/api/health", async () => { + return { status: "ok", message: "Server is alive" }; + }); + + // Global preHandler to block requests during migrations + // Excludes /api/health and /api/ready endpoints + server.addHook("preHandler", async (request, reply) => { + if (request.url === "/api/health" || request.url === "/api/ready") { + return; + } + if (!readinessState.isReady) { + return reply.code(503).send({ + status: "unavailable", + message: "Server is starting up, migrations in progress. Please try again in a moment." + }); + } + }); + + // Readiness check - returns 503 until migrations are complete + server.get("/api/ready", async (request, reply) => { + const cfg = getConfig(); + + // Calculate event loop statistics + const meanLagMs = histogram.mean / 1e6; + const maxLagMs = histogram.max / 1e6; + const p99LagMs = histogram.percentile(99) / 1e6; + + request.log.info( + `Event loop stats - Mean: ${meanLagMs.toFixed(2)}ms, Max: ${maxLagMs.toFixed(2)}ms, p99: ${p99LagMs.toFixed(2)}ms` + ); + + request.log.info(`Raw event loop stats: ${JSON.stringify(histogram, null, 2)}`); + + if (!readinessState.isReady) { + return reply.code(503).send({ + date: new Date(), + message: "Server is starting up, migrations in progress", + emailConfigured: cfg.isSmtpConfigured, + redisConfigured: cfg.isRedisConfigured, + secretScanningConfigured: cfg.isSecretScanningConfigured, + samlDefaultOrgSlug: cfg.samlDefaultOrgSlug, + auditLogStorageDisabled: Boolean(cfg.DISABLE_AUDIT_LOG_STORAGE) + }); + } + + const serverCfg = await getServerCfg(); + + return { + date: new Date(), + message: "Ok", + emailConfigured: cfg.isSmtpConfigured, + inviteOnlySignup: Boolean(serverCfg.allowSignUp), + redisConfigured: cfg.isRedisConfigured, + secretScanningConfigured: cfg.isSecretScanningConfigured, + samlDefaultOrgSlug: cfg.samlDefaultOrgSlug, + auditLogStorageDisabled: Boolean(cfg.DISABLE_AUDIT_LOG_STORAGE) + }; + }); + await server.register(registerRoutes, { smtp, queue, @@ -171,3 +242,8 @@ export const main = async ({ process.exit(1); } }; + +// Function to mark server as ready after migrations +export const markServerReady = () => { + readinessState.isReady = true; +};