Merge pull request #4760 from Infisical/misc/added-health-and-ready-probe-endpoints

misc: added health and ready probe endpoints

backend/e2e-test/vitest-environment-knex.ts

@@ -8,7 +8,7 @@ import path from "path";
 import { seedData1 } from "@app/db/seed-data";
 import { getDatabaseCredentials, getHsmConfig, initEnvConfig } from "@app/lib/config/env";
 import { initLogger } from "@app/lib/logger";
-import { main } from "@app/server/app";
+import { main, markServerReady } from "@app/server/app";
 import { AuthMethod, AuthTokenType } from "@app/services/auth/auth-type";
 
 import { mockSmtpServer } from "./mocks/smtp";
@@ -83,7 +83,7 @@ export default {
 
       await queue.initialize();
 
-      const server = await main({
+      const { server, completeServerInitialization } = await main({
         db,
         smtp,
         logger,
@@ -96,6 +96,10 @@ export default {
         envConfig: envCfg
       });
 
+      await completeServerInitialization();
+
+      markServerReady();
+
       await bootstrapCheck({ db });
 
       // @ts-expect-error type

backend/src/auto-start-migrations.ts

@@ -12,6 +12,7 @@ type TArgs = {
   auditLogDb?: Knex;
   applicationDb: Knex;
   logger: Logger;
+  onMigrationLockAcquired?: () => void;
 };
 
 const isProduction = process.env.NODE_ENV === "production";
@@ -30,7 +31,7 @@ const migrationStatusCheckErrorHandler = (err: Error) => {
   throw err;
 };
 
-export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs) => {
+export const runMigrations = async ({ applicationDb, auditLogDb, logger, onMigrationLockAcquired }: TArgs) => {
   try {
     // akhilmhdh(Feb 10 2025): 2 years  from now remove this
     if (isProduction) {
@@ -85,6 +86,13 @@ export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs
 
     await applicationDb.transaction(async (tx) => {
       await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]);
+
+      // Signal that this container is running migrations so that it can be marked as healthy/alive
+      // This is to prevent the container from being killed by the orchestrator
+      if (onMigrationLockAcquired) {
+        onMigrationLockAcquired();
+      }
+
       logger.info("Running application migrations.");
 
       const didPreviousInstanceRunMigration = !(await applicationDb.migrate

backend/src/ee/services/hsm/hsm-fns.ts

@@ -84,7 +84,7 @@ export const isHsmActiveAndEnabled = async ({
 
   rootKmsConfigEncryptionStrategy = (rootKmsConfig?.encryptionStrategy || null) as RootKeyEncryptionStrategy | null;
   if (
-    rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.HSM &&
+    (rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.HSM || isHsmConfigured) &&
     licenseService &&
     !licenseService.onPremFeatures.hsm
   ) {

backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts

@@ -42,168 +42,172 @@ export const secretRotationV2QueueServiceFactory = async ({
   smtpService,
   notificationService
 }: TSecretRotationV2QueueServiceFactoryDep) => {
-  const appCfg = getConfig();
+  const init = async () => {
+    const appCfg = getConfig();
 
-  if (appCfg.isRotationDevelopmentMode) {
+    if (appCfg.isRotationDevelopmentMode) {
-    logger.warn("Secret Rotation V2 is in development mode.");
+      logger.warn("Secret Rotation V2 is in development mode.");
-  }
+    }
 
-  await queueService.startPg<QueueName.SecretRotationV2>(
+    await queueService.startPg<QueueName.SecretRotationV2>(
-    QueueJobs.SecretRotationV2QueueRotations,
+      QueueJobs.SecretRotationV2QueueRotations,
-    async () => {
+      async () => {
-      try {
+        try {
-        const rotateBy = getNextUtcRotationInterval();
+          const rotateBy = getNextUtcRotationInterval();
 
-        const currentTime = new Date();
+          const currentTime = new Date();
 
-        const secretRotations = await secretRotationV2DAL.findSecretRotationsToQueue(rotateBy);
+          const secretRotations = await secretRotationV2DAL.findSecretRotationsToQueue(rotateBy);
 
-        logger.info(
-          `secretRotationV2Queue: Queue Rotations [currentTime=${currentTime.toISOString()}] [rotateBy=${rotateBy.toISOString()}] [count=${
-            secretRotations.length
-          }]`
-        );
-
-        for await (const rotation of secretRotations) {
           logger.info(
-            `secretRotationV2Queue: Queue Rotation [rotationId=${rotation.id}] [lastRotatedAt=${new Date(
+            `secretRotationV2Queue: Queue Rotations [currentTime=${currentTime.toISOString()}] [rotateBy=${rotateBy.toISOString()}] [count=${
-              rotation.lastRotatedAt
+              secretRotations.length
-            ).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]`
+            }]`
           );
 
-          const data = {
+          for await (const rotation of secretRotations) {
-            rotationId: rotation.id,
+            logger.info(
-            queuedAt: currentTime
+              `secretRotationV2Queue: Queue Rotation [rotationId=${rotation.id}] [lastRotatedAt=${new Date(
-          } as TSecretRotationRotateSecretsJobPayload;
+                rotation.lastRotatedAt
-
+              ).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]`
-          if (appCfg.isTestMode) {
-            logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode");
-            await rotateSecretsFns({
-              job: {
-                id: uuidv4(),
-                data,
-                retryCount: 0,
-                retryLimit: 0
-              },
-              secretRotationV2DAL,
-              secretRotationV2Service
-            });
-          } else {
-            await queueService.queuePg(
-              QueueJobs.SecretRotationV2RotateSecrets,
-              {
-                rotationId: rotation.id,
-                queuedAt: currentTime
-              },
-              getSecretRotationRotateSecretJobOptions(rotation)
             );
+
+            const data = {
+              rotationId: rotation.id,
+              queuedAt: currentTime
+            } as TSecretRotationRotateSecretsJobPayload;
+
+            if (appCfg.isTestMode) {
+              logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode");
+              await rotateSecretsFns({
+                job: {
+                  id: uuidv4(),
+                  data,
+                  retryCount: 0,
+                  retryLimit: 0
+                },
+                secretRotationV2DAL,
+                secretRotationV2Service
+              });
+            } else {
+              await queueService.queuePg(
+                QueueJobs.SecretRotationV2RotateSecrets,
+                {
+                  rotationId: rotation.id,
+                  queuedAt: currentTime
+                },
+                getSecretRotationRotateSecretJobOptions(rotation)
+              );
+            }
           }
+        } catch (error) {
+          logger.error(error, "secretRotationV2Queue: Queue Rotations Error:");
+          throw error;
         }
-      } catch (error) {
+      },
-        logger.error(error, "secretRotationV2Queue: Queue Rotations Error:");
+      {
-        throw error;
+        batchSize: 1,
+        workerCount: 1,
+        pollingIntervalSeconds: appCfg.isRotationDevelopmentMode ? 0.5 : 30
       }
-    },
+    );
-    {
-      batchSize: 1,
-      workerCount: 1,
-      pollingIntervalSeconds: appCfg.isRotationDevelopmentMode ? 0.5 : 30
-    }
-  );
 
-  await queueService.startPg<QueueName.SecretRotationV2>(
+    await queueService.startPg<QueueName.SecretRotationV2>(
-    QueueJobs.SecretRotationV2RotateSecrets,
+      QueueJobs.SecretRotationV2RotateSecrets,
-    async ([job]) => {
+      async ([job]) => {
-      await rotateSecretsFns({
+        await rotateSecretsFns({
-        job: {
+          job: {
-          ...job,
+            ...job,
-          data: job.data as TSecretRotationRotateSecretsJobPayload
+            data: job.data as TSecretRotationRotateSecretsJobPayload
-        },
+          },
-        secretRotationV2DAL,
+          secretRotationV2DAL,
-        secretRotationV2Service
+          secretRotationV2Service
-      });
-    },
-    {
-      batchSize: 1,
-      workerCount: 2,
-      pollingIntervalSeconds: 0.5
-    }
-  );
-
-  await queueService.startPg<QueueName.SecretRotationV2>(
-    QueueJobs.SecretRotationV2SendNotification,
-    async ([job]) => {
-      const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload;
-      try {
-        const {
-          name: rotationName,
-          type,
-          projectId,
-          lastRotationAttemptedAt,
-          folder,
-          environment,
-          id: rotationId
-        } = secretRotation;
-
-        logger.info(`secretRotationV2Queue: Sending Status Notification [rotationId=${rotationId}]`);
-
-        const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
-        const project = await projectDAL.findById(projectId);
-
-        const projectAdmins = projectMembers.filter((member) =>
-          member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
-        );
-
-        const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation];
-
-        const rotationPath = `/projects/secret-management/${projectId}/secrets/${environment.slug}`;
-
-        await notificationService.createUserNotifications(
-          projectAdmins.map((admin) => ({
-            userId: admin.userId,
-            orgId: project.orgId,
-            type: NotificationType.SECRET_ROTATION_FAILED,
-            title: "Secret Rotation Failed",
-            body: `Your **${rotationType}** rotation **${rotationName}** failed to rotate.`,
-            link: rotationPath
-          }))
-        );
-
-        await smtpService.sendMail({
-          recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
-          template: SmtpTemplates.SecretRotationFailed,
-          subjectLine: `Secret Rotation Failed`,
-          substitutions: {
-            rotationName,
-            rotationType,
-            content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date(
-              lastRotationAttemptedAt
-            ).toISOString()}. Please check the rotation status in Infisical for more details.`,
-            secretPath: folder.path,
-            environment: environment.name,
-            projectName: project.name,
-            rotationUrl: encodeURI(`${appCfg.SITE_URL}${rotationPath}`)
-          }
         });
-      } catch (error) {
+      },
-        logger.error(
+      {
-          error,
+        batchSize: 1,
-          `secretRotationV2Queue: Failed to Send Status Notification [rotationId=${secretRotation.id}]`
+        workerCount: 2,
-        );
+        pollingIntervalSeconds: 0.5
-        throw error;
       }
-    },
+    );
-    {
-      batchSize: 1,
-      workerCount: 2,
-      pollingIntervalSeconds: 1
-    }
-  );
 
-  await queueService.schedulePg(
+    await queueService.startPg<QueueName.SecretRotationV2>(
-    QueueJobs.SecretRotationV2QueueRotations,
+      QueueJobs.SecretRotationV2SendNotification,
-    appCfg.isRotationDevelopmentMode ? "* * * * *" : "0 0 * * *",
+      async ([job]) => {
-    undefined,
+        const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload;
-    { tz: "UTC" }
+        try {
-  );
+          const {
+            name: rotationName,
+            type,
+            projectId,
+            lastRotationAttemptedAt,
+            folder,
+            environment,
+            id: rotationId
+          } = secretRotation;
+
+          logger.info(`secretRotationV2Queue: Sending Status Notification [rotationId=${rotationId}]`);
+
+          const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
+          const project = await projectDAL.findById(projectId);
+
+          const projectAdmins = projectMembers.filter((member) =>
+            member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
+          );
+
+          const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation];
+
+          const rotationPath = `/projects/secret-management/${projectId}/secrets/${environment.slug}`;
+
+          await notificationService.createUserNotifications(
+            projectAdmins.map((admin) => ({
+              userId: admin.userId,
+              orgId: project.orgId,
+              type: NotificationType.SECRET_ROTATION_FAILED,
+              title: "Secret Rotation Failed",
+              body: `Your **${rotationType}** rotation **${rotationName}** failed to rotate.`,
+              link: rotationPath
+            }))
+          );
+
+          await smtpService.sendMail({
+            recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
+            template: SmtpTemplates.SecretRotationFailed,
+            subjectLine: `Secret Rotation Failed`,
+            substitutions: {
+              rotationName,
+              rotationType,
+              content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date(
+                lastRotationAttemptedAt
+              ).toISOString()}. Please check the rotation status in Infisical for more details.`,
+              secretPath: folder.path,
+              environment: environment.name,
+              projectName: project.name,
+              rotationUrl: encodeURI(`${appCfg.SITE_URL}${rotationPath}`)
+            }
+          });
+        } catch (error) {
+          logger.error(
+            error,
+            `secretRotationV2Queue: Failed to Send Status Notification [rotationId=${secretRotation.id}]`
+          );
+          throw error;
+        }
+      },
+      {
+        batchSize: 1,
+        workerCount: 2,
+        pollingIntervalSeconds: 1
+      }
+    );
+
+    await queueService.schedulePg(
+      QueueJobs.SecretRotationV2QueueRotations,
+      appCfg.isRotationDevelopmentMode ? "* * * * *" : "0 0 * * *",
+      undefined,
+      { tz: "UTC" }
+    );
+  };
+
+  return { init };
 };

backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts

@@ -141,202 +141,6 @@ export const secretScanningV2QueueServiceFactory = async ({
     }
   };
 
-  await queueService.startPg<QueueName.SecretScanningV2>(
-    QueueJobs.SecretScanningV2FullScan,
-    async ([job]) => {
-      const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan;
-      const { retryCount, retryLimit } = job;
-
-      const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
-
-      const tempFolder = await createTempFolder();
-
-      const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
-
-      if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
-
-      const resource = await secretScanningV2DAL.resources.findById(resourceId);
-
-      if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
-
-      let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
-
-      try {
-        try {
-          lock = await keyStore.acquireLock(
-            [KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
-            60 * 1000 * 5
-          );
-        } catch (e) {
-          throw new Error("Failed to acquire scanning lock.");
-        }
-
-        await secretScanningV2DAL.scans.update(
-          { id: scanId },
-          {
-            status: SecretScanningScanStatus.Scanning
-          }
-        );
-
-        let connection: TAppConnection | null = null;
-        if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
-
-        const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
-          kmsService,
-          appConnectionDAL
-        });
-
-        const findingsPath = join(tempFolder, "findings.json");
-
-        const scanPath = await factory.getFullScanPath({
-          dataSource: {
-            ...dataSource,
-            connection
-          } as TSecretScanningDataSourceWithConnection,
-          resourceName: resource.name,
-          tempFolder
-        });
-
-        const config = await secretScanningV2DAL.configs.findOne({
-          projectId: dataSource.projectId
-        });
-
-        let configPath: string | undefined;
-
-        if (config && config.content) {
-          configPath = join(tempFolder, "infisical-scan.toml");
-          await writeTextToFile(configPath, config.content);
-        }
-
-        let findingsPayload: TFindingsPayload;
-        switch (resource.type) {
-          case SecretScanningResource.Repository:
-          case SecretScanningResource.Project:
-            findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
-            break;
-          default:
-            throw new Error("Unhandled resource type");
-        }
-
-        const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
-          let findings: TSecretScanningFindings[] = [];
-          if (findingsPayload.length) {
-            findings = await secretScanningV2DAL.findings.upsert(
-              findingsPayload.map((finding) => ({
-                ...finding,
-                projectId: dataSource.projectId,
-                dataSourceName: dataSource.name,
-                dataSourceType: dataSource.type,
-                resourceName: resource.name,
-                resourceType: resource.type,
-                scanId
-              })),
-              ["projectId", "fingerprint"],
-              tx,
-              ["resourceName", "dataSourceName"]
-            );
-          }
-
-          await secretScanningV2DAL.scans.update(
-            { id: scanId },
-            {
-              status: SecretScanningScanStatus.Completed,
-              statusMessage: null
-            }
-          );
-
-          return findings;
-        });
-
-        const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
-
-        if (newFindings.length) {
-          await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
-            status: SecretScanningScanStatus.Completed,
-            resourceName: resource.name,
-            isDiffScan: false,
-            dataSource,
-            numberOfSecrets: newFindings.length,
-            scanId
-          });
-        }
-
-        await auditLogService.createAuditLog({
-          projectId: dataSource.projectId,
-          actor: {
-            type: ActorType.PLATFORM,
-            metadata: {}
-          },
-          event: {
-            type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
-            metadata: {
-              dataSourceId: dataSource.id,
-              dataSourceType: dataSource.type,
-              resourceId: resource.id,
-              resourceType: resource.type,
-              scanId,
-              scanStatus: SecretScanningScanStatus.Completed,
-              scanType: SecretScanningScanType.FullScan,
-              numberOfSecretsDetected: findingsPayload.length
-            }
-          }
-        });
-
-        logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`);
-      } catch (error) {
-        if (retryCount === retryLimit) {
-          const errorMessage = parseScanErrorMessage(error);
-
-          await secretScanningV2DAL.scans.update(
-            { id: scanId },
-            {
-              status: SecretScanningScanStatus.Failed,
-              statusMessage: errorMessage
-            }
-          );
-
-          await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
-            status: SecretScanningScanStatus.Failed,
-            resourceName: resource.name,
-            dataSource,
-            errorMessage
-          });
-
-          await auditLogService.createAuditLog({
-            projectId: dataSource.projectId,
-            actor: {
-              type: ActorType.PLATFORM,
-              metadata: {}
-            },
-            event: {
-              type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
-              metadata: {
-                dataSourceId: dataSource.id,
-                dataSourceType: dataSource.type,
-                resourceId: resource.id,
-                resourceType: resource.type,
-                scanId,
-                scanStatus: SecretScanningScanStatus.Failed,
-                scanType: SecretScanningScanType.FullScan
-              }
-            }
-          });
-        }
-
-        logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
-        throw error;
-      } finally {
-        await deleteTempFolder(tempFolder);
-        await lock?.release();
-      }
-    },
-    {
-      batchSize: 1,
-      workerCount: 2,
-      pollingIntervalSeconds: 1
-    }
-  );
-
   const queueResourceDiffScan = async ({
     payload,
     dataSourceId,
@@ -391,148 +195,127 @@ export const secretScanningV2QueueServiceFactory = async ({
     }
   };
 
-  await queueService.startPg<QueueName.SecretScanningV2>(
+  const init = async () => {
-    QueueJobs.SecretScanningV2DiffScan,
+    await queueService.startPg<QueueName.SecretScanningV2>(
-    async ([job]) => {
+      QueueJobs.SecretScanningV2FullScan,
-      const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan;
+      async ([job]) => {
-      const { retryCount, retryLimit } = job;
+        const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan;
+        const { retryCount, retryLimit } = job;
 
-      const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
+        const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
 
-      const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
+        const tempFolder = await createTempFolder();
 
-      if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
+        const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
 
-      const resource = await secretScanningV2DAL.resources.findById(resourceId);
+        if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
 
-      if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
+        const resource = await secretScanningV2DAL.resources.findById(resourceId);
 
-      const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
+        if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
-        kmsService,
-        appConnectionDAL
-      });
 
-      const tempFolder = await createTempFolder();
+        let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
 
-      try {
+        try {
-        await secretScanningV2DAL.scans.update(
+          try {
-          { id: scanId },
+            lock = await keyStore.acquireLock(
-          {
+              [KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
-            status: SecretScanningScanStatus.Scanning
+              60 * 1000 * 5
-          }
-        );
-
-        let connection: TAppConnection | null = null;
-        if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
-
-        const config = await secretScanningV2DAL.configs.findOne({
-          projectId: dataSource.projectId
-        });
-
-        let configPath: string | undefined;
-
-        if (config && config.content) {
-          configPath = join(tempFolder, "infisical-scan.toml");
-          await writeTextToFile(configPath, config.content);
-        }
-
-        const findingsPayload = await factory.getDiffScanFindingsPayload({
-          dataSource: {
-            ...dataSource,
-            connection
-          } as TSecretScanningDataSourceWithConnection,
-          resourceName: resource.name,
-          payload,
-          configPath
-        });
-
-        const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
-          let findings: TSecretScanningFindings[] = [];
-
-          if (findingsPayload.length) {
-            findings = await secretScanningV2DAL.findings.upsert(
-              findingsPayload.map((finding) => ({
-                ...finding,
-                projectId: dataSource.projectId,
-                dataSourceName: dataSource.name,
-                dataSourceType: dataSource.type,
-                resourceName: resource.name,
-                resourceType: resource.type,
-                scanId
-              })),
-              ["projectId", "fingerprint"],
-              tx,
-              ["resourceName", "dataSourceName"]
             );
+          } catch (e) {
+            throw new Error("Failed to acquire scanning lock.");
           }
 
           await secretScanningV2DAL.scans.update(
             { id: scanId },
             {
-              status: SecretScanningScanStatus.Completed
+              status: SecretScanningScanStatus.Scanning
             }
           );
 
-          return findings;
+          let connection: TAppConnection | null = null;
-        });
+          if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
 
-        const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
+          const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
-
+            kmsService,
-        if (newFindings.length) {
+            appConnectionDAL
-          const finding = newFindings[0] as TSecretScanningFinding;
-          await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
-            status: SecretScanningScanStatus.Completed,
-            resourceName: resource.name,
-            isDiffScan: true,
-            dataSource,
-            numberOfSecrets: newFindings.length,
-            scanId,
-            authorName: finding?.details?.author,
-            authorEmail: finding?.details?.email
           });
-        }
 
-        await auditLogService.createAuditLog({
+          const findingsPath = join(tempFolder, "findings.json");
-          projectId: dataSource.projectId,
+
-          actor: {
+          const scanPath = await factory.getFullScanPath({
-            type: ActorType.PLATFORM,
+            dataSource: {
-            metadata: {}
+              ...dataSource,
-          },
+              connection
-          event: {
+            } as TSecretScanningDataSourceWithConnection,
-            type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
+            resourceName: resource.name,
-            metadata: {
+            tempFolder
-              dataSourceId: dataSource.id,
+          });
-              dataSourceType: dataSource.type,
+
-              resourceId,
+          const config = await secretScanningV2DAL.configs.findOne({
-              resourceType: resource.type,
+            projectId: dataSource.projectId
-              scanId,
+          });
-              scanStatus: SecretScanningScanStatus.Completed,
+
-              scanType: SecretScanningScanType.DiffScan,
+          let configPath: string | undefined;
-              numberOfSecretsDetected: findingsPayload.length
+
-            }
+          if (config && config.content) {
+            configPath = join(tempFolder, "infisical-scan.toml");
+            await writeTextToFile(configPath, config.content);
           }
-        });
 
-        logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`);
+          let findingsPayload: TFindingsPayload;
-      } catch (error) {
+          switch (resource.type) {
-        if (retryCount === retryLimit) {
+            case SecretScanningResource.Repository:
-          const errorMessage = parseScanErrorMessage(error);
+            case SecretScanningResource.Project:
+              findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
+              break;
+            default:
+              throw new Error("Unhandled resource type");
+          }
 
-          await secretScanningV2DAL.scans.update(
+          const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
-            { id: scanId },
+            let findings: TSecretScanningFindings[] = [];
-            {
+            if (findingsPayload.length) {
-              status: SecretScanningScanStatus.Failed,
+              findings = await secretScanningV2DAL.findings.upsert(
-              statusMessage: errorMessage
+                findingsPayload.map((finding) => ({
+                  ...finding,
+                  projectId: dataSource.projectId,
+                  dataSourceName: dataSource.name,
+                  dataSourceType: dataSource.type,
+                  resourceName: resource.name,
+                  resourceType: resource.type,
+                  scanId
+                })),
+                ["projectId", "fingerprint"],
+                tx,
+                ["resourceName", "dataSourceName"]
+              );
             }
-          );
 
-          await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
+            await secretScanningV2DAL.scans.update(
-            status: SecretScanningScanStatus.Failed,
+              { id: scanId },
-            resourceName: resource.name,
+              {
-            dataSource,
+                status: SecretScanningScanStatus.Completed,
-            errorMessage
+                statusMessage: null
+              }
+            );
+
+            return findings;
           });
 
+          const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
+
+          if (newFindings.length) {
+            await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
+              status: SecretScanningScanStatus.Completed,
+              resourceName: resource.name,
+              isDiffScan: false,
+              dataSource,
+              numberOfSecrets: newFindings.length,
+              scanId
+            });
+          }
+
           await auditLogService.createAuditLog({
             projectId: dataSource.projectId,
             actor: {
@@ -547,128 +330,348 @@ export const secretScanningV2QueueServiceFactory = async ({
                 resourceId: resource.id,
                 resourceType: resource.type,
                 scanId,
-                scanStatus: SecretScanningScanStatus.Failed,
+                scanStatus: SecretScanningScanStatus.Completed,
-                scanType: SecretScanningScanType.DiffScan
+                scanType: SecretScanningScanType.FullScan,
+                numberOfSecretsDetected: findingsPayload.length
               }
             }
           });
+
+          logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`);
+        } catch (error) {
+          if (retryCount === retryLimit) {
+            const errorMessage = parseScanErrorMessage(error);
+
+            await secretScanningV2DAL.scans.update(
+              { id: scanId },
+              {
+                status: SecretScanningScanStatus.Failed,
+                statusMessage: errorMessage
+              }
+            );
+
+            await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
+              status: SecretScanningScanStatus.Failed,
+              resourceName: resource.name,
+              dataSource,
+              errorMessage
+            });
+
+            await auditLogService.createAuditLog({
+              projectId: dataSource.projectId,
+              actor: {
+                type: ActorType.PLATFORM,
+                metadata: {}
+              },
+              event: {
+                type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
+                metadata: {
+                  dataSourceId: dataSource.id,
+                  dataSourceType: dataSource.type,
+                  resourceId: resource.id,
+                  resourceType: resource.type,
+                  scanId,
+                  scanStatus: SecretScanningScanStatus.Failed,
+                  scanType: SecretScanningScanType.FullScan
+                }
+              }
+            });
+          }
+
+          logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
+          throw error;
+        } finally {
+          await deleteTempFolder(tempFolder);
+          await lock?.release();
         }
-
+      },
-        logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
+      {
-        throw error;
+        batchSize: 1,
-      } finally {
+        workerCount: 2,
-        await deleteTempFolder(tempFolder);
+        pollingIntervalSeconds: 1
       }
-    },
+    );
-    {
-      batchSize: 1,
-      workerCount: 2,
-      pollingIntervalSeconds: 1
-    }
-  );
 
-  await queueService.startPg<QueueName.SecretScanningV2>(
+    await queueService.startPg<QueueName.SecretScanningV2>(
-    QueueJobs.SecretScanningV2SendNotification,
+      QueueJobs.SecretScanningV2DiffScan,
-    async ([job]) => {
+      async ([job]) => {
-      const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification;
+        const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan;
+        const { retryCount, retryLimit } = job;
 
-      const appCfg = getConfig();
+        const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
 
-      if (!appCfg.isSmtpConfigured) return;
+        const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
 
-      try {
+        if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
-        const { projectId } = dataSource;
 
-        logger.info(
+        const resource = await secretScanningV2DAL.resources.findById(resourceId);
-          `secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
-        );
 
-        const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
+        if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
-        const project = await projectDAL.findById(projectId);
 
-        const recipients = projectMembers.filter((member) => {
+        const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
-          const isAdmin = member.roles.some((role) => role.role === ProjectMembershipRole.Admin);
+          kmsService,
-          const isCompleted = payload.status === SecretScanningScanStatus.Completed;
+          appConnectionDAL
-          // We assume that the committer is one of the project members
-          const isCommitter = isCompleted && payload.authorEmail === member.user.email;
-          return isAdmin || isCommitter;
         });
 
-        const timestamp = new Date().toISOString();
+        const tempFolder = await createTempFolder();
 
-        const subjectLine =
+        try {
-          payload.status === SecretScanningScanStatus.Completed
+          await secretScanningV2DAL.scans.update(
-            ? "Incident Alert: Secret(s) Leaked"
+            { id: scanId },
-            : `Secret Scanning Failed`;
+            {
+              status: SecretScanningScanStatus.Scanning
+            }
+          );
 
-        await notificationService.createUserNotifications(
+          let connection: TAppConnection | null = null;
-          recipients.map((member) => ({
+          if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
-            userId: member.userId,
-            orgId: project.orgId,
-            type:
-              payload.status === SecretScanningScanStatus.Completed
-                ? NotificationType.SECRET_SCANNING_SECRETS_DETECTED
-                : NotificationType.SECRET_SCANNING_SCAN_FAILED,
-            title: subjectLine,
-            body:
-              payload.status === SecretScanningScanStatus.Completed
-                ? `Uncovered **${payload.numberOfSecrets}** secret(s) ${payload.isDiffScan ? " from a recent commit to" : " in"} **${resourceName}**.`
-                : `Encountered an error while attempting to scan the resource **${resourceName}**: ${payload.errorMessage}`,
-            link:
-              payload.status === SecretScanningScanStatus.Completed
-                ? `/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
-                : `/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
-          }))
-        );
 
-        await smtpService.sendMail({
+          const config = await secretScanningV2DAL.configs.findOne({
-          recipients: recipients.map((member) => member.user.email!).filter(Boolean),
+            projectId: dataSource.projectId
-          template:
+          });
-            payload.status === SecretScanningScanStatus.Completed
+
-              ? SmtpTemplates.SecretScanningV2SecretsDetected
+          let configPath: string | undefined;
-              : SmtpTemplates.SecretScanningV2ScanFailed,
+
-          subjectLine,
+          if (config && config.content) {
-          substitutions:
+            configPath = join(tempFolder, "infisical-scan.toml");
-            payload.status === SecretScanningScanStatus.Completed
+            await writeTextToFile(configPath, config.content);
-              ? {
+          }
-                  authorName: payload.authorName,
+
-                  authorEmail: payload.authorEmail,
+          const findingsPayload = await factory.getDiffScanFindingsPayload({
-                  resourceName,
+            dataSource: {
-                  numberOfSecrets: payload.numberOfSecrets,
+              ...dataSource,
-                  isDiffScan: payload.isDiffScan,
+              connection
-                  url: encodeURI(
+            } as TSecretScanningDataSourceWithConnection,
-                    `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
+            resourceName: resource.name,
-                  ),
+            payload,
-                  timestamp
+            configPath
-                }
+          });
-              : {
+
+          const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
+            let findings: TSecretScanningFindings[] = [];
+
+            if (findingsPayload.length) {
+              findings = await secretScanningV2DAL.findings.upsert(
+                findingsPayload.map((finding) => ({
+                  ...finding,
+                  projectId: dataSource.projectId,
                   dataSourceName: dataSource.name,
-                  resourceName,
+                  dataSourceType: dataSource.type,
-                  projectName: project.name,
+                  resourceName: resource.name,
-                  timestamp,
+                  resourceType: resource.type,
-                  errorMessage: payload.errorMessage,
+                  scanId
-                  url: encodeURI(
+                })),
-                    `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
+                ["projectId", "fingerprint"],
-                  )
+                tx,
+                ["resourceName", "dataSourceName"]
+              );
+            }
+
+            await secretScanningV2DAL.scans.update(
+              { id: scanId },
+              {
+                status: SecretScanningScanStatus.Completed
+              }
+            );
+
+            return findings;
+          });
+
+          const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
+
+          if (newFindings.length) {
+            const finding = newFindings[0] as TSecretScanningFinding;
+            await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
+              status: SecretScanningScanStatus.Completed,
+              resourceName: resource.name,
+              isDiffScan: true,
+              dataSource,
+              numberOfSecrets: newFindings.length,
+              scanId,
+              authorName: finding?.details?.author,
+              authorEmail: finding?.details?.email
+            });
+          }
+
+          await auditLogService.createAuditLog({
+            projectId: dataSource.projectId,
+            actor: {
+              type: ActorType.PLATFORM,
+              metadata: {}
+            },
+            event: {
+              type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
+              metadata: {
+                dataSourceId: dataSource.id,
+                dataSourceType: dataSource.type,
+                resourceId,
+                resourceType: resource.type,
+                scanId,
+                scanStatus: SecretScanningScanStatus.Completed,
+                scanType: SecretScanningScanType.DiffScan,
+                numberOfSecretsDetected: findingsPayload.length
+              }
+            }
+          });
+
+          logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`);
+        } catch (error) {
+          if (retryCount === retryLimit) {
+            const errorMessage = parseScanErrorMessage(error);
+
+            await secretScanningV2DAL.scans.update(
+              { id: scanId },
+              {
+                status: SecretScanningScanStatus.Failed,
+                statusMessage: errorMessage
+              }
+            );
+
+            await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
+              status: SecretScanningScanStatus.Failed,
+              resourceName: resource.name,
+              dataSource,
+              errorMessage
+            });
+
+            await auditLogService.createAuditLog({
+              projectId: dataSource.projectId,
+              actor: {
+                type: ActorType.PLATFORM,
+                metadata: {}
+              },
+              event: {
+                type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
+                metadata: {
+                  dataSourceId: dataSource.id,
+                  dataSourceType: dataSource.type,
+                  resourceId: resource.id,
+                  resourceType: resource.type,
+                  scanId,
+                  scanStatus: SecretScanningScanStatus.Failed,
+                  scanType: SecretScanningScanType.DiffScan
                 }
-        });
+              }
-      } catch (error) {
+            });
-        logger.error(
+          }
-          error,
+
-          `secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
+          logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
-        );
+          throw error;
-        throw error;
+        } finally {
+          await deleteTempFolder(tempFolder);
+        }
+      },
+      {
+        batchSize: 1,
+        workerCount: 2,
+        pollingIntervalSeconds: 1
       }
-    },
+    );
-    {
+
-      batchSize: 1,
+    await queueService.startPg<QueueName.SecretScanningV2>(
-      workerCount: 2,
+      QueueJobs.SecretScanningV2SendNotification,
-      pollingIntervalSeconds: 1
+      async ([job]) => {
-    }
+        const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification;
-  );
+
+        const appCfg = getConfig();
+
+        if (!appCfg.isSmtpConfigured) return;
+
+        try {
+          const { projectId } = dataSource;
+
+          logger.info(
+            `secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
+          );
+
+          const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
+          const project = await projectDAL.findById(projectId);
+
+          const recipients = projectMembers.filter((member) => {
+            const isAdmin = member.roles.some((role) => role.role === ProjectMembershipRole.Admin);
+            const isCompleted = payload.status === SecretScanningScanStatus.Completed;
+            // We assume that the committer is one of the project members
+            const isCommitter = isCompleted && payload.authorEmail === member.user.email;
+            return isAdmin || isCommitter;
+          });
+
+          const timestamp = new Date().toISOString();
+
+          const subjectLine =
+            payload.status === SecretScanningScanStatus.Completed
+              ? "Incident Alert: Secret(s) Leaked"
+              : `Secret Scanning Failed`;
+
+          await notificationService.createUserNotifications(
+            recipients.map((member) => ({
+              userId: member.userId,
+              orgId: project.orgId,
+              type:
+                payload.status === SecretScanningScanStatus.Completed
+                  ? NotificationType.SECRET_SCANNING_SECRETS_DETECTED
+                  : NotificationType.SECRET_SCANNING_SCAN_FAILED,
+              title: subjectLine,
+              body:
+                payload.status === SecretScanningScanStatus.Completed
+                  ? `Uncovered **${payload.numberOfSecrets}** secret(s) ${payload.isDiffScan ? " from a recent commit to" : " in"} **${resourceName}**.`
+                  : `Encountered an error while attempting to scan the resource **${resourceName}**: ${payload.errorMessage}`,
+              link:
+                payload.status === SecretScanningScanStatus.Completed
+                  ? `/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
+                  : `/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
+            }))
+          );
+
+          await smtpService.sendMail({
+            recipients: recipients.map((member) => member.user.email!).filter(Boolean),
+            template:
+              payload.status === SecretScanningScanStatus.Completed
+                ? SmtpTemplates.SecretScanningV2SecretsDetected
+                : SmtpTemplates.SecretScanningV2ScanFailed,
+            subjectLine,
+            substitutions:
+              payload.status === SecretScanningScanStatus.Completed
+                ? {
+                    authorName: payload.authorName,
+                    authorEmail: payload.authorEmail,
+                    resourceName,
+                    numberOfSecrets: payload.numberOfSecrets,
+                    isDiffScan: payload.isDiffScan,
+                    url: encodeURI(
+                      `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
+                    ),
+                    timestamp
+                  }
+                : {
+                    dataSourceName: dataSource.name,
+                    resourceName,
+                    projectName: project.name,
+                    timestamp,
+                    errorMessage: payload.errorMessage,
+                    url: encodeURI(
+                      `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
+                    )
+                  }
+          });
+        } catch (error) {
+          logger.error(
+            error,
+            `secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
+          );
+          throw error;
+        }
+      },
+      {
+        batchSize: 1,
+        workerCount: 2,
+        pollingIntervalSeconds: 1
+      }
+    );
+  };
 
   return {
     queueDataSourceFullScan,
-    queueResourceDiffScan
+    queueResourceDiffScan,
+    init
   };
 };

backend/src/main.ts

@@ -16,7 +16,7 @@ import { buildRedisFromConfig } from "./lib/config/redis";
 import { removeTemporaryBaseDirectory } from "./lib/files";
 import { initLogger } from "./lib/logger";
 import { queueServiceFactory } from "./queue";
-import { main } from "./server/app";
+import { main, markRunningMigrations, markServerReady } from "./server/app";
 import { bootstrapCheck } from "./server/boot-strap-check";
 import { kmsRootConfigDALFactory } from "./services/kms/kms-root-config-dal";
 import { smtpServiceFactory } from "./services/smtp/smtp-service";
@@ -59,8 +59,6 @@ const run = async () => {
       })
     : undefined;
 
-  await runMigrations({ applicationDb: db, auditLogDb, logger });
-
   const smtp = smtpServiceFactory(formatSmtpConfig());
 
   const queue = queueServiceFactory(envConfig, {
@@ -74,7 +72,7 @@ const run = async () => {
   const keyStore = keyStoreFactory(envConfig, keyValueStoreDAL);
   const redis = buildRedisFromConfig(envConfig);
 
-  const server = await main({
+  const { server, completeServerInitialization } = await main({
     db,
     auditLogDb,
     superAdminDAL,
@@ -87,7 +85,6 @@ const run = async () => {
     redis,
     envConfig
   });
-  const bootstrap = await bootstrapCheck({ db });
 
   // eslint-disable-next-line
   process.on("SIGINT", async () => {
@@ -121,12 +118,46 @@ const run = async () => {
 
   await server.listen({
     port: envConfig.PORT,
-    host: envConfig.HOST,
+    host: envConfig.HOST
-    listenTextResolver: (address) => {
+  });
-      void bootstrap();
+
-      return address;
+  logger.info(`Server listening on ${envConfig.HOST}:${envConfig.PORT}`);
+  logger.info("Running migrations...");
+
+  // Run migrations while server is up
+  // All containers start as NOT HEALTHY (waiting for migrations)
+  // Container that acquires lock: becomes HEALTHY (running migrations) + NOT READY (no traffic)
+  // Other containers waiting: stay NOT HEALTHY (waiting) + NOT READY (no traffic)
+  await runMigrations({
+    applicationDb: db,
+    auditLogDb,
+    logger,
+    onMigrationLockAcquired: () => {
+      // Called after successfully acquiring the lock
+      // This container is now the migration runner
+      markRunningMigrations();
+      logger.info("Migration lock acquired! This container is running migrations.");
     }
   });
+
+  logger.info("Migrations complete. Completing server initialization...");
+
+  try {
+    await completeServerInitialization();
+  } catch (error) {
+    logger.error(error, "Failed to complete server initialization");
+    await server.close();
+    await queue.shutdown();
+    process.exit(1);
+  }
+
+  logger.info("Server initialization complete. Marking server as READY...");
+
+  markServerReady();
+
+  logger.info("Server is ready to accept traffic");
+  const bootstrap = await bootstrapCheck({ db });
+  void bootstrap();
 };
 
 void run();

backend/src/server/app.ts

@@ -1,5 +1,6 @@
 /* eslint-disable import/extensions */
 import path from "node:path";
+import { monitorEventLoopDelay } from "perf_hooks";
 
 import type { FastifyCookieOptions } from "@fastify/cookie";
 import cookie from "@fastify/cookie";
@@ -24,6 +25,7 @@ import { TQueueServiceFactory } from "@app/queue";
 import { TKmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal";
 import { TSmtpService } from "@app/services/smtp/smtp-service";
 import { TSuperAdminDALFactory } from "@app/services/super-admin/super-admin-dal";
+import { getServerCfg } from "@app/services/super-admin/super-admin-service";
 
 import { globalRateLimiterCfg } from "./config/rateLimiter";
 import { addErrorsToResponseSchemas } from "./plugins/add-errors-to-response-schemas";
@@ -36,6 +38,15 @@ import { registerServeUI } from "./plugins/serve-ui";
 import { fastifySwagger } from "./plugins/swagger";
 import { registerRoutes } from "./routes";
 
+const histogram = monitorEventLoopDelay({ resolution: 20 });
+histogram.enable();
+
+const serverState = {
+  isReady: false,
+  isRunningMigrations: false,
+  isWaitingForMigrations: true // Start as true - containers are unhealthy until they acquire migration lock or complete
+};
+
 type TMain = {
   auditLogDb?: Knex;
   db: Knex;
@@ -145,7 +156,72 @@ export const main = async ({
       })
     });
 
-    await server.register(registerRoutes, {
+    // Health check - returns 200 only if doing useful work (running migrations or ready)
+    // Returns 503 if waiting for another container to finish migrations
+    server.get("/api/health", async (_, reply) => {
+      if (serverState.isWaitingForMigrations) {
+        return reply.code(503).send({
+          status: "waiting",
+          message: "Waiting for migrations to complete in another container"
+        });
+      }
+      return { status: "ok", message: "Server is alive" };
+    });
+
+    // Global preHandler to block requests during migrations
+    server.addHook("preHandler", async (request, reply) => {
+      if (request.url === "/api/health" || request.url === "/api/ready") {
+        return;
+      }
+      if (!serverState.isReady) {
+        return reply.code(503).send({
+          status: "unavailable",
+          message: "Server is starting up, migrations in progress. Please try again in a moment."
+        });
+      }
+    });
+
+    // Readiness check - returns 503 until migrations are complete
+    server.get("/api/ready", async (request, reply) => {
+      const cfg = getConfig();
+
+      const meanLagMs = histogram.mean / 1e6;
+      const maxLagMs = histogram.max / 1e6;
+      const p99LagMs = histogram.percentile(99) / 1e6;
+
+      request.log.info(
+        `Event loop stats - Mean: ${meanLagMs.toFixed(2)}ms, Max: ${maxLagMs.toFixed(2)}ms, p99: ${p99LagMs.toFixed(2)}ms`
+      );
+
+      request.log.info(`Raw event loop stats: ${JSON.stringify(histogram, null, 2)}`);
+
+      if (!serverState.isReady) {
+        return reply.code(503).send({
+          date: new Date(),
+          message: "Server is starting up, migrations in progress",
+          emailConfigured: cfg.isSmtpConfigured,
+          redisConfigured: cfg.isRedisConfigured,
+          secretScanningConfigured: cfg.isSecretScanningConfigured,
+          samlDefaultOrgSlug: cfg.samlDefaultOrgSlug,
+          auditLogStorageDisabled: Boolean(cfg.DISABLE_AUDIT_LOG_STORAGE)
+        });
+      }
+
+      const serverCfg = await getServerCfg();
+
+      return {
+        date: new Date(),
+        message: "Ok",
+        emailConfigured: cfg.isSmtpConfigured,
+        inviteOnlySignup: Boolean(serverCfg.allowSignUp),
+        redisConfigured: cfg.isRedisConfigured,
+        secretScanningConfigured: cfg.isSecretScanningConfigured,
+        samlDefaultOrgSlug: cfg.samlDefaultOrgSlug,
+        auditLogStorageDisabled: Boolean(cfg.DISABLE_AUDIT_LOG_STORAGE)
+      };
+    });
+
+    const completeServerInitialization = await registerRoutes(server, {
       smtp,
       queue,
       db,
@@ -164,10 +240,21 @@ export const main = async ({
 
     await server.ready();
     server.swagger();
-    return server;
+    return { server, completeServerInitialization };
   } catch (err) {
     server.log.error(err);
     await queue.shutdown();
     process.exit(1);
   }
 };
+
+export const markServerReady = () => {
+  serverState.isReady = true;
+  serverState.isRunningMigrations = false;
+  serverState.isWaitingForMigrations = false;
+};
+
+export const markRunningMigrations = () => {
+  serverState.isRunningMigrations = true;
+  serverState.isWaitingForMigrations = false;
+};

backend/src/server/routes/index.ts

@@ -2208,7 +2208,7 @@ export const registerRoutes = async (
     internalCaFns
   });
 
-  await secretRotationV2QueueServiceFactory({
+  const secretRotationV2Queue = await secretRotationV2QueueServiceFactory({
     secretRotationV2Service,
     secretRotationV2DAL,
     queueService,
@@ -2305,8 +2305,6 @@ export const registerRoutes = async (
   // If FIPS is enabled, we check to ensure that the users license includes FIPS mode.
   crypto.verifyFipsLicense(licenseService);
 
-  await superAdminService.initServerCfg();
-
   // Start HSM service if it's configured/enabled.
   await hsmService.startService();
 
@@ -2331,21 +2329,60 @@ export const registerRoutes = async (
     }
   }
 
-  await telemetryQueue.startTelemetryCheck();
+  const completeServerInitialization = async () => {
-  await telemetryQueue.startAggregatedEventsJob();
+    await superAdminService.initServerCfg();
-  await dailyResourceCleanUp.init();
+
-  await healthAlert.init();
+    await telemetryQueue.startTelemetryCheck();
-  await pkiSyncCleanup.init();
+    await telemetryQueue.startAggregatedEventsJob();
-  await pamAccountRotation.init();
+    await dailyResourceCleanUp.init();
-  await dailyReminderQueueService.startDailyRemindersJob();
+    await healthAlert.init();
-  await dailyReminderQueueService.startSecretReminderMigrationJob();
+    await pkiSyncCleanup.init();
-  await dailyExpiringPkiItemAlert.startSendingAlerts();
+    await pamAccountRotation.init();
-  await pkiSubscriberQueue.startDailyAutoRenewalJob();
+    await dailyReminderQueueService.startDailyRemindersJob();
-  await certificateV3Queue.init();
+    await dailyReminderQueueService.startSecretReminderMigrationJob();
-  await kmsService.startService(hsmStatus);
+    await dailyExpiringPkiItemAlert.startSendingAlerts();
-  await microsoftTeamsService.start();
+    await pkiSubscriberQueue.startDailyAutoRenewalJob();
-  await dynamicSecretQueueService.init();
+    await certificateV3Queue.init();
-  await eventBusService.init();
+    await kmsService.startService(hsmStatus);
+    await microsoftTeamsService.start();
+    await dynamicSecretQueueService.init();
+    await secretScanningV2Queue.init();
+    await secretRotationV2Queue.init();
+    await notificationQueue.init();
+    await eventBusService.init();
+
+    const cronJobs: CronJob[] = [];
+    if (appCfg.isProductionMode) {
+      const rateLimitSyncJob = await rateLimitService.initializeBackgroundSync();
+      if (rateLimitSyncJob) {
+        cronJobs.push(rateLimitSyncJob);
+      }
+      const licenseSyncJob = await licenseService.initializeBackgroundSync();
+      if (licenseSyncJob) {
+        cronJobs.push(licenseSyncJob);
+      }
+
+      const microsoftTeamsSyncJob = await microsoftTeamsService.initializeBackgroundSync();
+      if (microsoftTeamsSyncJob) {
+        cronJobs.push(microsoftTeamsSyncJob);
+      }
+
+      const adminIntegrationsSyncJob = await superAdminService.initializeAdminIntegrationConfigSync();
+      if (adminIntegrationsSyncJob) {
+        cronJobs.push(adminIntegrationsSyncJob);
+      }
+    }
+
+    const configSyncJob = await superAdminService.initializeEnvConfigSync();
+    if (configSyncJob) {
+      cronJobs.push(configSyncJob);
+    }
+
+    const oauthConfigSyncJob = await initializeOauthConfigSync();
+    if (oauthConfigSyncJob) {
+      cronJobs.push(oauthConfigSyncJob);
+    }
+  };
 
   // inject all services
   server.decorate<FastifyZodProvider["services"]>("services", {
@@ -2475,38 +2512,6 @@ export const registerRoutes = async (
     convertor: convertorService
   });
 
-  const cronJobs: CronJob[] = [];
-  if (appCfg.isProductionMode) {
-    const rateLimitSyncJob = await rateLimitService.initializeBackgroundSync();
-    if (rateLimitSyncJob) {
-      cronJobs.push(rateLimitSyncJob);
-    }
-    const licenseSyncJob = await licenseService.initializeBackgroundSync();
-    if (licenseSyncJob) {
-      cronJobs.push(licenseSyncJob);
-    }
-
-    const microsoftTeamsSyncJob = await microsoftTeamsService.initializeBackgroundSync();
-    if (microsoftTeamsSyncJob) {
-      cronJobs.push(microsoftTeamsSyncJob);
-    }
-
-    const adminIntegrationsSyncJob = await superAdminService.initializeAdminIntegrationConfigSync();
-    if (adminIntegrationsSyncJob) {
-      cronJobs.push(adminIntegrationsSyncJob);
-    }
-  }
-
-  const configSyncJob = await superAdminService.initializeEnvConfigSync();
-  if (configSyncJob) {
-    cronJobs.push(configSyncJob);
-  }
-
-  const oauthConfigSyncJob = await initializeOauthConfigSync();
-  if (oauthConfigSyncJob) {
-    cronJobs.push(oauthConfigSyncJob);
-  }
-
   server.decorate<FastifyZodProvider["store"]>("store", {
     user: userDAL,
     kmipClient: kmipClientDAL
@@ -2595,9 +2600,10 @@ export const registerRoutes = async (
   await server.register(registerV4Routes, { prefix: "/api/v4" });
 
   server.addHook("onClose", async () => {
-    cronJobs.forEach((job) => job.stop());
     await telemetryService.flushAll();
     await eventBusService.close();
     sseService.close();
   });
+
+  return completeServerInitialization;
 };

backend/src/services/notification/notification-queue.ts

@@ -10,6 +10,7 @@ type TNotificationQueueServiceFactoryDep = {
 
 export type TNotificationQueueServiceFactory = {
   pushUserNotifications: (data: TCreateUserNotificationDTO[]) => Promise<void>;
+  init: () => Promise<void>;
 };
 
 export const notificationQueueServiceFactory = async ({
@@ -20,20 +21,23 @@ export const notificationQueueServiceFactory = async ({
     await queueService.queuePg(QueueJobs.UserNotification, { notifications: data });
   };
 
-  await queueService.startPg(
+  const init = async () => {
-    QueueJobs.UserNotification,
+    await queueService.startPg(
-    async ([job]) => {
+      QueueJobs.UserNotification,
-      const { notifications } = job.data as { notifications: TCreateUserNotificationDTO[] };
+      async ([job]) => {
-      await userNotificationDAL.batchInsert(notifications);
+        const { notifications } = job.data as { notifications: TCreateUserNotificationDTO[] };
-    },
+        await userNotificationDAL.batchInsert(notifications);
-    {
+      },
-      batchSize: 1,
+      {
-      workerCount: 2,
+        batchSize: 1,
-      pollingIntervalSeconds: 1
+        workerCount: 2,
-    }
+        pollingIntervalSeconds: 1
-  );
+      }
+    );
+  };
 
   return {
-    pushUserNotifications
+    pushUserNotifications,
+    init
   };
 };

docs/self-hosting/guides/production-hardening.mdx

@@ -137,6 +137,33 @@ Configure database read replicas for high availability PostgreSQL setups:
 DB_READ_REPLICAS='[{"DB_CONNECTION_URI":"postgresql://user:pass@replica:5432/db?sslmode=require"}]'
 ```
 
+### Health Check Endpoints
+
+Infisical provides two health check endpoints for proper container orchestration and load balancer integration:
+
+#### `/api/health` - Container Health Check
+
+Determines whether the application container should be kept alive or terminated.
+
+- Returns `200` if the application is running and operational
+- Returns `200` even during startup tasks
+- Returns `503` only if the application has crashed or is unable to start
+
+**Use for**: Docker health checks, Kubernetes liveness probes, ECS task health checks.
+
+#### `/api/ready` - Traffic Readiness Check
+
+Determines whether the application instance is ready to receive production traffic.
+
+- Returns `200` when the application is fully ready to serve requests
+- Returns `503` during startup tasks (e.g., database migrations, initialization)
+
+**Use for**: Load balancer health checks, Kubernetes readiness probes, ALB target health checks.
+
+#### Why Two Endpoints?
+
+Using both endpoints together enables zero-downtime deployments: containers stay alive during startup tasks (`/api/health` returns `200`) while load balancers avoid sending traffic to instances that aren't ready (`/api/ready` returns `503`). This ensures existing instances continue serving traffic until new instances complete their initialization.
+
 ### Operational Security
 
 #### User Access Management
@@ -207,14 +234,17 @@ docker run --memory=1g --cpus=0.5 infisical/infisical:latest
 
 #### Health Monitoring
 
-**Configure health checks**. Set up Docker health checks:
+**Configure health checks**. Set up Docker health checks using the appropriate endpoint:
 
 ```dockerfile
 # In Dockerfile or docker-compose.yml
+# Use /api/health for container health (keeps container alive during startup)
 HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
-  CMD curl -f http://localhost:8080/api/status || exit 1
+  CMD curl -f http://localhost:8080/api/health || exit 1
 ```
 
+**Note**: Use `/api/health` for container health checks and `/api/ready` for load balancer readiness checks. See [Health Check Endpoints](#health-check-endpoints) for detailed information.
+
 #### Network Security
 
 **Host firewall configuration**. Configure host-level firewall for Docker deployments:
@@ -433,26 +463,32 @@ stringData:
 
 #### Health Monitoring
 
-**Set up health checks**. Configure readiness and liveness probes:
+**Set up health checks**. Configure readiness and liveness probes using the appropriate endpoints:
 
 ```yaml
 # Health check configuration
 containers:
   - name: infisical
+    # Use /api/ready for readiness (traffic routing)
     readinessProbe:
       httpGet:
-        path: /api/status
+        path: /api/ready
         port: 8080
       initialDelaySeconds: 10
       periodSeconds: 5
+      failureThreshold: 3
+    # Use /api/health for liveness (container restart)
     livenessProbe:
       httpGet:
-        path: /api/status
+        path: /api/health
         port: 8080
       initialDelaySeconds: 30
       periodSeconds: 10
+      failureThreshold: 3
 ```
 
+**Important**: The `readinessProbe` uses `/api/ready` to ensure traffic is only sent to pods that are fully initialized. The `livenessProbe` uses `/api/health` to keep the container alive during startup. See [Health Check Endpoints](#health-check-endpoints) for detailed information.
+
 #### Infrastructure Considerations
 
 **Use managed databases (if possible)**. For production deployments, consider using managed PostgreSQL and Redis services instead of in-cluster instances when feasible, as they typically provide better security, backup, and maintenance capabilities.