Merge pull request #4773 from Infisical/revert-4760-misc/added-health-and-ready-probe-endpoints

Revert "misc: added health and ready probe endpoints"

backend/e2e-test/vitest-environment-knex.ts

@@ -8,7 +8,7 @@ import path from "path";
 import { seedData1 } from "@app/db/seed-data";
 import { getDatabaseCredentials, getHsmConfig, initEnvConfig } from "@app/lib/config/env";
 import { initLogger } from "@app/lib/logger";
-import { main, markServerReady } from "@app/server/app";
+import { main } from "@app/server/app";
 import { AuthMethod, AuthTokenType } from "@app/services/auth/auth-type";
 
 import { mockSmtpServer } from "./mocks/smtp";
@@ -83,7 +83,7 @@ export default {
 
       await queue.initialize();
 
-      const { server, completeServerInitialization } = await main({
+      const server = await main({
         db,
         smtp,
         logger,
@@ -96,10 +96,6 @@ export default {
         envConfig: envCfg
       });
 
-      await completeServerInitialization();
-
-      markServerReady();
-
       await bootstrapCheck({ db });
 
       // @ts-expect-error type

backend/src/auto-start-migrations.ts

@@ -12,7 +12,6 @@ type TArgs = {
   auditLogDb?: Knex;
   applicationDb: Knex;
   logger: Logger;
-  onMigrationLockAcquired?: () => void;
 };
 
 const isProduction = process.env.NODE_ENV === "production";
@@ -31,7 +30,7 @@ const migrationStatusCheckErrorHandler = (err: Error) => {
   throw err;
 };
 
-export const runMigrations = async ({ applicationDb, auditLogDb, logger, onMigrationLockAcquired }: TArgs) => {
+export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs) => {
   try {
     // akhilmhdh(Feb 10 2025): 2 years  from now remove this
     if (isProduction) {
@@ -86,13 +85,6 @@ export const runMigrations = async ({ applicationDb, auditLogDb, logger, onMigra
 
     await applicationDb.transaction(async (tx) => {
       await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]);
-
-      // Signal that this container is running migrations so that it can be marked as healthy/alive
-      // This is to prevent the container from being killed by the orchestrator
-      if (onMigrationLockAcquired) {
-        onMigrationLockAcquired();
-      }
-
       logger.info("Running application migrations.");
 
       const didPreviousInstanceRunMigration = !(await applicationDb.migrate

backend/src/ee/services/hsm/hsm-fns.ts

@@ -84,7 +84,7 @@ export const isHsmActiveAndEnabled = async ({
 
   rootKmsConfigEncryptionStrategy = (rootKmsConfig?.encryptionStrategy || null) as RootKeyEncryptionStrategy | null;
   if (
-    (rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.HSM || isHsmConfigured) &&
+    rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.HSM &&
     licenseService &&
     !licenseService.onPremFeatures.hsm
   ) {

backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts

@@ -42,172 +42,168 @@ export const secretRotationV2QueueServiceFactory = async ({
   smtpService,
   notificationService
 }: TSecretRotationV2QueueServiceFactoryDep) => {
-  const init = async () => {
+  const appCfg = getConfig();
-    const appCfg = getConfig();
 
-    if (appCfg.isRotationDevelopmentMode) {
+  if (appCfg.isRotationDevelopmentMode) {
-      logger.warn("Secret Rotation V2 is in development mode.");
+    logger.warn("Secret Rotation V2 is in development mode.");
-    }
+  }
 
-    await queueService.startPg<QueueName.SecretRotationV2>(
+  await queueService.startPg<QueueName.SecretRotationV2>(
-      QueueJobs.SecretRotationV2QueueRotations,
+    QueueJobs.SecretRotationV2QueueRotations,
-      async () => {
+    async () => {
-        try {
+      try {
-          const rotateBy = getNextUtcRotationInterval();
+        const rotateBy = getNextUtcRotationInterval();
 
-          const currentTime = new Date();
+        const currentTime = new Date();
 
-          const secretRotations = await secretRotationV2DAL.findSecretRotationsToQueue(rotateBy);
+        const secretRotations = await secretRotationV2DAL.findSecretRotationsToQueue(rotateBy);
 
+        logger.info(
+          `secretRotationV2Queue: Queue Rotations [currentTime=${currentTime.toISOString()}] [rotateBy=${rotateBy.toISOString()}] [count=${
+            secretRotations.length
+          }]`
+        );
+
+        for await (const rotation of secretRotations) {
           logger.info(
-            `secretRotationV2Queue: Queue Rotations [currentTime=${currentTime.toISOString()}] [rotateBy=${rotateBy.toISOString()}] [count=${
+            `secretRotationV2Queue: Queue Rotation [rotationId=${rotation.id}] [lastRotatedAt=${new Date(
-              secretRotations.length
+              rotation.lastRotatedAt
-            }]`
+            ).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]`
           );
 
-          for await (const rotation of secretRotations) {
+          const data = {
-            logger.info(
+            rotationId: rotation.id,
-              `secretRotationV2Queue: Queue Rotation [rotationId=${rotation.id}] [lastRotatedAt=${new Date(
+            queuedAt: currentTime
-                rotation.lastRotatedAt
+          } as TSecretRotationRotateSecretsJobPayload;
-              ).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]`
+
+          if (appCfg.isTestMode) {
+            logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode");
+            await rotateSecretsFns({
+              job: {
+                id: uuidv4(),
+                data,
+                retryCount: 0,
+                retryLimit: 0
+              },
+              secretRotationV2DAL,
+              secretRotationV2Service
+            });
+          } else {
+            await queueService.queuePg(
+              QueueJobs.SecretRotationV2RotateSecrets,
+              {
+                rotationId: rotation.id,
+                queuedAt: currentTime
+              },
+              getSecretRotationRotateSecretJobOptions(rotation)
             );
-
-            const data = {
-              rotationId: rotation.id,
-              queuedAt: currentTime
-            } as TSecretRotationRotateSecretsJobPayload;
-
-            if (appCfg.isTestMode) {
-              logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode");
-              await rotateSecretsFns({
-                job: {
-                  id: uuidv4(),
-                  data,
-                  retryCount: 0,
-                  retryLimit: 0
-                },
-                secretRotationV2DAL,
-                secretRotationV2Service
-              });
-            } else {
-              await queueService.queuePg(
-                QueueJobs.SecretRotationV2RotateSecrets,
-                {
-                  rotationId: rotation.id,
-                  queuedAt: currentTime
-                },
-                getSecretRotationRotateSecretJobOptions(rotation)
-              );
-            }
           }
-        } catch (error) {
-          logger.error(error, "secretRotationV2Queue: Queue Rotations Error:");
-          throw error;
         }
-      },
+      } catch (error) {
-      {
+        logger.error(error, "secretRotationV2Queue: Queue Rotations Error:");
-        batchSize: 1,
+        throw error;
-        workerCount: 1,
-        pollingIntervalSeconds: appCfg.isRotationDevelopmentMode ? 0.5 : 30
       }
-    );
+    },
+    {
+      batchSize: 1,
+      workerCount: 1,
+      pollingIntervalSeconds: appCfg.isRotationDevelopmentMode ? 0.5 : 30
+    }
+  );
 
-    await queueService.startPg<QueueName.SecretRotationV2>(
+  await queueService.startPg<QueueName.SecretRotationV2>(
-      QueueJobs.SecretRotationV2RotateSecrets,
+    QueueJobs.SecretRotationV2RotateSecrets,
-      async ([job]) => {
+    async ([job]) => {
-        await rotateSecretsFns({
+      await rotateSecretsFns({
-          job: {
+        job: {
-            ...job,
+          ...job,
-            data: job.data as TSecretRotationRotateSecretsJobPayload
+          data: job.data as TSecretRotationRotateSecretsJobPayload
-          },
+        },
-          secretRotationV2DAL,
+        secretRotationV2DAL,
-          secretRotationV2Service
+        secretRotationV2Service
+      });
+    },
+    {
+      batchSize: 1,
+      workerCount: 2,
+      pollingIntervalSeconds: 0.5
+    }
+  );
+
+  await queueService.startPg<QueueName.SecretRotationV2>(
+    QueueJobs.SecretRotationV2SendNotification,
+    async ([job]) => {
+      const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload;
+      try {
+        const {
+          name: rotationName,
+          type,
+          projectId,
+          lastRotationAttemptedAt,
+          folder,
+          environment,
+          id: rotationId
+        } = secretRotation;
+
+        logger.info(`secretRotationV2Queue: Sending Status Notification [rotationId=${rotationId}]`);
+
+        const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
+        const project = await projectDAL.findById(projectId);
+
+        const projectAdmins = projectMembers.filter((member) =>
+          member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
+        );
+
+        const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation];
+
+        const rotationPath = `/projects/secret-management/${projectId}/secrets/${environment.slug}`;
+
+        await notificationService.createUserNotifications(
+          projectAdmins.map((admin) => ({
+            userId: admin.userId,
+            orgId: project.orgId,
+            type: NotificationType.SECRET_ROTATION_FAILED,
+            title: "Secret Rotation Failed",
+            body: `Your **${rotationType}** rotation **${rotationName}** failed to rotate.`,
+            link: rotationPath
+          }))
+        );
+
+        await smtpService.sendMail({
+          recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
+          template: SmtpTemplates.SecretRotationFailed,
+          subjectLine: `Secret Rotation Failed`,
+          substitutions: {
+            rotationName,
+            rotationType,
+            content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date(
+              lastRotationAttemptedAt
+            ).toISOString()}. Please check the rotation status in Infisical for more details.`,
+            secretPath: folder.path,
+            environment: environment.name,
+            projectName: project.name,
+            rotationUrl: encodeURI(`${appCfg.SITE_URL}${rotationPath}`)
+          }
         });
-      },
+      } catch (error) {
-      {
+        logger.error(
-        batchSize: 1,
+          error,
-        workerCount: 2,
+          `secretRotationV2Queue: Failed to Send Status Notification [rotationId=${secretRotation.id}]`
-        pollingIntervalSeconds: 0.5
+        );
+        throw error;
       }
-    );
+    },
+    {
+      batchSize: 1,
+      workerCount: 2,
+      pollingIntervalSeconds: 1
+    }
+  );
 
-    await queueService.startPg<QueueName.SecretRotationV2>(
+  await queueService.schedulePg(
-      QueueJobs.SecretRotationV2SendNotification,
+    QueueJobs.SecretRotationV2QueueRotations,
-      async ([job]) => {
+    appCfg.isRotationDevelopmentMode ? "* * * * *" : "0 0 * * *",
-        const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload;
+    undefined,
-        try {
+    { tz: "UTC" }
-          const {
+  );
-            name: rotationName,
-            type,
-            projectId,
-            lastRotationAttemptedAt,
-            folder,
-            environment,
-            id: rotationId
-          } = secretRotation;
-
-          logger.info(`secretRotationV2Queue: Sending Status Notification [rotationId=${rotationId}]`);
-
-          const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
-          const project = await projectDAL.findById(projectId);
-
-          const projectAdmins = projectMembers.filter((member) =>
-            member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
-          );
-
-          const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation];
-
-          const rotationPath = `/projects/secret-management/${projectId}/secrets/${environment.slug}`;
-
-          await notificationService.createUserNotifications(
-            projectAdmins.map((admin) => ({
-              userId: admin.userId,
-              orgId: project.orgId,
-              type: NotificationType.SECRET_ROTATION_FAILED,
-              title: "Secret Rotation Failed",
-              body: `Your **${rotationType}** rotation **${rotationName}** failed to rotate.`,
-              link: rotationPath
-            }))
-          );
-
-          await smtpService.sendMail({
-            recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
-            template: SmtpTemplates.SecretRotationFailed,
-            subjectLine: `Secret Rotation Failed`,
-            substitutions: {
-              rotationName,
-              rotationType,
-              content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date(
-                lastRotationAttemptedAt
-              ).toISOString()}. Please check the rotation status in Infisical for more details.`,
-              secretPath: folder.path,
-              environment: environment.name,
-              projectName: project.name,
-              rotationUrl: encodeURI(`${appCfg.SITE_URL}${rotationPath}`)
-            }
-          });
-        } catch (error) {
-          logger.error(
-            error,
-            `secretRotationV2Queue: Failed to Send Status Notification [rotationId=${secretRotation.id}]`
-          );
-          throw error;
-        }
-      },
-      {
-        batchSize: 1,
-        workerCount: 2,
-        pollingIntervalSeconds: 1
-      }
-    );
-
-    await queueService.schedulePg(
-      QueueJobs.SecretRotationV2QueueRotations,
-      appCfg.isRotationDevelopmentMode ? "* * * * *" : "0 0 * * *",
-      undefined,
-      { tz: "UTC" }
-    );
-  };
-
-  return { init };
 };

backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts

@@ -141,6 +141,202 @@ export const secretScanningV2QueueServiceFactory = async ({
     }
   };
 
+  await queueService.startPg<QueueName.SecretScanningV2>(
+    QueueJobs.SecretScanningV2FullScan,
+    async ([job]) => {
+      const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan;
+      const { retryCount, retryLimit } = job;
+
+      const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
+
+      const tempFolder = await createTempFolder();
+
+      const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
+
+      if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
+
+      const resource = await secretScanningV2DAL.resources.findById(resourceId);
+
+      if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
+
+      let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
+
+      try {
+        try {
+          lock = await keyStore.acquireLock(
+            [KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
+            60 * 1000 * 5
+          );
+        } catch (e) {
+          throw new Error("Failed to acquire scanning lock.");
+        }
+
+        await secretScanningV2DAL.scans.update(
+          { id: scanId },
+          {
+            status: SecretScanningScanStatus.Scanning
+          }
+        );
+
+        let connection: TAppConnection | null = null;
+        if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
+
+        const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
+          kmsService,
+          appConnectionDAL
+        });
+
+        const findingsPath = join(tempFolder, "findings.json");
+
+        const scanPath = await factory.getFullScanPath({
+          dataSource: {
+            ...dataSource,
+            connection
+          } as TSecretScanningDataSourceWithConnection,
+          resourceName: resource.name,
+          tempFolder
+        });
+
+        const config = await secretScanningV2DAL.configs.findOne({
+          projectId: dataSource.projectId
+        });
+
+        let configPath: string | undefined;
+
+        if (config && config.content) {
+          configPath = join(tempFolder, "infisical-scan.toml");
+          await writeTextToFile(configPath, config.content);
+        }
+
+        let findingsPayload: TFindingsPayload;
+        switch (resource.type) {
+          case SecretScanningResource.Repository:
+          case SecretScanningResource.Project:
+            findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
+            break;
+          default:
+            throw new Error("Unhandled resource type");
+        }
+
+        const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
+          let findings: TSecretScanningFindings[] = [];
+          if (findingsPayload.length) {
+            findings = await secretScanningV2DAL.findings.upsert(
+              findingsPayload.map((finding) => ({
+                ...finding,
+                projectId: dataSource.projectId,
+                dataSourceName: dataSource.name,
+                dataSourceType: dataSource.type,
+                resourceName: resource.name,
+                resourceType: resource.type,
+                scanId
+              })),
+              ["projectId", "fingerprint"],
+              tx,
+              ["resourceName", "dataSourceName"]
+            );
+          }
+
+          await secretScanningV2DAL.scans.update(
+            { id: scanId },
+            {
+              status: SecretScanningScanStatus.Completed,
+              statusMessage: null
+            }
+          );
+
+          return findings;
+        });
+
+        const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
+
+        if (newFindings.length) {
+          await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
+            status: SecretScanningScanStatus.Completed,
+            resourceName: resource.name,
+            isDiffScan: false,
+            dataSource,
+            numberOfSecrets: newFindings.length,
+            scanId
+          });
+        }
+
+        await auditLogService.createAuditLog({
+          projectId: dataSource.projectId,
+          actor: {
+            type: ActorType.PLATFORM,
+            metadata: {}
+          },
+          event: {
+            type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
+            metadata: {
+              dataSourceId: dataSource.id,
+              dataSourceType: dataSource.type,
+              resourceId: resource.id,
+              resourceType: resource.type,
+              scanId,
+              scanStatus: SecretScanningScanStatus.Completed,
+              scanType: SecretScanningScanType.FullScan,
+              numberOfSecretsDetected: findingsPayload.length
+            }
+          }
+        });
+
+        logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`);
+      } catch (error) {
+        if (retryCount === retryLimit) {
+          const errorMessage = parseScanErrorMessage(error);
+
+          await secretScanningV2DAL.scans.update(
+            { id: scanId },
+            {
+              status: SecretScanningScanStatus.Failed,
+              statusMessage: errorMessage
+            }
+          );
+
+          await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
+            status: SecretScanningScanStatus.Failed,
+            resourceName: resource.name,
+            dataSource,
+            errorMessage
+          });
+
+          await auditLogService.createAuditLog({
+            projectId: dataSource.projectId,
+            actor: {
+              type: ActorType.PLATFORM,
+              metadata: {}
+            },
+            event: {
+              type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
+              metadata: {
+                dataSourceId: dataSource.id,
+                dataSourceType: dataSource.type,
+                resourceId: resource.id,
+                resourceType: resource.type,
+                scanId,
+                scanStatus: SecretScanningScanStatus.Failed,
+                scanType: SecretScanningScanType.FullScan
+              }
+            }
+          });
+        }
+
+        logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
+        throw error;
+      } finally {
+        await deleteTempFolder(tempFolder);
+        await lock?.release();
+      }
+    },
+    {
+      batchSize: 1,
+      workerCount: 2,
+      pollingIntervalSeconds: 1
+    }
+  );
+
   const queueResourceDiffScan = async ({
     payload,
     dataSourceId,
@@ -195,126 +391,147 @@ export const secretScanningV2QueueServiceFactory = async ({
     }
   };
 
-  const init = async () => {
+  await queueService.startPg<QueueName.SecretScanningV2>(
-    await queueService.startPg<QueueName.SecretScanningV2>(
+    QueueJobs.SecretScanningV2DiffScan,
-      QueueJobs.SecretScanningV2FullScan,
+    async ([job]) => {
-      async ([job]) => {
+      const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan;
-        const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan;
+      const { retryCount, retryLimit } = job;
-        const { retryCount, retryLimit } = job;
 
-        const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
+      const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
 
-        const tempFolder = await createTempFolder();
+      const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
 
-        const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
+      if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
 
-        if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
+      const resource = await secretScanningV2DAL.resources.findById(resourceId);
 
-        const resource = await secretScanningV2DAL.resources.findById(resourceId);
+      if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
 
-        if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
+      const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
+        kmsService,
+        appConnectionDAL
+      });
 
-        let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
+      const tempFolder = await createTempFolder();
 
-        try {
+      try {
-          try {
+        await secretScanningV2DAL.scans.update(
-            lock = await keyStore.acquireLock(
+          { id: scanId },
-              [KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
+          {
-              60 * 1000 * 5
+            status: SecretScanningScanStatus.Scanning
+          }
+        );
+
+        let connection: TAppConnection | null = null;
+        if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
+
+        const config = await secretScanningV2DAL.configs.findOne({
+          projectId: dataSource.projectId
+        });
+
+        let configPath: string | undefined;
+
+        if (config && config.content) {
+          configPath = join(tempFolder, "infisical-scan.toml");
+          await writeTextToFile(configPath, config.content);
+        }
+
+        const findingsPayload = await factory.getDiffScanFindingsPayload({
+          dataSource: {
+            ...dataSource,
+            connection
+          } as TSecretScanningDataSourceWithConnection,
+          resourceName: resource.name,
+          payload,
+          configPath
+        });
+
+        const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
+          let findings: TSecretScanningFindings[] = [];
+
+          if (findingsPayload.length) {
+            findings = await secretScanningV2DAL.findings.upsert(
+              findingsPayload.map((finding) => ({
+                ...finding,
+                projectId: dataSource.projectId,
+                dataSourceName: dataSource.name,
+                dataSourceType: dataSource.type,
+                resourceName: resource.name,
+                resourceType: resource.type,
+                scanId
+              })),
+              ["projectId", "fingerprint"],
+              tx,
+              ["resourceName", "dataSourceName"]
             );
-          } catch (e) {
-            throw new Error("Failed to acquire scanning lock.");
           }
 
           await secretScanningV2DAL.scans.update(
             { id: scanId },
             {
-              status: SecretScanningScanStatus.Scanning
+              status: SecretScanningScanStatus.Completed
             }
           );
 
-          let connection: TAppConnection | null = null;
+          return findings;
-          if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
+        });
 
-          const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
+        const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
-            kmsService,
-            appConnectionDAL
-          });
 
-          const findingsPath = join(tempFolder, "findings.json");
+        if (newFindings.length) {
-
+          const finding = newFindings[0] as TSecretScanningFinding;
-          const scanPath = await factory.getFullScanPath({
+          await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
-            dataSource: {
+            status: SecretScanningScanStatus.Completed,
-              ...dataSource,
-              connection
-            } as TSecretScanningDataSourceWithConnection,
             resourceName: resource.name,
-            tempFolder
+            isDiffScan: true,
+            dataSource,
+            numberOfSecrets: newFindings.length,
+            scanId,
+            authorName: finding?.details?.author,
+            authorEmail: finding?.details?.email
           });
+        }
 
-          const config = await secretScanningV2DAL.configs.findOne({
+        await auditLogService.createAuditLog({
-            projectId: dataSource.projectId
+          projectId: dataSource.projectId,
-          });
+          actor: {
-
+            type: ActorType.PLATFORM,
-          let configPath: string | undefined;
+            metadata: {}
-
+          },
-          if (config && config.content) {
+          event: {
-            configPath = join(tempFolder, "infisical-scan.toml");
+            type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
-            await writeTextToFile(configPath, config.content);
+            metadata: {
-          }
+              dataSourceId: dataSource.id,
-
+              dataSourceType: dataSource.type,
-          let findingsPayload: TFindingsPayload;
+              resourceId,
-          switch (resource.type) {
+              resourceType: resource.type,
-            case SecretScanningResource.Repository:
+              scanId,
-            case SecretScanningResource.Project:
+              scanStatus: SecretScanningScanStatus.Completed,
-              findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
+              scanType: SecretScanningScanType.DiffScan,
-              break;
+              numberOfSecretsDetected: findingsPayload.length
-            default:
-              throw new Error("Unhandled resource type");
-          }
-
-          const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
-            let findings: TSecretScanningFindings[] = [];
-            if (findingsPayload.length) {
-              findings = await secretScanningV2DAL.findings.upsert(
-                findingsPayload.map((finding) => ({
-                  ...finding,
-                  projectId: dataSource.projectId,
-                  dataSourceName: dataSource.name,
-                  dataSourceType: dataSource.type,
-                  resourceName: resource.name,
-                  resourceType: resource.type,
-                  scanId
-                })),
-                ["projectId", "fingerprint"],
-                tx,
-                ["resourceName", "dataSourceName"]
-              );
             }
-
-            await secretScanningV2DAL.scans.update(
-              { id: scanId },
-              {
-                status: SecretScanningScanStatus.Completed,
-                statusMessage: null
-              }
-            );
-
-            return findings;
-          });
-
-          const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
-
-          if (newFindings.length) {
-            await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
-              status: SecretScanningScanStatus.Completed,
-              resourceName: resource.name,
-              isDiffScan: false,
-              dataSource,
-              numberOfSecrets: newFindings.length,
-              scanId
-            });
           }
+        });
+
+        logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`);
+      } catch (error) {
+        if (retryCount === retryLimit) {
+          const errorMessage = parseScanErrorMessage(error);
+
+          await secretScanningV2DAL.scans.update(
+            { id: scanId },
+            {
+              status: SecretScanningScanStatus.Failed,
+              statusMessage: errorMessage
+            }
+          );
+
+          await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
+            status: SecretScanningScanStatus.Failed,
+            resourceName: resource.name,
+            dataSource,
+            errorMessage
+          });
 
           await auditLogService.createAuditLog({
             projectId: dataSource.projectId,
@@ -330,348 +547,128 @@ export const secretScanningV2QueueServiceFactory = async ({
                 resourceId: resource.id,
                 resourceType: resource.type,
                 scanId,
-                scanStatus: SecretScanningScanStatus.Completed,
+                scanStatus: SecretScanningScanStatus.Failed,
-                scanType: SecretScanningScanType.FullScan,
+                scanType: SecretScanningScanType.DiffScan
-                numberOfSecretsDetected: findingsPayload.length
               }
             }
           });
-
-          logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`);
-        } catch (error) {
-          if (retryCount === retryLimit) {
-            const errorMessage = parseScanErrorMessage(error);
-
-            await secretScanningV2DAL.scans.update(
-              { id: scanId },
-              {
-                status: SecretScanningScanStatus.Failed,
-                statusMessage: errorMessage
-              }
-            );
-
-            await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
-              status: SecretScanningScanStatus.Failed,
-              resourceName: resource.name,
-              dataSource,
-              errorMessage
-            });
-
-            await auditLogService.createAuditLog({
-              projectId: dataSource.projectId,
-              actor: {
-                type: ActorType.PLATFORM,
-                metadata: {}
-              },
-              event: {
-                type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
-                metadata: {
-                  dataSourceId: dataSource.id,
-                  dataSourceType: dataSource.type,
-                  resourceId: resource.id,
-                  resourceType: resource.type,
-                  scanId,
-                  scanStatus: SecretScanningScanStatus.Failed,
-                  scanType: SecretScanningScanType.FullScan
-                }
-              }
-            });
-          }
-
-          logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
-          throw error;
-        } finally {
-          await deleteTempFolder(tempFolder);
-          await lock?.release();
         }
-      },
+
-      {
+        logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
-        batchSize: 1,
+        throw error;
-        workerCount: 2,
+      } finally {
-        pollingIntervalSeconds: 1
+        await deleteTempFolder(tempFolder);
       }
-    );
+    },
+    {
+      batchSize: 1,
+      workerCount: 2,
+      pollingIntervalSeconds: 1
+    }
+  );
 
-    await queueService.startPg<QueueName.SecretScanningV2>(
+  await queueService.startPg<QueueName.SecretScanningV2>(
-      QueueJobs.SecretScanningV2DiffScan,
+    QueueJobs.SecretScanningV2SendNotification,
-      async ([job]) => {
+    async ([job]) => {
-        const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan;
+      const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification;
-        const { retryCount, retryLimit } = job;
 
-        const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
+      const appCfg = getConfig();
 
-        const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
+      if (!appCfg.isSmtpConfigured) return;
 
-        if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
+      try {
+        const { projectId } = dataSource;
 
-        const resource = await secretScanningV2DAL.resources.findById(resourceId);
+        logger.info(
+          `secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
+        );
 
-        if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
+        const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
+        const project = await projectDAL.findById(projectId);
 
-        const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
+        const recipients = projectMembers.filter((member) => {
-          kmsService,
+          const isAdmin = member.roles.some((role) => role.role === ProjectMembershipRole.Admin);
-          appConnectionDAL
+          const isCompleted = payload.status === SecretScanningScanStatus.Completed;
+          // We assume that the committer is one of the project members
+          const isCommitter = isCompleted && payload.authorEmail === member.user.email;
+          return isAdmin || isCommitter;
         });
 
-        const tempFolder = await createTempFolder();
+        const timestamp = new Date().toISOString();
 
-        try {
+        const subjectLine =
-          await secretScanningV2DAL.scans.update(
+          payload.status === SecretScanningScanStatus.Completed
-            { id: scanId },
+            ? "Incident Alert: Secret(s) Leaked"
-            {
+            : `Secret Scanning Failed`;
-              status: SecretScanningScanStatus.Scanning
-            }
-          );
 
-          let connection: TAppConnection | null = null;
+        await notificationService.createUserNotifications(
-          if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
+          recipients.map((member) => ({
+            userId: member.userId,
+            orgId: project.orgId,
+            type:
+              payload.status === SecretScanningScanStatus.Completed
+                ? NotificationType.SECRET_SCANNING_SECRETS_DETECTED
+                : NotificationType.SECRET_SCANNING_SCAN_FAILED,
+            title: subjectLine,
+            body:
+              payload.status === SecretScanningScanStatus.Completed
+                ? `Uncovered **${payload.numberOfSecrets}** secret(s) ${payload.isDiffScan ? " from a recent commit to" : " in"} **${resourceName}**.`
+                : `Encountered an error while attempting to scan the resource **${resourceName}**: ${payload.errorMessage}`,
+            link:
+              payload.status === SecretScanningScanStatus.Completed
+                ? `/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
+                : `/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
+          }))
+        );
 
-          const config = await secretScanningV2DAL.configs.findOne({
+        await smtpService.sendMail({
-            projectId: dataSource.projectId
+          recipients: recipients.map((member) => member.user.email!).filter(Boolean),
-          });
+          template:
-
-          let configPath: string | undefined;
-
-          if (config && config.content) {
-            configPath = join(tempFolder, "infisical-scan.toml");
-            await writeTextToFile(configPath, config.content);
-          }
-
-          const findingsPayload = await factory.getDiffScanFindingsPayload({
-            dataSource: {
-              ...dataSource,
-              connection
-            } as TSecretScanningDataSourceWithConnection,
-            resourceName: resource.name,
-            payload,
-            configPath
-          });
-
-          const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
-            let findings: TSecretScanningFindings[] = [];
-
-            if (findingsPayload.length) {
-              findings = await secretScanningV2DAL.findings.upsert(
-                findingsPayload.map((finding) => ({
-                  ...finding,
-                  projectId: dataSource.projectId,
-                  dataSourceName: dataSource.name,
-                  dataSourceType: dataSource.type,
-                  resourceName: resource.name,
-                  resourceType: resource.type,
-                  scanId
-                })),
-                ["projectId", "fingerprint"],
-                tx,
-                ["resourceName", "dataSourceName"]
-              );
-            }
-
-            await secretScanningV2DAL.scans.update(
-              { id: scanId },
-              {
-                status: SecretScanningScanStatus.Completed
-              }
-            );
-
-            return findings;
-          });
-
-          const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
-
-          if (newFindings.length) {
-            const finding = newFindings[0] as TSecretScanningFinding;
-            await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
-              status: SecretScanningScanStatus.Completed,
-              resourceName: resource.name,
-              isDiffScan: true,
-              dataSource,
-              numberOfSecrets: newFindings.length,
-              scanId,
-              authorName: finding?.details?.author,
-              authorEmail: finding?.details?.email
-            });
-          }
-
-          await auditLogService.createAuditLog({
-            projectId: dataSource.projectId,
-            actor: {
-              type: ActorType.PLATFORM,
-              metadata: {}
-            },
-            event: {
-              type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
-              metadata: {
-                dataSourceId: dataSource.id,
-                dataSourceType: dataSource.type,
-                resourceId,
-                resourceType: resource.type,
-                scanId,
-                scanStatus: SecretScanningScanStatus.Completed,
-                scanType: SecretScanningScanType.DiffScan,
-                numberOfSecretsDetected: findingsPayload.length
-              }
-            }
-          });
-
-          logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`);
-        } catch (error) {
-          if (retryCount === retryLimit) {
-            const errorMessage = parseScanErrorMessage(error);
-
-            await secretScanningV2DAL.scans.update(
-              { id: scanId },
-              {
-                status: SecretScanningScanStatus.Failed,
-                statusMessage: errorMessage
-              }
-            );
-
-            await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
-              status: SecretScanningScanStatus.Failed,
-              resourceName: resource.name,
-              dataSource,
-              errorMessage
-            });
-
-            await auditLogService.createAuditLog({
-              projectId: dataSource.projectId,
-              actor: {
-                type: ActorType.PLATFORM,
-                metadata: {}
-              },
-              event: {
-                type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
-                metadata: {
-                  dataSourceId: dataSource.id,
-                  dataSourceType: dataSource.type,
-                  resourceId: resource.id,
-                  resourceType: resource.type,
-                  scanId,
-                  scanStatus: SecretScanningScanStatus.Failed,
-                  scanType: SecretScanningScanType.DiffScan
-                }
-              }
-            });
-          }
-
-          logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
-          throw error;
-        } finally {
-          await deleteTempFolder(tempFolder);
-        }
-      },
-      {
-        batchSize: 1,
-        workerCount: 2,
-        pollingIntervalSeconds: 1
-      }
-    );
-
-    await queueService.startPg<QueueName.SecretScanningV2>(
-      QueueJobs.SecretScanningV2SendNotification,
-      async ([job]) => {
-        const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification;
-
-        const appCfg = getConfig();
-
-        if (!appCfg.isSmtpConfigured) return;
-
-        try {
-          const { projectId } = dataSource;
-
-          logger.info(
-            `secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
-          );
-
-          const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
-          const project = await projectDAL.findById(projectId);
-
-          const recipients = projectMembers.filter((member) => {
-            const isAdmin = member.roles.some((role) => role.role === ProjectMembershipRole.Admin);
-            const isCompleted = payload.status === SecretScanningScanStatus.Completed;
-            // We assume that the committer is one of the project members
-            const isCommitter = isCompleted && payload.authorEmail === member.user.email;
-            return isAdmin || isCommitter;
-          });
-
-          const timestamp = new Date().toISOString();
-
-          const subjectLine =
             payload.status === SecretScanningScanStatus.Completed
-              ? "Incident Alert: Secret(s) Leaked"
+              ? SmtpTemplates.SecretScanningV2SecretsDetected
-              : `Secret Scanning Failed`;
+              : SmtpTemplates.SecretScanningV2ScanFailed,
-
+          subjectLine,
-          await notificationService.createUserNotifications(
+          substitutions:
-            recipients.map((member) => ({
+            payload.status === SecretScanningScanStatus.Completed
-              userId: member.userId,
+              ? {
-              orgId: project.orgId,
+                  authorName: payload.authorName,
-              type:
+                  authorEmail: payload.authorEmail,
-                payload.status === SecretScanningScanStatus.Completed
+                  resourceName,
-                  ? NotificationType.SECRET_SCANNING_SECRETS_DETECTED
+                  numberOfSecrets: payload.numberOfSecrets,
-                  : NotificationType.SECRET_SCANNING_SCAN_FAILED,
+                  isDiffScan: payload.isDiffScan,
-              title: subjectLine,
+                  url: encodeURI(
-              body:
+                    `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
-                payload.status === SecretScanningScanStatus.Completed
+                  ),
-                  ? `Uncovered **${payload.numberOfSecrets}** secret(s) ${payload.isDiffScan ? " from a recent commit to" : " in"} **${resourceName}**.`
+                  timestamp
-                  : `Encountered an error while attempting to scan the resource **${resourceName}**: ${payload.errorMessage}`,
+                }
-              link:
+              : {
-                payload.status === SecretScanningScanStatus.Completed
+                  dataSourceName: dataSource.name,
-                  ? `/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
+                  resourceName,
-                  : `/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
+                  projectName: project.name,
-            }))
+                  timestamp,
-          );
+                  errorMessage: payload.errorMessage,
-
+                  url: encodeURI(
-          await smtpService.sendMail({
+                    `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
-            recipients: recipients.map((member) => member.user.email!).filter(Boolean),
+                  )
-            template:
+                }
-              payload.status === SecretScanningScanStatus.Completed
+        });
-                ? SmtpTemplates.SecretScanningV2SecretsDetected
+      } catch (error) {
-                : SmtpTemplates.SecretScanningV2ScanFailed,
+        logger.error(
-            subjectLine,
+          error,
-            substitutions:
+          `secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
-              payload.status === SecretScanningScanStatus.Completed
+        );
-                ? {
+        throw error;
-                    authorName: payload.authorName,
-                    authorEmail: payload.authorEmail,
-                    resourceName,
-                    numberOfSecrets: payload.numberOfSecrets,
-                    isDiffScan: payload.isDiffScan,
-                    url: encodeURI(
-                      `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
-                    ),
-                    timestamp
-                  }
-                : {
-                    dataSourceName: dataSource.name,
-                    resourceName,
-                    projectName: project.name,
-                    timestamp,
-                    errorMessage: payload.errorMessage,
-                    url: encodeURI(
-                      `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
-                    )
-                  }
-          });
-        } catch (error) {
-          logger.error(
-            error,
-            `secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
-          );
-          throw error;
-        }
-      },
-      {
-        batchSize: 1,
-        workerCount: 2,
-        pollingIntervalSeconds: 1
       }
-    );
+    },
-  };
+    {
+      batchSize: 1,
+      workerCount: 2,
+      pollingIntervalSeconds: 1
+    }
+  );
 
   return {
     queueDataSourceFullScan,
-    queueResourceDiffScan,
+    queueResourceDiffScan
-    init
   };
 };

backend/src/main.ts

@@ -16,7 +16,7 @@ import { buildRedisFromConfig } from "./lib/config/redis";
 import { removeTemporaryBaseDirectory } from "./lib/files";
 import { initLogger } from "./lib/logger";
 import { queueServiceFactory } from "./queue";
-import { main, markRunningMigrations, markServerReady } from "./server/app";
+import { main } from "./server/app";
 import { bootstrapCheck } from "./server/boot-strap-check";
 import { kmsRootConfigDALFactory } from "./services/kms/kms-root-config-dal";
 import { smtpServiceFactory } from "./services/smtp/smtp-service";
@@ -59,6 +59,8 @@ const run = async () => {
       })
     : undefined;
 
+  await runMigrations({ applicationDb: db, auditLogDb, logger });
+
   const smtp = smtpServiceFactory(formatSmtpConfig());
 
   const queue = queueServiceFactory(envConfig, {
@@ -72,7 +74,7 @@ const run = async () => {
   const keyStore = keyStoreFactory(envConfig, keyValueStoreDAL);
   const redis = buildRedisFromConfig(envConfig);
 
-  const { server, completeServerInitialization } = await main({
+  const server = await main({
     db,
     auditLogDb,
     superAdminDAL,
@@ -85,6 +87,7 @@ const run = async () => {
     redis,
     envConfig
   });
+  const bootstrap = await bootstrapCheck({ db });
 
   // eslint-disable-next-line
   process.on("SIGINT", async () => {
@@ -118,46 +121,12 @@ const run = async () => {
 
   await server.listen({
     port: envConfig.PORT,
-    host: envConfig.HOST
+    host: envConfig.HOST,
-  });
+    listenTextResolver: (address) => {
-
+      void bootstrap();
-  logger.info(`Server listening on ${envConfig.HOST}:${envConfig.PORT}`);
+      return address;
-  logger.info("Running migrations...");
-
-  // Run migrations while server is up
-  // All containers start as NOT HEALTHY (waiting for migrations)
-  // Container that acquires lock: becomes HEALTHY (running migrations) + NOT READY (no traffic)
-  // Other containers waiting: stay NOT HEALTHY (waiting) + NOT READY (no traffic)
-  await runMigrations({
-    applicationDb: db,
-    auditLogDb,
-    logger,
-    onMigrationLockAcquired: () => {
-      // Called after successfully acquiring the lock
-      // This container is now the migration runner
-      markRunningMigrations();
-      logger.info("Migration lock acquired! This container is running migrations.");
     }
   });
-
-  logger.info("Migrations complete. Completing server initialization...");
-
-  try {
-    await completeServerInitialization();
-  } catch (error) {
-    logger.error(error, "Failed to complete server initialization");
-    await server.close();
-    await queue.shutdown();
-    process.exit(1);
-  }
-
-  logger.info("Server initialization complete. Marking server as READY...");
-
-  markServerReady();
-
-  logger.info("Server is ready to accept traffic");
-  const bootstrap = await bootstrapCheck({ db });
-  void bootstrap();
 };
 
 void run();

backend/src/server/app.ts

@@ -1,6 +1,5 @@
 /* eslint-disable import/extensions */
 import path from "node:path";
-import { monitorEventLoopDelay } from "perf_hooks";
 
 import type { FastifyCookieOptions } from "@fastify/cookie";
 import cookie from "@fastify/cookie";
@@ -25,7 +24,6 @@ import { TQueueServiceFactory } from "@app/queue";
 import { TKmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal";
 import { TSmtpService } from "@app/services/smtp/smtp-service";
 import { TSuperAdminDALFactory } from "@app/services/super-admin/super-admin-dal";
-import { getServerCfg } from "@app/services/super-admin/super-admin-service";
 
 import { globalRateLimiterCfg } from "./config/rateLimiter";
 import { addErrorsToResponseSchemas } from "./plugins/add-errors-to-response-schemas";
@@ -38,15 +36,6 @@ import { registerServeUI } from "./plugins/serve-ui";
 import { fastifySwagger } from "./plugins/swagger";
 import { registerRoutes } from "./routes";
 
-const histogram = monitorEventLoopDelay({ resolution: 20 });
-histogram.enable();
-
-const serverState = {
-  isReady: false,
-  isRunningMigrations: false,
-  isWaitingForMigrations: true // Start as true - containers are unhealthy until they acquire migration lock or complete
-};
-
 type TMain = {
   auditLogDb?: Knex;
   db: Knex;
@@ -156,72 +145,7 @@ export const main = async ({
       })
     });
 
-    // Health check - returns 200 only if doing useful work (running migrations or ready)
+    await server.register(registerRoutes, {
-    // Returns 503 if waiting for another container to finish migrations
-    server.get("/api/health", async (_, reply) => {
-      if (serverState.isWaitingForMigrations) {
-        return reply.code(503).send({
-          status: "waiting",
-          message: "Waiting for migrations to complete in another container"
-        });
-      }
-      return { status: "ok", message: "Server is alive" };
-    });
-
-    // Global preHandler to block requests during migrations
-    server.addHook("preHandler", async (request, reply) => {
-      if (request.url === "/api/health" || request.url === "/api/ready") {
-        return;
-      }
-      if (!serverState.isReady) {
-        return reply.code(503).send({
-          status: "unavailable",
-          message: "Server is starting up, migrations in progress. Please try again in a moment."
-        });
-      }
-    });
-
-    // Readiness check - returns 503 until migrations are complete
-    server.get("/api/ready", async (request, reply) => {
-      const cfg = getConfig();
-
-      const meanLagMs = histogram.mean / 1e6;
-      const maxLagMs = histogram.max / 1e6;
-      const p99LagMs = histogram.percentile(99) / 1e6;
-
-      request.log.info(
-        `Event loop stats - Mean: ${meanLagMs.toFixed(2)}ms, Max: ${maxLagMs.toFixed(2)}ms, p99: ${p99LagMs.toFixed(2)}ms`
-      );
-
-      request.log.info(`Raw event loop stats: ${JSON.stringify(histogram, null, 2)}`);
-
-      if (!serverState.isReady) {
-        return reply.code(503).send({
-          date: new Date(),
-          message: "Server is starting up, migrations in progress",
-          emailConfigured: cfg.isSmtpConfigured,
-          redisConfigured: cfg.isRedisConfigured,
-          secretScanningConfigured: cfg.isSecretScanningConfigured,
-          samlDefaultOrgSlug: cfg.samlDefaultOrgSlug,
-          auditLogStorageDisabled: Boolean(cfg.DISABLE_AUDIT_LOG_STORAGE)
-        });
-      }
-
-      const serverCfg = await getServerCfg();
-
-      return {
-        date: new Date(),
-        message: "Ok",
-        emailConfigured: cfg.isSmtpConfigured,
-        inviteOnlySignup: Boolean(serverCfg.allowSignUp),
-        redisConfigured: cfg.isRedisConfigured,
-        secretScanningConfigured: cfg.isSecretScanningConfigured,
-        samlDefaultOrgSlug: cfg.samlDefaultOrgSlug,
-        auditLogStorageDisabled: Boolean(cfg.DISABLE_AUDIT_LOG_STORAGE)
-      };
-    });
-
-    const completeServerInitialization = await registerRoutes(server, {
       smtp,
       queue,
       db,
@@ -240,21 +164,10 @@ export const main = async ({
 
     await server.ready();
     server.swagger();
-    return { server, completeServerInitialization };
+    return server;
   } catch (err) {
     server.log.error(err);
     await queue.shutdown();
     process.exit(1);
   }
 };
-
-export const markServerReady = () => {
-  serverState.isReady = true;
-  serverState.isRunningMigrations = false;
-  serverState.isWaitingForMigrations = false;
-};
-
-export const markRunningMigrations = () => {
-  serverState.isRunningMigrations = true;
-  serverState.isWaitingForMigrations = false;
-};

backend/src/server/routes/index.ts

@@ -2208,7 +2208,7 @@ export const registerRoutes = async (
     internalCaFns
   });
 
-  const secretRotationV2Queue = await secretRotationV2QueueServiceFactory({
+  await secretRotationV2QueueServiceFactory({
     secretRotationV2Service,
     secretRotationV2DAL,
     queueService,
@@ -2305,6 +2305,8 @@ export const registerRoutes = async (
   // If FIPS is enabled, we check to ensure that the users license includes FIPS mode.
   crypto.verifyFipsLicense(licenseService);
 
+  await superAdminService.initServerCfg();
+
   // Start HSM service if it's configured/enabled.
   await hsmService.startService();
 
@@ -2329,60 +2331,21 @@ export const registerRoutes = async (
     }
   }
 
-  const completeServerInitialization = async () => {
+  await telemetryQueue.startTelemetryCheck();
-    await superAdminService.initServerCfg();
+  await telemetryQueue.startAggregatedEventsJob();
-
+  await dailyResourceCleanUp.init();
-    await telemetryQueue.startTelemetryCheck();
+  await healthAlert.init();
-    await telemetryQueue.startAggregatedEventsJob();
+  await pkiSyncCleanup.init();
-    await dailyResourceCleanUp.init();
+  await pamAccountRotation.init();
-    await healthAlert.init();
+  await dailyReminderQueueService.startDailyRemindersJob();
-    await pkiSyncCleanup.init();
+  await dailyReminderQueueService.startSecretReminderMigrationJob();
-    await pamAccountRotation.init();
+  await dailyExpiringPkiItemAlert.startSendingAlerts();
-    await dailyReminderQueueService.startDailyRemindersJob();
+  await pkiSubscriberQueue.startDailyAutoRenewalJob();
-    await dailyReminderQueueService.startSecretReminderMigrationJob();
+  await certificateV3Queue.init();
-    await dailyExpiringPkiItemAlert.startSendingAlerts();
+  await kmsService.startService(hsmStatus);
-    await pkiSubscriberQueue.startDailyAutoRenewalJob();
+  await microsoftTeamsService.start();
-    await certificateV3Queue.init();
+  await dynamicSecretQueueService.init();
-    await kmsService.startService(hsmStatus);
+  await eventBusService.init();
-    await microsoftTeamsService.start();
-    await dynamicSecretQueueService.init();
-    await secretScanningV2Queue.init();
-    await secretRotationV2Queue.init();
-    await notificationQueue.init();
-    await eventBusService.init();
-
-    const cronJobs: CronJob[] = [];
-    if (appCfg.isProductionMode) {
-      const rateLimitSyncJob = await rateLimitService.initializeBackgroundSync();
-      if (rateLimitSyncJob) {
-        cronJobs.push(rateLimitSyncJob);
-      }
-      const licenseSyncJob = await licenseService.initializeBackgroundSync();
-      if (licenseSyncJob) {
-        cronJobs.push(licenseSyncJob);
-      }
-
-      const microsoftTeamsSyncJob = await microsoftTeamsService.initializeBackgroundSync();
-      if (microsoftTeamsSyncJob) {
-        cronJobs.push(microsoftTeamsSyncJob);
-      }
-
-      const adminIntegrationsSyncJob = await superAdminService.initializeAdminIntegrationConfigSync();
-      if (adminIntegrationsSyncJob) {
-        cronJobs.push(adminIntegrationsSyncJob);
-      }
-    }
-
-    const configSyncJob = await superAdminService.initializeEnvConfigSync();
-    if (configSyncJob) {
-      cronJobs.push(configSyncJob);
-    }
-
-    const oauthConfigSyncJob = await initializeOauthConfigSync();
-    if (oauthConfigSyncJob) {
-      cronJobs.push(oauthConfigSyncJob);
-    }
-  };
 
   // inject all services
   server.decorate<FastifyZodProvider["services"]>("services", {
@@ -2512,6 +2475,38 @@ export const registerRoutes = async (
     convertor: convertorService
   });
 
+  const cronJobs: CronJob[] = [];
+  if (appCfg.isProductionMode) {
+    const rateLimitSyncJob = await rateLimitService.initializeBackgroundSync();
+    if (rateLimitSyncJob) {
+      cronJobs.push(rateLimitSyncJob);
+    }
+    const licenseSyncJob = await licenseService.initializeBackgroundSync();
+    if (licenseSyncJob) {
+      cronJobs.push(licenseSyncJob);
+    }
+
+    const microsoftTeamsSyncJob = await microsoftTeamsService.initializeBackgroundSync();
+    if (microsoftTeamsSyncJob) {
+      cronJobs.push(microsoftTeamsSyncJob);
+    }
+
+    const adminIntegrationsSyncJob = await superAdminService.initializeAdminIntegrationConfigSync();
+    if (adminIntegrationsSyncJob) {
+      cronJobs.push(adminIntegrationsSyncJob);
+    }
+  }
+
+  const configSyncJob = await superAdminService.initializeEnvConfigSync();
+  if (configSyncJob) {
+    cronJobs.push(configSyncJob);
+  }
+
+  const oauthConfigSyncJob = await initializeOauthConfigSync();
+  if (oauthConfigSyncJob) {
+    cronJobs.push(oauthConfigSyncJob);
+  }
+
   server.decorate<FastifyZodProvider["store"]>("store", {
     user: userDAL,
     kmipClient: kmipClientDAL
@@ -2600,10 +2595,9 @@ export const registerRoutes = async (
   await server.register(registerV4Routes, { prefix: "/api/v4" });
 
   server.addHook("onClose", async () => {
+    cronJobs.forEach((job) => job.stop());
     await telemetryService.flushAll();
     await eventBusService.close();
     sseService.close();
   });
-
-  return completeServerInitialization;
 };

backend/src/services/notification/notification-queue.ts

@@ -10,7 +10,6 @@ type TNotificationQueueServiceFactoryDep = {
 
 export type TNotificationQueueServiceFactory = {
   pushUserNotifications: (data: TCreateUserNotificationDTO[]) => Promise<void>;
-  init: () => Promise<void>;
 };
 
 export const notificationQueueServiceFactory = async ({
@@ -21,23 +20,20 @@ export const notificationQueueServiceFactory = async ({
     await queueService.queuePg(QueueJobs.UserNotification, { notifications: data });
   };
 
-  const init = async () => {
+  await queueService.startPg(
-    await queueService.startPg(
+    QueueJobs.UserNotification,
-      QueueJobs.UserNotification,
+    async ([job]) => {
-      async ([job]) => {
+      const { notifications } = job.data as { notifications: TCreateUserNotificationDTO[] };
-        const { notifications } = job.data as { notifications: TCreateUserNotificationDTO[] };
+      await userNotificationDAL.batchInsert(notifications);
-        await userNotificationDAL.batchInsert(notifications);
+    },
-      },
+    {
-      {
+      batchSize: 1,
-        batchSize: 1,
+      workerCount: 2,
-        workerCount: 2,
+      pollingIntervalSeconds: 1
-        pollingIntervalSeconds: 1
+    }
-      }
+  );
-    );
-  };
 
   return {
-    pushUserNotifications,
+    pushUserNotifications
-    init
   };
 };

docs/self-hosting/guides/production-hardening.mdx

@@ -137,33 +137,6 @@ Configure database read replicas for high availability PostgreSQL setups:
 DB_READ_REPLICAS='[{"DB_CONNECTION_URI":"postgresql://user:pass@replica:5432/db?sslmode=require"}]'
 ```
 
-### Health Check Endpoints
-
-Infisical provides two health check endpoints for proper container orchestration and load balancer integration:
-
-#### `/api/health` - Container Health Check
-
-Determines whether the application container should be kept alive or terminated.
-
-- Returns `200` if the application is running and operational
-- Returns `200` even during startup tasks
-- Returns `503` only if the application has crashed or is unable to start
-
-**Use for**: Docker health checks, Kubernetes liveness probes, ECS task health checks.
-
-#### `/api/ready` - Traffic Readiness Check
-
-Determines whether the application instance is ready to receive production traffic.
-
-- Returns `200` when the application is fully ready to serve requests
-- Returns `503` during startup tasks (e.g., database migrations, initialization)
-
-**Use for**: Load balancer health checks, Kubernetes readiness probes, ALB target health checks.
-
-#### Why Two Endpoints?
-
-Using both endpoints together enables zero-downtime deployments: containers stay alive during startup tasks (`/api/health` returns `200`) while load balancers avoid sending traffic to instances that aren't ready (`/api/ready` returns `503`). This ensures existing instances continue serving traffic until new instances complete their initialization.
-
 ### Operational Security
 
 #### User Access Management
@@ -234,17 +207,14 @@ docker run --memory=1g --cpus=0.5 infisical/infisical:latest
 
 #### Health Monitoring
 
-**Configure health checks**. Set up Docker health checks using the appropriate endpoint:
+**Configure health checks**. Set up Docker health checks:
 
 ```dockerfile
 # In Dockerfile or docker-compose.yml
-# Use /api/health for container health (keeps container alive during startup)
 HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
-  CMD curl -f http://localhost:8080/api/health || exit 1
+  CMD curl -f http://localhost:8080/api/status || exit 1
 ```
 
-**Note**: Use `/api/health` for container health checks and `/api/ready` for load balancer readiness checks. See [Health Check Endpoints](#health-check-endpoints) for detailed information.
-
 #### Network Security
 
 **Host firewall configuration**. Configure host-level firewall for Docker deployments:
@@ -463,32 +433,26 @@ stringData:
 
 #### Health Monitoring
 
-**Set up health checks**. Configure readiness and liveness probes using the appropriate endpoints:
+**Set up health checks**. Configure readiness and liveness probes:
 
 ```yaml
 # Health check configuration
 containers:
   - name: infisical
-    # Use /api/ready for readiness (traffic routing)
     readinessProbe:
       httpGet:
-        path: /api/ready
+        path: /api/status
         port: 8080
       initialDelaySeconds: 10
       periodSeconds: 5
-      failureThreshold: 3
-    # Use /api/health for liveness (container restart)
     livenessProbe:
       httpGet:
-        path: /api/health
+        path: /api/status
         port: 8080
       initialDelaySeconds: 30
       periodSeconds: 10
-      failureThreshold: 3
 ```
 
-**Important**: The `readinessProbe` uses `/api/ready` to ensure traffic is only sent to pods that are fully initialized. The `livenessProbe` uses `/api/health` to keep the container alive during startup. See [Health Check Endpoints](#health-check-endpoints) for detailed information.
-
 #### Infrastructure Considerations
 
 **Use managed databases (if possible)**. For production deployments, consider using managed PostgreSQL and Redis services instead of in-cluster instances when feasible, as they typically provide better security, backup, and maintenance capabilities.