Merge branch 'main' into ENG-4006

backend/e2e-test/vitest-environment-knex.ts

@@ -8,7 +8,7 @@ import path from "path";
 import { seedData1 } from "@app/db/seed-data";
 import { getDatabaseCredentials, getHsmConfig, initEnvConfig } from "@app/lib/config/env";
 import { initLogger } from "@app/lib/logger";
-import { main } from "@app/server/app";
+import { main, markServerReady } from "@app/server/app";
 import { AuthMethod, AuthTokenType } from "@app/services/auth/auth-type";
 
 import { mockSmtpServer } from "./mocks/smtp";
@@ -83,7 +83,7 @@ export default {
 
       await queue.initialize();
 
-      const server = await main({
+      const { server, completeServerInitialization } = await main({
         db,
         smtp,
         logger,
@@ -96,6 +96,10 @@ export default {
         envConfig: envCfg
       });
 
+      await completeServerInitialization();
+
+      markServerReady();
+
       await bootstrapCheck({ db });
 
       // @ts-expect-error type

backend/src/auto-start-migrations.ts

@@ -12,6 +12,7 @@ type TArgs = {
   auditLogDb?: Knex;
   applicationDb: Knex;
   logger: Logger;
+  onMigrationLockAcquired?: () => void;
 };
 
 const isProduction = process.env.NODE_ENV === "production";
@@ -30,7 +31,7 @@ const migrationStatusCheckErrorHandler = (err: Error) => {
   throw err;
 };
 
-export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs) => {
+export const runMigrations = async ({ applicationDb, auditLogDb, logger, onMigrationLockAcquired }: TArgs) => {
   try {
     // akhilmhdh(Feb 10 2025): 2 years  from now remove this
     if (isProduction) {
@@ -85,6 +86,13 @@ export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs
 
     await applicationDb.transaction(async (tx) => {
       await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]);
+
+      // Signal that this container is running migrations so that it can be marked as healthy/alive
+      // This is to prevent the container from being killed by the orchestrator
+      if (onMigrationLockAcquired) {
+        onMigrationLockAcquired();
+      }
+
       logger.info("Running application migrations.");
 
       const didPreviousInstanceRunMigration = !(await applicationDb.migrate

backend/src/ee/routes/v1/dynamic-secret-lease-router.ts

@@ -2,7 +2,6 @@ import { z } from "zod";
 
 import { DynamicSecretLeasesSchema } from "@app/db/schemas";
 import { ApiDocsTags, DYNAMIC_SECRET_LEASES } from "@app/lib/api-docs";
-import { daysToMillisecond } from "@app/lib/dates";
 import { removeTrailingSlash } from "@app/lib/fn";
 import { ms } from "@app/lib/ms";
 import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
@@ -32,8 +31,8 @@ export const registerDynamicSecretLeaseRouter = async (server: FastifyZodProvide
             const valMs = ms(val);
             if (valMs < 60 * 1000)
               ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be a greater than 1min" });
-            if (valMs > daysToMillisecond(1))
-              ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" });
+            if (valMs > ms("10y"))
+              ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than 10 years" });
           }),
         path: z.string().trim().default("/").transform(removeTrailingSlash).describe(DYNAMIC_SECRET_LEASES.CREATE.path),
         environmentSlug: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.CREATE.environmentSlug),
@@ -127,8 +126,8 @@ export const registerDynamicSecretLeaseRouter = async (server: FastifyZodProvide
             const valMs = ms(val);
             if (valMs < 60 * 1000)
               ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be a greater than 1min" });
-            if (valMs > daysToMillisecond(1))
-              ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" });
+            if (valMs > ms("10y"))
+              ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than 10 years" });
           }),
         projectSlug: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.RENEW.projectSlug),
         path: z

backend/src/ee/routes/v1/dynamic-secret-lease-routers/kubernetes-lease-router.ts

@@ -2,7 +2,6 @@ import { z } from "zod";
 
 import { DynamicSecretLeasesSchema } from "@app/db/schemas";
 import { ApiDocsTags, DYNAMIC_SECRET_LEASES } from "@app/lib/api-docs";
-import { daysToMillisecond } from "@app/lib/dates";
 import { removeTrailingSlash } from "@app/lib/fn";
 import { ms } from "@app/lib/ms";
 import { writeLimit } from "@app/server/config/rateLimiter";
@@ -32,8 +31,8 @@ export const registerKubernetesDynamicSecretLeaseRouter = async (server: Fastify
             const valMs = ms(val);
             if (valMs < 60 * 1000)
               ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be greater than 1min" });
-            if (valMs > daysToMillisecond(1))
-              ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" });
+            if (valMs > ms("10y"))
+              ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than 10 years" });
           }),
         path: z.string().trim().default("/").transform(removeTrailingSlash).describe(DYNAMIC_SECRET_LEASES.CREATE.path),
         environmentSlug: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.CREATE.environmentSlug),

backend/src/ee/routes/v1/dynamic-secret-router.ts

@@ -3,7 +3,6 @@ import { z } from "zod";
 import { DynamicSecretLeasesSchema } from "@app/db/schemas";
 import { DynamicSecretProviderSchema } from "@app/ee/services/dynamic-secret/providers/models";
 import { ApiDocsTags, DYNAMIC_SECRETS } from "@app/lib/api-docs";
-import { daysToMillisecond } from "@app/lib/dates";
 import { removeTrailingSlash } from "@app/lib/fn";
 import { ms } from "@app/lib/ms";
 import { isValidHandleBarTemplate } from "@app/lib/template/validate-handlebars";
@@ -60,8 +59,8 @@ export const registerDynamicSecretRouter = async (server: FastifyZodProvider) =>
             const valMs = ms(val);
             if (valMs < 60 * 1000)
               ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be a greater than 1min" });
-            if (valMs > daysToMillisecond(1))
-              ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" });
+            if (valMs > ms("10y"))
+              ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than 10 years" });
           }),
         maxTTL: z
           .string()
@@ -72,8 +71,8 @@ export const registerDynamicSecretRouter = async (server: FastifyZodProvider) =>
             const valMs = ms(val);
             if (valMs < 60 * 1000)
               ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be a greater than 1min" });
-            if (valMs > daysToMillisecond(1))
-              ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" });
+            if (valMs > ms("10y"))
+              ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than 10 years" });
           })
           .nullable(),
         path: z.string().describe(DYNAMIC_SECRETS.CREATE.path).trim().default("/").transform(removeTrailingSlash),
@@ -130,8 +129,8 @@ export const registerDynamicSecretRouter = async (server: FastifyZodProvider) =>
               const valMs = ms(val);
               if (valMs < 60 * 1000)
                 ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be a greater than 1min" });
-              if (valMs > daysToMillisecond(1))
-                ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" });
+              if (valMs > ms("10y"))
+                ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than 10 years" });
             }),
           maxTTL: z
             .string()
@@ -142,8 +141,8 @@ export const registerDynamicSecretRouter = async (server: FastifyZodProvider) =>
               const valMs = ms(val);
               if (valMs < 60 * 1000)
                 ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be a greater than 1min" });
-              if (valMs > daysToMillisecond(1))
-                ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" });
+              if (valMs > ms("10y"))
+                ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than 10 years" });
             })
             .nullable(),
           newName: z.string().describe(DYNAMIC_SECRETS.UPDATE.newName).optional(),

backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts

@@ -112,7 +112,7 @@ export const dynamicSecretServiceFactory = ({
 
     const existingDynamicSecret = await dynamicSecretDAL.findOne({ name, folderId: folder.id });
     if (existingDynamicSecret)
-      throw new BadRequestError({ message: "Provided dynamic secret already exist under the folder" });
+      throw new BadRequestError({ message: "Provided dynamic secret already exists under the folder" });
 
     const selectedProvider = dynamicSecretProviders[provider.type];
     const inputs = await selectedProvider.validateProviderInputs(provider.inputs, { projectId });
@@ -265,7 +265,7 @@ export const dynamicSecretServiceFactory = ({
     if (newName) {
       const existingDynamicSecret = await dynamicSecretDAL.findOne({ name: newName, folderId: folder.id });
       if (existingDynamicSecret)
-        throw new BadRequestError({ message: "Provided dynamic secret already exist under the folder" });
+        throw new BadRequestError({ message: "Provided dynamic secret already exists under the folder" });
     }
     const { encryptor: secretManagerEncryptor, decryptor: secretManagerDecryptor } =
       await kmsService.createCipherPairWithDataKey({

backend/src/ee/services/hsm/hsm-fns.ts

@@ -84,7 +84,7 @@ export const isHsmActiveAndEnabled = async ({
 
   rootKmsConfigEncryptionStrategy = (rootKmsConfig?.encryptionStrategy || null) as RootKeyEncryptionStrategy | null;
   if (
-    rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.HSM &&
+    (rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.HSM || isHsmConfigured) &&
     licenseService &&
     !licenseService.onPremFeatures.hsm
   ) {

backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts

@@ -1517,7 +1517,7 @@ export const secretApprovalRequestServiceFactory = ({
         }))
       );
       if (secrets.length)
-        throw new BadRequestError({ message: `Secret already exist: ${secrets.map((el) => el.key).join(",")}` });
+        throw new BadRequestError({ message: `Secret already exists: ${secrets.map((el) => el.key).join(",")}` });
 
       commits.push(
         ...createdSecrets.map((createdSecret) => ({

backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts

@@ -42,168 +42,172 @@ export const secretRotationV2QueueServiceFactory = async ({
   smtpService,
   notificationService
 }: TSecretRotationV2QueueServiceFactoryDep) => {
-  const appCfg = getConfig();
+  const init = async () => {
+    const appCfg = getConfig();
 
-  if (appCfg.isRotationDevelopmentMode) {
-    logger.warn("Secret Rotation V2 is in development mode.");
-  }
+    if (appCfg.isRotationDevelopmentMode) {
+      logger.warn("Secret Rotation V2 is in development mode.");
+    }
 
-  await queueService.startPg<QueueName.SecretRotationV2>(
-    QueueJobs.SecretRotationV2QueueRotations,
-    async () => {
-      try {
-        const rotateBy = getNextUtcRotationInterval();
+    await queueService.startPg<QueueName.SecretRotationV2>(
+      QueueJobs.SecretRotationV2QueueRotations,
+      async () => {
+        try {
+          const rotateBy = getNextUtcRotationInterval();
 
-        const currentTime = new Date();
+          const currentTime = new Date();
 
-        const secretRotations = await secretRotationV2DAL.findSecretRotationsToQueue(rotateBy);
+          const secretRotations = await secretRotationV2DAL.findSecretRotationsToQueue(rotateBy);
 
-        logger.info(
-          `secretRotationV2Queue: Queue Rotations [currentTime=${currentTime.toISOString()}] [rotateBy=${rotateBy.toISOString()}] [count=${
-            secretRotations.length
-          }]`
-        );
-
-        for await (const rotation of secretRotations) {
           logger.info(
-            `secretRotationV2Queue: Queue Rotation [rotationId=${rotation.id}] [lastRotatedAt=${new Date(
-              rotation.lastRotatedAt
-            ).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]`
+            `secretRotationV2Queue: Queue Rotations [currentTime=${currentTime.toISOString()}] [rotateBy=${rotateBy.toISOString()}] [count=${
+              secretRotations.length
+            }]`
           );
 
-          const data = {
-            rotationId: rotation.id,
-            queuedAt: currentTime
-          } as TSecretRotationRotateSecretsJobPayload;
-
-          if (appCfg.isTestMode) {
-            logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode");
-            await rotateSecretsFns({
-              job: {
-                id: uuidv4(),
-                data,
-                retryCount: 0,
-                retryLimit: 0
-              },
-              secretRotationV2DAL,
-              secretRotationV2Service
-            });
-          } else {
-            await queueService.queuePg(
-              QueueJobs.SecretRotationV2RotateSecrets,
-              {
-                rotationId: rotation.id,
-                queuedAt: currentTime
-              },
-              getSecretRotationRotateSecretJobOptions(rotation)
+          for await (const rotation of secretRotations) {
+            logger.info(
+              `secretRotationV2Queue: Queue Rotation [rotationId=${rotation.id}] [lastRotatedAt=${new Date(
+                rotation.lastRotatedAt
+              ).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]`
             );
+
+            const data = {
+              rotationId: rotation.id,
+              queuedAt: currentTime
+            } as TSecretRotationRotateSecretsJobPayload;
+
+            if (appCfg.isTestMode) {
+              logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode");
+              await rotateSecretsFns({
+                job: {
+                  id: uuidv4(),
+                  data,
+                  retryCount: 0,
+                  retryLimit: 0
+                },
+                secretRotationV2DAL,
+                secretRotationV2Service
+              });
+            } else {
+              await queueService.queuePg(
+                QueueJobs.SecretRotationV2RotateSecrets,
+                {
+                  rotationId: rotation.id,
+                  queuedAt: currentTime
+                },
+                getSecretRotationRotateSecretJobOptions(rotation)
+              );
+            }
           }
+        } catch (error) {
+          logger.error(error, "secretRotationV2Queue: Queue Rotations Error:");
+          throw error;
         }
-      } catch (error) {
-        logger.error(error, "secretRotationV2Queue: Queue Rotations Error:");
-        throw error;
+      },
+      {
+        batchSize: 1,
+        workerCount: 1,
+        pollingIntervalSeconds: appCfg.isRotationDevelopmentMode ? 0.5 : 30
       }
-    },
-    {
-      batchSize: 1,
-      workerCount: 1,
-      pollingIntervalSeconds: appCfg.isRotationDevelopmentMode ? 0.5 : 30
-    }
-  );
+    );
 
-  await queueService.startPg<QueueName.SecretRotationV2>(
-    QueueJobs.SecretRotationV2RotateSecrets,
-    async ([job]) => {
-      await rotateSecretsFns({
-        job: {
-          ...job,
-          data: job.data as TSecretRotationRotateSecretsJobPayload
-        },
-        secretRotationV2DAL,
-        secretRotationV2Service
-      });
-    },
-    {
-      batchSize: 1,
-      workerCount: 2,
-      pollingIntervalSeconds: 0.5
-    }
-  );
-
-  await queueService.startPg<QueueName.SecretRotationV2>(
-    QueueJobs.SecretRotationV2SendNotification,
-    async ([job]) => {
-      const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload;
-      try {
-        const {
-          name: rotationName,
-          type,
-          projectId,
-          lastRotationAttemptedAt,
-          folder,
-          environment,
-          id: rotationId
-        } = secretRotation;
-
-        logger.info(`secretRotationV2Queue: Sending Status Notification [rotationId=${rotationId}]`);
-
-        const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
-        const project = await projectDAL.findById(projectId);
-
-        const projectAdmins = projectMembers.filter((member) =>
-          member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
-        );
-
-        const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation];
-
-        const rotationPath = `/projects/secret-management/${projectId}/secrets/${environment.slug}`;
-
-        await notificationService.createUserNotifications(
-          projectAdmins.map((admin) => ({
-            userId: admin.userId,
-            orgId: project.orgId,
-            type: NotificationType.SECRET_ROTATION_FAILED,
-            title: "Secret Rotation Failed",
-            body: `Your **${rotationType}** rotation **${rotationName}** failed to rotate.`,
-            link: rotationPath
-          }))
-        );
-
-        await smtpService.sendMail({
-          recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
-          template: SmtpTemplates.SecretRotationFailed,
-          subjectLine: `Secret Rotation Failed`,
-          substitutions: {
-            rotationName,
-            rotationType,
-            content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date(
-              lastRotationAttemptedAt
-            ).toISOString()}. Please check the rotation status in Infisical for more details.`,
-            secretPath: folder.path,
-            environment: environment.name,
-            projectName: project.name,
-            rotationUrl: encodeURI(`${appCfg.SITE_URL}${rotationPath}`)
-          }
+    await queueService.startPg<QueueName.SecretRotationV2>(
+      QueueJobs.SecretRotationV2RotateSecrets,
+      async ([job]) => {
+        await rotateSecretsFns({
+          job: {
+            ...job,
+            data: job.data as TSecretRotationRotateSecretsJobPayload
+          },
+          secretRotationV2DAL,
+          secretRotationV2Service
         });
-      } catch (error) {
-        logger.error(
-          error,
-          `secretRotationV2Queue: Failed to Send Status Notification [rotationId=${secretRotation.id}]`
-        );
-        throw error;
+      },
+      {
+        batchSize: 1,
+        workerCount: 2,
+        pollingIntervalSeconds: 0.5
       }
-    },
-    {
-      batchSize: 1,
-      workerCount: 2,
-      pollingIntervalSeconds: 1
-    }
-  );
+    );
 
-  await queueService.schedulePg(
-    QueueJobs.SecretRotationV2QueueRotations,
-    appCfg.isRotationDevelopmentMode ? "* * * * *" : "0 0 * * *",
-    undefined,
-    { tz: "UTC" }
-  );
+    await queueService.startPg<QueueName.SecretRotationV2>(
+      QueueJobs.SecretRotationV2SendNotification,
+      async ([job]) => {
+        const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload;
+        try {
+          const {
+            name: rotationName,
+            type,
+            projectId,
+            lastRotationAttemptedAt,
+            folder,
+            environment,
+            id: rotationId
+          } = secretRotation;
+
+          logger.info(`secretRotationV2Queue: Sending Status Notification [rotationId=${rotationId}]`);
+
+          const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
+          const project = await projectDAL.findById(projectId);
+
+          const projectAdmins = projectMembers.filter((member) =>
+            member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
+          );
+
+          const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation];
+
+          const rotationPath = `/projects/secret-management/${projectId}/secrets/${environment.slug}`;
+
+          await notificationService.createUserNotifications(
+            projectAdmins.map((admin) => ({
+              userId: admin.userId,
+              orgId: project.orgId,
+              type: NotificationType.SECRET_ROTATION_FAILED,
+              title: "Secret Rotation Failed",
+              body: `Your **${rotationType}** rotation **${rotationName}** failed to rotate.`,
+              link: rotationPath
+            }))
+          );
+
+          await smtpService.sendMail({
+            recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
+            template: SmtpTemplates.SecretRotationFailed,
+            subjectLine: `Secret Rotation Failed`,
+            substitutions: {
+              rotationName,
+              rotationType,
+              content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date(
+                lastRotationAttemptedAt
+              ).toISOString()}. Please check the rotation status in Infisical for more details.`,
+              secretPath: folder.path,
+              environment: environment.name,
+              projectName: project.name,
+              rotationUrl: encodeURI(`${appCfg.SITE_URL}${rotationPath}`)
+            }
+          });
+        } catch (error) {
+          logger.error(
+            error,
+            `secretRotationV2Queue: Failed to Send Status Notification [rotationId=${secretRotation.id}]`
+          );
+          throw error;
+        }
+      },
+      {
+        batchSize: 1,
+        workerCount: 2,
+        pollingIntervalSeconds: 1
+      }
+    );
+
+    await queueService.schedulePg(
+      QueueJobs.SecretRotationV2QueueRotations,
+      appCfg.isRotationDevelopmentMode ? "* * * * *" : "0 0 * * *",
+      undefined,
+      { tz: "UTC" }
+    );
+  };
+
+  return { init };
 };

backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts

@@ -141,202 +141,6 @@ export const secretScanningV2QueueServiceFactory = async ({
     }
   };
 
-  await queueService.startPg<QueueName.SecretScanningV2>(
-    QueueJobs.SecretScanningV2FullScan,
-    async ([job]) => {
-      const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan;
-      const { retryCount, retryLimit } = job;
-
-      const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
-
-      const tempFolder = await createTempFolder();
-
-      const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
-
-      if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
-
-      const resource = await secretScanningV2DAL.resources.findById(resourceId);
-
-      if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
-
-      let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
-
-      try {
-        try {
-          lock = await keyStore.acquireLock(
-            [KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
-            60 * 1000 * 5
-          );
-        } catch (e) {
-          throw new Error("Failed to acquire scanning lock.");
-        }
-
-        await secretScanningV2DAL.scans.update(
-          { id: scanId },
-          {
-            status: SecretScanningScanStatus.Scanning
-          }
-        );
-
-        let connection: TAppConnection | null = null;
-        if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
-
-        const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
-          kmsService,
-          appConnectionDAL
-        });
-
-        const findingsPath = join(tempFolder, "findings.json");
-
-        const scanPath = await factory.getFullScanPath({
-          dataSource: {
-            ...dataSource,
-            connection
-          } as TSecretScanningDataSourceWithConnection,
-          resourceName: resource.name,
-          tempFolder
-        });
-
-        const config = await secretScanningV2DAL.configs.findOne({
-          projectId: dataSource.projectId
-        });
-
-        let configPath: string | undefined;
-
-        if (config && config.content) {
-          configPath = join(tempFolder, "infisical-scan.toml");
-          await writeTextToFile(configPath, config.content);
-        }
-
-        let findingsPayload: TFindingsPayload;
-        switch (resource.type) {
-          case SecretScanningResource.Repository:
-          case SecretScanningResource.Project:
-            findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
-            break;
-          default:
-            throw new Error("Unhandled resource type");
-        }
-
-        const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
-          let findings: TSecretScanningFindings[] = [];
-          if (findingsPayload.length) {
-            findings = await secretScanningV2DAL.findings.upsert(
-              findingsPayload.map((finding) => ({
-                ...finding,
-                projectId: dataSource.projectId,
-                dataSourceName: dataSource.name,
-                dataSourceType: dataSource.type,
-                resourceName: resource.name,
-                resourceType: resource.type,
-                scanId
-              })),
-              ["projectId", "fingerprint"],
-              tx,
-              ["resourceName", "dataSourceName"]
-            );
-          }
-
-          await secretScanningV2DAL.scans.update(
-            { id: scanId },
-            {
-              status: SecretScanningScanStatus.Completed,
-              statusMessage: null
-            }
-          );
-
-          return findings;
-        });
-
-        const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
-
-        if (newFindings.length) {
-          await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
-            status: SecretScanningScanStatus.Completed,
-            resourceName: resource.name,
-            isDiffScan: false,
-            dataSource,
-            numberOfSecrets: newFindings.length,
-            scanId
-          });
-        }
-
-        await auditLogService.createAuditLog({
-          projectId: dataSource.projectId,
-          actor: {
-            type: ActorType.PLATFORM,
-            metadata: {}
-          },
-          event: {
-            type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
-            metadata: {
-              dataSourceId: dataSource.id,
-              dataSourceType: dataSource.type,
-              resourceId: resource.id,
-              resourceType: resource.type,
-              scanId,
-              scanStatus: SecretScanningScanStatus.Completed,
-              scanType: SecretScanningScanType.FullScan,
-              numberOfSecretsDetected: findingsPayload.length
-            }
-          }
-        });
-
-        logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`);
-      } catch (error) {
-        if (retryCount === retryLimit) {
-          const errorMessage = parseScanErrorMessage(error);
-
-          await secretScanningV2DAL.scans.update(
-            { id: scanId },
-            {
-              status: SecretScanningScanStatus.Failed,
-              statusMessage: errorMessage
-            }
-          );
-
-          await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
-            status: SecretScanningScanStatus.Failed,
-            resourceName: resource.name,
-            dataSource,
-            errorMessage
-          });
-
-          await auditLogService.createAuditLog({
-            projectId: dataSource.projectId,
-            actor: {
-              type: ActorType.PLATFORM,
-              metadata: {}
-            },
-            event: {
-              type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
-              metadata: {
-                dataSourceId: dataSource.id,
-                dataSourceType: dataSource.type,
-                resourceId: resource.id,
-                resourceType: resource.type,
-                scanId,
-                scanStatus: SecretScanningScanStatus.Failed,
-                scanType: SecretScanningScanType.FullScan
-              }
-            }
-          });
-        }
-
-        logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
-        throw error;
-      } finally {
-        await deleteTempFolder(tempFolder);
-        await lock?.release();
-      }
-    },
-    {
-      batchSize: 1,
-      workerCount: 2,
-      pollingIntervalSeconds: 1
-    }
-  );
-
   const queueResourceDiffScan = async ({
     payload,
     dataSourceId,
@@ -391,148 +195,127 @@ export const secretScanningV2QueueServiceFactory = async ({
     }
   };
 
-  await queueService.startPg<QueueName.SecretScanningV2>(
-    QueueJobs.SecretScanningV2DiffScan,
-    async ([job]) => {
-      const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan;
-      const { retryCount, retryLimit } = job;
+  const init = async () => {
+    await queueService.startPg<QueueName.SecretScanningV2>(
+      QueueJobs.SecretScanningV2FullScan,
+      async ([job]) => {
+        const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan;
+        const { retryCount, retryLimit } = job;
 
-      const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
+        const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
 
-      const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
+        const tempFolder = await createTempFolder();
 
-      if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
+        const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
 
-      const resource = await secretScanningV2DAL.resources.findById(resourceId);
+        if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
 
-      if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
+        const resource = await secretScanningV2DAL.resources.findById(resourceId);
 
-      const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
-        kmsService,
-        appConnectionDAL
-      });
+        if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
 
-      const tempFolder = await createTempFolder();
+        let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
 
-      try {
-        await secretScanningV2DAL.scans.update(
-          { id: scanId },
-          {
-            status: SecretScanningScanStatus.Scanning
-          }
-        );
-
-        let connection: TAppConnection | null = null;
-        if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
-
-        const config = await secretScanningV2DAL.configs.findOne({
-          projectId: dataSource.projectId
-        });
-
-        let configPath: string | undefined;
-
-        if (config && config.content) {
-          configPath = join(tempFolder, "infisical-scan.toml");
-          await writeTextToFile(configPath, config.content);
-        }
-
-        const findingsPayload = await factory.getDiffScanFindingsPayload({
-          dataSource: {
-            ...dataSource,
-            connection
-          } as TSecretScanningDataSourceWithConnection,
-          resourceName: resource.name,
-          payload,
-          configPath
-        });
-
-        const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
-          let findings: TSecretScanningFindings[] = [];
-
-          if (findingsPayload.length) {
-            findings = await secretScanningV2DAL.findings.upsert(
-              findingsPayload.map((finding) => ({
-                ...finding,
-                projectId: dataSource.projectId,
-                dataSourceName: dataSource.name,
-                dataSourceType: dataSource.type,
-                resourceName: resource.name,
-                resourceType: resource.type,
-                scanId
-              })),
-              ["projectId", "fingerprint"],
-              tx,
-              ["resourceName", "dataSourceName"]
+        try {
+          try {
+            lock = await keyStore.acquireLock(
+              [KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
+              60 * 1000 * 5
             );
+          } catch (e) {
+            throw new Error("Failed to acquire scanning lock.");
           }
 
           await secretScanningV2DAL.scans.update(
             { id: scanId },
             {
-              status: SecretScanningScanStatus.Completed
+              status: SecretScanningScanStatus.Scanning
             }
           );
 
-          return findings;
-        });
+          let connection: TAppConnection | null = null;
+          if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
 
-        const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
-
-        if (newFindings.length) {
-          const finding = newFindings[0] as TSecretScanningFinding;
-          await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
-            status: SecretScanningScanStatus.Completed,
-            resourceName: resource.name,
-            isDiffScan: true,
-            dataSource,
-            numberOfSecrets: newFindings.length,
-            scanId,
-            authorName: finding?.details?.author,
-            authorEmail: finding?.details?.email
+          const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
+            kmsService,
+            appConnectionDAL
           });
-        }
 
-        await auditLogService.createAuditLog({
-          projectId: dataSource.projectId,
-          actor: {
-            type: ActorType.PLATFORM,
-            metadata: {}
-          },
-          event: {
-            type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
-            metadata: {
-              dataSourceId: dataSource.id,
-              dataSourceType: dataSource.type,
-              resourceId,
-              resourceType: resource.type,
-              scanId,
-              scanStatus: SecretScanningScanStatus.Completed,
-              scanType: SecretScanningScanType.DiffScan,
-              numberOfSecretsDetected: findingsPayload.length
-            }
+          const findingsPath = join(tempFolder, "findings.json");
+
+          const scanPath = await factory.getFullScanPath({
+            dataSource: {
+              ...dataSource,
+              connection
+            } as TSecretScanningDataSourceWithConnection,
+            resourceName: resource.name,
+            tempFolder
+          });
+
+          const config = await secretScanningV2DAL.configs.findOne({
+            projectId: dataSource.projectId
+          });
+
+          let configPath: string | undefined;
+
+          if (config && config.content) {
+            configPath = join(tempFolder, "infisical-scan.toml");
+            await writeTextToFile(configPath, config.content);
           }
-        });
 
-        logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`);
-      } catch (error) {
-        if (retryCount === retryLimit) {
-          const errorMessage = parseScanErrorMessage(error);
+          let findingsPayload: TFindingsPayload;
+          switch (resource.type) {
+            case SecretScanningResource.Repository:
+            case SecretScanningResource.Project:
+              findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
+              break;
+            default:
+              throw new Error("Unhandled resource type");
+          }
 
-          await secretScanningV2DAL.scans.update(
-            { id: scanId },
-            {
-              status: SecretScanningScanStatus.Failed,
-              statusMessage: errorMessage
+          const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
+            let findings: TSecretScanningFindings[] = [];
+            if (findingsPayload.length) {
+              findings = await secretScanningV2DAL.findings.upsert(
+                findingsPayload.map((finding) => ({
+                  ...finding,
+                  projectId: dataSource.projectId,
+                  dataSourceName: dataSource.name,
+                  dataSourceType: dataSource.type,
+                  resourceName: resource.name,
+                  resourceType: resource.type,
+                  scanId
+                })),
+                ["projectId", "fingerprint"],
+                tx,
+                ["resourceName", "dataSourceName"]
+              );
             }
-          );
 
-          await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
-            status: SecretScanningScanStatus.Failed,
-            resourceName: resource.name,
-            dataSource,
-            errorMessage
+            await secretScanningV2DAL.scans.update(
+              { id: scanId },
+              {
+                status: SecretScanningScanStatus.Completed,
+                statusMessage: null
+              }
+            );
+
+            return findings;
           });
 
+          const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
+
+          if (newFindings.length) {
+            await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
+              status: SecretScanningScanStatus.Completed,
+              resourceName: resource.name,
+              isDiffScan: false,
+              dataSource,
+              numberOfSecrets: newFindings.length,
+              scanId
+            });
+          }
+
           await auditLogService.createAuditLog({
             projectId: dataSource.projectId,
             actor: {
@@ -547,128 +330,348 @@ export const secretScanningV2QueueServiceFactory = async ({
                 resourceId: resource.id,
                 resourceType: resource.type,
                 scanId,
-                scanStatus: SecretScanningScanStatus.Failed,
-                scanType: SecretScanningScanType.DiffScan
+                scanStatus: SecretScanningScanStatus.Completed,
+                scanType: SecretScanningScanType.FullScan,
+                numberOfSecretsDetected: findingsPayload.length
               }
             }
           });
+
+          logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`);
+        } catch (error) {
+          if (retryCount === retryLimit) {
+            const errorMessage = parseScanErrorMessage(error);
+
+            await secretScanningV2DAL.scans.update(
+              { id: scanId },
+              {
+                status: SecretScanningScanStatus.Failed,
+                statusMessage: errorMessage
+              }
+            );
+
+            await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
+              status: SecretScanningScanStatus.Failed,
+              resourceName: resource.name,
+              dataSource,
+              errorMessage
+            });
+
+            await auditLogService.createAuditLog({
+              projectId: dataSource.projectId,
+              actor: {
+                type: ActorType.PLATFORM,
+                metadata: {}
+              },
+              event: {
+                type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
+                metadata: {
+                  dataSourceId: dataSource.id,
+                  dataSourceType: dataSource.type,
+                  resourceId: resource.id,
+                  resourceType: resource.type,
+                  scanId,
+                  scanStatus: SecretScanningScanStatus.Failed,
+                  scanType: SecretScanningScanType.FullScan
+                }
+              }
+            });
+          }
+
+          logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
+          throw error;
+        } finally {
+          await deleteTempFolder(tempFolder);
+          await lock?.release();
         }
-
-        logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
-        throw error;
-      } finally {
-        await deleteTempFolder(tempFolder);
+      },
+      {
+        batchSize: 1,
+        workerCount: 2,
+        pollingIntervalSeconds: 1
       }
-    },
-    {
-      batchSize: 1,
-      workerCount: 2,
-      pollingIntervalSeconds: 1
-    }
-  );
+    );
 
-  await queueService.startPg<QueueName.SecretScanningV2>(
-    QueueJobs.SecretScanningV2SendNotification,
-    async ([job]) => {
-      const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification;
+    await queueService.startPg<QueueName.SecretScanningV2>(
+      QueueJobs.SecretScanningV2DiffScan,
+      async ([job]) => {
+        const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan;
+        const { retryCount, retryLimit } = job;
 
-      const appCfg = getConfig();
+        const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
 
-      if (!appCfg.isSmtpConfigured) return;
+        const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
 
-      try {
-        const { projectId } = dataSource;
+        if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
 
-        logger.info(
-          `secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
-        );
+        const resource = await secretScanningV2DAL.resources.findById(resourceId);
 
-        const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
-        const project = await projectDAL.findById(projectId);
+        if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
 
-        const recipients = projectMembers.filter((member) => {
-          const isAdmin = member.roles.some((role) => role.role === ProjectMembershipRole.Admin);
-          const isCompleted = payload.status === SecretScanningScanStatus.Completed;
-          // We assume that the committer is one of the project members
-          const isCommitter = isCompleted && payload.authorEmail === member.user.email;
-          return isAdmin || isCommitter;
+        const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
+          kmsService,
+          appConnectionDAL
         });
 
-        const timestamp = new Date().toISOString();
+        const tempFolder = await createTempFolder();
 
-        const subjectLine =
-          payload.status === SecretScanningScanStatus.Completed
-            ? "Incident Alert: Secret(s) Leaked"
-            : `Secret Scanning Failed`;
+        try {
+          await secretScanningV2DAL.scans.update(
+            { id: scanId },
+            {
+              status: SecretScanningScanStatus.Scanning
+            }
+          );
 
-        await notificationService.createUserNotifications(
-          recipients.map((member) => ({
-            userId: member.userId,
-            orgId: project.orgId,
-            type:
-              payload.status === SecretScanningScanStatus.Completed
-                ? NotificationType.SECRET_SCANNING_SECRETS_DETECTED
-                : NotificationType.SECRET_SCANNING_SCAN_FAILED,
-            title: subjectLine,
-            body:
-              payload.status === SecretScanningScanStatus.Completed
-                ? `Uncovered **${payload.numberOfSecrets}** secret(s) ${payload.isDiffScan ? " from a recent commit to" : " in"} **${resourceName}**.`
-                : `Encountered an error while attempting to scan the resource **${resourceName}**: ${payload.errorMessage}`,
-            link:
-              payload.status === SecretScanningScanStatus.Completed
-                ? `/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
-                : `/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
-          }))
-        );
+          let connection: TAppConnection | null = null;
+          if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
 
-        await smtpService.sendMail({
-          recipients: recipients.map((member) => member.user.email!).filter(Boolean),
-          template:
-            payload.status === SecretScanningScanStatus.Completed
-              ? SmtpTemplates.SecretScanningV2SecretsDetected
-              : SmtpTemplates.SecretScanningV2ScanFailed,
-          subjectLine,
-          substitutions:
-            payload.status === SecretScanningScanStatus.Completed
-              ? {
-                  authorName: payload.authorName,
-                  authorEmail: payload.authorEmail,
-                  resourceName,
-                  numberOfSecrets: payload.numberOfSecrets,
-                  isDiffScan: payload.isDiffScan,
-                  url: encodeURI(
-                    `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
-                  ),
-                  timestamp
-                }
-              : {
+          const config = await secretScanningV2DAL.configs.findOne({
+            projectId: dataSource.projectId
+          });
+
+          let configPath: string | undefined;
+
+          if (config && config.content) {
+            configPath = join(tempFolder, "infisical-scan.toml");
+            await writeTextToFile(configPath, config.content);
+          }
+
+          const findingsPayload = await factory.getDiffScanFindingsPayload({
+            dataSource: {
+              ...dataSource,
+              connection
+            } as TSecretScanningDataSourceWithConnection,
+            resourceName: resource.name,
+            payload,
+            configPath
+          });
+
+          const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
+            let findings: TSecretScanningFindings[] = [];
+
+            if (findingsPayload.length) {
+              findings = await secretScanningV2DAL.findings.upsert(
+                findingsPayload.map((finding) => ({
+                  ...finding,
+                  projectId: dataSource.projectId,
                   dataSourceName: dataSource.name,
-                  resourceName,
-                  projectName: project.name,
-                  timestamp,
-                  errorMessage: payload.errorMessage,
-                  url: encodeURI(
-                    `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
-                  )
+                  dataSourceType: dataSource.type,
+                  resourceName: resource.name,
+                  resourceType: resource.type,
+                  scanId
+                })),
+                ["projectId", "fingerprint"],
+                tx,
+                ["resourceName", "dataSourceName"]
+              );
+            }
+
+            await secretScanningV2DAL.scans.update(
+              { id: scanId },
+              {
+                status: SecretScanningScanStatus.Completed
+              }
+            );
+
+            return findings;
+          });
+
+          const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
+
+          if (newFindings.length) {
+            const finding = newFindings[0] as TSecretScanningFinding;
+            await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
+              status: SecretScanningScanStatus.Completed,
+              resourceName: resource.name,
+              isDiffScan: true,
+              dataSource,
+              numberOfSecrets: newFindings.length,
+              scanId,
+              authorName: finding?.details?.author,
+              authorEmail: finding?.details?.email
+            });
+          }
+
+          await auditLogService.createAuditLog({
+            projectId: dataSource.projectId,
+            actor: {
+              type: ActorType.PLATFORM,
+              metadata: {}
+            },
+            event: {
+              type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
+              metadata: {
+                dataSourceId: dataSource.id,
+                dataSourceType: dataSource.type,
+                resourceId,
+                resourceType: resource.type,
+                scanId,
+                scanStatus: SecretScanningScanStatus.Completed,
+                scanType: SecretScanningScanType.DiffScan,
+                numberOfSecretsDetected: findingsPayload.length
+              }
+            }
+          });
+
+          logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`);
+        } catch (error) {
+          if (retryCount === retryLimit) {
+            const errorMessage = parseScanErrorMessage(error);
+
+            await secretScanningV2DAL.scans.update(
+              { id: scanId },
+              {
+                status: SecretScanningScanStatus.Failed,
+                statusMessage: errorMessage
+              }
+            );
+
+            await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
+              status: SecretScanningScanStatus.Failed,
+              resourceName: resource.name,
+              dataSource,
+              errorMessage
+            });
+
+            await auditLogService.createAuditLog({
+              projectId: dataSource.projectId,
+              actor: {
+                type: ActorType.PLATFORM,
+                metadata: {}
+              },
+              event: {
+                type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
+                metadata: {
+                  dataSourceId: dataSource.id,
+                  dataSourceType: dataSource.type,
+                  resourceId: resource.id,
+                  resourceType: resource.type,
+                  scanId,
+                  scanStatus: SecretScanningScanStatus.Failed,
+                  scanType: SecretScanningScanType.DiffScan
                 }
-        });
-      } catch (error) {
-        logger.error(
-          error,
-          `secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
-        );
-        throw error;
+              }
+            });
+          }
+
+          logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
+          throw error;
+        } finally {
+          await deleteTempFolder(tempFolder);
+        }
+      },
+      {
+        batchSize: 1,
+        workerCount: 2,
+        pollingIntervalSeconds: 1
       }
-    },
-    {
-      batchSize: 1,
-      workerCount: 2,
-      pollingIntervalSeconds: 1
-    }
-  );
+    );
+
+    await queueService.startPg<QueueName.SecretScanningV2>(
+      QueueJobs.SecretScanningV2SendNotification,
+      async ([job]) => {
+        const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification;
+
+        const appCfg = getConfig();
+
+        if (!appCfg.isSmtpConfigured) return;
+
+        try {
+          const { projectId } = dataSource;
+
+          logger.info(
+            `secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
+          );
+
+          const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
+          const project = await projectDAL.findById(projectId);
+
+          const recipients = projectMembers.filter((member) => {
+            const isAdmin = member.roles.some((role) => role.role === ProjectMembershipRole.Admin);
+            const isCompleted = payload.status === SecretScanningScanStatus.Completed;
+            // We assume that the committer is one of the project members
+            const isCommitter = isCompleted && payload.authorEmail === member.user.email;
+            return isAdmin || isCommitter;
+          });
+
+          const timestamp = new Date().toISOString();
+
+          const subjectLine =
+            payload.status === SecretScanningScanStatus.Completed
+              ? "Incident Alert: Secret(s) Leaked"
+              : `Secret Scanning Failed`;
+
+          await notificationService.createUserNotifications(
+            recipients.map((member) => ({
+              userId: member.userId,
+              orgId: project.orgId,
+              type:
+                payload.status === SecretScanningScanStatus.Completed
+                  ? NotificationType.SECRET_SCANNING_SECRETS_DETECTED
+                  : NotificationType.SECRET_SCANNING_SCAN_FAILED,
+              title: subjectLine,
+              body:
+                payload.status === SecretScanningScanStatus.Completed
+                  ? `Uncovered **${payload.numberOfSecrets}** secret(s) ${payload.isDiffScan ? " from a recent commit to" : " in"} **${resourceName}**.`
+                  : `Encountered an error while attempting to scan the resource **${resourceName}**: ${payload.errorMessage}`,
+              link:
+                payload.status === SecretScanningScanStatus.Completed
+                  ? `/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
+                  : `/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
+            }))
+          );
+
+          await smtpService.sendMail({
+            recipients: recipients.map((member) => member.user.email!).filter(Boolean),
+            template:
+              payload.status === SecretScanningScanStatus.Completed
+                ? SmtpTemplates.SecretScanningV2SecretsDetected
+                : SmtpTemplates.SecretScanningV2ScanFailed,
+            subjectLine,
+            substitutions:
+              payload.status === SecretScanningScanStatus.Completed
+                ? {
+                    authorName: payload.authorName,
+                    authorEmail: payload.authorEmail,
+                    resourceName,
+                    numberOfSecrets: payload.numberOfSecrets,
+                    isDiffScan: payload.isDiffScan,
+                    url: encodeURI(
+                      `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
+                    ),
+                    timestamp
+                  }
+                : {
+                    dataSourceName: dataSource.name,
+                    resourceName,
+                    projectName: project.name,
+                    timestamp,
+                    errorMessage: payload.errorMessage,
+                    url: encodeURI(
+                      `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
+                    )
+                  }
+          });
+        } catch (error) {
+          logger.error(
+            error,
+            `secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
+          );
+          throw error;
+        }
+      },
+      {
+        batchSize: 1,
+        workerCount: 2,
+        pollingIntervalSeconds: 1
+      }
+    );
+  };
 
   return {
     queueDataSourceFullScan,
-    queueResourceDiffScan
+    queueResourceDiffScan,
+    init
   };
 };

backend/src/main.ts

@@ -16,7 +16,7 @@ import { buildRedisFromConfig } from "./lib/config/redis";
 import { removeTemporaryBaseDirectory } from "./lib/files";
 import { initLogger } from "./lib/logger";
 import { queueServiceFactory } from "./queue";
-import { main } from "./server/app";
+import { main, markRunningMigrations, markServerReady } from "./server/app";
 import { bootstrapCheck } from "./server/boot-strap-check";
 import { kmsRootConfigDALFactory } from "./services/kms/kms-root-config-dal";
 import { smtpServiceFactory } from "./services/smtp/smtp-service";
@@ -59,8 +59,6 @@ const run = async () => {
       })
     : undefined;
 
-  await runMigrations({ applicationDb: db, auditLogDb, logger });
-
   const smtp = smtpServiceFactory(formatSmtpConfig());
 
   const queue = queueServiceFactory(envConfig, {
@@ -74,7 +72,7 @@ const run = async () => {
   const keyStore = keyStoreFactory(envConfig, keyValueStoreDAL);
   const redis = buildRedisFromConfig(envConfig);
 
-  const server = await main({
+  const { server, completeServerInitialization } = await main({
     db,
     auditLogDb,
     superAdminDAL,
@@ -87,7 +85,6 @@ const run = async () => {
     redis,
     envConfig
   });
-  const bootstrap = await bootstrapCheck({ db });
 
   // eslint-disable-next-line
   process.on("SIGINT", async () => {
@@ -121,12 +118,46 @@ const run = async () => {
 
   await server.listen({
     port: envConfig.PORT,
-    host: envConfig.HOST,
-    listenTextResolver: (address) => {
-      void bootstrap();
-      return address;
+    host: envConfig.HOST
+  });
+
+  logger.info(`Server listening on ${envConfig.HOST}:${envConfig.PORT}`);
+  logger.info("Running migrations...");
+
+  // Run migrations while server is up
+  // All containers start as NOT HEALTHY (waiting for migrations)
+  // Container that acquires lock: becomes HEALTHY (running migrations) + NOT READY (no traffic)
+  // Other containers waiting: stay NOT HEALTHY (waiting) + NOT READY (no traffic)
+  await runMigrations({
+    applicationDb: db,
+    auditLogDb,
+    logger,
+    onMigrationLockAcquired: () => {
+      // Called after successfully acquiring the lock
+      // This container is now the migration runner
+      markRunningMigrations();
+      logger.info("Migration lock acquired! This container is running migrations.");
     }
   });
+
+  logger.info("Migrations complete. Completing server initialization...");
+
+  try {
+    await completeServerInitialization();
+  } catch (error) {
+    logger.error(error, "Failed to complete server initialization");
+    await server.close();
+    await queue.shutdown();
+    process.exit(1);
+  }
+
+  logger.info("Server initialization complete. Marking server as READY...");
+
+  markServerReady();
+
+  logger.info("Server is ready to accept traffic");
+  const bootstrap = await bootstrapCheck({ db });
+  void bootstrap();
 };
 
 void run();

backend/src/server/app.ts

@@ -1,5 +1,6 @@
 /* eslint-disable import/extensions */
 import path from "node:path";
+import { monitorEventLoopDelay } from "perf_hooks";
 
 import type { FastifyCookieOptions } from "@fastify/cookie";
 import cookie from "@fastify/cookie";
@@ -24,6 +25,7 @@ import { TQueueServiceFactory } from "@app/queue";
 import { TKmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal";
 import { TSmtpService } from "@app/services/smtp/smtp-service";
 import { TSuperAdminDALFactory } from "@app/services/super-admin/super-admin-dal";
+import { getServerCfg } from "@app/services/super-admin/super-admin-service";
 
 import { globalRateLimiterCfg } from "./config/rateLimiter";
 import { addErrorsToResponseSchemas } from "./plugins/add-errors-to-response-schemas";
@@ -36,6 +38,15 @@ import { registerServeUI } from "./plugins/serve-ui";
 import { fastifySwagger } from "./plugins/swagger";
 import { registerRoutes } from "./routes";
 
+const histogram = monitorEventLoopDelay({ resolution: 20 });
+histogram.enable();
+
+const serverState = {
+  isReady: false,
+  isRunningMigrations: false,
+  isWaitingForMigrations: true // Start as true - containers are unhealthy until they acquire migration lock or complete
+};
+
 type TMain = {
   auditLogDb?: Knex;
   db: Knex;
@@ -145,7 +156,72 @@ export const main = async ({
       })
     });
 
-    await server.register(registerRoutes, {
+    // Health check - returns 200 only if doing useful work (running migrations or ready)
+    // Returns 503 if waiting for another container to finish migrations
+    server.get("/api/health", async (_, reply) => {
+      if (serverState.isWaitingForMigrations) {
+        return reply.code(503).send({
+          status: "waiting",
+          message: "Waiting for migrations to complete in another container"
+        });
+      }
+      return { status: "ok", message: "Server is alive" };
+    });
+
+    // Global preHandler to block requests during migrations
+    server.addHook("preHandler", async (request, reply) => {
+      if (request.url === "/api/health" || request.url === "/api/ready") {
+        return;
+      }
+      if (!serverState.isReady) {
+        return reply.code(503).send({
+          status: "unavailable",
+          message: "Server is starting up, migrations in progress. Please try again in a moment."
+        });
+      }
+    });
+
+    // Readiness check - returns 503 until migrations are complete
+    server.get("/api/ready", async (request, reply) => {
+      const cfg = getConfig();
+
+      const meanLagMs = histogram.mean / 1e6;
+      const maxLagMs = histogram.max / 1e6;
+      const p99LagMs = histogram.percentile(99) / 1e6;
+
+      request.log.info(
+        `Event loop stats - Mean: ${meanLagMs.toFixed(2)}ms, Max: ${maxLagMs.toFixed(2)}ms, p99: ${p99LagMs.toFixed(2)}ms`
+      );
+
+      request.log.info(`Raw event loop stats: ${JSON.stringify(histogram, null, 2)}`);
+
+      if (!serverState.isReady) {
+        return reply.code(503).send({
+          date: new Date(),
+          message: "Server is starting up, migrations in progress",
+          emailConfigured: cfg.isSmtpConfigured,
+          redisConfigured: cfg.isRedisConfigured,
+          secretScanningConfigured: cfg.isSecretScanningConfigured,
+          samlDefaultOrgSlug: cfg.samlDefaultOrgSlug,
+          auditLogStorageDisabled: Boolean(cfg.DISABLE_AUDIT_LOG_STORAGE)
+        });
+      }
+
+      const serverCfg = await getServerCfg();
+
+      return {
+        date: new Date(),
+        message: "Ok",
+        emailConfigured: cfg.isSmtpConfigured,
+        inviteOnlySignup: Boolean(serverCfg.allowSignUp),
+        redisConfigured: cfg.isRedisConfigured,
+        secretScanningConfigured: cfg.isSecretScanningConfigured,
+        samlDefaultOrgSlug: cfg.samlDefaultOrgSlug,
+        auditLogStorageDisabled: Boolean(cfg.DISABLE_AUDIT_LOG_STORAGE)
+      };
+    });
+
+    const completeServerInitialization = await registerRoutes(server, {
       smtp,
       queue,
       db,
@@ -164,10 +240,21 @@ export const main = async ({
 
     await server.ready();
     server.swagger();
-    return server;
+    return { server, completeServerInitialization };
   } catch (err) {
     server.log.error(err);
     await queue.shutdown();
     process.exit(1);
   }
 };
+
+export const markServerReady = () => {
+  serverState.isReady = true;
+  serverState.isRunningMigrations = false;
+  serverState.isWaitingForMigrations = false;
+};
+
+export const markRunningMigrations = () => {
+  serverState.isRunningMigrations = true;
+  serverState.isWaitingForMigrations = false;
+};

backend/src/server/routes/index.ts

@@ -2208,7 +2208,7 @@ export const registerRoutes = async (
     internalCaFns
   });
 
-  await secretRotationV2QueueServiceFactory({
+  const secretRotationV2Queue = await secretRotationV2QueueServiceFactory({
     secretRotationV2Service,
     secretRotationV2DAL,
     queueService,
@@ -2305,8 +2305,6 @@ export const registerRoutes = async (
   // If FIPS is enabled, we check to ensure that the users license includes FIPS mode.
   crypto.verifyFipsLicense(licenseService);
 
-  await superAdminService.initServerCfg();
-
   // Start HSM service if it's configured/enabled.
   await hsmService.startService();
 
@@ -2331,21 +2329,60 @@ export const registerRoutes = async (
     }
   }
 
-  await telemetryQueue.startTelemetryCheck();
-  await telemetryQueue.startAggregatedEventsJob();
-  await dailyResourceCleanUp.init();
-  await healthAlert.init();
-  await pkiSyncCleanup.init();
-  await pamAccountRotation.init();
-  await dailyReminderQueueService.startDailyRemindersJob();
-  await dailyReminderQueueService.startSecretReminderMigrationJob();
-  await dailyExpiringPkiItemAlert.startSendingAlerts();
-  await pkiSubscriberQueue.startDailyAutoRenewalJob();
-  await certificateV3Queue.init();
-  await kmsService.startService(hsmStatus);
-  await microsoftTeamsService.start();
-  await dynamicSecretQueueService.init();
-  await eventBusService.init();
+  const completeServerInitialization = async () => {
+    await superAdminService.initServerCfg();
+
+    await telemetryQueue.startTelemetryCheck();
+    await telemetryQueue.startAggregatedEventsJob();
+    await dailyResourceCleanUp.init();
+    await healthAlert.init();
+    await pkiSyncCleanup.init();
+    await pamAccountRotation.init();
+    await dailyReminderQueueService.startDailyRemindersJob();
+    await dailyReminderQueueService.startSecretReminderMigrationJob();
+    await dailyExpiringPkiItemAlert.startSendingAlerts();
+    await pkiSubscriberQueue.startDailyAutoRenewalJob();
+    await certificateV3Queue.init();
+    await kmsService.startService(hsmStatus);
+    await microsoftTeamsService.start();
+    await dynamicSecretQueueService.init();
+    await secretScanningV2Queue.init();
+    await secretRotationV2Queue.init();
+    await notificationQueue.init();
+    await eventBusService.init();
+
+    const cronJobs: CronJob[] = [];
+    if (appCfg.isProductionMode) {
+      const rateLimitSyncJob = await rateLimitService.initializeBackgroundSync();
+      if (rateLimitSyncJob) {
+        cronJobs.push(rateLimitSyncJob);
+      }
+      const licenseSyncJob = await licenseService.initializeBackgroundSync();
+      if (licenseSyncJob) {
+        cronJobs.push(licenseSyncJob);
+      }
+
+      const microsoftTeamsSyncJob = await microsoftTeamsService.initializeBackgroundSync();
+      if (microsoftTeamsSyncJob) {
+        cronJobs.push(microsoftTeamsSyncJob);
+      }
+
+      const adminIntegrationsSyncJob = await superAdminService.initializeAdminIntegrationConfigSync();
+      if (adminIntegrationsSyncJob) {
+        cronJobs.push(adminIntegrationsSyncJob);
+      }
+    }
+
+    const configSyncJob = await superAdminService.initializeEnvConfigSync();
+    if (configSyncJob) {
+      cronJobs.push(configSyncJob);
+    }
+
+    const oauthConfigSyncJob = await initializeOauthConfigSync();
+    if (oauthConfigSyncJob) {
+      cronJobs.push(oauthConfigSyncJob);
+    }
+  };
 
   // inject all services
   server.decorate<FastifyZodProvider["services"]>("services", {
@@ -2475,38 +2512,6 @@ export const registerRoutes = async (
     convertor: convertorService
   });
 
-  const cronJobs: CronJob[] = [];
-  if (appCfg.isProductionMode) {
-    const rateLimitSyncJob = await rateLimitService.initializeBackgroundSync();
-    if (rateLimitSyncJob) {
-      cronJobs.push(rateLimitSyncJob);
-    }
-    const licenseSyncJob = await licenseService.initializeBackgroundSync();
-    if (licenseSyncJob) {
-      cronJobs.push(licenseSyncJob);
-    }
-
-    const microsoftTeamsSyncJob = await microsoftTeamsService.initializeBackgroundSync();
-    if (microsoftTeamsSyncJob) {
-      cronJobs.push(microsoftTeamsSyncJob);
-    }
-
-    const adminIntegrationsSyncJob = await superAdminService.initializeAdminIntegrationConfigSync();
-    if (adminIntegrationsSyncJob) {
-      cronJobs.push(adminIntegrationsSyncJob);
-    }
-  }
-
-  const configSyncJob = await superAdminService.initializeEnvConfigSync();
-  if (configSyncJob) {
-    cronJobs.push(configSyncJob);
-  }
-
-  const oauthConfigSyncJob = await initializeOauthConfigSync();
-  if (oauthConfigSyncJob) {
-    cronJobs.push(oauthConfigSyncJob);
-  }
-
   server.decorate<FastifyZodProvider["store"]>("store", {
     user: userDAL,
     kmipClient: kmipClientDAL
@@ -2595,9 +2600,10 @@ export const registerRoutes = async (
   await server.register(registerV4Routes, { prefix: "/api/v4" });
 
   server.addHook("onClose", async () => {
-    cronJobs.forEach((job) => job.stop());
     await telemetryService.flushAll();
     await eventBusService.close();
     sseService.close();
   });
+
+  return completeServerInitialization;
 };

backend/src/server/routes/v3/external-migration-router.ts

@@ -393,6 +393,56 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider
     }
   });
 
+  server.route({
+    method: "GET",
+    url: "/vault/kubernetes-roles",
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      querystring: z.object({
+        namespace: z.string(),
+        mountPath: z.string()
+      }),
+      response: {
+        200: z.object({
+          roles: z.array(
+            z.object({
+              name: z.string(),
+              mountPath: z.string(),
+              allowed_kubernetes_namespaces: z.array(z.string()).nullish(),
+              allowed_kubernetes_namespace_selector: z.string().nullish(),
+              token_max_ttl: z.number().nullish(),
+              token_default_ttl: z.number().nullish(),
+              token_default_audiences: z.array(z.string()).nullish(),
+              service_account_name: z.string().nullish(),
+              kubernetes_role_name: z.string().nullish(),
+              kubernetes_role_type: z.string().nullish(),
+              generated_role_rules: z.string().nullish(),
+              name_template: z.string().nullish(),
+              extra_annotations: z.record(z.string()).nullish(),
+              extra_labels: z.record(z.string()).nullish(),
+              config: z.object({
+                kubernetes_host: z.string(),
+                kubernetes_ca_cert: z.string().nullish()
+              })
+            })
+          )
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const roles = await server.services.migration.getVaultKubernetesRoles({
+        actor: req.permission,
+        namespace: req.query.namespace,
+        mountPath: req.query.mountPath
+      });
+
+      return { roles };
+    }
+  });
+
   server.route({
     method: "GET",
     url: "/vault/secret-paths",

backend/src/services/app-connection/gitlab/gitlab-connection-enums.ts

@@ -5,5 +5,6 @@ export enum GitLabConnectionMethod {
 
 export enum GitLabAccessTokenType {
   Project = "project",
-  Personal = "personal"
+  Personal = "personal",
+  Group = "group"
 }

backend/src/services/app-connection/hc-vault/hc-vault-connection-fns.ts

@@ -21,6 +21,8 @@ import {
   THCVaultKubernetesAuthConfig,
   THCVaultKubernetesAuthRole,
   THCVaultKubernetesAuthRoleWithConfig,
+  THCVaultKubernetesRole,
+  THCVaultKubernetesSecretsConfig,
   THCVaultMount,
   THCVaultMountResponse
 } from "./hc-vault-connection-types";
@@ -816,3 +818,122 @@ export const getHCVaultKubernetesAuthRoles = async (
     });
   }
 };
+
+export const getHCVaultKubernetesRoles = async (
+  namespace: string,
+  mountPath: string,
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+): Promise<THCVaultKubernetesRole[]> => {
+  // Remove trailing slash from mount path
+  const cleanMountPath = mountPath.endsWith("/") ? mountPath.slice(0, -1) : mountPath;
+
+  try {
+    const instanceUrl = await getHCVaultInstanceUrl(connection);
+    const accessToken = await getHCVaultAccessToken(connection, gatewayService);
+    // 1. Get the Kubernetes secrets engine configuration for this mount
+    const { data: configResponse } = await requestWithHCVaultGateway<{ data: THCVaultKubernetesSecretsConfig }>(
+      connection,
+      gatewayService,
+      {
+        url: `${instanceUrl}/v1/${cleanMountPath}/config`,
+        method: "GET",
+        headers: {
+          "X-Vault-Token": accessToken,
+          "X-Vault-Namespace": namespace
+        }
+      }
+    );
+
+    const kubernetesConfig = configResponse.data;
+
+    // 2. List all roles in this mount
+    let roleNames: string[] = [];
+    try {
+      const { data: roleListResponse } = await requestWithHCVaultGateway<{ data: { keys: string[] } }>(
+        connection,
+        gatewayService,
+        {
+          url: `${instanceUrl}/v1/${cleanMountPath}/roles?list=true`,
+          method: "GET",
+          headers: {
+            "X-Vault-Token": accessToken,
+            "X-Vault-Namespace": namespace
+          }
+        }
+      );
+      roleNames = roleListResponse.data.keys || [];
+    } catch (error) {
+      // Vault returns 404 when no roles are configured yet
+      if (error && typeof error === "object" && "response" in error) {
+        const axiosError = error as { response?: { status?: number } };
+        if (axiosError.response?.status === 404) {
+          return [];
+        }
+      }
+
+      throw error;
+    }
+
+    if (!roleNames || roleNames.length === 0) {
+      return [];
+    }
+
+    // 3. Fetch details for each role with concurrency control
+    const limiter = createConcurrencyLimiter(HC_VAULT_CONCURRENCY_LIMIT);
+
+    const roleDetailsPromises = roleNames.map((roleName) =>
+      limiter(async () => {
+        const { data: roleResponse } = await requestWithHCVaultGateway<{
+          data: {
+            allowed_kubernetes_namespaces?: string[];
+            allowed_kubernetes_namespace_selector?: string;
+            token_max_ttl?: number;
+            token_default_ttl?: number;
+            token_default_audiences?: string[];
+            service_account_name?: string;
+            kubernetes_role_name?: string;
+            kubernetes_role_type?: string;
+            generated_role_rules?: string;
+            name_template?: string;
+            extra_annotations?: Record<string, string>;
+            extra_labels?: Record<string, string>;
+          };
+        }>(connection, gatewayService, {
+          url: `${instanceUrl}/v1/${cleanMountPath}/roles/${roleName}`,
+          method: "GET",
+          headers: {
+            "X-Vault-Token": accessToken,
+            "X-Vault-Namespace": namespace
+          }
+        });
+
+        // 4. Merge the role with the config
+        return {
+          ...roleResponse.data,
+          name: roleName,
+          config: kubernetesConfig,
+          mountPath: cleanMountPath
+        } as THCVaultKubernetesRole;
+      })
+    );
+
+    const roles = await Promise.all(roleDetailsPromises);
+
+    return roles;
+  } catch (error: unknown) {
+    logger.error(error, "Unable to list HC Vault Kubernetes secrets engine roles");
+
+    if (error instanceof AxiosError) {
+      const errorMessage =
+        (error.response?.data as { errors?: string[] })?.errors?.[0] || error.message || "Unknown error";
+      throw new BadRequestError({
+        message: `Failed to list Kubernetes secrets engine roles: ${errorMessage}`
+      });
+    }
+
+    throw new BadRequestError({
+      message: "Unable to list Kubernetes secrets engine roles from HashiCorp Vault"
+    });
+  }
+};

backend/src/services/app-connection/hc-vault/hc-vault-connection-types.ts

@@ -95,3 +95,26 @@ export type THCVaultKubernetesAuthRoleWithConfig = THCVaultKubernetesAuthRole &
   config: THCVaultKubernetesAuthConfig;
   mountPath: string;
 };
+
+export type THCVaultKubernetesSecretsConfig = {
+  kubernetes_host: string;
+  kubernetes_ca_cert?: string;
+};
+
+export type THCVaultKubernetesRole = {
+  name: string;
+  allowed_kubernetes_namespaces?: string[];
+  allowed_kubernetes_namespace_selector?: string;
+  token_max_ttl?: number;
+  token_default_ttl?: number;
+  token_default_audiences?: string[];
+  service_account_name?: string;
+  kubernetes_role_name?: string;
+  kubernetes_role_type?: string;
+  generated_role_rules?: string;
+  name_template?: string;
+  extra_annotations?: Record<string, string>;
+  extra_labels?: Record<string, string>;
+  config: THCVaultKubernetesSecretsConfig;
+  mountPath: string;
+};

backend/src/services/external-migration/external-migration-fns/import.ts

@@ -82,7 +82,7 @@ export const importDataIntoInfisicalFn = async ({
 
         if (existingEnv) {
           throw new BadRequestError({
-            message: `Environment with slug '${slug}' already exist`,
+            message: `Environment with slug '${slug}' already exists`,
             name: "CreateEnvironment"
           });
         }
@@ -312,7 +312,7 @@ export const importDataIntoInfisicalFn = async ({
           );
           if (secretsByKeys.length) {
             throw new BadRequestError({
-              message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}`
+              message: `Secret already exists: ${secretsByKeys.map((el) => el.key).join(",")}`
             });
           }
           await fnSecretBulkInsert({

backend/src/services/external-migration/external-migration-service.ts

@@ -19,6 +19,7 @@ import {
   convertVaultValueToString,
   getHCVaultAuthMounts,
   getHCVaultKubernetesAuthRoles,
+  getHCVaultKubernetesRoles,
   getHCVaultSecretsForPath,
   HCVaultAuthType,
   listHCVaultMounts,
@@ -762,6 +763,56 @@ export const externalMigrationServiceFactory = ({
     return roles;
   };
 
+  const getVaultKubernetesRoles = async ({
+    actor,
+    namespace,
+    mountPath
+  }: {
+    actor: OrgServiceActor;
+    namespace: string;
+    mountPath: string;
+  }) => {
+    const { hasRole } = await permissionService.getOrgPermission({
+      scope: OrganizationActionScope.Any,
+      actor: actor.type,
+      actorId: actor.id,
+      orgId: actor.orgId,
+      actorAuthMethod: actor.authMethod,
+      actorOrgId: actor.orgId
+    });
+
+    if (!hasRole(OrgMembershipRole.Admin)) {
+      throw new ForbiddenRequestError({ message: "Only admins can get Kubernetes roles" });
+    }
+
+    const vaultConfig = await vaultExternalMigrationConfigDAL.findOne({
+      orgId: actor.orgId,
+      namespace
+    });
+
+    if (!vaultConfig) {
+      throw new NotFoundError({ message: "Vault migration config not found for this namespace" });
+    }
+
+    if (!vaultConfig.connection) {
+      throw new BadRequestError({ message: "Vault migration connection is not configured for this namespace" });
+    }
+
+    const credentials = await decryptAppConnectionCredentials({
+      orgId: vaultConfig.orgId,
+      encryptedCredentials: vaultConfig.connection.encryptedCredentials,
+      kmsService,
+      projectId: null
+    });
+
+    const connection = {
+      ...vaultConfig.connection,
+      credentials
+    } as THCVaultConnection;
+
+    return getHCVaultKubernetesRoles(namespace, mountPath, connection, gatewayService);
+  };
+
   return {
     importEnvKeyData,
     importVaultData,
@@ -776,6 +827,7 @@ export const externalMigrationServiceFactory = ({
     getVaultAuthMounts,
     getVaultSecretPaths,
     importVaultSecrets,
-    getVaultKubernetesAuthRoles
+    getVaultKubernetesAuthRoles,
+    getVaultKubernetesRoles
   };
 };

backend/src/services/notification/notification-queue.ts

@@ -10,6 +10,7 @@ type TNotificationQueueServiceFactoryDep = {
 
 export type TNotificationQueueServiceFactory = {
   pushUserNotifications: (data: TCreateUserNotificationDTO[]) => Promise<void>;
+  init: () => Promise<void>;
 };
 
 export const notificationQueueServiceFactory = async ({
@@ -20,20 +21,23 @@ export const notificationQueueServiceFactory = async ({
     await queueService.queuePg(QueueJobs.UserNotification, { notifications: data });
   };
 
-  await queueService.startPg(
-    QueueJobs.UserNotification,
-    async ([job]) => {
-      const { notifications } = job.data as { notifications: TCreateUserNotificationDTO[] };
-      await userNotificationDAL.batchInsert(notifications);
-    },
-    {
-      batchSize: 1,
-      workerCount: 2,
-      pollingIntervalSeconds: 1
-    }
-  );
+  const init = async () => {
+    await queueService.startPg(
+      QueueJobs.UserNotification,
+      async ([job]) => {
+        const { notifications } = job.data as { notifications: TCreateUserNotificationDTO[] };
+        await userNotificationDAL.batchInsert(notifications);
+      },
+      {
+        batchSize: 1,
+        workerCount: 2,
+        pollingIntervalSeconds: 1
+      }
+    );
+  };
 
   return {
-    pushUserNotifications
+    pushUserNotifications,
+    init
   };
 };

backend/src/services/org/org-service.ts

@@ -517,7 +517,7 @@ export const orgServiceFactory = ({
     if (slug) {
       const existingOrg = await orgDAL.findOne({ slug, rootOrgId: null });
       if (existingOrg && existingOrg?.id !== orgId)
-        throw new BadRequestError({ message: `Organization with slug ${slug} already exist` });
+        throw new BadRequestError({ message: `Organization with slug ${slug} already exists` });
     }
 
     if (googleSsoAuthEnforced) {
@@ -1149,7 +1149,7 @@ export const orgServiceFactory = ({
     const doesIncidentContactExist = await incidentContactDAL.findOne(orgId, { email });
     if (doesIncidentContactExist) {
       throw new BadRequestError({
-        message: "Incident contact already exist",
+        message: "Incident contact already exists",
         name: "Incident contact exist"
       });
     }

backend/src/services/project-env/project-env-service.ts

@@ -76,7 +76,7 @@ export const projectEnvServiceFactory = ({
       const existingEnv = envs.find(({ slug: envSlug }) => envSlug === slug);
       if (existingEnv)
         throw new BadRequestError({
-          message: "Environment with slug already exist",
+          message: "Environment with slug already exists",
           name: "CreateEnvironment"
         });
 
@@ -171,7 +171,7 @@ export const projectEnvServiceFactory = ({
         const existingEnv = await projectEnvDAL.findOne({ slug, projectId });
         if (existingEnv && existingEnv.id !== id) {
           throw new BadRequestError({
-            message: "Environment with slug already exist",
+            message: "Environment with slug already exists",
             name: "UpdateEnvironment"
           });
         }

backend/src/services/project/project-service.ts

@@ -10,7 +10,8 @@ import {
   ProjectType,
   ProjectVersion,
   TableName,
-  TProjectEnvironments
+  TProjectEnvironments,
+  TProjects
 } from "@app/db/schemas";
 import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission";
@@ -40,7 +41,8 @@ import { TSshHostGroupDALFactory } from "@app/ee/services/ssh-host-group/ssh-hos
 import { PgSqlLock, TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig } from "@app/lib/config/env";
 import { crypto } from "@app/lib/crypto/cryptography";
-import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors";
+import { DatabaseErrorCode } from "@app/lib/error-codes";
+import { BadRequestError, DatabaseError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors";
 import { groupBy } from "@app/lib/fn";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
 import { TProjectPermission } from "@app/lib/types";
@@ -303,19 +305,34 @@ export const projectServiceFactory = ({
           });
       }
 
-      const project = await projectDAL.create(
-        {
-          name: workspaceName,
-          type,
-          description: workspaceDescription,
-          orgId: organization.id,
-          slug: projectSlug || slugify(`${workspaceName}-${alphaNumericNanoId(4)}`),
-          kmsSecretManagerKeyId: kmsKeyId,
-          version: ProjectVersion.V3,
-          pitVersionLimit: 10
-        },
-        tx
-      );
+      const slug = projectSlug || slugify(`${workspaceName}-${alphaNumericNanoId(4)}`);
+
+      let project: TProjects;
+      try {
+        project = await projectDAL.create(
+          {
+            name: workspaceName,
+            type,
+            description: workspaceDescription,
+            orgId: organization.id,
+            slug,
+            kmsSecretManagerKeyId: kmsKeyId,
+            version: ProjectVersion.V3,
+            pitVersionLimit: 10
+          },
+          tx
+        );
+      } catch (err) {
+        if (
+          err instanceof DatabaseError &&
+          (err.error as { code: string })?.code === DatabaseErrorCode.UniqueViolation
+        ) {
+          throw new BadRequestError({
+            message: `A project with the slug "${slug}" already exists in your organization. Please choose a different name or slug.`
+          });
+        }
+        throw err;
+      }
 
       if (type === ProjectType.SSH) {
         await bootstrapSshProject({
@@ -579,39 +596,36 @@ export const projectServiceFactory = ({
     });
     ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.Settings);
 
-    if (update.slug) {
-      const existingProject = await projectDAL.findOne({
-        slug: update.slug,
-        orgId: actorOrgId
-      });
-      if (existingProject && existingProject.id !== project.id) {
-        throw new BadRequestError({
-          message: `Failed to update project slug. The project "${existingProject.name}" with the slug "${existingProject.slug}" already exists in your organization. Please choose a unique slug for your project.`
-        });
-      }
-    }
-
     if (update.secretDetectionIgnoreValues && !hasRole(ProjectMembershipRole.Admin)) {
       throw new ForbiddenRequestError({
         message: "Only admins can update secret detection ignore values"
       });
     }
 
-    const updatedProject = await projectDAL.updateById(project.id, {
-      name: update.name,
-      description: update.description,
-      autoCapitalization: update.autoCapitalization,
-      enforceCapitalization: update.autoCapitalization,
-      hasDeleteProtection: update.hasDeleteProtection,
-      slug: update.slug,
-      secretSharing: update.secretSharing,
-      defaultProduct: update.defaultProduct,
-      showSnapshotsLegacy: update.showSnapshotsLegacy,
-      secretDetectionIgnoreValues: update.secretDetectionIgnoreValues,
-      pitVersionLimit: update.pitVersionLimit
-    });
+    try {
+      const updatedProject = await projectDAL.updateById(project.id, {
+        name: update.name,
+        description: update.description,
+        autoCapitalization: update.autoCapitalization,
+        enforceCapitalization: update.autoCapitalization,
+        hasDeleteProtection: update.hasDeleteProtection,
+        slug: update.slug,
+        secretSharing: update.secretSharing,
+        defaultProduct: update.defaultProduct,
+        showSnapshotsLegacy: update.showSnapshotsLegacy,
+        secretDetectionIgnoreValues: update.secretDetectionIgnoreValues,
+        pitVersionLimit: update.pitVersionLimit
+      });
 
-    return updatedProject;
+      return updatedProject;
+    } catch (err) {
+      if (err instanceof DatabaseError && (err.error as { code: string })?.code === DatabaseErrorCode.UniqueViolation) {
+        throw new BadRequestError({
+          message: `Failed to update project. A project with the slug "${update.slug}" already exists in your organization. Please choose a different slug.`
+        });
+      }
+      throw err;
+    }
   };
 
   const toggleAutoCapitalization = async ({

backend/src/services/secret-tag/secret-tag-service.ts

@@ -35,7 +35,7 @@ export const secretTagServiceFactory = ({ secretTagDAL, permissionService }: TSe
     ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Tags);
 
     const existingTag = await secretTagDAL.findOne({ slug, projectId });
-    if (existingTag) throw new BadRequestError({ message: "Tag already exist" });
+    if (existingTag) throw new BadRequestError({ message: "Tag already exists" });
 
     const newTag = await secretTagDAL.create({
       projectId,
@@ -53,7 +53,7 @@ export const secretTagServiceFactory = ({ secretTagDAL, permissionService }: TSe
 
     if (slug) {
       const existingTag = await secretTagDAL.findOne({ slug, projectId: tag.projectId });
-      if (existingTag && existingTag.id !== tag.id) throw new BadRequestError({ message: "Tag already exist" });
+      if (existingTag && existingTag.id !== tag.id) throw new BadRequestError({ message: "Tag already exists" });
     }
 
     const { permission } = await permissionService.getProjectPermission({

backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts

@@ -282,7 +282,7 @@ export const secretV2BridgeServiceFactory = ({
       folderId
     });
     if (inputSecret.type === SecretType.Shared && doesSecretExist)
-      throw new BadRequestError({ message: "Secret already exist" });
+      throw new BadRequestError({ message: "Secret already exists" });
 
     // if user creating personal check its shared also exist
     if (inputSecret.type === SecretType.Personal && !doesSecretExist) {
@@ -527,7 +527,7 @@ export const secretV2BridgeServiceFactory = ({
         type: SecretType.Shared,
         folderId
       });
-      if (doesNewNameSecretExist) throw new BadRequestError({ message: "Secret with the new name already exist" });
+      if (doesNewNameSecretExist) throw new BadRequestError({ message: "Secret with the new name already exists" });
       ForbiddenError.from(permission).throwUnlessCan(
         ProjectPermissionSecretActions.Edit,
         subject(ProjectPermissionSub.Secrets, {
@@ -1674,7 +1674,7 @@ export const secretV2BridgeServiceFactory = ({
       }
     });
     if (secrets.length)
-      throw new BadRequestError({ message: `Secret already exist: ${secrets.map((el) => el.key).join(",")}` });
+      throw new BadRequestError({ message: `Secret already exists: ${secrets.map((el) => el.key).join(",")}` });
 
     const project = await projectDAL.findById(projectId);
     await scanSecretPolicyViolations(projectId, secretPath, inputSecrets, project.secretDetectionIgnoreValues || []);

backend/src/services/secret/secret-fns.ts

@@ -525,7 +525,7 @@ export const fnSecretBlindIndexCheck = async ({
   );
 
   if (isNew) {
-    if (secrets.length) throw new BadRequestError({ message: "Secret already exist" });
+    if (secrets.length) throw new BadRequestError({ message: "Secret already exists" });
   } else {
     const secretKeysInDB = unique(secrets, (el) => el.secretBlindIndex as string).map(
       (el) => blindIndex2KeyName[el.secretBlindIndex as string]
@@ -819,7 +819,7 @@ export const createManySecretsRawFnFactory = ({
       );
       if (secretsStoredInDB.length)
         throw new BadRequestError({
-          message: `Secret already exist: ${secretsStoredInDB.map((el) => el.key).join(",")}`
+          message: `Secret already exists: ${secretsStoredInDB.map((el) => el.key).join(",")}`
         });
 
       const inputSecrets = secrets.map((secret) => {

backend/src/services/secret/secret-service.ts

@@ -2751,7 +2751,7 @@ export const secretServiceFactory = ({
     const existingSecretTags = await secretDAL.getSecretTags(secret.id);
 
     if (existingSecretTags.some((tag) => tagSlugs.includes(tag.slug))) {
-      throw new BadRequestError({ message: "One or more tags already exist on the secret" });
+      throw new BadRequestError({ message: "One or more tags already exists on the secret" });
     }
 
     const combinedTags = new Set([...existingSecretTags.map((tag) => tag.id), ...tags.map((el) => el.id)]);