feat: added secret approval policy in replication

backend/e2e-test/mocks/keystore.ts

@@ -25,6 +25,9 @@ export const mockKeyStore = (): TKeyStoreFactory => {
     },
     incrementBy: async () => {
       return 1;
+    },
+    acquireLock: () => {
+      throw new Error("Not implemented");
     }
   };
 };

backend/src/db/migrations/20240430024805_secret-replication.ts

@@ -20,6 +20,12 @@ export async function up(knex: Knex): Promise<void> {
       t.boolean("isReplicated");
     });
   }
+
+  if (await knex.schema.hasTable(TableName.SecretApprovalRequestSecret)) {
+    await knex.schema.alterTable(TableName.SecretApprovalRequestSecret, (t) => {
+      t.boolean("isReplicated");
+    });
+  }
 }
 
 export async function down(knex: Knex): Promise<void> {
@@ -36,7 +42,13 @@ export async function down(knex: Knex): Promise<void> {
   }
 
   if (await knex.schema.hasTable(TableName.SecretVersion)) {
-    await knex.schema.alterTable(TableName.Secret, (t) => {
+    await knex.schema.alterTable(TableName.SecretVersion, (t) => {
+      t.dropColumns("isReplicated");
+    });
+  }
+
+  if (await knex.schema.hasTable(TableName.SecretApprovalRequestSecret)) {
+    await knex.schema.alterTable(TableName.SecretApprovalRequestSecret, (t) => {
       t.dropColumns("isReplicated");
     });
   }

backend/src/db/schemas/secret-approval-requests-secrets.ts

@@ -31,7 +31,8 @@ export const SecretApprovalRequestsSecretsSchema = z.object({
   requestId: z.string().uuid(),
   op: z.string(),
   secretId: z.string().uuid().nullable().optional(),
-  secretVersion: z.string().uuid().nullable().optional()
+  secretVersion: z.string().uuid().nullable().optional(),
+  isReplicated: z.boolean().nullable().optional()
 });
 
 export type TSecretApprovalRequestsSecrets = z.infer<typeof SecretApprovalRequestsSecretsSchema>;

backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts

@@ -16,8 +16,15 @@ import { TProjectDALFactory } from "@app/services/project/project-dal";
 import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
 import { TSecretDALFactory } from "@app/services/secret/secret-dal";
 import { getAllNestedSecretReferences } from "@app/services/secret/secret-fns";
+import {
+  fnSecretBlindIndexCheck,
+  fnSecretBlindIndexCheckV2,
+  fnSecretBulkDelete,
+  fnSecretBulkInsert,
+  fnSecretBulkUpdate
+} from "@app/services/secret/secret-fns";
 import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
-import { TSecretServiceFactory } from "@app/services/secret/secret-service";
+import { SecretOperations } from "@app/services/secret/secret-types";
 import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
 import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
 import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
@@ -32,7 +39,6 @@ import { TSecretApprovalRequestReviewerDALFactory } from "./secret-approval-requ
 import { TSecretApprovalRequestSecretDALFactory } from "./secret-approval-request-secret-dal";
 import {
   ApprovalStatus,
-  CommitType,
   RequestState,
   TApprovalRequestCountDTO,
   TGenerateSecretApprovalRequestDTO,
@@ -48,7 +54,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
   secretApprovalRequestDAL: TSecretApprovalRequestDALFactory;
   secretApprovalRequestSecretDAL: TSecretApprovalRequestSecretDALFactory;
   secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory;
-  folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findById" | "findSecretPathByFolderIds">;
+  folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findSecretPathByFolderIds">;
   secretDAL: TSecretDALFactory;
   secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret">;
   secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
@@ -56,16 +62,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
   secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany" | "insertMany">;
   secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
   projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
-  projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
+  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "removeSecretReminder">;
-  secretService: Pick<
-    TSecretServiceFactory,
-    | "fnSecretBulkInsert"
-    | "fnSecretBulkUpdate"
-    | "fnSecretBlindIndexCheck"
-    | "fnSecretBulkDelete"
-    | "fnSecretBlindIndexCheckV2"
-  >;
-  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
 };
 
 export type TSecretApprovalRequestServiceFactory = ReturnType<typeof secretApprovalRequestServiceFactory>;
@@ -82,7 +79,6 @@ export const secretApprovalRequestServiceFactory = ({
   projectDAL,
   permissionService,
   snapshotService,
-  secretService,
   secretVersionDAL,
   secretQueueService,
   projectBotService
@@ -302,11 +298,12 @@ export const secretApprovalRequestServiceFactory = ({
     const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
     if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" });
 
-    const conflicts: Array<{ secretId: string; op: CommitType }> = [];
+    const conflicts: Array<{ secretId: string; op: SecretOperations }> = [];
-    let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Create);
+    let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create);
     if (secretCreationCommits.length) {
-      const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({
+      const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
         folderId,
+        secretDAL,
         inputSecrets: secretCreationCommits.map(({ secretBlindIndex }) => {
           if (!secretBlindIndex) {
             throw new BadRequestError({
@@ -319,17 +316,19 @@ export const secretApprovalRequestServiceFactory = ({
       secretCreationCommits
         .filter(({ secretBlindIndex }) => conflictGroupByBlindIndex[secretBlindIndex || ""])
         .forEach((el) => {
-          conflicts.push({ op: CommitType.Create, secretId: el.id });
+          conflicts.push({ op: SecretOperations.Create, secretId: el.id });
         });
       secretCreationCommits = secretCreationCommits.filter(
         ({ secretBlindIndex }) => !conflictGroupByBlindIndex[secretBlindIndex || ""]
       );
     }
 
-    let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Update);
+    let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update);
     if (secretUpdationCommits.length) {
-      const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({
+      const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
         folderId,
+        secretDAL,
+        userId: "",
         inputSecrets: secretUpdationCommits
           .filter(({ secretBlindIndex, secret }) => secret && secret.secretBlindIndex !== secretBlindIndex)
           .map(({ secretBlindIndex }) => {
@@ -347,7 +346,7 @@ export const secretApprovalRequestServiceFactory = ({
             (secretBlindIndex && conflictGroupByBlindIndex[secretBlindIndex]) || !secretId
         )
         .forEach((el) => {
-          conflicts.push({ op: CommitType.Update, secretId: el.id });
+          conflicts.push({ op: SecretOperations.Update, secretId: el.id });
         });
 
       secretUpdationCommits = secretUpdationCommits.filter(
@@ -356,11 +355,11 @@ export const secretApprovalRequestServiceFactory = ({
       );
     }
 
-    const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Delete);
+    const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete);
-    const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
+
     const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
       const newSecrets = secretCreationCommits.length
-        ? await secretService.fnSecretBulkInsert({
+        ? await fnSecretBulkInsert({
             tx,
             folderId,
             inputSecrets: secretCreationCommits.map((el) => ({
@@ -380,7 +379,8 @@ export const secretApprovalRequestServiceFactory = ({
                 "secretReminderRepeatDays",
                 "algorithm",
                 "keyEncoding",
-                "secretBlindIndex"
+                "secretBlindIndex",
+                "isReplicated"
               ]),
               tags: el?.tags.map(({ id }) => id),
               version: 1,
@@ -403,7 +403,7 @@ export const secretApprovalRequestServiceFactory = ({
           })
         : [];
       const updatedSecrets = secretUpdationCommits.length
-        ? await secretService.fnSecretBulkUpdate({
+        ? await fnSecretBulkUpdate({
             folderId,
             projectId,
             tx,
@@ -425,6 +425,7 @@ export const secretApprovalRequestServiceFactory = ({
                   "secretKeyTag",
                   "secretKeyIV",
                   "metadata",
+                  "isReplicated",
                   "skipMultilineEncoding",
                   "secretReminderNote",
                   "secretReminderRepeatDays",
@@ -449,11 +450,13 @@ export const secretApprovalRequestServiceFactory = ({
           })
         : [];
       const deletedSecret = secretDeletionCommits.length
-        ? await secretService.fnSecretBulkDelete({
+        ? await fnSecretBulkDelete({
             projectId,
             folderId,
             tx,
             actorId: "",
+            secretDAL,
+            secretQueueService,
             inputSecrets: secretDeletionCommits.map(({ secretBlindIndex }) => {
               if (!secretBlindIndex) {
                 throw new BadRequestError({
@@ -480,12 +483,33 @@ export const secretApprovalRequestServiceFactory = ({
       };
     });
     await snapshotService.performSnapshot(folderId);
-    const folder = await folderDAL.findById(folderId);
+    const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]);
     // TODO(akhilmhdh-pg):  change query to do secret path from folder
     await secretQueueService.syncSecrets({
       projectId,
-      secretPath: "/",
+      secretPath: folder.path,
-      environment: folder?.environment.envSlug as string
+      environmentSlug: folder.environmentSlug,
+      folderId: folder.id,
+      membershipId: membership.id,
+      environmentId: folder.envId,
+      secrets: mergeStatus.secrets.created
+        .map(({ id, version }) => ({
+          operation: SecretOperations.Create,
+          version,
+          id
+        }))
+        .concat(
+          mergeStatus.secrets.updated.map(({ id, version }) => ({
+            operation: SecretOperations.Update,
+            version,
+            id
+          })),
+          mergeStatus.secrets.deleted.map(({ id, version }) => ({
+            operation: SecretOperations.Delete,
+            version,
+            id
+          }))
+        )
     });
     return mergeStatus;
   };
@@ -533,9 +557,9 @@ export const secretApprovalRequestServiceFactory = ({
     const commits: Omit<TSecretApprovalRequestsSecretsInsert, "requestId">[] = [];
     const commitTagIds: Record<string, string[]> = {};
     // for created secret approval change
-    const createdSecrets = data[CommitType.Create];
+    const createdSecrets = data[SecretOperations.Create];
     if (createdSecrets && createdSecrets?.length) {
-      const { keyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({
+      const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
         inputSecrets: createdSecrets,
         folderId,
         isNew: true,
@@ -546,7 +570,7 @@ export const secretApprovalRequestServiceFactory = ({
       commits.push(
         ...createdSecrets.map(({ secretName, ...el }) => ({
           ...el,
-          op: CommitType.Create as const,
+          op: SecretOperations.Create as const,
           version: 1,
           secretBlindIndex: keyName2BlindIndex[secretName],
           algorithm: SecretEncryptionAlgo.AES_256_GCM,
@@ -558,12 +582,12 @@ export const secretApprovalRequestServiceFactory = ({
       });
     }
     // not secret approval for update operations
-    const updatedSecrets = data[CommitType.Update];
+    const updatedSecrets = data[SecretOperations.Update];
     if (updatedSecrets && updatedSecrets?.length) {
       // get all blind index
       // Find all those secrets
       // if not throw not found
-      const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await secretService.fnSecretBlindIndexCheck({
+      const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await fnSecretBlindIndexCheck({
         inputSecrets: updatedSecrets,
         folderId,
         isNew: false,
@@ -574,7 +598,7 @@ export const secretApprovalRequestServiceFactory = ({
       // now find any secret that needs to update its name
       // same process as above
       const nameUpdatedSecrets = updatedSecrets.filter(({ newSecretName }) => Boolean(newSecretName));
-      const { keyName2BlindIndex: newKeyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({
+      const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({
         inputSecrets: nameUpdatedSecrets,
         folderId,
         isNew: true,
@@ -599,7 +623,7 @@ export const secretApprovalRequestServiceFactory = ({
           return {
             ...latestSecretVersions[secretId],
             ...el,
-            op: CommitType.Update as const,
+            op: SecretOperations.Update as const,
             secret: secretId,
             secretVersion: latestSecretVersions[secretId].id,
             secretBlindIndex,
@@ -609,12 +633,12 @@ export const secretApprovalRequestServiceFactory = ({
       );
     }
     // deleted secrets
-    const deletedSecrets = data[CommitType.Delete];
+    const deletedSecrets = data[SecretOperations.Delete];
     if (deletedSecrets && deletedSecrets.length) {
       // get all blind index
       // Find all those secrets
       // if not throw not found
-      const { keyName2BlindIndex, secrets } = await secretService.fnSecretBlindIndexCheck({
+      const { keyName2BlindIndex, secrets } = await fnSecretBlindIndexCheck({
         inputSecrets: deletedSecrets,
         folderId,
         isNew: false,
@@ -635,7 +659,7 @@ export const secretApprovalRequestServiceFactory = ({
           if (!latestSecretVersions[secretId].secretBlindIndex)
             throw new BadRequestError({ message: "Failed to find secret blind index" });
           return {
-            op: CommitType.Delete as const,
+            op: SecretOperations.Delete as const,
             ...latestSecretVersions[secretId],
             secretBlindIndex: latestSecretVersions[secretId].secretBlindIndex as string,
             secret: secretId,

backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts

@@ -1,11 +1,6 @@
 import { TImmutableDBKeys, TSecretApprovalPolicies, TSecretApprovalRequestsSecrets } from "@app/db/schemas";
 import { TProjectPermission } from "@app/lib/types";
-
+import { SecretOperations } from "@app/services/secret/secret-types";
-export enum CommitType {
-  Create = "create",
-  Update = "update",
-  Delete = "delete"
-}
 
 export enum RequestState {
   Open = "open",
@@ -18,14 +13,14 @@ export enum ApprovalStatus {
   REJECTED = "rejected"
 }
 
-type TApprovalCreateSecret = Omit<
+export type TApprovalCreateSecret = Omit<
   TSecretApprovalRequestsSecrets,
   TImmutableDBKeys | "version" | "algorithm" | "keyEncoding" | "requestId" | "op" | "secretVersion" | "secretBlindIndex"
 > & {
   secretName: string;
   tagIds?: string[];
 };
-type TApprovalUpdateSecret = Partial<TApprovalCreateSecret> & {
+export type TApprovalUpdateSecret = Partial<TApprovalCreateSecret> & {
   secretName: string;
   newSecretName?: string;
   tagIds?: string[];
@@ -36,9 +31,9 @@ export type TGenerateSecretApprovalRequestDTO = {
   secretPath: string;
   policy: TSecretApprovalPolicies;
   data: {
-    [CommitType.Create]?: TApprovalCreateSecret[];
+    [SecretOperations.Create]?: TApprovalCreateSecret[];
-    [CommitType.Update]?: TApprovalUpdateSecret[];
+    [SecretOperations.Update]?: TApprovalUpdateSecret[];
-    [CommitType.Delete]?: { secretName: string }[];
+    [SecretOperations.Delete]?: { secretName: string }[];
   };
 } & TProjectPermission;
 

backend/src/queue/queue-service.ts

@@ -7,7 +7,7 @@ import {
   TScanFullRepoEventPayload,
   TScanPushEventPayload
 } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
-import { TSyncSecretReplicationDTO } from "@app/services/secret-replication/secret-replication-types";
+import { TSyncSecretsDTO } from "@app/services/secret/secret-types";
 
 export enum QueueName {
   SecretRotation = "secret-rotation",
@@ -23,7 +23,8 @@ export enum QueueName {
   SecretPushEventScan = "secret-push-event-scan",
   UpgradeProjectToGhost = "upgrade-project-to-ghost",
   DynamicSecretRevocation = "dynamic-secret-revocation",
-  SecretReplication = "secret-replication"
+  SecretReplication = "secret-replication",
+  SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication
 }
 
 export enum QueueJobs {
@@ -40,7 +41,8 @@ export enum QueueJobs {
   UpgradeProjectToGhost = "upgrade-project-to-ghost-job",
   DynamicSecretRevocation = "dynamic-secret-revocation",
   DynamicSecretPruning = "dynamic-secret-pruning",
-  SecretReplication = "secret-replication"
+  SecretReplication = "secret-replication",
+  SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication
 }
 
 export type TQueueJobTypes = {
@@ -121,7 +123,11 @@ export type TQueueJobTypes = {
       };
   [QueueName.SecretReplication]: {
     name: QueueJobs.SecretReplication;
-    payload: TSyncSecretReplicationDTO;
+    payload: Omit<TSyncSecretsDTO, "environmentSlug">;
+  };
+  [QueueName.SecretSync]: {
+    name: QueueJobs.SecretSync;
+    payload: TSyncSecretsDTO;
   };
 };
 

backend/src/server/routes/index.ts

@@ -243,8 +243,8 @@ export const registerRoutes = async (
   const sapApproverDAL = secretApprovalPolicyApproverDALFactory(db);
   const secretApprovalPolicyDAL = secretApprovalPolicyDALFactory(db);
   const secretApprovalRequestDAL = secretApprovalRequestDALFactory(db);
-  const sarReviewerDAL = secretApprovalRequestReviewerDALFactory(db);
+  const secretApprovalRequestReviewerDAL = secretApprovalRequestReviewerDALFactory(db);
-  const sarSecretDAL = secretApprovalRequestSecretDALFactory(db);
+  const secretApprovalRequestSecretDAL = secretApprovalRequestSecretDALFactory(db);
 
   const secretRotationDAL = secretRotationDALFactory(db);
   const snapshotDAL = snapshotDALFactory(db);
@@ -492,7 +492,7 @@ export const registerRoutes = async (
     projectBotDAL,
     projectMembershipDAL,
     secretApprovalRequestDAL,
-    secretApprovalSecretDAL: sarSecretDAL,
+    secretApprovalSecretDAL: secretApprovalRequestSecretDAL,
     projectUserMembershipRoleDAL
   });
 
@@ -603,17 +603,6 @@ export const registerRoutes = async (
     secretDAL,
     secretBlindIndexDAL
   });
-  const secretReplicationService = secretReplicationServiceFactory({
-    secretTagDAL,
-    secretVersionTagDAL,
-    secretDAL,
-    secretVersionDAL,
-    secretImportDAL,
-    keyStore,
-    queueService,
-    secretReplicationDAL,
-    folderDAL
-  });
   const secretService = secretServiceFactory({
     folderDAL,
     secretVersionDAL,
@@ -625,7 +614,6 @@ export const registerRoutes = async (
     secretTagDAL,
     snapshotService,
     secretQueueService,
-    secretReplicationService,
     secretImportDAL,
     projectEnvDAL,
     projectBotService
@@ -636,19 +624,18 @@ export const registerRoutes = async (
     secretSharingDAL
   });
 
-  const sarService = secretApprovalRequestServiceFactory({
+  const secretApprovalRequestService = secretApprovalRequestServiceFactory({
     permissionService,
     projectBotService,
     folderDAL,
     secretDAL,
     secretTagDAL,
-    secretApprovalRequestSecretDAL: sarSecretDAL,
+    secretApprovalRequestSecretDAL,
-    secretApprovalRequestReviewerDAL: sarReviewerDAL,
+    secretApprovalRequestReviewerDAL,
     projectDAL,
     secretVersionDAL,
     secretBlindIndexDAL,
     secretApprovalRequestDAL,
-    secretService,
     snapshotService,
     secretVersionTagDAL,
     secretQueueService
@@ -677,6 +664,21 @@ export const registerRoutes = async (
     accessApprovalPolicyApproverDAL
   });
 
+  const secretReplicationService = secretReplicationServiceFactory({
+    secretTagDAL,
+    secretVersionTagDAL,
+    secretDAL,
+    secretVersionDAL,
+    secretImportDAL,
+    keyStore,
+    queueService,
+    secretReplicationDAL,
+    folderDAL,
+    secretApprovalPolicyService,
+    secretBlindIndexDAL,
+    secretApprovalRequestDAL,
+    secretApprovalRequestSecretDAL
+  });
   const secretRotationQueue = secretRotationQueueFactory({
     telemetryService,
     secretRotationDAL,
@@ -858,10 +860,10 @@ export const registerRoutes = async (
     identityGcpAuth: identityGcpAuthService,
     identityAwsAuth: identityAwsAuthService,
     identityAzureAuth: identityAzureAuthService,
-    secretApprovalPolicy: sapService,
     accessApprovalPolicy: accessApprovalPolicyService,
     accessApprovalRequest: accessApprovalRequestService,
-    secretApprovalRequest: sarService,
+    secretApprovalPolicy: secretApprovalPolicyService,
+    secretApprovalRequest: secretApprovalRequestService,
     secretRotation: secretRotationService,
     dynamicSecret: dynamicSecretService,
     dynamicSecretLease: dynamicSecretLeaseService,

backend/src/server/routes/v3/secret-router.ts

@@ -9,7 +9,6 @@ import {
   ServiceTokenScopes
 } from "@app/db/schemas";
 import { EventType } from "@app/ee/services/audit-log/audit-log-types";
-import { CommitType } from "@app/ee/services/secret-approval-request/secret-approval-request-types";
 import { RAW_SECRETS, SECRETS } from "@app/lib/api-docs";
 import { BadRequestError } from "@app/lib/errors";
 import { removeTrailingSlash } from "@app/lib/fn";
@@ -19,6 +18,7 @@ import { getUserAgentType } from "@app/server/plugins/audit-log";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { ActorType, AuthMode } from "@app/services/auth/auth-type";
 import { ProjectFilterType } from "@app/services/project/project-types";
+import { SecretOperations } from "@app/services/secret/secret-types";
 import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
 
 import { secretRawSchema } from "../sanitizedSchemas";
@@ -902,7 +902,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
             projectId,
             policy,
             data: {
-              [CommitType.Create]: [
+              [SecretOperations.Create]: [
                 {
                   secretName: req.params.secretName,
                   secretValueCiphertext,
@@ -1084,7 +1084,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
             projectId,
             policy,
             data: {
-              [CommitType.Update]: [
+              [SecretOperations.Update]: [
                 {
                   secretName: req.params.secretName,
                   newSecretName,
@@ -1234,7 +1234,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
             projectId,
             policy,
             data: {
-              [CommitType.Delete]: [
+              [SecretOperations.Delete]: [
                 {
                   secretName: req.params.secretName
                 }
@@ -1364,7 +1364,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
             projectId,
             policy,
             data: {
-              [CommitType.Create]: inputSecrets
+              [SecretOperations.Create]: inputSecrets
             }
           });
 
@@ -1491,7 +1491,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
             projectId,
             policy,
             data: {
-              [CommitType.Update]: inputSecrets.filter(({ type }) => type === "shared")
+              [SecretOperations.Update]: inputSecrets.filter(({ type }) => type === "shared")
             }
           });
 
@@ -1606,7 +1606,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
             projectId,
             policy,
             data: {
-              [CommitType.Delete]: inputSecrets.filter(({ type }) => type === "shared")
+              [SecretOperations.Delete]: inputSecrets.filter(({ type }) => type === "shared")
             }
           });
           await server.services.auditLog.createAuditLog({

backend/src/services/secret-import/secret-import-service.ts

@@ -111,7 +111,8 @@ export const secretImportServiceFactory = ({
     await secretQueueService.syncSecrets({
       secretPath: secImport.importPath,
       projectId,
-      environment: importEnv.slug
+      environmentSlug: importEnv.slug,
+      excludeReplication: true
     });
 
     return { ...secImport, importEnv };
@@ -223,7 +224,8 @@ export const secretImportServiceFactory = ({
     await secretQueueService.syncSecrets({
       secretPath: path,
       projectId,
-      environment
+      environmentSlug: environment,
+      excludeReplication: true
     });
 
     return secImport;

backend/src/services/secret-replication/secret-replication-service.ts

@@ -1,28 +1,40 @@
+import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
+import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
+import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
 import { TKeyStoreFactory } from "@app/keystore/keystore";
 import { groupBy } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
+import { alphaNumericNanoId } from "@app/lib/nanoid";
 import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
 
 import { TSecretDALFactory } from "../secret/secret-dal";
 import { fnSecretBulkInsert, fnSecretBulkUpdate } from "../secret/secret-fns";
+import { SecretOperations, TSyncSecretsDTO } from "../secret/secret-types";
 import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
 import { TSecretVersionTagDALFactory } from "../secret/secret-version-tag-dal";
+import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
 import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
 import { TSecretReplicationDALFactory } from "./secret-replication-dal";
-import { SecretReplicationOperations, TSyncSecretReplicationDTO } from "./secret-replication-types";
 
 type TSecretReplicationServiceFactoryDep = {
   secretReplicationDAL: TSecretReplicationDALFactory;
   secretDAL: Pick<TSecretDALFactory, "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete">;
+  secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany">;
   secretImportDAL: Pick<TSecretImportDALFactory, "find">;
-  folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds">;
+  folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds" | "findBySecretPath">;
-  secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update">;
-  secretTagDAL: Pick<TSecretTagDALFactory, "find" | "saveTagsToSecret" | "deleteTagsManySecret">;
   secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "find" | "insertMany">;
   queueService: Pick<TQueueServiceFactory, "start" | "listen" | "queue" | "stopJobById">;
+  secretApprovalPolicyService: Pick<TSecretApprovalPolicyServiceFactory, "getSecretApprovalPolicy">;
   keyStore: Pick<TKeyStoreFactory, "acquireLock">;
+  secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret" | "find">;
+  secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "create" | "transaction">;
+  secretApprovalRequestSecretDAL: Pick<
+    TSecretApprovalRequestSecretDALFactory,
+    "insertMany" | "insertApprovalSecretTags"
+  >;
 };
 
 export type TSecretReplicationServiceFactory = ReturnType<typeof secretReplicationServiceFactory>;
@@ -44,18 +56,21 @@ export const secretReplicationServiceFactory = ({
   keyStore,
   secretVersionTagDAL,
   secretTagDAL,
-  folderDAL
+  folderDAL,
+  secretApprovalPolicyService,
+  secretApprovalRequestSecretDAL,
+  secretApprovalRequestDAL
 }: TSecretReplicationServiceFactoryDep) => {
   queueService.start(QueueName.SecretReplication, async (job) => {
     logger.info(job.data, "Replication started");
-    const { secrets, folderId, secretPath, environmentId, projectId } = job.data;
+    const { secrets, folderId, secretPath, environmentId, projectId, membershipId } = job.data;
     const secretImports = await secretImportDAL.find({
       importPath: secretPath,
       importEnv: environmentId,
       isReplication: true
     });
     console.log(">>>> Secret Imports replics ", secretImports.length, secretPath, environmentId);
-    if (!secretImports.length) return;
+    if (!secretImports.length || !secrets.length) return;
 
     // unfiltered secrets to be replicated
     console.log(secrets.length);
@@ -75,7 +90,8 @@ export const secretReplicationServiceFactory = ({
     try {
       /*  eslint-disable no-await-in-loop */
       for (const secretImport of secretImports) {
-        const importFolderId = secretImport.folderId;
+        const [importedFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]);
+        const importFolderId = importedFolder.id;
 
         const localSecrets = await secretDAL.find({
           $in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) },
@@ -86,14 +102,14 @@ export const secretReplicationServiceFactory = ({
 
         const locallyCreatedSecrets = secrets.filter(({ operation, id }) => {
           return (
-            (operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) &&
+            (operation === SecretOperations.Create || operation === SecretOperations.Update) &&
             !localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
           );
         });
 
         const locallyUpdatedSecrets = secrets.filter(
           ({ operation, id }) =>
-            (operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) &&
+            (operation === SecretOperations.Create || operation === SecretOperations.Update) &&
             localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
         );
 
@@ -102,35 +118,48 @@ export const secretReplicationServiceFactory = ({
         console.log("update", locallyUpdatedSecrets);
         console.log("local board", localSecrets);
 
-        const locallyDeletedSecrets = secrets
+        const locallyDeletedSecrets = secrets.filter(
-          .filter(
+          ({ operation, id }) =>
-            ({ operation, id }) =>
+            operation === SecretOperations.Delete &&
-              operation === SecretReplicationOperations.Delete &&
+            Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) &&
-              Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) &&
+            localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
-              localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
+        );
-          )
-          .map(
-            ({ id }) =>
-              localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string][0]
-          );
 
-        let nestedImportSecrets: TSyncSecretReplicationDTO["secrets"] = [];
+        const policy = await secretApprovalPolicyService.getSecretApprovalPolicy(
-        await secretReplicationDAL.transaction(async (tx) => {
+          projectId,
-          if (locallyCreatedSecrets.length) {
+          importedFolder.environmentSlug,
-            const newSecrets = await fnSecretBulkInsert({
+          importedFolder.path
-              folderId: importFolderId,
+        );
-              secretVersionDAL,
+        // this means it should be a approval request rather than direct replication
-              secretDAL,
+        if (policy) {
-              tx,
+          const localSecretsLatestVersions = localSecrets.map(({ id }) => id);
-              secretTagDAL,
+          const latestSecretVersions = await secretVersionDAL.findLatestVersionMany(
-              secretVersionTagDAL,
+            importFolderId,
-              inputSecrets: locallyCreatedSecrets.map(({ id }) => {
+            localSecretsLatestVersions
+          );
+          await secretApprovalRequestDAL.transaction(async (tx) => {
+            const approvalRequestDoc = await secretApprovalRequestDAL.create(
+              {
+                folderId: importFolderId,
+                slug: alphaNumericNanoId(),
+                policyId: policy.id,
+                status: "open",
+                hasMerged: false,
+                committerId: membershipId
+              },
+              tx
+            );
+            const commits = locallyCreatedSecrets
+              .concat(locallyUpdatedSecrets)
+              .concat(locallyDeletedSecrets)
+              .map(({ id, operation }) => {
                 const doc = replicatedSecretsGroupBySecretId[id][0];
+                const localSecret = localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0];
                 return {
+                  op: operation,
                   keyEncoding: doc.keyEncoding,
                   algorithm: doc.algorithm,
-                  folderId,
+                  requestId: approvalRequestDoc.id,
-                  type: doc.type,
                   metadata: doc.metadata,
                   secretKeyIV: doc.secretKeyIV,
                   secretKeyTag: doc.secretKeyTag,
@@ -143,31 +172,31 @@ export const secretReplicationServiceFactory = ({
                   secretCommentTag: doc.secretCommentTag,
                   secretCommentCiphertext: doc.secretCommentCiphertext,
                   isReplicated: true,
-                  skipMultilineEncoding: doc.skipMultilineEncoding
+                  skipMultilineEncoding: doc.skipMultilineEncoding,
+                  // except create operation other two needs the secret id and version id
+                  ...(operation !== SecretOperations.Create
+                    ? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id }
+                    : {})
                 };
-              })
+              });
-            });
+            const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx);
-            nestedImportSecrets = nestedImportSecrets.concat(
+
-              ...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Create, version, id }))
+            return { ...approvalRequestDoc, commits: approvalCommits };
-            );
+          });
-          }
+        } else {
-          if (locallyUpdatedSecrets.length) {
+          let nestedImportSecrets: TSyncSecretsDTO["secrets"] = [];
-            const newSecrets = await fnSecretBulkUpdate({
+          await secretReplicationDAL.transaction(async (tx) => {
-              projectId,
+            if (locallyCreatedSecrets.length) {
-              folderId: importFolderId,
+              const newSecrets = await fnSecretBulkInsert({
-              secretVersionDAL,
+                folderId: importFolderId,
-              secretDAL,
+                secretVersionDAL,
-              tx,
+                secretDAL,
-              secretTagDAL,
+                tx,
-              secretVersionTagDAL,
+                secretTagDAL,
-              inputSecrets: locallyUpdatedSecrets.map(({ id }) => {
+                secretVersionTagDAL,
-                const doc = replicatedSecretsGroupBySecretId[id][0];
+                inputSecrets: locallyCreatedSecrets.map(({ id }) => {
-                return {
+                  const doc = replicatedSecretsGroupBySecretId[id][0];
-                  filter: {
+                  return {
-                    folderId: importFolderId,
-                    id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
-                  },
-                  data: {
                     keyEncoding: doc.keyEncoding,
                     algorithm: doc.algorithm,
                     type: doc.type,
@@ -184,39 +213,80 @@ export const secretReplicationServiceFactory = ({
                     secretCommentCiphertext: doc.secretCommentCiphertext,
                     isReplicated: true,
                     skipMultilineEncoding: doc.skipMultilineEncoding
-                  }
+                  };
-                };
+                })
-              })
+              });
-            });
+              nestedImportSecrets = nestedImportSecrets.concat(
-            nestedImportSecrets = nestedImportSecrets.concat(
+                newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id }))
-              ...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Update, version, id }))
+              );
-            );
+            }
-          }
+            if (locallyUpdatedSecrets.length) {
-          if (locallyDeletedSecrets.length) {
+              const newSecrets = await fnSecretBulkUpdate({
-            const newSecrets = await secretDAL.delete(
+                projectId,
-              {
+                folderId: importFolderId,
-                $in: {
+                secretVersionDAL,
-                  id: locallyDeletedSecrets.map(({ id }) => id)
+                secretDAL,
+                tx,
+                secretTagDAL,
+                secretVersionTagDAL,
+                inputSecrets: locallyUpdatedSecrets.map(({ id }) => {
+                  const doc = replicatedSecretsGroupBySecretId[id][0];
+                  return {
+                    filter: {
+                      folderId: importFolderId,
+                      id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
+                    },
+                    data: {
+                      keyEncoding: doc.keyEncoding,
+                      algorithm: doc.algorithm,
+                      type: doc.type,
+                      metadata: doc.metadata,
+                      secretKeyIV: doc.secretKeyIV,
+                      secretKeyTag: doc.secretKeyTag,
+                      secretKeyCiphertext: doc.secretKeyCiphertext,
+                      secretValueIV: doc.secretValueIV,
+                      secretValueTag: doc.secretValueTag,
+                      secretValueCiphertext: doc.secretValueCiphertext,
+                      secretBlindIndex: doc.secretBlindIndex,
+                      secretCommentIV: doc.secretCommentIV,
+                      secretCommentTag: doc.secretCommentTag,
+                      secretCommentCiphertext: doc.secretCommentCiphertext,
+                      isReplicated: true,
+                      skipMultilineEncoding: doc.skipMultilineEncoding
+                    }
+                  };
+                })
+              });
+              nestedImportSecrets = nestedImportSecrets.concat(
+                newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Update, version, id }))
+              );
+            }
+            if (locallyDeletedSecrets.length) {
+              const newSecrets = await secretDAL.delete(
+                {
+                  $in: {
+                    id: locallyDeletedSecrets.map(({ id }) => id)
+                  },
+                  isReplicated: true,
+                  folderId: importFolderId
                 },
-                isReplicated: true,
+                tx
-                folderId: importFolderId
+              );
-              },
+              nestedImportSecrets = nestedImportSecrets.concat(
-              tx
+                newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Delete, version, id }))
-            );
+              );
-            nestedImportSecrets = nestedImportSecrets.concat(
+            }
-              ...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Delete, version, id }))
+          });
-            );
+          console.log("Environment ID -> slug", importedFolder.envId, importedFolder.environmentSlug);
-          }
+          await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, {
-        });
+            folderId: importedFolder.id,
-        const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]);
+            projectId,
-        console.log("Environment ID -> slug", folder.envId, folder.environmentSlug);
+            secrets: nestedImportSecrets,
-        await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, {
+            secretPath: importedFolder.path,
-          folderId: folder.id,
+            environmentId: importedFolder.envId,
-          projectId,
+            membershipId
-          secrets: nestedImportSecrets,
+          });
-          secretPath: folder.path,
+        }
-          environmentId: folder.envId
-        });
       }
       await secretVersionDAL.update({ $in: { id: replicatedSecrets.map(({ id }) => id) } }, { isReplicated: true });
       /*  eslint-enable no-await-in-loop */
@@ -228,20 +298,4 @@ export const secretReplicationServiceFactory = ({
   queueService.listen(QueueName.SecretReplication, "failed", async (job, err) => {
     logger.error(err, "Failed to replicate secret", job?.data);
   });
-
-  const replicate = async (data: TSyncSecretReplicationDTO) => {
-    await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, data, {
-      attempts: 3,
-      backoff: {
-        type: "exponential",
-        delay: 1000
-      },
-      removeOnComplete: true,
-      removeOnFail: true
-    });
-  };
-
-  return {
-    replicate
-  };
 };

backend/src/services/secret-replication/secret-replication-types.ts

@@ -1,17 +1,3 @@
-export enum SecretReplicationOperations {
-  Create = "create",
-  Update = "update",
-  Delete = "delete"
-}
-
 export type TSyncSecretReplicationDTO = {
-  secretPath: string;
+  id: string;
-  projectId: string;
-  environmentId: string;
-  folderId: string;
-  secrets: {
-    operation: SecretReplicationOperations;
-    id: string;
-    version: number;
-  }[];
 };

backend/src/services/secret/secret-fns.ts

@@ -32,6 +32,8 @@ import {
   TCreateManySecretsRawFn,
   TCreateManySecretsRawFnFactory,
   TFnSecretBlindIndexCheck,
+  TFnSecretBlindIndexCheckV2,
+  TFnSecretBulkDelete,
   TFnSecretBulkInsert,
   TFnSecretBulkUpdate,
   TUpdateManySecretsRawFn,
@@ -396,6 +398,30 @@ export const decryptSecretRaw = (
   };
 };
 
+// this is used when secret blind index already exist
+// mainly for secret approval
+export const fnSecretBlindIndexCheckV2 = async ({
+  inputSecrets,
+  folderId,
+  userId,
+  secretDAL
+}: TFnSecretBlindIndexCheckV2) => {
+  if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
+    throw new BadRequestError({ message: "Missing user id for personal secret" });
+  }
+  const secrets = await secretDAL.findByBlindIndexes(
+    folderId,
+    inputSecrets.map(({ secretBlindIndex, type }) => ({
+      blindIndex: secretBlindIndex,
+      type: type || SecretType.Shared
+    })),
+    userId
+  );
+  const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string);
+
+  return { secsGroupedByBlindIndex, secrets };
+};
+
 /**
  * Grabs and processes nested secret references from a string
  *
@@ -598,6 +624,35 @@ export const fnSecretBulkUpdate = async ({
   return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
 };
 
+export const fnSecretBulkDelete = async ({
+  folderId,
+  inputSecrets,
+  tx,
+  actorId,
+  secretDAL,
+  secretQueueService
+}: TFnSecretBulkDelete) => {
+  const deletedSecrets = await secretDAL.deleteMany(
+    inputSecrets.map(({ type, secretBlindIndex }) => ({
+      blindIndex: secretBlindIndex,
+      type
+    })),
+    folderId,
+    actorId,
+    tx
+  );
+
+  await Promise.allSettled(
+    deletedSecrets
+      .filter(({ secretReminderRepeatDays }) => Boolean(secretReminderRepeatDays))
+      .map(({ id, secretReminderRepeatDays }) =>
+        secretQueueService.removeSecretReminder({ secretId: id, repeatDays: secretReminderRepeatDays as number })
+      )
+  );
+
+  return deletedSecrets;
+};
+
 export const createManySecretsRawFnFactory = ({
   projectDAL,
   projectBotDAL,

backend/src/services/secret/secret-queue.ts

@@ -28,7 +28,12 @@ import { TWebhookDALFactory } from "../webhook/webhook-dal";
 import { fnTriggerWebhook } from "../webhook/webhook-fns";
 import { TSecretDALFactory } from "./secret-dal";
 import { interpolateSecrets } from "./secret-fns";
-import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types";
+import {
+  TCreateSecretReminderDTO,
+  THandleReminderDTO,
+  TRemoveSecretReminderDTO,
+  TSyncSecretsDTO
+} from "./secret-types";
 
 export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
 type TSecretQueueFactoryDep = {
@@ -116,21 +121,17 @@ export const secretQueueFactory = ({
     });
   };
 
-  const syncSecrets = async ({
+  const syncSecrets = async <T extends boolean = false>({_deDupeQueue:deDupeQueue = {},_depth = 0, ...dto}: TSyncSecretsDTO<T>) => {
-    deDupeQueue = {},
+    logger.info(
-    ...dto
+      `syncSecrets: syncing project secrets where [projectId=${dto.projectId}]  [environment=${dto.environmentSlug}] [path=${dto.secretPath}]`
-  }: TGetSecrets & { depth?: number; deDupeQueue?: Record<string, boolean> }) => {
+    );
-    const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath);
+    const deDuplicationKey = uniqueIntegrationKey(dto.environmentSlug, dto.secretPath);
     if (deDupeQueue?.[deDuplicationKey]) {
       return;
     }
     // eslint-disable-next-line
     deDupeQueue[deDuplicationKey] = true;
-    logger.info(
+    await queueService.queue(QueueName.SecretSync, QueueJobs.SecretSync, dto as TSyncSecretsDTO, {
-      `syncSecrets: syncing project secrets where [projectId=${dto.projectId}]  [environment=${dto.environment}] [path=${dto.secretPath}]`
-    );
-    await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, {
-      jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`,
       removeOnFail: true,
       removeOnComplete: true,
       delay: 1000,
@@ -140,7 +141,6 @@ export const secretQueueFactory = ({
         delay: 3000
       }
     });
-    await syncIntegrations({ ...dto, deDupeQueue });
   };
 
   const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
@@ -333,6 +333,52 @@ export const secretQueueFactory = ({
     return content;
   };
 
+  queueService.start(QueueName.SecretSync, async (job) => {
+    const {
+      secretPath,
+      environmentId,
+      projectId,
+      environmentSlug: environment,
+      secrets,
+      folderId,
+      excludeReplication,
+      membershipId
+    } = job.data;
+    await queueService.queue(
+      QueueName.SecretWebhook,
+      QueueJobs.SecWebhook,
+      { environment, projectId, secretPath },
+      {
+        jobId: `secret-webhook-${environmentId}-${projectId}-${secretPath}`,
+        removeOnFail: { count: 5 },
+        removeOnComplete: true,
+        delay: 1000,
+        attempts: 5,
+        backoff: {
+          type: "exponential",
+          delay: 3000
+        }
+      }
+    );
+    await syncIntegrations({ secretPath, projectId, environment });
+    if (!excludeReplication) {
+      await queueService.queue(
+        QueueName.SecretReplication,
+        QueueJobs.SecretReplication,
+        { environmentId, projectId, secretPath, folderId, secrets, membershipId },
+        {
+          attempts: 3,
+          backoff: {
+            type: "exponential",
+            delay: 1000
+          },
+          removeOnComplete: true,
+          removeOnFail: true
+        }
+      );
+    }
+  });
+
   queueService.start(QueueName.IntegrationSync, async (job) => {
     const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
 
@@ -375,11 +421,12 @@ export const secretQueueFactory = ({
             )
             .map(({ folderId }) =>
               syncSecrets({
-                depth: depth + 1,
+                _depth: depth + 1,
                 projectId,
                 secretPath: foldersGroupedById[folderId][0].path,
-                environment: foldersGroupedById[folderId][0].environmentSlug,
+                environmentSlug: foldersGroupedById[folderId][0].environmentSlug,
-                deDupeQueue
+                _deDupeQueue: deDupeQueue,
+                excludeReplication: true
               })
             )
         );
@@ -412,11 +459,12 @@ export const secretQueueFactory = ({
             )
             .map(({ folderId }) =>
               syncSecrets({
-                depth: depth + 1,
+                _depth: depth + 1,
                 projectId,
                 secretPath: referencedFoldersGroupedById[folderId][0].path,
-                environment: referencedFoldersGroupedById[folderId][0].environmentSlug,
+                environmentSlug: referencedFoldersGroupedById[folderId][0].environmentSlug,
-                deDupeQueue
+                _deDupeQueue: deDupeQueue,
+                excludeReplication: true
               })
             )
         );
@@ -546,7 +594,7 @@ export const secretQueueFactory = ({
 
   return {
     // depth is internal only field thus no need to make it available outside
-    syncSecrets: (dto: TGetSecrets) => syncSecrets(dto),
+    syncSecrets,
     syncIntegrations,
     addSecretReminder,
     removeSecretReminder,

backend/src/services/secret/secret-service.ts

@@ -30,13 +30,12 @@ import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
 import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
-import { TSecretReplicationServiceFactory } from "../secret-replication/secret-replication-service";
-import { SecretReplicationOperations } from "../secret-replication/secret-replication-types";
 import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
 import { TSecretDALFactory } from "./secret-dal";
 import {
   decryptSecretRaw,
   fnSecretBlindIndexCheck,
+  fnSecretBulkDelete,
   fnSecretBulkInsert,
   fnSecretBulkUpdate,
   getAllNestedSecretReferences,
@@ -45,6 +44,7 @@ import {
 } from "./secret-fns";
 import { TSecretQueueFactory } from "./secret-queue";
 import {
+  SecretOperations,
   TAttachSecretTagsDTO,
   TBackFillSecretReferencesDTO,
   TCreateBulkSecretDTO,
@@ -55,8 +55,6 @@ import {
   TDeleteManySecretRawDTO,
   TDeleteSecretDTO,
   TDeleteSecretRawDTO,
-  TFnSecretBlindIndexCheckV2,
-  TFnSecretBulkDelete,
   TGetASecretDTO,
   TGetASecretRawDTO,
   TGetSecretsDTO,
@@ -87,7 +85,6 @@ type TSecretServiceFactoryDep = {
   projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
   secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
-  secretReplicationService: Pick<TSecretReplicationServiceFactory, "replicate">;
 };
 
 export type TSecretServiceFactory = ReturnType<typeof secretServiceFactory>;
@@ -104,8 +101,7 @@ export const secretServiceFactory = ({
   projectDAL,
   projectBotService,
   secretImportDAL,
-  secretVersionTagDAL,
+  secretVersionTagDAL
-  secretReplicationService
 }: TSecretServiceFactoryDep) => {
   const getSecretReference = async (projectId: string) => {
     // if bot key missing means e2e still exist
@@ -143,53 +139,6 @@ export const secretServiceFactory = ({
     return secretBlindIndex;
   };
 
-  const fnSecretBulkDelete = async ({ folderId, inputSecrets, tx, actorId }: TFnSecretBulkDelete) => {
-    const deletedSecrets = await secretDAL.deleteMany(
-      inputSecrets.map(({ type, secretBlindIndex }) => ({
-        blindIndex: secretBlindIndex,
-        type
-      })),
-      folderId,
-      actorId,
-      tx
-    );
-
-    for (const s of deletedSecrets) {
-      if (s.secretReminderRepeatDays) {
-        // eslint-disable-next-line no-await-in-loop
-        await secretQueueService
-          .removeSecretReminder({
-            secretId: s.id,
-            repeatDays: s.secretReminderRepeatDays
-          })
-          .catch((err) => {
-            logger.error(err, `Failed to delete secret reminder for secret with ID ${s?.id}`);
-          });
-      }
-    }
-
-    return deletedSecrets;
-  };
-
-  // this is used when secret blind index already exist
-  // mainly for secret approval
-  const fnSecretBlindIndexCheckV2 = async ({ inputSecrets, folderId, userId }: TFnSecretBlindIndexCheckV2) => {
-    if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
-      throw new BadRequestError({ message: "Missing user id for personal secret" });
-    }
-    const secrets = await secretDAL.findByBlindIndexes(
-      folderId,
-      inputSecrets.map(({ secretBlindIndex, type }) => ({
-        blindIndex: secretBlindIndex,
-        type: type || SecretType.Shared
-      })),
-      userId
-    );
-    const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string);
-
-    return { secsGroupedByBlindIndex, secrets };
-  };
-
   const createSecret = async ({
     path,
     actor,
@@ -200,7 +149,7 @@ export const secretServiceFactory = ({
     projectId,
     ...inputSecret
   }: TCreateSecretDTO) => {
-    const { permission } = await permissionService.getProjectPermission(
+    const { permission, membership } = await permissionService.getProjectPermission(
       actor,
       actorId,
       projectId,
@@ -287,16 +236,17 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
+    await secretQueueService.syncSecrets({
-    // TODO(akhilmhdh-pg): licence check, posthog service and snapshot
-    await secretReplicationService.replicate({
-      folderId,
-      projectId,
-      environmentId: folder.envId,
       secretPath: path,
+      // if secret service reached means there was no secret policy
+      // TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
+      membershipId: membership?.id as string,
+      projectId,
+      environmentSlug: folder.environment.slug,
+      environmentId: folder.envId,
       secrets: [
         {
-          operation: SecretReplicationOperations.Create,
+          operation: SecretOperations.Create,
           id: secret[0].id,
           version: 1
         }
@@ -315,7 +265,7 @@ export const secretServiceFactory = ({
     projectId,
     ...inputSecret
   }: TUpdateSecretDTO) => {
-    const { permission } = await permissionService.getProjectPermission(
+    const { permission, membership } = await permissionService.getProjectPermission(
       actor,
       actorId,
       projectId,
@@ -430,15 +380,18 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
+    await secretQueueService.syncSecrets({
-    await secretReplicationService.replicate({
+      // if secret service reached means there was no secret policy
-      folderId,
+      // TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
-      projectId,
+      membershipId: membership?.id as string,
-      environmentId: folder.envId,
       secretPath: path,
+      folderId: folder.id,
+      projectId,
+      environmentSlug: folder.environment.slug,
+      environmentId: folder.envId,
       secrets: [
         {
-          operation: SecretReplicationOperations.Update,
+          operation: SecretOperations.Update,
           id: updatedSecret[0].id,
           version: updatedSecret[0].version
         }
@@ -457,7 +410,7 @@ export const secretServiceFactory = ({
     projectId,
     ...inputSecret
   }: TDeleteSecretDTO) => {
-    const { permission } = await permissionService.getProjectPermission(
+    const { permission, membership } = await permissionService.getProjectPermission(
       actor,
       actorId,
       projectId,
@@ -499,6 +452,8 @@ export const secretServiceFactory = ({
         projectId,
         folderId,
         actorId,
+        secretDAL,
+        secretQueueService,
         inputSecrets: [
           {
             type: inputSecret.type as SecretType,
@@ -510,15 +465,18 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
+    await secretQueueService.syncSecrets({
-    await secretReplicationService.replicate({
+      // if secret service reached means there was no secret policy
-      folderId,
+      // TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
-      projectId,
+      membershipId: membership?.id as string,
-      environmentId: folder.envId,
       secretPath: path,
+      folderId: folder.id,
+      projectId,
+      environmentSlug: folder.environment.slug,
+      environmentId: folder.envId,
       secrets: [
         {
-          operation: SecretReplicationOperations.Delete,
+          operation: SecretOperations.Delete,
           id: deletedSecret[0].id,
           version: deletedSecret[0].version
         }
@@ -744,7 +702,7 @@ export const secretServiceFactory = ({
     projectId,
     secrets: inputSecrets
   }: TCreateBulkSecretDTO) => {
-    const { permission } = await permissionService.getProjectPermission(
+    const { permission, membership } = await permissionService.getProjectPermission(
       actor,
       actorId,
       projectId,
@@ -808,13 +766,16 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
+    await secretQueueService.syncSecrets({
-    await secretReplicationService.replicate({
+      // if secret service reached means there was no secret policy
-      folderId,
+      // TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
-      projectId,
+      membershipId: membership?.id as string,
-      environmentId: folder.envId,
       secretPath: path,
-      secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Create }))
+      folderId: folder.id,
+      projectId,
+      environmentSlug: folder.environment.slug,
+      environmentId: folder.envId,
+      secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Create }))
     });
 
     return newSecrets;
@@ -830,7 +791,7 @@ export const secretServiceFactory = ({
     projectId,
     secrets: inputSecrets
   }: TUpdateBulkSecretDTO) => {
-    const { permission } = await permissionService.getProjectPermission(
+    const { permission, membership } = await permissionService.getProjectPermission(
       actor,
       actorId,
       projectId,
@@ -915,13 +876,16 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
+    await secretQueueService.syncSecrets({
-    await secretReplicationService.replicate({
+      // if secret service reached means there was no secret policy
-      folderId,
+      // TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
-      projectId,
+      membershipId: membership?.id as string,
-      environmentId: folder.envId,
       secretPath: path,
-      secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Update }))
+      folderId: folder.id,
+      projectId,
+      environmentSlug: folder.environment.slug,
+      environmentId: folder.envId,
+      secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Update }))
     });
 
     return secrets;
@@ -937,7 +901,7 @@ export const secretServiceFactory = ({
     actorAuthMethod,
     actorOrgId
   }: TDeleteBulkSecretDTO) => {
-    const { permission } = await permissionService.getProjectPermission(
+    const { permission, membership } = await permissionService.getProjectPermission(
       actor,
       actorId,
       projectId,
@@ -972,6 +936,8 @@ export const secretServiceFactory = ({
 
     const secretsDeleted = await secretDAL.transaction(async (tx) =>
       fnSecretBulkDelete({
+        secretDAL,
+        secretQueueService,
         inputSecrets: inputSecrets.map(({ type, secretName }) => ({
           secretBlindIndex: keyName2BlindIndex[secretName],
           type
@@ -984,13 +950,16 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
+    await secretQueueService.syncSecrets({
-    await secretReplicationService.replicate({
+      // if secret service reached means there was no secret policy
-      folderId,
+      // TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
-      projectId,
+      membershipId: membership?.id as string,
-      environmentId: folder.envId,
       secretPath: path,
-      secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Delete }))
+      folderId: folder.id,
+      projectId,
+      environmentSlug: folder.environment.slug,
+      environmentId: folder.envId,
+      secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretOperations.Delete }))
     });
 
     return secretsDeleted;
@@ -1171,9 +1140,6 @@ export const secretServiceFactory = ({
       skipMultilineEncoding
     });
 
-    await snapshotService.performSnapshot(secret.folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return decryptSecretRaw(secret, botKey);
   };
 
@@ -1212,8 +1178,6 @@ export const secretServiceFactory = ({
     });
 
     await snapshotService.performSnapshot(secret.folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return decryptSecretRaw(secret, botKey);
   };
 
@@ -1243,9 +1207,6 @@ export const secretServiceFactory = ({
       actorAuthMethod
     });
 
-    await snapshotService.performSnapshot(secret.folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return decryptSecretRaw(secret, botKey);
   };
 
@@ -1294,12 +1255,7 @@ export const secretServiceFactory = ({
       })
     });
 
-    await snapshotService.performSnapshot(secrets[0].folderId);
+    return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
-    return secrets.map((secret) =>
-      decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
-    );
   };
 
   const updateManySecretsRaw = async ({
@@ -1348,12 +1304,7 @@ export const secretServiceFactory = ({
       })
     });
 
-    await snapshotService.performSnapshot(secrets[0].folderId);
+    return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
-    return secrets.map((secret) =>
-      decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
-    );
   };
 
   const deleteManySecretsRaw = async ({
@@ -1384,12 +1335,7 @@ export const secretServiceFactory = ({
       secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared }))
     });
 
-    await snapshotService.performSnapshot(secrets[0].folderId);
+    return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
-    return secrets.map((secret) =>
-      decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
-    );
   };
 
   const getSecretVersions = async ({
@@ -1510,7 +1456,12 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folder.id);
-    await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment });
+    await secretQueueService.syncSecrets({
+      secretPath,
+      projectId: project.id,
+      environmentSlug: environment,
+      excludeReplication: true
+    });
 
     return {
       ...updatedSecret[0],
@@ -1612,7 +1563,12 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folder.id);
-    await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment });
+    await secretQueueService.syncSecrets({
+      secretPath,
+      projectId: project.id,
+      environmentSlug: environment,
+      excludeReplication: true
+    });
 
     return {
       ...updatedSecret[0],
@@ -1685,13 +1641,6 @@ export const secretServiceFactory = ({
     createManySecretsRaw,
     updateManySecretsRaw,
     deleteManySecretsRaw,
-    getSecretVersions,
+    getSecretVersions
-    backfillSecretReferences,
-    // external services function
-    fnSecretBulkDelete,
-    fnSecretBulkUpdate,
-    fnSecretBlindIndexCheck,
-    fnSecretBulkInsert,
-    fnSecretBlindIndexCheckV2
   };
 };

backend/src/services/secret/secret-types.ts

@@ -264,6 +264,10 @@ export type TFnSecretBulkDelete = {
   inputSecrets: Array<{ type: SecretType; secretBlindIndex: string }>;
   actorId: string;
   tx?: Knex;
+  secretDAL: Pick<TSecretDALFactory, "deleteMany">;
+  secretQueueService: {
+    removeSecretReminder: (data: TRemoveSecretReminderDTO) => Promise<void>;
+  };
 };
 
 export type TFnSecretBlindIndexCheck = {
@@ -277,6 +281,7 @@ export type TFnSecretBlindIndexCheck = {
 
 // when blind index is already present
 export type TFnSecretBlindIndexCheckV2 = {
+  secretDAL: Pick<TSecretDALFactory, "findByBlindIndexes">;
   folderId: string;
   userId?: string;
   inputSecrets: Array<{ secretBlindIndex: string; type?: SecretType }>;
@@ -363,3 +368,30 @@ export type TUpdateManySecretsRawFn = {
   }[];
   userId?: string;
 };
+
+export enum SecretOperations {
+  Create = "create",
+  Update = "update",
+  Delete = "delete"
+}
+
+export type TSyncSecretsDTO<T extends boolean = false> = {
+  _depth?: number;
+  _deDupeQueue?: Record<string, boolean>;
+  secretPath: string;
+  projectId: string;
+  environmentSlug: string;
+  // cases for just doing sync integration and webhook
+  excludeReplication?: T;
+} & (T extends true
+  ? object
+  : {
+      environmentId: string;
+      folderId: string;
+      membershipId: string;
+      secrets: {
+        operation: SecretOperations;
+        id: string;
+        version: number;
+      }[];
+    });

backend/src/services/secret/secret-version-dal.ts

@@ -89,6 +89,7 @@ export const secretVersionDALFactory = (db: TDbClient) => {
 
   const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => {
     try {
+      if (!secretIds.length) return {};
       const docs: Array<TSecretVersions & { max: number }> = await (tx || db)(TableName.SecretVersion)
         .where("folderId", folderId)
         .whereIn(`${TableName.SecretVersion}.secretId`, secretIds)