Merge remote-tracking branch 'origin/main' into feat/add-version-limits

2026-10-15 14:27:23 +00:00 · 2024-06-08 01:37:14 +08:00
parent ee152f2d20 faa1572faf
commit 7fe7056af4
126 changed files with 3741 additions and 729 deletions
@@ -15,9 +15,16 @@ import { ActorType } from "@app/services/auth/auth-type";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
 import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
 import { TSecretDALFactory } from "@app/services/secret/secret-dal";
-import { getAllNestedSecretReferences } from "@app/services/secret/secret-fns";
+import {
+  fnSecretBlindIndexCheck,
+  fnSecretBlindIndexCheckV2,
+  fnSecretBulkDelete,
+  fnSecretBulkInsert,
+  fnSecretBulkUpdate,
+  getAllNestedSecretReferences
+} from "@app/services/secret/secret-fns";
 import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
-import { TSecretServiceFactory } from "@app/services/secret/secret-service";
+import { SecretOperations } from "@app/services/secret/secret-types";
 import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
 import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
 import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
@@ -32,7 +39,6 @@ import { TSecretApprovalRequestReviewerDALFactory } from "./secret-approval-requ
 import { TSecretApprovalRequestSecretDALFactory } from "./secret-approval-request-secret-dal";
 import {
  ApprovalStatus,
-  CommitType,
  RequestState,
  TApprovalRequestCountDTO,
  TGenerateSecretApprovalRequestDTO,
@@ -45,10 +51,11 @@ import {

 type TSecretApprovalRequestServiceFactoryDep = {
  permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
+  projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
  secretApprovalRequestDAL: TSecretApprovalRequestDALFactory;
  secretApprovalRequestSecretDAL: TSecretApprovalRequestSecretDALFactory;
  secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory;
-  folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findById" | "findSecretPathByFolderIds">;
+  folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findSecretPathByFolderIds">;
  secretDAL: TSecretDALFactory;
  secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret">;
  secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
@@ -56,16 +63,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
  secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany" | "insertMany">;
  secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
  projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
-  projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
-  secretService: Pick<
-    TSecretServiceFactory,
-    | "fnSecretBulkInsert"
-    | "fnSecretBulkUpdate"
-    | "fnSecretBlindIndexCheck"
-    | "fnSecretBulkDelete"
-    | "fnSecretBlindIndexCheckV2"
-  >;
-  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
+  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "removeSecretReminder">;
 };

 export type TSecretApprovalRequestServiceFactory = ReturnType<typeof secretApprovalRequestServiceFactory>;
@@ -82,7 +80,6 @@ export const secretApprovalRequestServiceFactory = ({
  projectDAL,
  permissionService,
  snapshotService,
-  secretService,
  secretVersionDAL,
  secretQueueService,
  projectBotService
@@ -302,11 +299,12 @@ export const secretApprovalRequestServiceFactory = ({
    const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
    if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" });

-    const conflicts: Array<{ secretId: string; op: CommitType }> = [];
-    let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Create);
+    const conflicts: Array<{ secretId: string; op: SecretOperations }> = [];
+    let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create);
    if (secretCreationCommits.length) {
-      const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({
+      const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
        folderId,
+        secretDAL,
        inputSecrets: secretCreationCommits.map(({ secretBlindIndex }) => {
          if (!secretBlindIndex) {
            throw new BadRequestError({
@@ -319,17 +317,19 @@ export const secretApprovalRequestServiceFactory = ({
      secretCreationCommits
        .filter(({ secretBlindIndex }) => conflictGroupByBlindIndex[secretBlindIndex || ""])
        .forEach((el) => {
-          conflicts.push({ op: CommitType.Create, secretId: el.id });
+          conflicts.push({ op: SecretOperations.Create, secretId: el.id });
        });
      secretCreationCommits = secretCreationCommits.filter(
        ({ secretBlindIndex }) => !conflictGroupByBlindIndex[secretBlindIndex || ""]
      );
    }

-    let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Update);
+    let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update);
    if (secretUpdationCommits.length) {
-      const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({
+      const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
        folderId,
+        secretDAL,
+        userId: "",
        inputSecrets: secretUpdationCommits
          .filter(({ secretBlindIndex, secret }) => secret && secret.secretBlindIndex !== secretBlindIndex)
          .map(({ secretBlindIndex }) => {
@@ -347,7 +347,7 @@ export const secretApprovalRequestServiceFactory = ({
            (secretBlindIndex && conflictGroupByBlindIndex[secretBlindIndex]) || !secretId
        )
        .forEach((el) => {
-          conflicts.push({ op: CommitType.Update, secretId: el.id });
+          conflicts.push({ op: SecretOperations.Update, secretId: el.id });
        });

      secretUpdationCommits = secretUpdationCommits.filter(
@@ -356,11 +356,11 @@ export const secretApprovalRequestServiceFactory = ({
      );
    }

-    const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Delete);
+    const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete);
    const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
    const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
      const newSecrets = secretCreationCommits.length
-        ? await secretService.fnSecretBulkInsert({
+        ? await fnSecretBulkInsert({
            tx,
            folderId,
            inputSecrets: secretCreationCommits.map((el) => ({
@@ -403,7 +403,7 @@ export const secretApprovalRequestServiceFactory = ({
          })
        : [];
      const updatedSecrets = secretUpdationCommits.length
-        ? await secretService.fnSecretBulkUpdate({
+        ? await fnSecretBulkUpdate({
            folderId,
            projectId,
            tx,
@@ -449,11 +449,13 @@ export const secretApprovalRequestServiceFactory = ({
          })
        : [];
      const deletedSecret = secretDeletionCommits.length
-        ? await secretService.fnSecretBulkDelete({
+        ? await fnSecretBulkDelete({
            projectId,
            folderId,
            tx,
            actorId: "",
+            secretDAL,
+            secretQueueService,
            inputSecrets: secretDeletionCommits.map(({ secretBlindIndex }) => {
              if (!secretBlindIndex) {
                throw new BadRequestError({
@@ -480,12 +482,14 @@ export const secretApprovalRequestServiceFactory = ({
      };
    });
    await snapshotService.performSnapshot(folderId);
-    const folder = await folderDAL.findById(folderId);
-    // TODO(akhilmhdh-pg):  change query to do secret path from folder
+    const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]);
+    if (!folder) throw new BadRequestError({ message: "Folder not found" });
    await secretQueueService.syncSecrets({
      projectId,
-      secretPath: "/",
-      environment: folder?.environment.envSlug as string
+      secretPath: folder.path,
+      environmentSlug: folder.environmentSlug,
+      actorId,
+      actor
    });
    return mergeStatus;
  };
@@ -533,9 +537,9 @@ export const secretApprovalRequestServiceFactory = ({
    const commits: Omit<TSecretApprovalRequestsSecretsInsert, "requestId">[] = [];
    const commitTagIds: Record<string, string[]> = {};
    // for created secret approval change
-    const createdSecrets = data[CommitType.Create];
+    const createdSecrets = data[SecretOperations.Create];
    if (createdSecrets && createdSecrets?.length) {
-      const { keyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({
+      const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
        inputSecrets: createdSecrets,
        folderId,
        isNew: true,
@@ -546,7 +550,7 @@ export const secretApprovalRequestServiceFactory = ({
      commits.push(
        ...createdSecrets.map(({ secretName, ...el }) => ({
          ...el,
-          op: CommitType.Create as const,
+          op: SecretOperations.Create as const,
          version: 1,
          secretBlindIndex: keyName2BlindIndex[secretName],
          algorithm: SecretEncryptionAlgo.AES_256_GCM,
@@ -558,12 +562,12 @@ export const secretApprovalRequestServiceFactory = ({
      });
    }
    // not secret approval for update operations
-    const updatedSecrets = data[CommitType.Update];
+    const updatedSecrets = data[SecretOperations.Update];
    if (updatedSecrets && updatedSecrets?.length) {
      // get all blind index
      // Find all those secrets
      // if not throw not found
-      const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await secretService.fnSecretBlindIndexCheck({
+      const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await fnSecretBlindIndexCheck({
        inputSecrets: updatedSecrets,
        folderId,
        isNew: false,
@@ -574,8 +578,8 @@ export const secretApprovalRequestServiceFactory = ({
      // now find any secret that needs to update its name
      // same process as above
      const nameUpdatedSecrets = updatedSecrets.filter(({ newSecretName }) => Boolean(newSecretName));
-      const { keyName2BlindIndex: newKeyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({
-        inputSecrets: nameUpdatedSecrets,
+      const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({
+        inputSecrets: nameUpdatedSecrets.map(({ newSecretName }) => ({ secretName: newSecretName as string })),
        folderId,
        isNew: true,
        blindIndexCfg,
@@ -592,14 +596,14 @@ export const secretApprovalRequestServiceFactory = ({
          const secretId = secsGroupedByBlindIndex[keyName2BlindIndex[secretName]][0].id;
          const secretBlindIndex =
            newSecretName && newKeyName2BlindIndex[newSecretName]
-              ? newKeyName2BlindIndex?.[secretName]
+              ? newKeyName2BlindIndex?.[newSecretName]
              : keyName2BlindIndex[secretName];
          // add tags
          if (tagIds?.length) commitTagIds[keyName2BlindIndex[secretName]] = tagIds;
          return {
            ...latestSecretVersions[secretId],
            ...el,
-            op: CommitType.Update as const,
+            op: SecretOperations.Update as const,
            secret: secretId,
            secretVersion: latestSecretVersions[secretId].id,
            secretBlindIndex,
@@ -609,12 +613,12 @@ export const secretApprovalRequestServiceFactory = ({
      );
    }
    // deleted secrets
-    const deletedSecrets = data[CommitType.Delete];
+    const deletedSecrets = data[SecretOperations.Delete];
    if (deletedSecrets && deletedSecrets.length) {
      // get all blind index
      // Find all those secrets
      // if not throw not found
-      const { keyName2BlindIndex, secrets } = await secretService.fnSecretBlindIndexCheck({
+      const { keyName2BlindIndex, secrets } = await fnSecretBlindIndexCheck({
        inputSecrets: deletedSecrets,
        folderId,
        isNew: false,
@@ -635,7 +639,7 @@ export const secretApprovalRequestServiceFactory = ({
          if (!latestSecretVersions[secretId].secretBlindIndex)
            throw new BadRequestError({ message: "Failed to find secret blind index" });
          return {
-            op: CommitType.Delete as const,
+            op: SecretOperations.Delete as const,
            ...latestSecretVersions[secretId],
            secretBlindIndex: latestSecretVersions[secretId].secretBlindIndex as string,
            secret: secretId,
@@ -1,11 +1,6 @@
 import { TImmutableDBKeys, TSecretApprovalPolicies, TSecretApprovalRequestsSecrets } from "@app/db/schemas";
 import { TProjectPermission } from "@app/lib/types";
-
-export enum CommitType {
-  Create = "create",
-  Update = "update",
-  Delete = "delete"
-}
+import { SecretOperations } from "@app/services/secret/secret-types";

 export enum RequestState {
  Open = "open",
@@ -18,14 +13,14 @@ export enum ApprovalStatus {
  REJECTED = "rejected"
 }

-type TApprovalCreateSecret = Omit<
+export type TApprovalCreateSecret = Omit<
  TSecretApprovalRequestsSecrets,
  TImmutableDBKeys | "version" | "algorithm" | "keyEncoding" | "requestId" | "op" | "secretVersion" | "secretBlindIndex"
 > & {
  secretName: string;
  tagIds?: string[];
 };
-type TApprovalUpdateSecret = Partial<TApprovalCreateSecret> & {
+export type TApprovalUpdateSecret = Partial<TApprovalCreateSecret> & {
  secretName: string;
  newSecretName?: string;
  tagIds?: string[];
@@ -36,9 +31,9 @@ export type TGenerateSecretApprovalRequestDTO = {
  secretPath: string;
  policy: TSecretApprovalPolicies;
  data: {
-    [CommitType.Create]?: TApprovalCreateSecret[];
-    [CommitType.Update]?: TApprovalUpdateSecret[];
-    [CommitType.Delete]?: { secretName: string }[];
+    [SecretOperations.Create]?: TApprovalCreateSecret[];
+    [SecretOperations.Update]?: TApprovalUpdateSecret[];
+    [SecretOperations.Delete]?: { secretName: string }[];
  };
 } & TProjectPermission;

@@ -0,0 +1 @@
+export const MAX_REPLICATION_DEPTH = 5;
@@ -0,0 +1,10 @@
+import { TDbClient } from "@app/db";
+import { TableName } from "@app/db/schemas";
+import { ormify } from "@app/lib/knex";
+
+export type TSecretReplicationDALFactory = ReturnType<typeof secretReplicationDALFactory>;
+
+export const secretReplicationDALFactory = (db: TDbClient) => {
+  const orm = ormify(db, TableName.SecretVersion);
+  return orm;
+};
@@ -0,0 +1,485 @@
+import { SecretType, TSecrets } from "@app/db/schemas";
+import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
+import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
+import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
+import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
+import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
+import { BadRequestError } from "@app/lib/errors";
+import { groupBy, unique } from "@app/lib/fn";
+import { logger } from "@app/lib/logger";
+import { alphaNumericNanoId } from "@app/lib/nanoid";
+import { QueueName, TQueueServiceFactory } from "@app/queue";
+import { ActorType } from "@app/services/auth/auth-type";
+import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
+import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal";
+import { TSecretDALFactory } from "@app/services/secret/secret-dal";
+import { fnSecretBulkInsert, fnSecretBulkUpdate } from "@app/services/secret/secret-fns";
+import { TSecretQueueFactory, uniqueSecretQueueKey } from "@app/services/secret/secret-queue";
+import { SecretOperations } from "@app/services/secret/secret-types";
+import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
+import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
+import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
+import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
+import { ReservedFolders } from "@app/services/secret-folder/secret-folder-types";
+import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
+import { fnSecretsFromImports } from "@app/services/secret-import/secret-import-fns";
+import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
+
+import { MAX_REPLICATION_DEPTH } from "./secret-replication-constants";
+
+type TSecretReplicationServiceFactoryDep = {
+  secretDAL: Pick<
+    TSecretDALFactory,
+    "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" | "transaction"
+  >;
+  secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany">;
+  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "updateById" | "findByFolderIds">;
+  folderDAL: Pick<
+    TSecretFolderDALFactory,
+    "findSecretPathByFolderIds" | "findBySecretPath" | "create" | "findOne" | "findByManySecretPath"
+  >;
+  secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "find" | "insertMany">;
+  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "replicateSecrets">;
+  queueService: Pick<TQueueServiceFactory, "start" | "listen" | "queue" | "stopJobById">;
+  secretApprovalPolicyService: Pick<TSecretApprovalPolicyServiceFactory, "getSecretApprovalPolicy">;
+  keyStore: Pick<TKeyStoreFactory, "acquireLock" | "setItemWithExpiry" | "getItem">;
+  secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret" | "find">;
+  secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "create" | "transaction">;
+  projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findOne">;
+  secretApprovalRequestSecretDAL: Pick<
+    TSecretApprovalRequestSecretDALFactory,
+    "insertMany" | "insertApprovalSecretTags"
+  >;
+  projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
+};
+
+export type TSecretReplicationServiceFactory = ReturnType<typeof secretReplicationServiceFactory>;
+const SECRET_IMPORT_SUCCESS_LOCK = 10;
+
+const keystoreReplicationSuccessKey = (jobId: string, secretImportId: string) => `${jobId}-${secretImportId}`;
+const getReplicationKeyLockPrefix = (projectId: string, environmentSlug: string, secretPath: string) =>
+  `REPLICATION_SECRET_${projectId}-${environmentSlug}-${secretPath}`;
+export const getReplicationFolderName = (importId: string) => `${ReservedFolders.SecretReplication}${importId}`;
+
+const getDecryptedKeyValue = (key: string, secret: TSecrets) => {
+  const secretKey = decryptSymmetric128BitHexKeyUTF8({
+    ciphertext: secret.secretKeyCiphertext,
+    iv: secret.secretKeyIV,
+    tag: secret.secretKeyTag,
+    key
+  });
+
+  const secretValue = decryptSymmetric128BitHexKeyUTF8({
+    ciphertext: secret.secretValueCiphertext,
+    iv: secret.secretValueIV,
+    tag: secret.secretValueTag,
+    key
+  });
+  return { key: secretKey, value: secretValue };
+};
+
+export const secretReplicationServiceFactory = ({
+  secretDAL,
+  queueService,
+  secretVersionDAL,
+  secretImportDAL,
+  keyStore,
+  secretVersionTagDAL,
+  secretTagDAL,
+  folderDAL,
+  secretApprovalPolicyService,
+  secretApprovalRequestSecretDAL,
+  secretApprovalRequestDAL,
+  secretQueueService,
+  projectMembershipDAL,
+  projectBotService
+}: TSecretReplicationServiceFactoryDep) => {
+  const getReplicatedSecrets = (
+    botKey: string,
+    localSecrets: TSecrets[],
+    importedSecrets: { secrets: TSecrets[] }[]
+  ) => {
+    const deDupe = new Set<string>();
+    const secrets = localSecrets
+      .filter(({ secretBlindIndex }) => Boolean(secretBlindIndex))
+      .map((el) => {
+        const decryptedSecret = getDecryptedKeyValue(botKey, el);
+        deDupe.add(decryptedSecret.key);
+        return { ...el, secretKey: decryptedSecret.key, secretValue: decryptedSecret.value };
+      });
+
+    for (let i = importedSecrets.length - 1; i >= 0; i = -1) {
+      importedSecrets[i].secrets.forEach((el) => {
+        const decryptedSecret = getDecryptedKeyValue(botKey, el);
+        if (deDupe.has(decryptedSecret.key) || !el.secretBlindIndex) {
+          return;
+        }
+        deDupe.add(decryptedSecret.key);
+        secrets.push({ ...el, secretKey: decryptedSecret.key, secretValue: decryptedSecret.value });
+      });
+    }
+    return secrets;
+  };
+
+  // IMPORTANT NOTE BEFORE READING THE FUNCTION
+  // SOURCE - Where secrets are copied from
+  // DESTINATION - Where the replicated imports that points to SOURCE from Destination
+  queueService.start(QueueName.SecretReplication, async (job) => {
+    logger.info(job.data, "Replication started");
+    const {
+      secretPath,
+      environmentSlug,
+      projectId,
+      actorId,
+      actor,
+      pickOnlyImportIds,
+      _deDupeReplicationQueue: deDupeReplicationQueue,
+      _deDupeQueue: deDupeQueue,
+      _depth: depth = 0
+    } = job.data;
+    if (depth > MAX_REPLICATION_DEPTH) return;
+
+    const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, secretPath);
+    if (!folder) return;
+
+    // the the replicated imports made to the source. These are the destinations
+    const destinationSecretImports = await secretImportDAL.find({
+      importPath: secretPath,
+      importEnv: folder.envId
+    });
+
+    // CASE: normal mode <- link import  <- replicated import
+    const nonReplicatedDestinationImports = destinationSecretImports.filter(({ isReplication }) => !isReplication);
+    if (nonReplicatedDestinationImports.length) {
+      // keep calling sync secret for all the imports made
+      const importedFolderIds = unique(nonReplicatedDestinationImports, (i) => i.folderId).map(
+        ({ folderId }) => folderId
+      );
+      const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds);
+      const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string);
+      await Promise.all(
+        nonReplicatedDestinationImports
+          .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0]?.path as string))
+          // filter out already synced ones
+          .filter(
+            ({ folderId }) =>
+              !deDupeQueue?.[
+                uniqueSecretQueueKey(
+                  foldersGroupedById[folderId][0]?.environmentSlug as string,
+                  foldersGroupedById[folderId][0]?.path as string
+                )
+              ]
+          )
+          .map(({ folderId }) =>
+            secretQueueService.replicateSecrets({
+              projectId,
+              secretPath: foldersGroupedById[folderId][0]?.path as string,
+              environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string,
+              actorId,
+              actor,
+              _depth: depth + 1,
+              _deDupeReplicationQueue: deDupeReplicationQueue,
+              _deDupeQueue: deDupeQueue
+            })
+          )
+      );
+    }
+
+    let destinationReplicatedSecretImports = destinationSecretImports.filter(({ isReplication }) =>
+      Boolean(isReplication)
+    );
+    destinationReplicatedSecretImports = pickOnlyImportIds
+      ? destinationReplicatedSecretImports.filter(({ id }) => pickOnlyImportIds?.includes(id))
+      : destinationReplicatedSecretImports;
+    if (!destinationReplicatedSecretImports.length) return;
+
+    const botKey = await projectBotService.getBotKey(projectId);
+
+    // these are the secrets to be added in replicated folders
+    const sourceLocalSecrets = await secretDAL.find({ folderId: folder.id, type: SecretType.Shared });
+    const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id });
+    const sourceImportedSecrets = await fnSecretsFromImports({
+      allowedImports: sourceSecretImports,
+      secretDAL,
+      folderDAL,
+      secretImportDAL
+    });
+    // secrets that gets replicated across imports
+    const sourceSecrets = getReplicatedSecrets(botKey, sourceLocalSecrets, sourceImportedSecrets);
+    const sourceSecretsGroupByBlindIndex = groupBy(sourceSecrets, (i) => i.secretBlindIndex as string);
+
+    const lock = await keyStore.acquireLock(
+      [getReplicationKeyLockPrefix(projectId, environmentSlug, secretPath)],
+      5000
+    );
+
+    try {
+      /*  eslint-disable no-await-in-loop */
+      for (const destinationSecretImport of destinationReplicatedSecretImports) {
+        try {
+          const hasJobCompleted = await keyStore.getItem(
+            keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id),
+            KeyStorePrefixes.SecretReplication
+          );
+          if (hasJobCompleted) {
+            logger.info(
+              { jobId: job.id, importId: destinationSecretImport.id },
+              "Skipping this job as this has been successfully replicated."
+            );
+            // eslint-disable-next-line
+            continue;
+          }
+
+          const [destinationFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [
+            destinationSecretImport.folderId
+          ]);
+          if (!destinationFolder) throw new BadRequestError({ message: "Imported folder not found" });
+
+          let destinationReplicationFolder = await folderDAL.findOne({
+            parentId: destinationFolder.id,
+            name: getReplicationFolderName(destinationSecretImport.id),
+            isReserved: true
+          });
+          if (!destinationReplicationFolder) {
+            destinationReplicationFolder = await folderDAL.create({
+              parentId: destinationFolder.id,
+              name: getReplicationFolderName(destinationSecretImport.id),
+              envId: destinationFolder.envId,
+              isReserved: true
+            });
+          }
+          const destinationReplicationFolderId = destinationReplicationFolder.id;
+
+          const destinationLocalSecretsFromDB = await secretDAL.find({
+            folderId: destinationReplicationFolderId
+          });
+          const destinationLocalSecrets = destinationLocalSecretsFromDB.map((el) => {
+            const decryptedSecret = getDecryptedKeyValue(botKey, el);
+            return { ...el, secretKey: decryptedSecret.key, secretValue: decryptedSecret.value };
+          });
+
+          const destinationLocalSecretsGroupedByBlindIndex = groupBy(
+            destinationLocalSecrets.filter(({ secretBlindIndex }) => Boolean(secretBlindIndex)),
+            (i) => i.secretBlindIndex as string
+          );
+
+          const locallyCreatedSecrets = sourceSecrets
+            .filter(
+              ({ secretBlindIndex }) => !destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0]
+            )
+            .map((el) => ({ ...el, operation: SecretOperations.Create })); // rewrite update ops to create
+
+          const locallyUpdatedSecrets = sourceSecrets
+            .filter(
+              ({ secretBlindIndex, secretKey, secretValue }) =>
+                destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0] &&
+                // if key or value changed
+                (destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0]?.secretKey !== secretKey ||
+                  destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0]?.secretValue !==
+                    secretValue)
+            )
+            .map((el) => ({ ...el, operation: SecretOperations.Update })); // rewrite update ops to create
+
+          const locallyDeletedSecrets = destinationLocalSecrets
+            .filter(({ secretBlindIndex }) => !sourceSecretsGroupByBlindIndex[secretBlindIndex as string]?.[0])
+            .map((el) => ({ ...el, operation: SecretOperations.Delete }));
+
+          const isEmtpy =
+            locallyCreatedSecrets.length + locallyUpdatedSecrets.length + locallyDeletedSecrets.length === 0;
+          // eslint-disable-next-line
+          if (isEmtpy) continue;
+
+          const policy = await secretApprovalPolicyService.getSecretApprovalPolicy(
+            projectId,
+            destinationFolder.environmentSlug,
+            destinationFolder.path
+          );
+          // this means it should be a approval request rather than direct replication
+          if (policy && actor === ActorType.USER) {
+            const membership = await projectMembershipDAL.findOne({ projectId, userId: actorId });
+            if (!membership) {
+              logger.error("Project membership not found in %s for user %s", projectId, actorId);
+              return;
+            }
+
+            const localSecretsLatestVersions = destinationLocalSecrets.map(({ id }) => id);
+            const latestSecretVersions = await secretVersionDAL.findLatestVersionMany(
+              destinationReplicationFolderId,
+              localSecretsLatestVersions
+            );
+            await secretApprovalRequestDAL.transaction(async (tx) => {
+              const approvalRequestDoc = await secretApprovalRequestDAL.create(
+                {
+                  folderId: destinationReplicationFolderId,
+                  slug: alphaNumericNanoId(),
+                  policyId: policy.id,
+                  status: "open",
+                  hasMerged: false,
+                  committerId: membership.id,
+                  isReplicated: true
+                },
+                tx
+              );
+              const commits = locallyCreatedSecrets
+                .concat(locallyUpdatedSecrets)
+                .concat(locallyDeletedSecrets)
+                .map((doc) => {
+                  const { operation } = doc;
+                  const localSecret = destinationLocalSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0];
+
+                  return {
+                    op: operation,
+                    keyEncoding: doc.keyEncoding,
+                    algorithm: doc.algorithm,
+                    requestId: approvalRequestDoc.id,
+                    metadata: doc.metadata,
+                    secretKeyIV: doc.secretKeyIV,
+                    secretKeyTag: doc.secretKeyTag,
+                    secretKeyCiphertext: doc.secretKeyCiphertext,
+                    secretValueIV: doc.secretValueIV,
+                    secretValueTag: doc.secretValueTag,
+                    secretValueCiphertext: doc.secretValueCiphertext,
+                    secretBlindIndex: doc.secretBlindIndex,
+                    secretCommentIV: doc.secretCommentIV,
+                    secretCommentTag: doc.secretCommentTag,
+                    secretCommentCiphertext: doc.secretCommentCiphertext,
+                    skipMultilineEncoding: doc.skipMultilineEncoding,
+                    // except create operation other two needs the secret id and version id
+                    ...(operation !== SecretOperations.Create
+                      ? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id }
+                      : {})
+                  };
+                });
+              const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx);
+
+              return { ...approvalRequestDoc, commits: approvalCommits };
+            });
+          } else {
+            await secretDAL.transaction(async (tx) => {
+              if (locallyCreatedSecrets.length) {
+                await fnSecretBulkInsert({
+                  folderId: destinationReplicationFolderId,
+                  secretVersionDAL,
+                  secretDAL,
+                  tx,
+                  secretTagDAL,
+                  secretVersionTagDAL,
+                  inputSecrets: locallyCreatedSecrets.map((doc) => {
+                    return {
+                      keyEncoding: doc.keyEncoding,
+                      algorithm: doc.algorithm,
+                      type: doc.type,
+                      metadata: doc.metadata,
+                      secretKeyIV: doc.secretKeyIV,
+                      secretKeyTag: doc.secretKeyTag,
+                      secretKeyCiphertext: doc.secretKeyCiphertext,
+                      secretValueIV: doc.secretValueIV,
+                      secretValueTag: doc.secretValueTag,
+                      secretValueCiphertext: doc.secretValueCiphertext,
+                      secretBlindIndex: doc.secretBlindIndex,
+                      secretCommentIV: doc.secretCommentIV,
+                      secretCommentTag: doc.secretCommentTag,
+                      secretCommentCiphertext: doc.secretCommentCiphertext,
+                      skipMultilineEncoding: doc.skipMultilineEncoding
+                    };
+                  })
+                });
+              }
+              if (locallyUpdatedSecrets.length) {
+                await fnSecretBulkUpdate({
+                  projectId,
+                  folderId: destinationReplicationFolderId,
+                  secretVersionDAL,
+                  secretDAL,
+                  tx,
+                  secretTagDAL,
+                  secretVersionTagDAL,
+                  inputSecrets: locallyUpdatedSecrets.map((doc) => {
+                    return {
+                      filter: {
+                        folderId: destinationReplicationFolderId,
+                        id: destinationLocalSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
+                      },
+                      data: {
+                        keyEncoding: doc.keyEncoding,
+                        algorithm: doc.algorithm,
+                        type: doc.type,
+                        metadata: doc.metadata,
+                        secretKeyIV: doc.secretKeyIV,
+                        secretKeyTag: doc.secretKeyTag,
+                        secretKeyCiphertext: doc.secretKeyCiphertext,
+                        secretValueIV: doc.secretValueIV,
+                        secretValueTag: doc.secretValueTag,
+                        secretValueCiphertext: doc.secretValueCiphertext,
+                        secretBlindIndex: doc.secretBlindIndex,
+                        secretCommentIV: doc.secretCommentIV,
+                        secretCommentTag: doc.secretCommentTag,
+                        secretCommentCiphertext: doc.secretCommentCiphertext,
+                        skipMultilineEncoding: doc.skipMultilineEncoding
+                      }
+                    };
+                  })
+                });
+              }
+              if (locallyDeletedSecrets.length) {
+                await secretDAL.delete(
+                  {
+                    $in: {
+                      id: locallyDeletedSecrets.map(({ id }) => id)
+                    },
+                    folderId: destinationReplicationFolderId
+                  },
+                  tx
+                );
+              }
+            });
+
+            await secretQueueService.syncSecrets({
+              projectId,
+              secretPath: destinationFolder.path,
+              environmentSlug: destinationFolder.environmentSlug,
+              actorId,
+              actor,
+              _depth: depth + 1,
+              _deDupeReplicationQueue: deDupeReplicationQueue,
+              _deDupeQueue: deDupeQueue
+            });
+          }
+
+          // this is used to avoid multiple times generating secret approval by failed one
+          await keyStore.setItemWithExpiry(
+            keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id),
+            SECRET_IMPORT_SUCCESS_LOCK,
+            1,
+            KeyStorePrefixes.SecretReplication
+          );
+
+          await secretImportDAL.updateById(destinationSecretImport.id, {
+            lastReplicated: new Date(),
+            replicationStatus: null,
+            isReplicationSuccess: true
+          });
+        } catch (err) {
+          logger.error(
+            err,
+            `Failed to replicate secret with import id=[${destinationSecretImport.id}] env=[${destinationSecretImport.importEnv.slug}] path=[${destinationSecretImport.importPath}]`
+          );
+          await secretImportDAL.updateById(destinationSecretImport.id, {
+            lastReplicated: new Date(),
+            replicationStatus: (err as Error)?.message.slice(0, 500),
+            isReplicationSuccess: false
+          });
+        }
+      }
+      /*  eslint-enable no-await-in-loop */
+    } finally {
+      await lock.release();
+      logger.info(job.data, "Replication finished");
+    }
+  });
+
+  queueService.listen(QueueName.SecretReplication, "failed", (job, err) => {
+    logger.error(err, "Failed to replicate secret", job?.data);
+  });
+};
@@ -0,0 +1,3 @@
+export type TSyncSecretReplicationDTO = {
+  id: string;
+};
@@ -219,7 +219,7 @@ export const secretSnapshotServiceFactory = ({
      const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id);
      // this will remove all secrets and folders on child
      // due to sql foreign key and link list connection removing the folders removes everything below too
-      const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId }, tx);
+      const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx);
      const deletedTopLevelFolders = groupBy(
        deletedFolders.filter(({ parentId }) => parentId === snapshot.folderId),
        (item) => item.id