Merge remote-tracking branch 'origin/main' into feat/add-version-limits

backend/src/services/integration-auth/integration-auth-service.ts

@@ -199,6 +199,7 @@ export const integrationAuthServiceFactory = ({
       projectId,
       namespace,
       integration,
+      url,
       algorithm: SecretEncryptionAlgo.AES_256_GCM,
       keyEncoding: SecretKeyEncoding.UTF8,
       ...(integration === Integrations.GCP_SECRET_MANAGER

backend/src/services/integration-auth/integration-list.ts

@@ -30,7 +30,8 @@ export enum Integrations {
   DIGITAL_OCEAN_APP_PLATFORM = "digital-ocean-app-platform",
   CLOUD_66 = "cloud-66",
   NORTHFLANK = "northflank",
-  HASURA_CLOUD = "hasura-cloud"
+  HASURA_CLOUD = "hasura-cloud",
+  RUNDECK = "rundeck"
 }
 
 export enum IntegrationType {
@@ -368,6 +369,15 @@ export const getIntegrationOptions = async () => {
       type: "pat",
       clientId: "",
       docsLink: ""
+    },
+    {
+      name: "Rundeck",
+      slug: "rundeck",
+      image: "Rundeck.svg",
+      isAvailable: true,
+      type: "pat",
+      clientId: "",
+      docsLink: ""
     }
   ];
 

backend/src/services/integration-auth/integration-sync-secret.ts

@@ -27,6 +27,7 @@ import { z } from "zod";
 import { SecretType, TIntegrationAuths, TIntegrations, TSecrets } from "@app/db/schemas";
 import { request } from "@app/lib/config/request";
 import { BadRequestError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
 import { TCreateManySecretsRawFn, TUpdateManySecretsRawFn } from "@app/services/secret/secret-types";
 
 import { TIntegrationDALFactory } from "../integration/integration-dal";
@@ -521,18 +522,42 @@ const syncSecretsAWSParameterStore = async ({
             .promise();
         }
         // case: secret exists in AWS parameter store
-      } else if (awsParameterStoreSecretsObj[key].Value !== secrets[key].value) {
-        // case: secret value doesn't match one in AWS parameter store
+      } else {
         // -> update secret
-        await ssm
-          .putParameter({
-            Name: `${integration.path}${key}`,
-            Type: "SecureString",
-            Value: secrets[key].value,
-            Overwrite: true
-            // Tags: metadata.secretAWSTag ? [{ Key: metadata.secretAWSTag.key, Value: metadata.secretAWSTag.value }] : []
-          })
-          .promise();
+        if (awsParameterStoreSecretsObj[key].Value !== secrets[key].value) {
+          await ssm
+            .putParameter({
+              Name: `${integration.path}${key}`,
+              Type: "SecureString",
+              Value: secrets[key].value,
+              Overwrite: true
+            })
+            .promise();
+        }
+
+        if (awsParameterStoreSecretsObj[key].Name) {
+          try {
+            await ssm
+              .addTagsToResource({
+                ResourceType: "Parameter",
+                ResourceId: awsParameterStoreSecretsObj[key].Name as string,
+                Tags: metadata.secretAWSTag
+                  ? metadata.secretAWSTag.map((tag: { key: string; value: string }) => ({
+                      Key: tag.key,
+                      Value: tag.value
+                    }))
+                  : []
+              })
+              .promise();
+          } catch (err) {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            if ((err as any).code === "AccessDeniedException") {
+              logger.error(
+                `AWS Parameter Store Error [integration=${integration.id}]: double check AWS account permissions (refer to the Infisical docs)`
+              );
+            }
+          }
+        }
       }
 
       await new Promise((resolve) => {
@@ -2725,6 +2750,20 @@ const syncSecretsCloudflarePages = async ({
       }
     }
   );
+
+  const metadata = z.record(z.any()).parse(integration.metadata);
+  if (metadata.shouldAutoRedeploy) {
+    await request.post(
+      `${IntegrationUrls.CLOUDFLARE_PAGES_API_URL}/client/v4/accounts/${accessId}/pages/projects/${integration.app}/deployments`,
+      {},
+      {
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+          Accept: "application/json"
+        }
+      }
+    );
+  }
 };
 
 /**
@@ -3330,6 +3369,82 @@ const syncSecretsHasuraCloud = async ({
   }
 };
 
+/** Sync/push [secrets] to Rundeck
+ * @param {Object} obj
+ * @param {TIntegrations} obj.integration - integration details
+ * @param {Object} obj.secrets - secrets to push to integration (object where keys are secret keys and values are secret values)
+ * @param {String} obj.accessToken - access token for Rundeck integration
+ */
+const syncSecretsRundeck = async ({
+  integration,
+  secrets,
+  accessToken
+}: {
+  integration: TIntegrations;
+  secrets: Record<string, { value: string; comment?: string }>;
+  accessToken: string;
+}) => {
+  interface RundeckSecretResource {
+    name: string;
+  }
+  interface RundeckSecretsGetRes {
+    resources: RundeckSecretResource[];
+  }
+
+  let existingRundeckSecrets: string[] = [];
+
+  try {
+    const listResult = await request.get<RundeckSecretsGetRes>(
+      `${integration.url}/api/44/storage/${integration.path}`,
+      {
+        headers: {
+          "X-Rundeck-Auth-Token": accessToken
+        }
+      }
+    );
+
+    existingRundeckSecrets = listResult.data.resources.map((res) => res.name);
+  } catch (err) {
+    logger.info("No existing rundeck secrets");
+  }
+
+  try {
+    for await (const [key, value] of Object.entries(secrets)) {
+      if (existingRundeckSecrets.includes(key)) {
+        await request.put(`${integration.url}/api/44/storage/${integration.path}/${key}`, value.value, {
+          headers: {
+            "X-Rundeck-Auth-Token": accessToken,
+            "Content-Type": "application/x-rundeck-data-password"
+          }
+        });
+      } else {
+        await request.post(`${integration.url}/api/44/storage/${integration.path}/${key}`, value.value, {
+          headers: {
+            "X-Rundeck-Auth-Token": accessToken,
+            "Content-Type": "application/x-rundeck-data-password"
+          }
+        });
+      }
+    }
+
+    for await (const existingSecret of existingRundeckSecrets) {
+      if (!(existingSecret in secrets)) {
+        await request.delete(`${integration.url}/api/44/storage/${integration.path}/${existingSecret}`, {
+          headers: {
+            "X-Rundeck-Auth-Token": accessToken
+          }
+        });
+      }
+    }
+  } catch (err: unknown) {
+    throw new Error(
+      `Ensure that the provided Rundeck URL is accessible by Infisical and that the linked API token has sufficient permissions.\n\n${
+        (err as Error).message
+      }`
+    );
+  }
+};
+
 /**
  * Sync/push [secrets] to [app] in integration named [integration]
  *
@@ -3596,6 +3711,13 @@ export const syncIntegrationSecrets = async ({
         accessToken
       });
       break;
+    case Integrations.RUNDECK:
+      await syncSecretsRundeck({
+        integration,
+        secrets,
+        accessToken
+      });
+      break;
     default:
       throw new BadRequestError({ message: "Invalid integration" });
   }

backend/src/services/integration/integration-service.ts

@@ -43,6 +43,7 @@ export const integrationServiceFactory = ({
     scope,
     actorId,
     region,
+    url,
     isActive,
     metadata,
     secretPath,
@@ -87,6 +88,7 @@ export const integrationServiceFactory = ({
       region,
       scope,
       owner,
+      url,
       appId,
       path,
       app,

backend/src/services/integration/integration-types.ts

@@ -12,6 +12,7 @@ export type TCreateIntegrationDTO = {
   targetService?: string;
   targetServiceId?: string;
   owner?: string;
+  url?: string;
   path?: string;
   region?: string;
   scope?: string;

backend/src/services/kms/kms-dal.ts

@@ -0,0 +1,10 @@
+import { TDbClient } from "@app/db";
+import { TableName } from "@app/db/schemas";
+import { ormify } from "@app/lib/knex";
+
+export type TKmsDALFactory = ReturnType<typeof kmsDALFactory>;
+
+export const kmsDALFactory = (db: TDbClient) => {
+  const kmsOrm = ormify(db, TableName.KmsKey);
+  return kmsOrm;
+};

backend/src/services/kms/kms-root-config-dal.ts

@@ -0,0 +1,10 @@
+import { TDbClient } from "@app/db";
+import { TableName } from "@app/db/schemas";
+import { ormify } from "@app/lib/knex";
+
+export type TKmsRootConfigDALFactory = ReturnType<typeof kmsRootConfigDALFactory>;
+
+export const kmsRootConfigDALFactory = (db: TDbClient) => {
+  const kmsOrm = ormify(db, TableName.KmsServerRootConfig);
+  return kmsOrm;
+};

backend/src/services/kms/kms-service.ts

@@ -0,0 +1,126 @@
+import { TKeyStoreFactory } from "@app/keystore/keystore";
+import { getConfig } from "@app/lib/config/env";
+import { randomSecureBytes } from "@app/lib/crypto";
+import { symmetricCipherService, SymmetricEncryption } from "@app/lib/crypto/cipher";
+import { BadRequestError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
+
+import { TKmsDALFactory } from "./kms-dal";
+import { TKmsRootConfigDALFactory } from "./kms-root-config-dal";
+import { TDecryptWithKmsDTO, TEncryptWithKmsDTO, TGenerateKMSDTO } from "./kms-types";
+
+type TKmsServiceFactoryDep = {
+  kmsDAL: TKmsDALFactory;
+  kmsRootConfigDAL: Pick<TKmsRootConfigDALFactory, "findById" | "create">;
+  keyStore: Pick<TKeyStoreFactory, "acquireLock" | "waitTillReady" | "setItemWithExpiry">;
+};
+
+export type TKmsServiceFactory = ReturnType<typeof kmsServiceFactory>;
+
+const KMS_ROOT_CONFIG_UUID = "00000000-0000-0000-0000-000000000000";
+
+const KMS_ROOT_CREATION_WAIT_KEY = "wait_till_ready_kms_root_key";
+const KMS_ROOT_CREATION_WAIT_TIME = 10;
+
+// akhilmhdh: Don't edit this value. This is measured for blob concatination in kms
+const KMS_VERSION = "v01";
+const KMS_VERSION_BLOB_LENGTH = 3;
+export const kmsServiceFactory = ({ kmsDAL, kmsRootConfigDAL, keyStore }: TKmsServiceFactoryDep) => {
+  let ROOT_ENCRYPTION_KEY = Buffer.alloc(0);
+
+  // this is used symmetric encryption
+  const generateKmsKey = async ({ scopeId, scopeType, isReserved = true }: TGenerateKMSDTO) => {
+    const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+    const kmsKeyMaterial = randomSecureBytes(32);
+    const encryptedKeyMaterial = cipher.encrypt(kmsKeyMaterial, ROOT_ENCRYPTION_KEY);
+
+    const { encryptedKey, ...doc } = await kmsDAL.create({
+      version: 1,
+      encryptedKey: encryptedKeyMaterial,
+      encryptionAlgorithm: SymmetricEncryption.AES_GCM_256,
+      isReserved,
+      orgId: scopeType === "org" ? scopeId : undefined,
+      projectId: scopeType === "project" ? scopeId : undefined
+    });
+    return doc;
+  };
+
+  const encrypt = async ({ kmsId, plainText }: TEncryptWithKmsDTO) => {
+    const kmsDoc = await kmsDAL.findById(kmsId);
+    if (!kmsDoc) throw new BadRequestError({ message: "KMS ID not found" });
+    // akhilmhdh: as more encryption are added do a check here on kmsDoc.encryptionAlgorithm
+    const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+
+    const kmsKey = cipher.decrypt(kmsDoc.encryptedKey, ROOT_ENCRYPTION_KEY);
+    const encryptedPlainTextBlob = cipher.encrypt(plainText, kmsKey);
+
+    // Buffer#1 encrypted text + Buffer#2 version number
+    const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3
+    const cipherTextBlob = Buffer.concat([encryptedPlainTextBlob, versionBlob]);
+    return { cipherTextBlob };
+  };
+
+  const decrypt = async ({ cipherTextBlob: versionedCipherTextBlob, kmsId }: TDecryptWithKmsDTO) => {
+    const kmsDoc = await kmsDAL.findById(kmsId);
+    if (!kmsDoc) throw new BadRequestError({ message: "KMS ID not found" });
+    // akhilmhdh: as more encryption are added do a check here on kmsDoc.encryptionAlgorithm
+    const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+    const kmsKey = cipher.decrypt(kmsDoc.encryptedKey, ROOT_ENCRYPTION_KEY);
+
+    const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH);
+    const decryptedBlob = cipher.decrypt(cipherTextBlob, kmsKey);
+    return decryptedBlob;
+  };
+
+  const startService = async () => {
+    const appCfg = getConfig();
+    // This will switch to a seal process and HMS flow in future
+    const encryptionKey = appCfg.ENCRYPTION_KEY || appCfg.ROOT_ENCRYPTION_KEY;
+    // if root key its base64 encoded
+    const isBase64 = Boolean(appCfg.ROOT_ENCRYPTION_KEY);
+    if (!encryptionKey) throw new Error("Root encryption key not found for KMS service.");
+    const encryptionKeyBuffer = Buffer.from(encryptionKey, isBase64 ? "base64" : "utf8");
+
+    const lock = await keyStore.acquireLock([`KMS_ROOT_CFG_LOCK`], 3000, { retryCount: 3 }).catch(() => null);
+    if (!lock) {
+      await keyStore.waitTillReady({
+        key: KMS_ROOT_CREATION_WAIT_KEY,
+        keyCheckCb: (val) => val === "true",
+        waitingCb: () => logger.info("KMS. Waiting for leader to finish creation of KMS Root Key")
+      });
+    }
+
+    // check if KMS root key was already generated and saved in DB
+    const kmsRootConfig = await kmsRootConfigDAL.findById(KMS_ROOT_CONFIG_UUID);
+    const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+    if (kmsRootConfig) {
+      if (lock) await lock.release();
+      logger.info("KMS: Encrypted ROOT Key found from DB. Decrypting.");
+      const decryptedRootKey = cipher.decrypt(kmsRootConfig.encryptedRootKey, encryptionKeyBuffer);
+      // set the flag so that other instancen nodes can start
+      await keyStore.setItemWithExpiry(KMS_ROOT_CREATION_WAIT_KEY, KMS_ROOT_CREATION_WAIT_TIME, "true");
+      logger.info("KMS: Loading ROOT Key into Memory.");
+      ROOT_ENCRYPTION_KEY = decryptedRootKey;
+      return;
+    }
+
+    logger.info("KMS: Generating ROOT Key");
+    const newRootKey = randomSecureBytes(32);
+    const encryptedRootKey = cipher.encrypt(newRootKey, encryptionKeyBuffer);
+    // @ts-expect-error id is kept as fixed for idempotence and to avoid race condition
+    await kmsRootConfigDAL.create({ encryptedRootKey, id: KMS_ROOT_CONFIG_UUID });
+
+    // set the flag so that other instancen nodes can start
+    await keyStore.setItemWithExpiry(KMS_ROOT_CREATION_WAIT_KEY, KMS_ROOT_CREATION_WAIT_TIME, "true");
+    logger.info("KMS: Saved and loaded ROOT Key into memory");
+    if (lock) await lock.release();
+    ROOT_ENCRYPTION_KEY = newRootKey;
+  };
+
+  return {
+    startService,
+    generateKmsKey,
+    encrypt,
+    decrypt
+  };
+};

backend/src/services/kms/kms-types.ts

@@ -0,0 +1,15 @@
+export type TGenerateKMSDTO = {
+  scopeType: "project" | "org";
+  scopeId: string;
+  isReserved?: boolean;
+};
+
+export type TEncryptWithKmsDTO = {
+  kmsId: string;
+  plainText: Buffer;
+};
+
+export type TDecryptWithKmsDTO = {
+  kmsId: string;
+  cipherTextBlob: Buffer;
+};

backend/src/services/secret-folder/secret-folder-dal.ts

@@ -169,6 +169,7 @@ const sqlFindSecretPathByFolderId = (db: Knex, projectId: string, folderIds: str
           // this is for root condition
           //  if the given folder id is root folder id then intial path is set as / instead of /root
           //  if not root folder the path here will be /<folder name>
+          depth: 1,
           path: db.raw(`CONCAT('/', (CASE WHEN "parentId" is NULL THEN '' ELSE ${TableName.SecretFolder}.name END))`),
           child: db.raw("NULL::uuid"),
           environmentSlug: `${TableName.Environment}.slug`
@@ -185,6 +186,7 @@ const sqlFindSecretPathByFolderId = (db: Knex, projectId: string, folderIds: str
               .select({
                 // then we join join this folder name behind previous as we are going from child to parent
                 // the root folder check is used to avoid last / and also root name in folders
+                depth: db.raw("parent.depth + 1"),
                 path: db.raw(
                   `CONCAT( CASE 
                   WHEN  ${TableName.SecretFolder}."parentId" is NULL THEN '' 
@@ -199,7 +201,7 @@ const sqlFindSecretPathByFolderId = (db: Knex, projectId: string, folderIds: str
         );
     })
     .select("*")
-    .from<TSecretFolders & { child: string | null; path: string; environmentSlug: string }>("parent");
+    .from<TSecretFolders & { child: string | null; path: string; environmentSlug: string; depth: number }>("parent");
 
 export type TSecretFolderDALFactory = ReturnType<typeof secretFolderDALFactory>;
 // never change this. If u do write a migration for it
@@ -260,12 +262,23 @@ export const secretFolderDALFactory = (db: TDbClient) => {
     try {
       const folders = await sqlFindSecretPathByFolderId(tx || db, projectId, folderIds);
 
+      //  travelling all the way from leaf node to root contains real path
       const rootFolders = groupBy(
         folders.filter(({ parentId }) => parentId === null),
         (i) => i.child || i.id // root condition then child and parent will null
       );
+      const actualFolders = groupBy(
+        folders.filter(({ depth }) => depth === 1),
+        (i) => i.id // root condition then child and parent will null
+      );
 
-      return folderIds.map((folderId) => rootFolders[folderId]?.[0]);
+      return folderIds.map((folderId) => {
+        if (!rootFolders[folderId]?.[0]) return;
+
+        const actualId = rootFolders[folderId][0].child || rootFolders[folderId][0].id;
+        const folder = actualFolders[actualId][0];
+        return { ...folder, path: rootFolders[folderId]?.[0].path };
+      });
     } catch (error) {
       throw new DatabaseError({ error, name: "Find by secret path" });
     }

backend/src/services/secret-folder/secret-folder-service.ts

@@ -253,7 +253,7 @@ export const secretFolderServiceFactory = ({
     const env = await projectEnvDAL.findOne({ projectId, slug: environment });
     if (!env) throw new BadRequestError({ message: "Environment not found", name: "Update folder" });
     const folder = await folderDAL
-      .findOne({ envId: env.id, id, parentId: parentFolder.id })
+      .findOne({ envId: env.id, id, parentId: parentFolder.id, isReserved: false })
       // now folder api accepts id based change
       // this is for cli backward compatiability and when cli removes this, we will remove this logic
       .catch(() => folderDAL.findOne({ envId: env.id, name: id, parentId: parentFolder.id }));
@@ -276,7 +276,11 @@ export const secretFolderServiceFactory = ({
     }
 
     const newFolder = await folderDAL.transaction(async (tx) => {
-      const [doc] = await folderDAL.update({ envId: env.id, id: folder.id, parentId: parentFolder.id }, { name }, tx);
+      const [doc] = await folderDAL.update(
+        { envId: env.id, id: folder.id, parentId: parentFolder.id, isReserved: false },
+        { name },
+        tx
+      );
       await folderVersionDAL.create(
         {
           name: doc.name,
@@ -324,7 +328,12 @@ export const secretFolderServiceFactory = ({
       if (!parentFolder) throw new BadRequestError({ message: "Secret path not found" });
 
       const [doc] = await folderDAL.delete(
-        { envId: env.id, [uuidValidate(idOrName) ? "id" : "name"]: idOrName, parentId: parentFolder.id },
+        {
+          envId: env.id,
+          [uuidValidate(idOrName) ? "id" : "name"]: idOrName,
+          parentId: parentFolder.id,
+          isReserved: false
+        },
         tx
       );
       if (!doc) throw new BadRequestError({ message: "Folder not found", name: "Delete folder" });
@@ -354,7 +363,7 @@ export const secretFolderServiceFactory = ({
     const parentFolder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!parentFolder) return [];
 
-    const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id });
+    const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id, isReserved: false });
 
     return folders;
   };

backend/src/services/secret-folder/secret-folder-types.ts

@@ -1,5 +1,9 @@
 import { TProjectPermission } from "@app/lib/types";
 
+export enum ReservedFolders {
+  SecretReplication = "__reserve_replication_"
+}
+
 export type TCreateFolderDTO = {
   environment: string;
   path: string;

backend/src/services/secret-folder/secret-folder-version-dal.ts

@@ -15,7 +15,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
     try {
       const docs = await (tx || db)(TableName.SecretFolderVersion)
         .join(TableName.SecretFolder, `${TableName.SecretFolderVersion}.folderId`, `${TableName.SecretFolder}.id`)
-        .where({ parentId: folderId })
+        .where({ parentId: folderId, isReserved: false })
         .join<TSecretFolderVersions>(
           (tx || db)(TableName.SecretFolderVersion)
             .groupBy("envId", "folderId")

backend/src/services/secret-import/secret-import-dal.ts

@@ -20,14 +20,14 @@ export const secretImportDALFactory = (db: TDbClient) => {
     return lastPos?.position || 0;
   };
 
-  const updateAllPosition = async (folderId: string, pos: number, targetPos: number, tx?: Knex) => {
+  const updateAllPosition = async (folderId: string, pos: number, targetPos: number, positionInc = 1, tx?: Knex) => {
     try {
       if (targetPos === -1) {
         // this means delete
         await (tx || db)(TableName.SecretImport)
           .where({ folderId })
           .andWhere("position", ">", pos)
-          .decrement("position", 1);
+          .decrement("position", positionInc);
         return;
       }
 
@@ -36,13 +36,13 @@ export const secretImportDALFactory = (db: TDbClient) => {
           .where({ folderId })
           .where("position", "<=", targetPos)
           .andWhere("position", ">", pos)
-          .decrement("position", 1);
+          .decrement("position", positionInc);
       } else {
         await (tx || db)(TableName.SecretImport)
           .where({ folderId })
           .where("position", ">=", targetPos)
           .andWhere("position", "<", pos)
-          .increment("position", 1);
+          .increment("position", positionInc);
       }
     } catch (error) {
       throw new DatabaseError({ error, name: "Update position" });
@@ -74,6 +74,7 @@ export const secretImportDALFactory = (db: TDbClient) => {
     try {
       const docs = await (tx || db)(TableName.SecretImport)
         .whereIn("folderId", folderIds)
+        .where("isReplication", false)
         .join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`)
         .select(
           db.ref("*").withSchema(TableName.SecretImport) as unknown as keyof TSecretImports,

backend/src/services/secret-import/secret-import-fns.ts

@@ -79,7 +79,7 @@ export const fnSecretsFromImports = async ({
   let secretsFromDeeperImports: TSecretImportSecrets[] = [];
   if (deeperImports.length) {
     secretsFromDeeperImports = await fnSecretsFromImports({
-      allowedImports: deeperImports,
+      allowedImports: deeperImports.filter(({ isReplication }) => !isReplication),
       secretImportDAL,
       folderDAL,
       secretDAL,

backend/src/services/secret-import/secret-import-service.ts

@@ -1,7 +1,12 @@
+import path from "node:path";
+
 import { ForbiddenError, subject } from "@casl/ability";
 
+import { TableName } from "@app/db/schemas";
+import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
+import { getReplicationFolderName } from "@app/ee/services/secret-replication/secret-replication-service";
 import { BadRequestError } from "@app/lib/errors";
 
 import { TProjectDALFactory } from "../project/project-dal";
@@ -16,6 +21,7 @@ import {
   TDeleteSecretImportDTO,
   TGetSecretImportsDTO,
   TGetSecretsFromImportDTO,
+  TResyncSecretImportReplicationDTO,
   TUpdateSecretImportDTO
 } from "./secret-import-types";
 
@@ -26,7 +32,8 @@ type TSecretImportServiceFactoryDep = {
   projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
   projectEnvDAL: TProjectEnvDALFactory;
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
-  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
+  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "replicateSecrets">;
+  licenseService: Pick<TLicenseServiceFactory, "getPlan">;
 };
 
 const ERR_SEC_IMP_NOT_FOUND = new BadRequestError({ message: "Secret import not found" });
@@ -40,7 +47,8 @@ export const secretImportServiceFactory = ({
   folderDAL,
   projectDAL,
   secretDAL,
-  secretQueueService
+  secretQueueService,
+  licenseService
 }: TSecretImportServiceFactoryDep) => {
   const createImport = async ({
     environment,
@@ -50,7 +58,8 @@ export const secretImportServiceFactory = ({
     actorOrgId,
     actorAuthMethod,
     projectId,
-    path
+    isReplication,
+    path: secretPath
   }: TCreateSecretImportDTO) => {
     const { permission } = await permissionService.getProjectPermission(
       actor,
@@ -63,7 +72,7 @@ export const secretImportServiceFactory = ({
     // check if user has permission to import into destination  path
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Create,
-      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
 
     // check if user has permission to import from target path
@@ -74,10 +83,18 @@ export const secretImportServiceFactory = ({
         secretPath: data.path
       })
     );
+    if (isReplication) {
+      const plan = await licenseService.getPlan(actorOrgId);
+      if (!plan.secretApproval) {
+        throw new BadRequestError({
+          message: "Failed to create secret replication due to plan restriction. Upgrade plan to create replication."
+        });
+      }
+    }
 
     await projectDAL.checkProjectUpgradeStatus(projectId);
 
-    const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create import" });
 
     const [importEnv] = await projectEnvDAL.findBySlugs(projectId, [data.environment]);
@@ -88,35 +105,62 @@ export const secretImportServiceFactory = ({
       const existingImport = await secretImportDAL.findOne({
         folderId: sourceFolder.id,
         importEnv: folder.environment.id,
-        importPath: path
+        importPath: secretPath
       });
       if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" });
     }
 
     const secImport = await secretImportDAL.transaction(async (tx) => {
       const lastPos = await secretImportDAL.findLastImportPosition(folder.id, tx);
-      return secretImportDAL.create(
+      const doc = await secretImportDAL.create(
         {
           folderId: folder.id,
           position: lastPos + 1,
           importEnv: importEnv.id,
-          importPath: data.path
+          importPath: data.path,
+          isReplication
         },
         tx
       );
+      if (doc.isReplication) {
+        await secretImportDAL.create(
+          {
+            folderId: folder.id,
+            position: lastPos + 2,
+            isReserved: true,
+            importEnv: folder.environment.id,
+            importPath: path.join(secretPath, getReplicationFolderName(doc.id))
+          },
+          tx
+        );
+      }
+      return doc;
     });
 
-    await secretQueueService.syncSecrets({
-      secretPath: secImport.importPath,
-      projectId,
-      environment: importEnv.slug
-    });
+    if (secImport.isReplication && sourceFolder) {
+      await secretQueueService.replicateSecrets({
+        secretPath: secImport.importPath,
+        projectId,
+        environmentSlug: importEnv.slug,
+        pickOnlyImportIds: [secImport.id],
+        actorId,
+        actor
+      });
+    } else {
+      await secretQueueService.syncSecrets({
+        secretPath,
+        projectId,
+        environmentSlug: environment,
+        actorId,
+        actor
+      });
+    }
 
     return { ...secImport, importEnv };
   };
 
   const updateImport = async ({
-    path,
+    path: secretPath,
     environment,
     projectId,
     actor,
@@ -135,10 +179,10 @@ export const secretImportServiceFactory = ({
     );
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Edit,
-      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
 
-    const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update import" });
 
     const secImpDoc = await secretImportDAL.findOne({ folderId: folder.id, id });
@@ -158,7 +202,7 @@ export const secretImportServiceFactory = ({
       const existingImport = await secretImportDAL.findOne({
         folderId: sourceFolder.id,
         importEnv: folder.environment.id,
-        importPath: path
+        importPath: secretPath
       });
       if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" });
     }
@@ -167,12 +211,31 @@ export const secretImportServiceFactory = ({
       const secImp = await secretImportDAL.findOne({ folderId: folder.id, id });
       if (!secImp) throw ERR_SEC_IMP_NOT_FOUND;
       if (data.position) {
-        await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, tx);
+        if (secImp.isReplication) {
+          await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, 2, tx);
+        } else {
+          await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, 1, tx);
+        }
+      }
+      if (secImp.isReplication) {
+        const replicationFolderPath = path.join(secretPath, getReplicationFolderName(secImp.id));
+        await secretImportDAL.update(
+          {
+            folderId: folder.id,
+            importEnv: folder.environment.id,
+            importPath: replicationFolderPath,
+            isReserved: true
+          },
+          { position: data?.position ? data.position + 1 : undefined },
+          tx
+        );
       }
       const [doc] = await secretImportDAL.update(
         { id, folderId: folder.id },
         {
-          position: data?.position,
+          // when moving replicated import, the position is meant for reserved import
+          // replicated one should always be behind the reserved import
+          position: data.position,
           importEnv: data?.environment ? importedEnv.id : undefined,
           importPath: data?.path
         },
@@ -184,7 +247,7 @@ export const secretImportServiceFactory = ({
   };
 
   const deleteImport = async ({
-    path,
+    path: secretPath,
     environment,
     projectId,
     actor,
@@ -202,16 +265,34 @@ export const secretImportServiceFactory = ({
     );
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Delete,
-      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
 
-    const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Delete import" });
 
     const secImport = await secretImportDAL.transaction(async (tx) => {
       const [doc] = await secretImportDAL.delete({ folderId: folder.id, id }, tx);
       if (!doc) throw new BadRequestError({ name: "Sec imp del", message: "Secret import doc not found" });
-      await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, tx);
+      if (doc.isReplication) {
+        const replicationFolderPath = path.join(secretPath, getReplicationFolderName(doc.id));
+        const replicatedFolder = await folderDAL.findBySecretPath(projectId, environment, replicationFolderPath, tx);
+        if (replicatedFolder) {
+          await secretImportDAL.delete(
+            {
+              folderId: folder.id,
+              importEnv: folder.environment.id,
+              importPath: replicationFolderPath,
+              isReserved: true
+            },
+            tx
+          );
+          await folderDAL.deleteById(replicatedFolder.id, tx);
+        }
+        await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, 2, tx);
+      } else {
+        await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, 1, tx);
+      }
 
       const importEnv = await projectEnvDAL.findById(doc.importEnv);
       if (!importEnv) throw new BadRequestError({ error: "Imported env not found", name: "Create import" });
@@ -219,16 +300,91 @@ export const secretImportServiceFactory = ({
     });
 
     await secretQueueService.syncSecrets({
-      secretPath: path,
+      secretPath,
       projectId,
-      environment
+      environmentSlug: environment,
+      actor,
+      actorId
     });
 
     return secImport;
   };
 
+  const resyncSecretImportReplication = async ({
+    environment,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod,
+    projectId,
+    path: secretPath,
+    id: secretImportDocId
+  }: TResyncSecretImportReplicationDTO) => {
+    const { permission, membership } = await permissionService.getProjectPermission(
+      actor,
+      actorId,
+      projectId,
+      actorAuthMethod,
+      actorOrgId
+    );
+
+    // check if user has permission to import into destination  path
+    ForbiddenError.from(permission).throwUnlessCan(
+      ProjectPermissionActions.Create,
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
+    );
+
+    const plan = await licenseService.getPlan(actorOrgId);
+    if (!plan.secretApproval) {
+      throw new BadRequestError({
+        message: "Failed to create secret replication due to plan restriction. Upgrade plan to create replication."
+      });
+    }
+
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
+    if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update import" });
+
+    const [secretImportDoc] = await secretImportDAL.find({
+      folderId: folder.id,
+      [`${TableName.SecretImport}.id` as "id"]: secretImportDocId
+    });
+    if (!secretImportDoc) throw new BadRequestError({ message: "Failed to find secret import" });
+
+    if (!secretImportDoc.isReplication) throw new BadRequestError({ message: "Import is not in replication mode" });
+
+    // check if user has permission to import from target path
+    ForbiddenError.from(permission).throwUnlessCan(
+      ProjectPermissionActions.Create,
+      subject(ProjectPermissionSub.Secrets, {
+        environment: secretImportDoc.importEnv.slug,
+        secretPath: secretImportDoc.importPath
+      })
+    );
+
+    await projectDAL.checkProjectUpgradeStatus(projectId);
+
+    const sourceFolder = await folderDAL.findBySecretPath(
+      projectId,
+      secretImportDoc.importEnv.slug,
+      secretImportDoc.importPath
+    );
+
+    if (membership && sourceFolder) {
+      await secretQueueService.replicateSecrets({
+        secretPath: secretImportDoc.importPath,
+        projectId,
+        environmentSlug: secretImportDoc.importEnv.slug,
+        pickOnlyImportIds: [secretImportDoc.id],
+        actorId,
+        actor
+      });
+    }
+
+    return { message: "replication started" };
+  };
+
   const getImports = async ({
-    path,
+    path: secretPath,
     environment,
     projectId,
     actor,
@@ -245,10 +401,10 @@ export const secretImportServiceFactory = ({
     );
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Read,
-      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
 
-    const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Get imports" });
 
     const secImports = await secretImportDAL.find({ folderId: folder.id });
@@ -256,7 +412,7 @@ export const secretImportServiceFactory = ({
   };
 
   const getSecretsFromImports = async ({
-    path,
+    path: secretPath,
     environment,
     projectId,
     actor,
@@ -273,13 +429,13 @@ export const secretImportServiceFactory = ({
     );
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Read,
-      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
-    const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) return [];
     // this will already order by position
     // so anything based on this order will also be in right position
-    const secretImports = await secretImportDAL.find({ folderId: folder.id });
+    const secretImports = await secretImportDAL.find({ folderId: folder.id, isReplication: false });
 
     const allowedImports = secretImports.filter(({ importEnv, importPath }) =>
       permission.can(
@@ -299,6 +455,7 @@ export const secretImportServiceFactory = ({
     deleteImport,
     getImports,
     getSecretsFromImports,
+    resyncSecretImportReplication,
     fnSecretsFromImports
   };
 };

backend/src/services/secret-import/secret-import-types.ts

@@ -7,6 +7,7 @@ export type TCreateSecretImportDTO = {
     environment: string;
     path: string;
   };
+  isReplication?: boolean;
 } & TProjectPermission;
 
 export type TUpdateSecretImportDTO = {
@@ -16,6 +17,12 @@ export type TUpdateSecretImportDTO = {
   data: Partial<{ environment: string; path: string; position: number }>;
 } & TProjectPermission;
 
+export type TResyncSecretImportReplicationDTO = {
+  environment: string;
+  path: string;
+  id: string;
+} & TProjectPermission;
+
 export type TDeleteSecretImportDTO = {
   environment: string;
   path: string;

backend/src/services/secret-sharing/secret-sharing-service.ts

@@ -16,17 +16,28 @@ export const secretSharingServiceFactory = ({
   secretSharingDAL
 }: TSecretSharingServiceFactoryDep) => {
   const createSharedSecret = async (createSharedSecretInput: TCreateSharedSecretDTO) => {
-    const { actor, actorId, orgId, actorAuthMethod, actorOrgId, name, encryptedValue, iv, tag, hashedHex, expiresAt } =
-      createSharedSecretInput;
-    const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
-    if (!permission) throw new UnauthorizedError({ name: "User not in org" });
-    const newSharedSecret = await secretSharingDAL.create({
-      name,
+    const {
+      actor,
+      actorId,
+      orgId,
+      actorAuthMethod,
+      actorOrgId,
       encryptedValue,
       iv,
       tag,
       hashedHex,
       expiresAt,
+      expiresAfterViews
+    } = createSharedSecretInput;
+    const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
+    if (!permission) throw new UnauthorizedError({ name: "User not in org" });
+    const newSharedSecret = await secretSharingDAL.create({
+      encryptedValue,
+      iv,
+      tag,
+      hashedHex,
+      expiresAt,
+      expiresAfterViews,
       userId: actorId,
       orgId
     });
@@ -43,9 +54,16 @@ export const secretSharingServiceFactory = ({
 
   const getActiveSharedSecretByIdAndHashedHex = async (sharedSecretId: string, hashedHex: string) => {
     const sharedSecret = await secretSharingDAL.findOne({ id: sharedSecretId, hashedHex });
-    if (sharedSecret && sharedSecret.expiresAt < new Date()) {
+    if (sharedSecret.expiresAt && sharedSecret.expiresAt < new Date()) {
       return;
     }
+    if (sharedSecret.expiresAfterViews != null && sharedSecret.expiresAfterViews >= 0) {
+      if (sharedSecret.expiresAfterViews === 0) {
+        await secretSharingDAL.deleteById(sharedSecretId);
+        return;
+      }
+      await secretSharingDAL.updateById(sharedSecretId, { $decr: { expiresAfterViews: 1 } });
+    }
     return sharedSecret;
   };
 

backend/src/services/secret-sharing/secret-sharing-types.ts

@@ -9,12 +9,12 @@ export type TSharedSecretPermission = {
 };
 
 export type TCreateSharedSecretDTO = {
-  name: string;
   encryptedValue: string;
   iv: string;
   tag: string;
   hashedHex: string;
   expiresAt: Date;
+  expiresAfterViews: number;
 } & TSharedSecretPermission;
 
 export type TDeleteSharedSecretDTO = {

backend/src/services/secret/secret-fns.ts

@@ -32,6 +32,8 @@ import {
   TCreateManySecretsRawFn,
   TCreateManySecretsRawFnFactory,
   TFnSecretBlindIndexCheck,
+  TFnSecretBlindIndexCheckV2,
+  TFnSecretBulkDelete,
   TFnSecretBulkInsert,
   TFnSecretBulkUpdate,
   TUpdateManySecretsRawFn,
@@ -149,7 +151,8 @@ export const recursivelyGetSecretPaths = ({
 
     // Fetch all folders in env once with a single query
     const folders = await folderDAL.find({
-      envId: env.id
+      envId: env.id,
+      isReserved: false
     });
 
     // Build the folder hierarchy map
@@ -396,6 +399,30 @@ export const decryptSecretRaw = (
   };
 };
 
+// this is used when secret blind index already exist
+// mainly for secret approval
+export const fnSecretBlindIndexCheckV2 = async ({
+  inputSecrets,
+  folderId,
+  userId,
+  secretDAL
+}: TFnSecretBlindIndexCheckV2) => {
+  if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
+    throw new BadRequestError({ message: "Missing user id for personal secret" });
+  }
+  const secrets = await secretDAL.findByBlindIndexes(
+    folderId,
+    inputSecrets.map(({ secretBlindIndex, type }) => ({
+      blindIndex: secretBlindIndex,
+      type: type || SecretType.Shared
+    })),
+    userId
+  );
+  const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string);
+
+  return { secsGroupedByBlindIndex, secrets };
+};
+
 /**
  * Grabs and processes nested secret references from a string
  *
@@ -598,6 +625,35 @@ export const fnSecretBulkUpdate = async ({
   return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
 };
 
+export const fnSecretBulkDelete = async ({
+  folderId,
+  inputSecrets,
+  tx,
+  actorId,
+  secretDAL,
+  secretQueueService
+}: TFnSecretBulkDelete) => {
+  const deletedSecrets = await secretDAL.deleteMany(
+    inputSecrets.map(({ type, secretBlindIndex }) => ({
+      blindIndex: secretBlindIndex,
+      type
+    })),
+    folderId,
+    actorId,
+    tx
+  );
+
+  await Promise.allSettled(
+    deletedSecrets
+      .filter(({ secretReminderRepeatDays }) => Boolean(secretReminderRepeatDays))
+      .map(({ id, secretReminderRepeatDays }) =>
+        secretQueueService.removeSecretReminder({ secretId: id, repeatDays: secretReminderRepeatDays as number })
+      )
+  );
+
+  return deletedSecrets;
+};
+
 export const createManySecretsRawFnFactory = ({
   projectDAL,
   projectBotDAL,

backend/src/services/secret/secret-queue.ts

@@ -28,7 +28,12 @@ import { TWebhookDALFactory } from "../webhook/webhook-dal";
 import { fnTriggerWebhook } from "../webhook/webhook-fns";
 import { TSecretDALFactory } from "./secret-dal";
 import { interpolateSecrets } from "./secret-fns";
-import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types";
+import {
+  TCreateSecretReminderDTO,
+  THandleReminderDTO,
+  TRemoveSecretReminderDTO,
+  TSyncSecretsDTO
+} from "./secret-types";
 
 export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
 type TSecretQueueFactoryDep = {
@@ -59,8 +64,10 @@ export type TGetSecrets = {
 };
 
 const MAX_SYNC_SECRET_DEPTH = 5;
-const uniqueIntegrationKey = (environment: string, secretPath: string) => `integration-${environment}-${secretPath}`;
+export const uniqueSecretQueueKey = (environment: string, secretPath: string) =>
+  `secret-queue-dedupe-${environment}-${secretPath}`;
 
+type TIntegrationSecret = Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean }>;
 export const secretQueueFactory = ({
   queueService,
   integrationDAL,
@@ -81,68 +88,6 @@ export const secretQueueFactory = ({
   secretTagDAL,
   secretVersionTagDAL
 }: TSecretQueueFactoryDep) => {
-  const createManySecretsRawFn = createManySecretsRawFnFactory({
-    projectDAL,
-    projectBotDAL,
-    secretDAL,
-    secretVersionDAL,
-    secretBlindIndexDAL,
-    secretTagDAL,
-    secretVersionTagDAL,
-    folderDAL
-  });
-
-  const updateManySecretsRawFn = updateManySecretsRawFnFactory({
-    projectDAL,
-    projectBotDAL,
-    secretDAL,
-    secretVersionDAL,
-    secretBlindIndexDAL,
-    secretTagDAL,
-    secretVersionTagDAL,
-    folderDAL
-  });
-
-  const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record<string, boolean> }) => {
-    await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
-      attempts: 3,
-      delay: 1000,
-      backoff: {
-        type: "exponential",
-        delay: 3000
-      },
-      removeOnComplete: true,
-      removeOnFail: true
-    });
-  };
-
-  const syncSecrets = async ({
-    deDupeQueue = {},
-    ...dto
-  }: TGetSecrets & { depth?: number; deDupeQueue?: Record<string, boolean> }) => {
-    const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath);
-    if (deDupeQueue?.[deDuplicationKey]) {
-      return;
-    }
-    // eslint-disable-next-line
-    deDupeQueue[deDuplicationKey] = true;
-    logger.info(
-      `syncSecrets: syncing project secrets where [projectId=${dto.projectId}]  [environment=${dto.environment}] [path=${dto.secretPath}]`
-    );
-    await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, {
-      jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`,
-      removeOnFail: true,
-      removeOnComplete: true,
-      delay: 1000,
-      attempts: 5,
-      backoff: {
-        type: "exponential",
-        delay: 3000
-      }
-    });
-    await syncIntegrations({ ...dto, deDupeQueue });
-  };
-
   const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
     const appCfg = getConfig();
     await queueService.stopRepeatableJob(
@@ -237,8 +182,27 @@ export const secretQueueFactory = ({
       }
     }
   };
+  const createManySecretsRawFn = createManySecretsRawFnFactory({
+    projectDAL,
+    projectBotDAL,
+    secretDAL,
+    secretVersionDAL,
+    secretBlindIndexDAL,
+    secretTagDAL,
+    secretVersionTagDAL,
+    folderDAL
+  });
 
-  type Content = Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean }>;
+  const updateManySecretsRawFn = updateManySecretsRawFnFactory({
+    projectDAL,
+    projectBotDAL,
+    secretDAL,
+    secretVersionDAL,
+    secretBlindIndexDAL,
+    secretTagDAL,
+    secretVersionTagDAL,
+    folderDAL
+  });
 
   /**
    * Return the secrets in a given [folderId] including secrets from
@@ -251,7 +215,7 @@ export const secretQueueFactory = ({
     key: string;
     depth: number;
   }) => {
-    let content: Content = {};
+    let content: TIntegrationSecret = {};
     if (dto.depth > MAX_SYNC_SECRET_DEPTH) {
       logger.info(
         `getIntegrationSecrets: secret depth exceeded for [projectId=${dto.projectId}] [folderId=${dto.folderId}] [depth=${dto.depth}]`
@@ -301,7 +265,7 @@ export const secretQueueFactory = ({
     await expandSecrets(content);
 
     // check if current folder has any imports from other folders
-    const secretImport = await secretImportDAL.find({ folderId: dto.folderId });
+    const secretImport = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false });
 
     // if no imports then return secrets in the current folder
     if (!secretImport) return content;
@@ -333,8 +297,122 @@ export const secretQueueFactory = ({
     return content;
   };
 
+  const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record<string, boolean> }) => {
+    await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
+      attempts: 3,
+      delay: 1000,
+      backoff: {
+        type: "exponential",
+        delay: 3000
+      },
+      removeOnComplete: true,
+      removeOnFail: true
+    });
+  };
+
+  const replicateSecrets = async (dto: Omit<TSyncSecretsDTO, "deDupeQueue">) => {
+    await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, dto, {
+      attempts: 3,
+      backoff: {
+        type: "exponential",
+        delay: 2000
+      },
+      removeOnComplete: true,
+      removeOnFail: true
+    });
+  };
+
+  const syncSecrets = async <T extends boolean = false>({
+    // seperate de-dupe queue for integration sync and replication sync
+    _deDupeQueue: deDupeQueue = {},
+    _depth: depth = 0,
+    _deDupeReplicationQueue: deDupeReplicationQueue = {},
+    ...dto
+  }: TSyncSecretsDTO<T>) => {
+    logger.info(
+      `syncSecrets: syncing project secrets where [projectId=${dto.projectId}]  [environment=${dto.environmentSlug}] [path=${dto.secretPath}]`
+    );
+    const deDuplicationKey = uniqueSecretQueueKey(dto.environmentSlug, dto.secretPath);
+    if (
+      !dto.excludeReplication
+        ? deDupeReplicationQueue?.[deDuplicationKey]
+        : deDupeQueue?.[deDuplicationKey] || depth > MAX_SYNC_SECRET_DEPTH
+    ) {
+      return;
+    }
+    // eslint-disable-next-line
+    deDupeQueue[deDuplicationKey] = true;
+    // eslint-disable-next-line
+    deDupeReplicationQueue[deDuplicationKey] = true;
+    await queueService.queue(
+      QueueName.SecretSync,
+      QueueJobs.SecretSync,
+      {
+        ...dto,
+        _deDupeQueue: deDupeQueue,
+        _deDupeReplicationQueue: deDupeReplicationQueue,
+        _depth: depth
+      } as TSyncSecretsDTO,
+      {
+        removeOnFail: true,
+        removeOnComplete: true,
+        delay: 1000,
+        attempts: 5,
+        backoff: {
+          type: "exponential",
+          delay: 3000
+        }
+      }
+    );
+  };
+
+  queueService.start(QueueName.SecretSync, async (job) => {
+    const {
+      _deDupeQueue: deDupeQueue,
+      _deDupeReplicationQueue: deDupeReplicationQueue,
+      _depth: depth,
+      secretPath,
+      projectId,
+      environmentSlug: environment,
+      excludeReplication,
+      actorId,
+      actor
+    } = job.data;
+
+    await queueService.queue(
+      QueueName.SecretWebhook,
+      QueueJobs.SecWebhook,
+      { environment, projectId, secretPath },
+      {
+        jobId: `secret-webhook-${environment}-${projectId}-${secretPath}`,
+        removeOnFail: { count: 5 },
+        removeOnComplete: true,
+        delay: 1000,
+        attempts: 5,
+        backoff: {
+          type: "exponential",
+          delay: 3000
+        }
+      }
+    );
+    await syncIntegrations({ secretPath, projectId, environment, deDupeQueue });
+    if (!excludeReplication) {
+      await replicateSecrets({
+        _deDupeReplicationQueue: deDupeReplicationQueue,
+        _depth: depth,
+        projectId,
+        secretPath,
+        actorId,
+        actor,
+        excludeReplication,
+        environmentSlug: environment
+      });
+    }
+  });
+
   queueService.start(QueueName.IntegrationSync, async (job) => {
     const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
+    if (depth > MAX_SYNC_SECRET_DEPTH) return;
 
     const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) {
@@ -348,7 +426,8 @@ export const secretQueueFactory = ({
       const linkSourceDto = {
         projectId,
         importEnv: folder.environment.id,
-        importPath: secretPath
+        importPath: secretPath,
+        isReplication: false
       };
       const imports = await secretImportDAL.find(linkSourceDto);
 
@@ -356,30 +435,31 @@ export const secretQueueFactory = ({
         // keep calling sync secret for all the imports made
         const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId);
         const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds);
-        const foldersGroupedById = groupBy(importedFolders, (i) => i.child || i.id);
+        const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string);
         logger.info(
           `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
         );
         await Promise.all(
           imports
-            .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0].path))
+            .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0]?.path as string))
             // filter out already synced ones
             .filter(
               ({ folderId }) =>
                 !deDupeQueue[
-                  uniqueIntegrationKey(
-                    foldersGroupedById[folderId][0].environmentSlug,
-                    foldersGroupedById[folderId][0].path
+                  uniqueSecretQueueKey(
+                    foldersGroupedById[folderId][0]?.environmentSlug as string,
+                    foldersGroupedById[folderId][0]?.path as string
                   )
                 ]
             )
             .map(({ folderId }) =>
               syncSecrets({
-                depth: depth + 1,
                 projectId,
-                secretPath: foldersGroupedById[folderId][0].path,
-                environment: foldersGroupedById[folderId][0].environmentSlug,
-                deDupeQueue
+                secretPath: foldersGroupedById[folderId][0]?.path as string,
+                environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string,
+                _deDupeQueue: deDupeQueue,
+                _depth: depth + 1,
+                excludeReplication: true
               })
             )
         );
@@ -393,30 +473,31 @@ export const secretQueueFactory = ({
       if (secretReferences.length) {
         const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId);
         const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds);
-        const referencedFoldersGroupedById = groupBy(referencedFolders, (i) => i.child || i.id);
+        const referencedFoldersGroupedById = groupBy(referencedFolders.filter(Boolean), (i) => i?.id as string);
         logger.info(
           `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
         );
         await Promise.all(
           secretReferences
-            .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0].path))
+            .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0]?.path))
             // filter out already synced ones
             .filter(
               ({ folderId }) =>
                 !deDupeQueue[
-                  uniqueIntegrationKey(
-                    referencedFoldersGroupedById[folderId][0].environmentSlug,
-                    referencedFoldersGroupedById[folderId][0].path
+                  uniqueSecretQueueKey(
+                    referencedFoldersGroupedById[folderId][0]?.environmentSlug as string,
+                    referencedFoldersGroupedById[folderId][0]?.path as string
                   )
                 ]
             )
             .map(({ folderId }) =>
               syncSecrets({
-                depth: depth + 1,
                 projectId,
-                secretPath: referencedFoldersGroupedById[folderId][0].path,
-                environment: referencedFoldersGroupedById[folderId][0].environmentSlug,
-                deDupeQueue
+                secretPath: referencedFoldersGroupedById[folderId][0]?.path as string,
+                environmentSlug: referencedFoldersGroupedById[folderId][0]?.environmentSlug as string,
+                _deDupeQueue: deDupeQueue,
+                _depth: depth + 1,
+                excludeReplication: true
               })
             )
         );
@@ -546,10 +627,11 @@ export const secretQueueFactory = ({
 
   return {
     // depth is internal only field thus no need to make it available outside
-    syncSecrets: (dto: TGetSecrets) => syncSecrets(dto),
+    syncSecrets,
     syncIntegrations,
     addSecretReminder,
     removeSecretReminder,
-    handleSecretReminder
+    handleSecretReminder,
+    replicateSecrets
   };
 };

backend/src/services/secret/secret-service.ts

@@ -35,6 +35,7 @@ import { TSecretDALFactory } from "./secret-dal";
 import {
   decryptSecretRaw,
   fnSecretBlindIndexCheck,
+  fnSecretBulkDelete,
   fnSecretBulkInsert,
   fnSecretBulkUpdate,
   getAllNestedSecretReferences,
@@ -53,8 +54,6 @@ import {
   TDeleteManySecretRawDTO,
   TDeleteSecretDTO,
   TDeleteSecretRawDTO,
-  TFnSecretBlindIndexCheckV2,
-  TFnSecretBulkDelete,
   TGetASecretDTO,
   TGetASecretRawDTO,
   TGetSecretsDTO,
@@ -139,53 +138,6 @@ export const secretServiceFactory = ({
     return secretBlindIndex;
   };
 
-  const fnSecretBulkDelete = async ({ folderId, inputSecrets, tx, actorId }: TFnSecretBulkDelete) => {
-    const deletedSecrets = await secretDAL.deleteMany(
-      inputSecrets.map(({ type, secretBlindIndex }) => ({
-        blindIndex: secretBlindIndex,
-        type
-      })),
-      folderId,
-      actorId,
-      tx
-    );
-
-    for (const s of deletedSecrets) {
-      if (s.secretReminderRepeatDays) {
-        // eslint-disable-next-line no-await-in-loop
-        await secretQueueService
-          .removeSecretReminder({
-            secretId: s.id,
-            repeatDays: s.secretReminderRepeatDays
-          })
-          .catch((err) => {
-            logger.error(err, `Failed to delete secret reminder for secret with ID ${s?.id}`);
-          });
-      }
-    }
-
-    return deletedSecrets;
-  };
-
-  // this is used when secret blind index already exist
-  // mainly for secret approval
-  const fnSecretBlindIndexCheckV2 = async ({ inputSecrets, folderId, userId }: TFnSecretBlindIndexCheckV2) => {
-    if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
-      throw new BadRequestError({ message: "Missing user id for personal secret" });
-    }
-    const secrets = await secretDAL.findByBlindIndexes(
-      folderId,
-      inputSecrets.map(({ secretBlindIndex, type }) => ({
-        blindIndex: secretBlindIndex,
-        type: type || SecretType.Shared
-      })),
-      userId
-    );
-    const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string);
-
-    return { secsGroupedByBlindIndex, secrets };
-  };
-
   const createSecret = async ({
     path,
     actor,
@@ -283,8 +235,13 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
-    // TODO(akhilmhdh-pg): licence check, posthog service and snapshot
+    await secretQueueService.syncSecrets({
+      secretPath: path,
+      actorId,
+      actor,
+      projectId,
+      environmentSlug: folder.environment.slug
+    });
     return { ...secret[0], environment, workspace: projectId, tags, secretPath: path };
   };
 
@@ -413,8 +370,13 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
-    // TODO(akhilmhdh-pg): licence check, posthog service and snapshot
+    await secretQueueService.syncSecrets({
+      actor,
+      actorId,
+      secretPath: path,
+      projectId,
+      environmentSlug: folder.environment.slug
+    });
     return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path };
   };
 
@@ -470,6 +432,8 @@ export const secretServiceFactory = ({
         projectId,
         folderId,
         actorId,
+        secretDAL,
+        secretQueueService,
         inputSecrets: [
           {
             type: inputSecret.type as SecretType,
@@ -481,8 +445,13 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
-
+    await secretQueueService.syncSecrets({
+      actor,
+      actorId,
+      secretPath: path,
+      projectId,
+      environmentSlug: folder.environment.slug
+    });
     // TODO(akhilmhdh-pg): licence check, posthog service and snapshot
     return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path };
   };
@@ -551,7 +520,8 @@ export const secretServiceFactory = ({
 
     if (includeImports) {
       const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId));
-      const allowedImports = secretImports.filter(({ importEnv, importPath }) =>
+      const allowedImports = secretImports.filter(({ importEnv, importPath, isReplication }) =>
+        !isReplication &&
         // if its service token allow full access over imported one
         actor === ActorType.SERVICE
           ? true
@@ -656,7 +626,7 @@ export const secretServiceFactory = ({
     // then search for imported secrets
     // here we consider the import order also thus starting from bottom
     if (!secret && includeImports) {
-      const secretImports = await secretImportDAL.find({ folderId });
+      const secretImports = await secretImportDAL.find({ folderId, isReplication: false });
       const allowedImports = secretImports.filter(({ importEnv, importPath }) =>
         // if its service token allow full access over imported one
         actor === ActorType.SERVICE
@@ -767,7 +737,13 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
+    await secretQueueService.syncSecrets({
+      actor,
+      actorId,
+      secretPath: path,
+      projectId,
+      environmentSlug: folder.environment.slug
+    });
 
     return newSecrets;
   };
@@ -867,7 +843,13 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
+    await secretQueueService.syncSecrets({
+      actor,
+      actorId,
+      secretPath: path,
+      projectId,
+      environmentSlug: folder.environment.slug
+    });
 
     return secrets;
   };
@@ -917,6 +899,8 @@ export const secretServiceFactory = ({
 
     const secretsDeleted = await secretDAL.transaction(async (tx) =>
       fnSecretBulkDelete({
+        secretDAL,
+        secretQueueService,
         inputSecrets: inputSecrets.map(({ type, secretName }) => ({
           secretBlindIndex: keyName2BlindIndex[secretName],
           type
@@ -929,7 +913,13 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
+    await secretQueueService.syncSecrets({
+      actor,
+      actorId,
+      secretPath: path,
+      projectId,
+      environmentSlug: folder.environment.slug
+    });
 
     return secretsDeleted;
   };
@@ -1109,9 +1099,6 @@ export const secretServiceFactory = ({
       skipMultilineEncoding
     });
 
-    await snapshotService.performSnapshot(secret.folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return decryptSecretRaw(secret, botKey);
   };
 
@@ -1150,8 +1137,6 @@ export const secretServiceFactory = ({
     });
 
     await snapshotService.performSnapshot(secret.folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return decryptSecretRaw(secret, botKey);
   };
 
@@ -1181,9 +1166,6 @@ export const secretServiceFactory = ({
       actorAuthMethod
     });
 
-    await snapshotService.performSnapshot(secret.folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return decryptSecretRaw(secret, botKey);
   };
 
@@ -1232,9 +1214,6 @@ export const secretServiceFactory = ({
       })
     });
 
-    await snapshotService.performSnapshot(secrets[0].folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return secrets.map((secret) =>
       decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
     );
@@ -1286,9 +1265,6 @@ export const secretServiceFactory = ({
       })
     });
 
-    await snapshotService.performSnapshot(secrets[0].folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return secrets.map((secret) =>
       decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
     );
@@ -1322,9 +1298,6 @@ export const secretServiceFactory = ({
       secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared }))
     });
 
-    await snapshotService.performSnapshot(secrets[0].folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return secrets.map((secret) =>
       decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
     );
@@ -1448,7 +1421,12 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folder.id);
-    await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment });
+    await secretQueueService.syncSecrets({
+      secretPath,
+      projectId: project.id,
+      environmentSlug: environment,
+      excludeReplication: true
+    });
 
     return {
       ...updatedSecret[0],
@@ -1550,7 +1528,12 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folder.id);
-    await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment });
+    await secretQueueService.syncSecrets({
+      secretPath,
+      projectId: project.id,
+      environmentSlug: environment,
+      excludeReplication: true
+    });
 
     return {
       ...updatedSecret[0],
@@ -1624,12 +1607,6 @@ export const secretServiceFactory = ({
     updateManySecretsRaw,
     deleteManySecretsRaw,
     getSecretVersions,
-    backfillSecretReferences,
-    // external services function
-    fnSecretBulkDelete,
-    fnSecretBulkUpdate,
-    fnSecretBlindIndexCheck,
-    fnSecretBulkInsert,
-    fnSecretBlindIndexCheckV2
+    backfillSecretReferences
   };
 };

backend/src/services/secret/secret-types.ts

@@ -11,6 +11,8 @@ import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/se
 import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
 import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
 
+import { ActorType } from "../auth/auth-type";
+
 type TPartialSecret = Pick<TSecrets, "id" | "secretReminderRepeatDays" | "secretReminderNote">;
 
 type TPartialInputSecret = Pick<TSecrets, "type" | "secretReminderNote" | "secretReminderRepeatDays" | "id">;
@@ -264,6 +266,10 @@ export type TFnSecretBulkDelete = {
   inputSecrets: Array<{ type: SecretType; secretBlindIndex: string }>;
   actorId: string;
   tx?: Knex;
+  secretDAL: Pick<TSecretDALFactory, "deleteMany">;
+  secretQueueService: {
+    removeSecretReminder: (data: TRemoveSecretReminderDTO) => Promise<void>;
+  };
 };
 
 export type TFnSecretBlindIndexCheck = {
@@ -277,6 +283,7 @@ export type TFnSecretBlindIndexCheck = {
 
 // when blind index is already present
 export type TFnSecretBlindIndexCheckV2 = {
+  secretDAL: Pick<TSecretDALFactory, "findByBlindIndexes">;
   folderId: string;
   userId?: string;
   inputSecrets: Array<{ secretBlindIndex: string; type?: SecretType }>;
@@ -363,3 +370,27 @@ export type TUpdateManySecretsRawFn = {
   }[];
   userId?: string;
 };
+
+export enum SecretOperations {
+  Create = "create",
+  Update = "update",
+  Delete = "delete"
+}
+
+export type TSyncSecretsDTO<T extends boolean = false> = {
+  _deDupeQueue?: Record<string, boolean>;
+  _deDupeReplicationQueue?: Record<string, boolean>;
+  _depth?: number;
+  secretPath: string;
+  projectId: string;
+  environmentSlug: string;
+  // cases for just doing sync integration and webhook
+  excludeReplication?: T;
+} & (T extends true
+  ? object
+  : {
+      actor: ActorType;
+      actorId: string;
+      // used for import creation to trigger replication
+      pickOnlyImportIds?: string[];
+    });

backend/src/services/secret/secret-version-dal.ts

@@ -89,6 +89,7 @@ export const secretVersionDALFactory = (db: TDbClient) => {
 
   const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => {
     try {
+      if (!secretIds.length) return {};
       const docs: Array<TSecretVersions & { max: number }> = await (tx || db)(TableName.SecretVersion)
         .where("folderId", folderId)
         .whereIn(`${TableName.SecretVersion}.secretId`, secretIds)