feat: added bridge logic in secret replication, snapshot and approval for raw endpoint

2026-10-14 08:29:55 +00:00 · 2024-07-30 23:11:04 +05:30
parent 9e314d7a09
commit ef2816b2ee
14 changed files with 2205 additions and 283 deletions
@@ -1,4 +1,4 @@
-import { SecretType, TSecrets } from "@app/db/schemas";
+import { SecretType, TSecrets, TSecretsV2 } from "@app/db/schemas";
 import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
 import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
 import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
@@ -13,18 +13,28 @@ import { ActorType } from "@app/services/auth/auth-type";
 import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
 import { TSecretDALFactory } from "@app/services/secret/secret-dal";
 import { fnSecretBulkInsert, fnSecretBulkUpdate } from "@app/services/secret/secret-fns";
+import {
+  fnSecretBulkInsert as fnSecretV2BridgeBulkInsert,
+  fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate,
+  getAllNestedSecretReferences,
+  getAllNestedSecretReferences as getAllNestedSecretReferencesV2Bridge
+} from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";
 import { TSecretQueueFactory, uniqueSecretQueueKey } from "@app/services/secret/secret-queue";
 import { SecretOperations } from "@app/services/secret/secret-types";
 import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
 import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
-import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
 import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
 import { ReservedFolders } from "@app/services/secret-folder/secret-folder-types";
 import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
-import { fnSecretsFromImports } from "@app/services/secret-import/secret-import-fns";
+import { fnSecretsFromImports, fnSecretsV2FromImports } from "@app/services/secret-import/secret-import-fns";
 import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";

 import { MAX_REPLICATION_DEPTH } from "./secret-replication-constants";
+import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
+import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
+import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal";
+import { TKmsServiceFactory } from "@app/services/kms/kms-service";
+import { KmsDataKey } from "@app/services/kms/kms-types";

 type TSecretReplicationServiceFactoryDep = {
  secretDAL: Pick<
@@ -32,24 +42,42 @@ type TSecretReplicationServiceFactoryDep = {
    "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" | "transaction"
  >;
  secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany">;
+  secretV2BridgeDAL: Pick<
+    TSecretV2BridgeDALFactory,
+    "find" | "findBySecretKeys" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" | "transaction"
+  >;
+  secretVersionV2BridgeDAL: Pick<
+    TSecretVersionV2DALFactory,
+    "find" | "insertMany" | "update" | "findLatestVersionMany"
+  >;
  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "updateById" | "findByFolderIds">;
  folderDAL: Pick<
    TSecretFolderDALFactory,
    "findSecretPathByFolderIds" | "findBySecretPath" | "create" | "findOne" | "findByManySecretPath"
  >;
  secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "find" | "insertMany">;
+  secretVersionV2TagBridgeDAL: Pick<TSecretVersionV2TagDALFactory, "find" | "insertMany">;
  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "replicateSecrets">;
  queueService: Pick<TQueueServiceFactory, "start" | "listen" | "queue" | "stopJobById">;
  secretApprovalPolicyService: Pick<TSecretApprovalPolicyServiceFactory, "getSecretApprovalPolicy">;
  keyStore: Pick<TKeyStoreFactory, "acquireLock" | "setItemWithExpiry" | "getItem">;
-  secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
-  secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret" | "find">;
+  secretTagDAL: Pick<
+    TSecretTagDALFactory,
+    | "findManyTagsById"
+    | "saveTagsToSecret"
+    | "deleteTagsManySecret"
+    | "find"
+    | "saveTagsToSecretV2"
+    | "deleteTagsToSecretV2"
+  >;
  secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "create" | "transaction">;
  secretApprovalRequestSecretDAL: Pick<
    TSecretApprovalRequestSecretDALFactory,
-    "insertMany" | "insertApprovalSecretTags"
+    "insertMany" | "insertApprovalSecretTags" | "insertV2Bridge"
  >;
+
  projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
+  kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
 };

 export type TSecretReplicationServiceFactory = ReturnType<typeof secretReplicationServiceFactory>;
@@ -90,9 +118,13 @@ export const secretReplicationServiceFactory = ({
  secretApprovalRequestSecretDAL,
  secretApprovalRequestDAL,
  secretQueueService,
-  projectBotService
+  projectBotService,
+  secretVersionV2TagBridgeDAL,
+  secretVersionV2BridgeDAL,
+  secretV2BridgeDAL,
+  kmsService
 }: TSecretReplicationServiceFactoryDep) => {
-  const getReplicatedSecrets = (
+  const $getReplicatedSecrets = (
    botKey: string,
    localSecrets: TSecrets[],
    importedSecrets: { secrets: TSecrets[] }[]
@@ -119,6 +151,25 @@ export const secretReplicationServiceFactory = ({
    return secrets;
  };

+  const $getReplicatedSecretsV2 = (
+    localSecrets: (TSecretsV2 & { secretKey: string; secretValue?: string })[],
+    importedSecrets: { secrets: (TSecretsV2 & { secretKey: string; secretValue?: string })[] }[]
+  ) => {
+    const deDupe = new Set<string>();
+    const secrets = [...localSecrets];
+
+    for (let i = importedSecrets.length - 1; i >= 0; i = -1) {
+      importedSecrets[i].secrets.forEach((el) => {
+        if (deDupe.has(el.key)) {
+          return;
+        }
+        deDupe.add(el.key);
+        secrets.push(el);
+      });
+    }
+    return secrets;
+  };
+
  // IMPORTANT NOTE BEFORE READING THE FUNCTION
  // SOURCE - Where secrets are copied from
  // DESTINATION - Where the replicated imports that points to SOURCE from Destination
@@ -139,6 +190,7 @@ export const secretReplicationServiceFactory = ({

    const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, secretPath);
    if (!folder) return;
+    const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);

    // the the replicated imports made to the source. These are the destinations
    const destinationSecretImports = await secretImportDAL.find({
@@ -191,8 +243,270 @@ export const secretReplicationServiceFactory = ({
      : destinationReplicatedSecretImports;
    if (!destinationReplicatedSecretImports.length) return;

-    const botKey = await projectBotService.getBotKey(projectId);
+    if (shouldUseSecretV2Bridge) {
+      const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
+        type: KmsDataKey.SecretManager,
+        projectId
+      });

+      // these are the secrets to be added in replicated folders
+      const sourceLocalSecrets = await secretV2BridgeDAL.find({ folderId: folder.id, type: SecretType.Shared });
+      const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id });
+      const sourceImportedSecrets = await fnSecretsV2FromImports({
+        allowedImports: sourceSecretImports,
+        secretDAL: secretV2BridgeDAL,
+        folderDAL,
+        secretImportDAL,
+        decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : undefined)
+      });
+      // secrets that gets replicated across imports
+      const sourceDecryptedLocalSecrets = sourceLocalSecrets.map((el) => ({
+        ...el,
+        secretKey: el.key,
+        secretValue: el.encryptedValue
+          ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString()
+          : undefined
+      }));
+      const sourceSecrets = $getReplicatedSecretsV2(sourceDecryptedLocalSecrets, sourceImportedSecrets);
+      const sourceSecretsGroupByKey = groupBy(sourceSecrets, (i) => i.key);
+
+      const lock = await keyStore.acquireLock(
+        [getReplicationKeyLockPrefix(projectId, environmentSlug, secretPath)],
+        5000
+      );
+
+      try {
+        /*  eslint-disable no-await-in-loop */
+        for (const destinationSecretImport of destinationReplicatedSecretImports) {
+          try {
+            const hasJobCompleted = await keyStore.getItem(
+              keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id),
+              KeyStorePrefixes.SecretReplication
+            );
+            if (hasJobCompleted) {
+              logger.info(
+                { jobId: job.id, importId: destinationSecretImport.id },
+                "Skipping this job as this has been successfully replicated."
+              );
+              // eslint-disable-next-line
+              continue;
+            }
+
+            const [destinationFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [
+              destinationSecretImport.folderId
+            ]);
+            if (!destinationFolder) throw new BadRequestError({ message: "Imported folder not found" });
+
+            let destinationReplicationFolder = await folderDAL.findOne({
+              parentId: destinationFolder.id,
+              name: getReplicationFolderName(destinationSecretImport.id),
+              isReserved: true
+            });
+            if (!destinationReplicationFolder) {
+              destinationReplicationFolder = await folderDAL.create({
+                parentId: destinationFolder.id,
+                name: getReplicationFolderName(destinationSecretImport.id),
+                envId: destinationFolder.envId,
+                isReserved: true
+              });
+            }
+            const destinationReplicationFolderId = destinationReplicationFolder.id;
+
+            const destinationLocalSecretsFromDB = await secretV2BridgeDAL.find({
+              folderId: destinationReplicationFolderId
+            });
+            const destinationLocalSecrets = destinationLocalSecretsFromDB.map((el) => ({
+              ...el,
+              secretKey: el.key,
+              secretValue: el.encryptedValue
+                ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString()
+                : undefined
+            }));
+
+            const destinationLocalSecretsGroupedByKey = groupBy(destinationLocalSecrets, (i) => i.key);
+
+            const locallyCreatedSecrets = sourceSecrets
+              .filter(({ key }) => !destinationLocalSecretsGroupedByKey[key]?.[0])
+              .map((el) => ({ ...el, operation: SecretOperations.Create })); // rewrite update ops to create
+
+            const locallyUpdatedSecrets = sourceSecrets
+              .filter(
+                ({ key, secretKey, secretValue }) =>
+                  destinationLocalSecretsGroupedByKey[key]?.[0] &&
+                  // if key or value changed
+                  (destinationLocalSecretsGroupedByKey[key]?.[0]?.secretKey !== secretKey ||
+                    destinationLocalSecretsGroupedByKey[key]?.[0]?.secretValue !== secretValue)
+              )
+              .map((el) => ({ ...el, operation: SecretOperations.Update })); // rewrite update ops to create
+
+            const locallyDeletedSecrets = destinationLocalSecrets
+              .filter(({ key }) => !sourceSecretsGroupByKey[key]?.[0])
+              .map((el) => ({ ...el, operation: SecretOperations.Delete }));
+
+            const isEmtpy =
+              locallyCreatedSecrets.length + locallyUpdatedSecrets.length + locallyDeletedSecrets.length === 0;
+            // eslint-disable-next-line
+            if (isEmtpy) continue;
+
+            const policy = await secretApprovalPolicyService.getSecretApprovalPolicy(
+              projectId,
+              destinationFolder.environmentSlug,
+              destinationFolder.path
+            );
+            // this means it should be a approval request rather than direct replication
+            if (policy && actor === ActorType.USER) {
+              const localSecretsLatestVersions = destinationLocalSecrets.map(({ id }) => id);
+              const latestSecretVersions = await secretVersionV2BridgeDAL.findLatestVersionMany(
+                destinationReplicationFolderId,
+                localSecretsLatestVersions
+              );
+              await secretApprovalRequestDAL.transaction(async (tx) => {
+                const approvalRequestDoc = await secretApprovalRequestDAL.create(
+                  {
+                    folderId: destinationReplicationFolderId,
+                    slug: alphaNumericNanoId(),
+                    policyId: policy.id,
+                    status: "open",
+                    hasMerged: false,
+                    committerUserId: actorId,
+                    isReplicated: true
+                  },
+                  tx
+                );
+                const commits = locallyCreatedSecrets
+                  .concat(locallyUpdatedSecrets)
+                  .concat(locallyDeletedSecrets)
+                  .map((doc) => {
+                    const { operation } = doc;
+                    const localSecret = destinationLocalSecretsGroupedByKey[doc.key]?.[0];
+
+                    return {
+                      op: operation,
+                      requestId: approvalRequestDoc.id,
+                      metadata: doc.metadata,
+                      key: doc.key,
+                      encryptedValue: doc.encryptedValue,
+                      encryptedComment: doc.encryptedComment,
+                      skipMultilineEncoding: doc.skipMultilineEncoding,
+                      // except create operation other two needs the secret id and version id
+                      ...(operation !== SecretOperations.Create
+                        ? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id }
+                        : {})
+                    };
+                  });
+                const approvalCommits = await secretApprovalRequestSecretDAL.insertV2Bridge(commits, tx);
+
+                return { ...approvalRequestDoc, commits: approvalCommits };
+              });
+            } else {
+              await secretDAL.transaction(async (tx) => {
+                if (locallyCreatedSecrets.length) {
+                  await fnSecretV2BridgeBulkInsert({
+                    folderId: destinationReplicationFolderId,
+                    secretVersionDAL: secretVersionV2BridgeDAL,
+                    secretDAL: secretV2BridgeDAL,
+                    tx,
+                    secretTagDAL,
+                    secretVersionTagDAL: secretVersionV2TagBridgeDAL,
+                    inputSecrets: locallyCreatedSecrets.map((doc) => {
+                      return {
+                        type: doc.type,
+                        metadata: doc.metadata,
+                        key: doc.key,
+                        encryptedValue: doc.encryptedValue,
+                        encryptedComment: doc.encryptedComment,
+                        skipMultilineEncoding: doc.skipMultilineEncoding,
+                        references: doc.secretValue ? getAllNestedSecretReferencesV2Bridge(doc.secretValue) : []
+                      };
+                    })
+                  });
+                }
+                if (locallyUpdatedSecrets.length) {
+                  await fnSecretV2BridgeBulkUpdate({
+                    folderId: destinationReplicationFolderId,
+                    secretVersionDAL: secretVersionV2BridgeDAL,
+                    secretDAL: secretV2BridgeDAL,
+                    tx,
+                    secretTagDAL,
+                    secretVersionTagDAL: secretVersionV2TagBridgeDAL,
+                    inputSecrets: locallyUpdatedSecrets.map((doc) => {
+                      return {
+                        filter: {
+                          folderId: destinationReplicationFolderId,
+                          id: destinationLocalSecretsGroupedByKey[doc.key][0].id
+                        },
+                        data: {
+                          type: doc.type,
+                          metadata: doc.metadata,
+                          key: doc.key,
+                          encryptedValue: doc.encryptedValue as Buffer,
+                          encryptedComment: doc.encryptedComment,
+                          skipMultilineEncoding: doc.skipMultilineEncoding,
+                          references: doc.secretValue ? getAllNestedSecretReferencesV2Bridge(doc.secretValue) : []
+                        }
+                      };
+                    })
+                  });
+                }
+                if (locallyDeletedSecrets.length) {
+                  await secretDAL.delete(
+                    {
+                      $in: {
+                        id: locallyDeletedSecrets.map(({ id }) => id)
+                      },
+                      folderId: destinationReplicationFolderId
+                    },
+                    tx
+                  );
+                }
+              });
+
+              await secretQueueService.syncSecrets({
+                projectId,
+                secretPath: destinationFolder.path,
+                environmentSlug: destinationFolder.environmentSlug,
+                actorId,
+                actor,
+                _depth: depth + 1,
+                _deDupeReplicationQueue: deDupeReplicationQueue,
+                _deDupeQueue: deDupeQueue
+              });
+            }
+
+            // this is used to avoid multiple times generating secret approval by failed one
+            await keyStore.setItemWithExpiry(
+              keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id),
+              SECRET_IMPORT_SUCCESS_LOCK,
+              1,
+              KeyStorePrefixes.SecretReplication
+            );
+
+            await secretImportDAL.updateById(destinationSecretImport.id, {
+              lastReplicated: new Date(),
+              replicationStatus: null,
+              isReplicationSuccess: true
+            });
+          } catch (err) {
+            logger.error(
+              err,
+              `Failed to replicate secret with import id=[${destinationSecretImport.id}] env=[${destinationSecretImport.importEnv.slug}] path=[${destinationSecretImport.importPath}]`
+            );
+            await secretImportDAL.updateById(destinationSecretImport.id, {
+              lastReplicated: new Date(),
+              replicationStatus: (err as Error)?.message.slice(0, 500),
+              isReplicationSuccess: false
+            });
+          }
+        }
+        /*  eslint-enable no-await-in-loop */
+      } finally {
+        await lock.release();
+        logger.info(job.data, "Replication finished");
+      }
+      return;
+    }
+
+    if (!botKey) throw new BadRequestError({ message: "Bot not found" });
    // these are the secrets to be added in replicated folders
    const sourceLocalSecrets = await secretDAL.find({ folderId: folder.id, type: SecretType.Shared });
    const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id });
@@ -203,7 +517,7 @@ export const secretReplicationServiceFactory = ({
      secretImportDAL
    });
    // secrets that gets replicated across imports
-    const sourceSecrets = getReplicatedSecrets(botKey, sourceLocalSecrets, sourceImportedSecrets);
+    const sourceSecrets = $getReplicatedSecrets(botKey, sourceLocalSecrets, sourceImportedSecrets);
    const sourceSecretsGroupByBlindIndex = groupBy(sourceSecrets, (i) => i.secretBlindIndex as string);

    const lock = await keyStore.acquireLock(
@@ -372,7 +686,8 @@ export const secretReplicationServiceFactory = ({
                      secretCommentIV: doc.secretCommentIV,
                      secretCommentTag: doc.secretCommentTag,
                      secretCommentCiphertext: doc.secretCommentCiphertext,
-                      skipMultilineEncoding: doc.skipMultilineEncoding
+                      skipMultilineEncoding: doc.skipMultilineEncoding,
+                      references: getAllNestedSecretReferences(doc.secretValue)
                    };
                  })
                });
@@ -407,7 +722,8 @@ export const secretReplicationServiceFactory = ({
                        secretCommentIV: doc.secretCommentIV,
                        secretCommentTag: doc.secretCommentTag,
                        secretCommentCiphertext: doc.secretCommentCiphertext,
-                        skipMultilineEncoding: doc.skipMultilineEncoding
+                        skipMultilineEncoding: doc.skipMultilineEncoding,
+                        references: getAllNestedSecretReferences(doc.secretValue)
                      }
                    };
                  })