Merge remote-tracking branch 'origin' into ldap

backend/src/services/secret/secret-fns.ts

@@ -1,12 +1,35 @@
 /* eslint-disable no-await-in-loop */
 import path from "path";
 
-import { SecretKeyEncoding, TSecretBlindIndexes, TSecrets } from "@app/db/schemas";
+import {
+  SecretEncryptionAlgo,
+  SecretKeyEncoding,
+  SecretType,
+  TableName,
+  TSecretBlindIndexes,
+  TSecrets
+} from "@app/db/schemas";
 import { getConfig } from "@app/lib/config/env";
-import { buildSecretBlindIndexFromName, decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
+import {
+  buildSecretBlindIndexFromName,
+  decryptSymmetric128BitHexKeyUTF8,
+  encryptSymmetric128BitHexKeyUTF8
+} from "@app/lib/crypto";
+import { BadRequestError } from "@app/lib/errors";
+import { groupBy, unique } from "@app/lib/fn";
 
+import { getBotKeyFnFactory } from "../project-bot/project-bot-fns";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretDALFactory } from "./secret-dal";
+import {
+  TCreateManySecretsRawFn,
+  TCreateManySecretsRawFnFactory,
+  TFnSecretBlindIndexCheck,
+  TFnSecretBulkInsert,
+  TFnSecretBulkUpdate,
+  TUpdateManySecretsRawFn,
+  TUpdateManySecretsRawFnFactory
+} from "./secret-types";
 
 export const generateSecretBlindIndexBySalt = async (secretName: string, secretBlindIndexDoc: TSecretBlindIndexes) => {
   const appCfg = getConfig();
@@ -228,3 +251,399 @@ export const decryptSecretRaw = (secret: TSecrets & { workspace: string; environ
     user: secret.userId
   };
 };
+
+/**
+ * Checks and handles secrets using a blind index method.
+ * The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes.
+ * For new secrets (isNew = true), it ensures they don't already exist in the database.
+ * For existing secrets, it verifies their presence in the database.
+ * If discrepancies are found, errors are thrown. The function returns mappings and the fetched secrets.
+ */
+export const fnSecretBlindIndexCheck = async ({
+  inputSecrets,
+  folderId,
+  isNew,
+  userId,
+  blindIndexCfg,
+  secretDAL
+}: TFnSecretBlindIndexCheck) => {
+  const blindIndex2KeyName: Record<string, string> = {}; // used at audit log point
+  const keyName2BlindIndex = await Promise.all(
+    inputSecrets.map(({ secretName }) => generateSecretBlindIndexBySalt(secretName, blindIndexCfg))
+  ).then((blindIndexes) =>
+    blindIndexes.reduce<Record<string, string>>((prev, curr, i) => {
+      // eslint-disable-next-line
+      prev[inputSecrets[i].secretName] = curr;
+      blindIndex2KeyName[curr] = inputSecrets[i].secretName;
+      return prev;
+    }, {})
+  );
+
+  if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
+    throw new BadRequestError({ message: "Missing user id for personal secret" });
+  }
+
+  const secrets = await secretDAL.findByBlindIndexes(
+    folderId,
+    inputSecrets.map(({ secretName, type }) => ({
+      blindIndex: keyName2BlindIndex[secretName],
+      type: type || SecretType.Shared
+    })),
+    userId
+  );
+
+  if (isNew) {
+    if (secrets.length) throw new BadRequestError({ message: "Secret already exist" });
+  } else {
+    const secretKeysInDB = unique(secrets, (el) => el.secretBlindIndex as string).map(
+      (el) => blindIndex2KeyName[el.secretBlindIndex as string]
+    );
+    const hasUnknownSecretsProvided = secretKeysInDB.length !== inputSecrets.length;
+    if (hasUnknownSecretsProvided) {
+      const keysMissingInDB = Object.keys(keyName2BlindIndex).filter((key) => !secretKeysInDB.includes(key));
+      throw new BadRequestError({
+        message: `Secret not found: blind index ${keysMissingInDB.join(",")}`
+      });
+    }
+  }
+
+  return { blindIndex2KeyName, keyName2BlindIndex, secrets };
+};
+
+// these functions are special functions shared by a couple of resources
+// used by secret approval, rotation or anywhere in which secret needs to modified
+export const fnSecretBulkInsert = async ({
+  // TODO: Pick types here
+  folderId,
+  inputSecrets,
+  secretDAL,
+  secretVersionDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  tx
+}: TFnSecretBulkInsert) => {
+  const newSecrets = await secretDAL.insertMany(
+    inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })),
+    tx
+  );
+  const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string);
+  const newSecretTags = inputSecrets.flatMap(({ tags: secretTags = [], secretBlindIndex }) =>
+    secretTags.map((tag) => ({
+      [`${TableName.SecretTag}Id` as const]: tag,
+      [`${TableName.Secret}Id` as const]: newSecretGroupByBlindIndex[secretBlindIndex as string][0].id
+    }))
+  );
+  const secretVersions = await secretVersionDAL.insertMany(
+    inputSecrets.map(({ tags, ...el }) => ({
+      ...el,
+      folderId,
+      secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id
+    })),
+    tx
+  );
+  if (newSecretTags.length) {
+    const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
+    const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
+    const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
+      [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
+      [`${TableName.SecretTag}Id` as const]: secret_tagsId
+    }));
+    await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
+  }
+
+  return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
+};
+
+export const fnSecretBulkUpdate = async ({
+  tx,
+  inputSecrets,
+  folderId,
+  projectId,
+  secretDAL,
+  secretVersionDAL,
+  secretTagDAL,
+  secretVersionTagDAL
+}: TFnSecretBulkUpdate) => {
+  const newSecrets = await secretDAL.bulkUpdate(
+    inputSecrets.map(({ filter, data: { tags, ...data } }) => ({
+      filter: { ...filter, folderId },
+      data
+    })),
+    tx
+  );
+  const secretVersions = await secretVersionDAL.insertMany(
+    newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({
+      ...el,
+      secretId: id
+    })),
+    tx
+  );
+  const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
+    tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
+  );
+  if (secsUpdatedTag.length) {
+    await secretTagDAL.deleteTagsManySecret(
+      projectId,
+      secsUpdatedTag.map(({ secretId }) => secretId),
+      tx
+    );
+    const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) =>
+      secretTags.map((tag) => ({
+        [`${TableName.SecretTag}Id` as const]: tag,
+        [`${TableName.Secret}Id` as const]: secretId
+      }))
+    );
+    if (newSecretTags.length) {
+      const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
+      const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
+      const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
+        [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
+        [`${TableName.SecretTag}Id` as const]: secret_tagsId
+      }));
+      await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
+    }
+  }
+
+  return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
+};
+
+export const createManySecretsRawFnFactory = ({
+  projectDAL,
+  projectBotDAL,
+  secretDAL,
+  secretVersionDAL,
+  secretBlindIndexDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  folderDAL
+}: TCreateManySecretsRawFnFactory) => {
+  const getBotKeyFn = getBotKeyFnFactory(projectBotDAL);
+  const createManySecretsRawFn = async ({
+    projectId,
+    environment,
+    path: secretPath,
+    secrets,
+    userId
+  }: TCreateManySecretsRawFn) => {
+    const botKey = await getBotKeyFn(projectId);
+    if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
+
+    await projectDAL.checkProjectUpgradeStatus(projectId);
+
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
+    if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
+    const folderId = folder.id;
+
+    const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
+    if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Create secret" });
+
+    // insert operation
+    const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
+      inputSecrets: secrets,
+      folderId,
+      isNew: true,
+      blindIndexCfg,
+      secretDAL
+    });
+
+    const inputSecrets = await Promise.all(
+      secrets.map(async (secret) => {
+        const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
+        const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
+        const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
+
+        if (secret.type === SecretType.Personal) {
+          if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
+          const sharedExist = await secretDAL.findOne({
+            secretBlindIndex: keyName2BlindIndex[secret.secretName],
+            folderId,
+            type: SecretType.Shared
+          });
+
+          if (!sharedExist)
+            throw new BadRequestError({
+              message: "Failed to create personal secret override for no corresponding shared secret"
+            });
+        }
+
+        const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
+        if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
+
+        return {
+          type: secret.type,
+          userId: secret.type === SecretType.Personal ? userId : null,
+          secretName: secret.secretName,
+          secretKeyCiphertext: secretKeyEncrypted.ciphertext,
+          secretKeyIV: secretKeyEncrypted.iv,
+          secretKeyTag: secretKeyEncrypted.tag,
+          secretValueCiphertext: secretValueEncrypted.ciphertext,
+          secretValueIV: secretValueEncrypted.iv,
+          secretValueTag: secretValueEncrypted.tag,
+          secretCommentCiphertext: secretCommentEncrypted.ciphertext,
+          secretCommentIV: secretCommentEncrypted.iv,
+          secretCommentTag: secretCommentEncrypted.tag,
+          skipMultilineEncoding: secret.skipMultilineEncoding,
+          tags: secret.tags
+        };
+      })
+    );
+
+    const newSecrets = await secretDAL.transaction(async (tx) =>
+      fnSecretBulkInsert({
+        inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({
+          ...el,
+          version: 0,
+          secretBlindIndex: keyName2BlindIndex[secretName],
+          algorithm: SecretEncryptionAlgo.AES_256_GCM,
+          keyEncoding: SecretKeyEncoding.UTF8
+        })),
+        folderId,
+        secretDAL,
+        secretVersionDAL,
+        secretTagDAL,
+        secretVersionTagDAL,
+        tx
+      })
+    );
+
+    return newSecrets;
+  };
+
+  return createManySecretsRawFn;
+};
+
+export const updateManySecretsRawFnFactory = ({
+  projectDAL,
+  projectBotDAL,
+  secretDAL,
+  secretVersionDAL,
+  secretBlindIndexDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  folderDAL
+}: TUpdateManySecretsRawFnFactory) => {
+  const getBotKeyFn = getBotKeyFnFactory(projectBotDAL);
+  const updateManySecretsRawFn = async ({
+    projectId,
+    environment,
+    path: secretPath,
+    secrets, // consider accepting instead ciphertext secrets
+    userId
+  }: TUpdateManySecretsRawFn): Promise<Array<TSecrets & { _id: string }>> => {
+    const botKey = await getBotKeyFn(projectId);
+    if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
+
+    await projectDAL.checkProjectUpgradeStatus(projectId);
+
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
+    if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Update secret" });
+    const folderId = folder.id;
+
+    const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
+    if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
+
+    const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
+      inputSecrets: secrets,
+      folderId,
+      isNew: false,
+      blindIndexCfg,
+      secretDAL,
+      userId
+    });
+
+    const inputSecrets = await Promise.all(
+      secrets.map(async (secret) => {
+        if (secret.newSecretName === "") {
+          throw new BadRequestError({ message: "New secret name cannot be empty" });
+        }
+
+        const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
+        const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
+        const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
+
+        if (secret.type === SecretType.Personal) {
+          if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
+
+          const sharedExist = await secretDAL.findOne({
+            secretBlindIndex: keyName2BlindIndex[secret.secretName],
+            folderId,
+            type: SecretType.Shared
+          });
+
+          if (!sharedExist)
+            throw new BadRequestError({
+              message: "Failed to update personal secret override for no corresponding shared secret"
+            });
+
+          if (secret.newSecretName)
+            throw new BadRequestError({ message: "Personal secret cannot change the key name" });
+        }
+
+        const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
+        if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
+
+        return {
+          type: secret.type,
+          userId: secret.type === SecretType.Personal ? userId : null,
+          secretName: secret.secretName,
+          newSecretName: secret.newSecretName,
+          secretKeyCiphertext: secretKeyEncrypted.ciphertext,
+          secretKeyIV: secretKeyEncrypted.iv,
+          secretKeyTag: secretKeyEncrypted.tag,
+          secretValueCiphertext: secretValueEncrypted.ciphertext,
+          secretValueIV: secretValueEncrypted.iv,
+          secretValueTag: secretValueEncrypted.tag,
+          secretCommentCiphertext: secretCommentEncrypted.ciphertext,
+          secretCommentIV: secretCommentEncrypted.iv,
+          secretCommentTag: secretCommentEncrypted.tag,
+          skipMultilineEncoding: secret.skipMultilineEncoding,
+          tags: secret.tags
+        };
+      })
+    );
+
+    const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
+    const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
+    if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
+
+    // now find any secret that needs to update its name
+    // same process as above
+    const nameUpdatedSecrets = inputSecrets.filter(({ newSecretName }) => Boolean(newSecretName));
+    const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({
+      inputSecrets: nameUpdatedSecrets,
+      folderId,
+      isNew: true,
+      blindIndexCfg,
+      secretDAL
+    });
+
+    const updatedSecrets = await secretDAL.transaction(async (tx) =>
+      fnSecretBulkUpdate({
+        folderId,
+        projectId,
+        tx,
+        inputSecrets: inputSecrets.map(({ secretName, newSecretName, ...el }) => ({
+          filter: { secretBlindIndex: keyName2BlindIndex[secretName], type: SecretType.Shared },
+          data: {
+            ...el,
+            folderId,
+            secretBlindIndex:
+              newSecretName && newKeyName2BlindIndex[newSecretName]
+                ? newKeyName2BlindIndex[newSecretName]
+                : keyName2BlindIndex[secretName],
+            algorithm: SecretEncryptionAlgo.AES_256_GCM,
+            keyEncoding: SecretKeyEncoding.UTF8
+          }
+        })),
+        secretDAL,
+        secretVersionDAL,
+        secretTagDAL,
+        secretVersionTagDAL
+      })
+    );
+
+    return updatedSecrets;
+  };
+
+  return updateManySecretsRawFn;
+};

backend/src/services/secret/secret-queue.ts

@@ -6,6 +6,12 @@ import { BadRequestError } from "@app/lib/errors";
 import { isSamePath } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
 import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
+import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
+import { createManySecretsRawFnFactory, updateManySecretsRawFnFactory } from "@app/services/secret/secret-fns";
+import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
+import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
+import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
+import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
 
 import { TIntegrationDALFactory } from "../integration/integration-dal";
 import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service";
@@ -29,18 +35,23 @@ export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
 
 type TSecretQueueFactoryDep = {
   queueService: TQueueServiceFactory;
-  integrationDAL: Pick<TIntegrationDALFactory, "findByProjectIdV2">;
+  integrationDAL: Pick<TIntegrationDALFactory, "findByProjectIdV2" | "updateById">;
   projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   integrationAuthService: Pick<TIntegrationAuthServiceFactory, "getIntegrationAccessToken">;
-  folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findByManySecretPath">;
-  secretDAL: Pick<TSecretDALFactory, "findByFolderId" | "find">;
+  folderDAL: TSecretFolderDALFactory;
+  secretDAL: TSecretDALFactory;
   secretImportDAL: Pick<TSecretImportDALFactory, "find">;
   webhookDAL: Pick<TWebhookDALFactory, "findAllWebhooks" | "transaction" | "update" | "bulkUpdate">;
   projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
-  projectDAL: Pick<TProjectDALFactory, "findById">;
+  projectDAL: TProjectDALFactory;
+  projectBotDAL: TProjectBotDALFactory;
   projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
   smtpService: TSmtpService;
   orgDAL: Pick<TOrgDALFactory, "findOrgByProjectId">;
+  secretVersionDAL: TSecretVersionDALFactory;
+  secretBlindIndexDAL: TSecretBlindIndexDALFactory;
+  secretTagDAL: TSecretTagDALFactory;
+  secretVersionTagDAL: TSecretVersionTagDALFactory;
 };
 
 export type TGetSecrets = {
@@ -62,8 +73,35 @@ export const secretQueueFactory = ({
   orgDAL,
   smtpService,
   projectDAL,
-  projectMembershipDAL
+  projectBotDAL,
+  projectMembershipDAL,
+  secretVersionDAL,
+  secretBlindIndexDAL,
+  secretTagDAL,
+  secretVersionTagDAL
 }: TSecretQueueFactoryDep) => {
+  const createManySecretsRawFn = createManySecretsRawFnFactory({
+    projectDAL,
+    projectBotDAL,
+    secretDAL,
+    secretVersionDAL,
+    secretBlindIndexDAL,
+    secretTagDAL,
+    secretVersionTagDAL,
+    folderDAL
+  });
+
+  const updateManySecretsRawFn = updateManySecretsRawFnFactory({
+    projectDAL,
+    projectBotDAL,
+    secretDAL,
+    secretVersionDAL,
+    secretBlindIndexDAL,
+    secretTagDAL,
+    secretVersionTagDAL,
+    folderDAL
+  });
+
   const syncIntegrations = async (dto: TGetSecrets) => {
     await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
       attempts: 5,
@@ -307,6 +345,9 @@ export const secretQueueFactory = ({
       }
 
       await syncIntegrationSecrets({
+        createManySecretsRawFn,
+        updateManySecretsRawFn,
+        integrationDAL,
         integration,
         integrationAuth,
         secrets: Object.keys(suffixedSecrets).length !== 0 ? suffixedSecrets : secrets,

backend/src/services/secret/secret-service.ts

@@ -1,13 +1,13 @@
 import { ForbiddenError, subject } from "@casl/ability";
 
-import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType, TableName } from "@app/db/schemas";
+import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType } from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
 import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
 import { getConfig } from "@app/lib/config/env";
 import { buildSecretBlindIndexFromName, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
 import { BadRequestError } from "@app/lib/errors";
-import { groupBy, pick, unique } from "@app/lib/fn";
+import { groupBy, pick } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
 
 import { ActorType } from "../auth/auth-type";
@@ -19,7 +19,7 @@ import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
 import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
 import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
 import { TSecretDALFactory } from "./secret-dal";
-import { decryptSecretRaw, generateSecretBlindIndexBySalt } from "./secret-fns";
+import { decryptSecretRaw, fnSecretBlindIndexCheck, fnSecretBulkInsert, fnSecretBulkUpdate } from "./secret-fns";
 import { TSecretQueueFactory } from "./secret-queue";
 import {
   TCreateBulkSecretDTO,
@@ -28,11 +28,8 @@ import {
   TDeleteBulkSecretDTO,
   TDeleteSecretDTO,
   TDeleteSecretRawDTO,
-  TFnSecretBlindIndexCheck,
   TFnSecretBlindIndexCheckV2,
   TFnSecretBulkDelete,
-  TFnSecretBulkInsert,
-  TFnSecretBulkUpdate,
   TGetASecretDTO,
   TGetASecretRawDTO,
   TGetSecretsDTO,
@@ -95,85 +92,6 @@ export const secretServiceFactory = ({
     return secretBlindIndex;
   };
 
-  // these functions are special functions shared by a couple of resources
-  // used by secret approval, rotation or anywhere in which secret needs to modified
-  const fnSecretBulkInsert = async ({ folderId, inputSecrets, tx }: TFnSecretBulkInsert) => {
-    const newSecrets = await secretDAL.insertMany(
-      inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })),
-      tx
-    );
-    const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string);
-    const newSecretTags = inputSecrets.flatMap(({ tags: secretTags = [], secretBlindIndex }) =>
-      secretTags.map((tag) => ({
-        [`${TableName.SecretTag}Id` as const]: tag,
-        [`${TableName.Secret}Id` as const]: newSecretGroupByBlindIndex[secretBlindIndex as string][0].id
-      }))
-    );
-    const secretVersions = await secretVersionDAL.insertMany(
-      inputSecrets.map(({ tags, ...el }) => ({
-        ...el,
-        folderId,
-        secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id
-      })),
-      tx
-    );
-    if (newSecretTags.length) {
-      const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
-      const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
-      const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
-        [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
-        [`${TableName.SecretTag}Id` as const]: secret_tagsId
-      }));
-      await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
-    }
-
-    return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
-  };
-
-  const fnSecretBulkUpdate = async ({ tx, inputSecrets, folderId, projectId }: TFnSecretBulkUpdate) => {
-    const newSecrets = await secretDAL.bulkUpdate(
-      inputSecrets.map(({ filter, data: { tags, ...data } }) => ({
-        filter: { ...filter, folderId },
-        data
-      })),
-      tx
-    );
-    const secretVersions = await secretVersionDAL.insertMany(
-      newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({
-        ...el,
-        secretId: id
-      })),
-      tx
-    );
-    const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
-      tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
-    );
-    if (secsUpdatedTag.length) {
-      await secretTagDAL.deleteTagsManySecret(
-        projectId,
-        secsUpdatedTag.map(({ secretId }) => secretId),
-        tx
-      );
-      const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) =>
-        secretTags.map((tag) => ({
-          [`${TableName.SecretTag}Id` as const]: tag,
-          [`${TableName.Secret}Id` as const]: secretId
-        }))
-      );
-      if (newSecretTags.length) {
-        const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
-        const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
-        const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
-          [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
-          [`${TableName.SecretTag}Id` as const]: secret_tagsId
-        }));
-        await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
-      }
-    }
-
-    return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
-  };
-
   const fnSecretBulkDelete = async ({ folderId, inputSecrets, tx, actorId }: TFnSecretBulkDelete) => {
     const deletedSecrets = await secretDAL.deleteMany(
       inputSecrets.map(({ type, secretBlindIndex }) => ({
@@ -202,63 +120,6 @@ export const secretServiceFactory = ({
     return deletedSecrets;
   };
 
-  /**
-   * Checks and handles secrets using a blind index method.
-   * The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes.
-   * For new secrets (isNew = true), it ensures they don't already exist in the database.
-   * For existing secrets, it verifies their presence in the database.
-   * If discrepancies are found, errors are thrown. The function returns mappings and the fetched secrets.
-   */
-  const fnSecretBlindIndexCheck = async ({
-    inputSecrets,
-    folderId,
-    isNew,
-    userId,
-    blindIndexCfg
-  }: TFnSecretBlindIndexCheck) => {
-    const blindIndex2KeyName: Record<string, string> = {}; // used at audit log point
-    const keyName2BlindIndex = await Promise.all(
-      inputSecrets.map(({ secretName }) => generateSecretBlindIndexBySalt(secretName, blindIndexCfg))
-    ).then((blindIndexes) =>
-      blindIndexes.reduce<Record<string, string>>((prev, curr, i) => {
-        // eslint-disable-next-line
-        prev[inputSecrets[i].secretName] = curr;
-        blindIndex2KeyName[curr] = inputSecrets[i].secretName;
-        return prev;
-      }, {})
-    );
-
-    if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
-      throw new BadRequestError({ message: "Missing user id for personal secret" });
-    }
-
-    const secrets = await secretDAL.findByBlindIndexes(
-      folderId,
-      inputSecrets.map(({ secretName, type }) => ({
-        blindIndex: keyName2BlindIndex[secretName],
-        type: type || SecretType.Shared
-      })),
-      userId
-    );
-
-    if (isNew) {
-      if (secrets.length) throw new BadRequestError({ message: "Secret already exist" });
-    } else {
-      const secretKeysInDB = unique(secrets, (el) => el.secretBlindIndex as string).map(
-        (el) => blindIndex2KeyName[el.secretBlindIndex as string]
-      );
-      const hasUnknownSecretsProvided = secretKeysInDB.length !== inputSecrets.length;
-      if (hasUnknownSecretsProvided) {
-        const keysMissingInDB = Object.keys(keyName2BlindIndex).filter((key) => !secretKeysInDB.includes(key));
-        throw new BadRequestError({
-          message: `Secret not found: blind index ${keysMissingInDB.join(",")}`
-        });
-      }
-    }
-
-    return { blindIndex2KeyName, keyName2BlindIndex, secrets };
-  };
-
   // this is used when secret blind index already exist
   // mainly for secret approval
   const fnSecretBlindIndexCheckV2 = async ({ inputSecrets, folderId, userId }: TFnSecretBlindIndexCheckV2) => {
@@ -311,7 +172,8 @@ export const secretServiceFactory = ({
       folderId,
       isNew: true,
       userId: actorId,
-      blindIndexCfg
+      blindIndexCfg,
+      secretDAL
     });
 
     // if user creating personal check its shared also exist
@@ -348,6 +210,10 @@ export const secretServiceFactory = ({
             tags: inputSecret.tags
           }
         ],
+        secretDAL,
+        secretVersionDAL,
+        secretTagDAL,
+        secretVersionTagDAL,
         tx
       })
     );
@@ -395,7 +261,8 @@ export const secretServiceFactory = ({
       folderId,
       isNew: false,
       blindIndexCfg,
-      userId: actorId
+      userId: actorId,
+      secretDAL
     });
     if (inputSecret.newSecretName && inputSecret.type === SecretType.Personal) {
       throw new BadRequestError({ message: "Personal secret cannot change the key name" });
@@ -407,7 +274,8 @@ export const secretServiceFactory = ({
         inputSecrets: [{ secretName: inputSecret.newSecretName }],
         folderId,
         isNew: true,
-        blindIndexCfg
+        blindIndexCfg,
+        secretDAL
       });
       newSecretNameBlindIndex = kN2NewBlindIndex[inputSecret.newSecretName];
     }
@@ -454,6 +322,10 @@ export const secretServiceFactory = ({
             }
           }
         ],
+        secretDAL,
+        secretVersionDAL,
+        secretTagDAL,
+        secretVersionTagDAL,
         tx
       })
     );
@@ -496,7 +368,8 @@ export const secretServiceFactory = ({
       inputSecrets: [{ secretName: inputSecret.secretName }],
       folderId,
       isNew: false,
-      blindIndexCfg
+      blindIndexCfg,
+      secretDAL
     });
 
     const deletedSecret = await secretDAL.transaction(async (tx) =>
@@ -679,13 +552,14 @@ export const secretServiceFactory = ({
     const folderId = folder.id;
 
     const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
-    if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
+    if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Create secret" });
 
     const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
       inputSecrets,
       folderId,
       isNew: true,
-      blindIndexCfg
+      blindIndexCfg,
+      secretDAL
     });
 
     // get all tags
@@ -704,6 +578,10 @@ export const secretServiceFactory = ({
           keyEncoding: SecretKeyEncoding.UTF8
         })),
         folderId,
+        secretDAL,
+        secretVersionDAL,
+        secretTagDAL,
+        secretVersionTagDAL,
         tx
       })
     );
@@ -732,7 +610,7 @@ export const secretServiceFactory = ({
     await projectDAL.checkProjectUpgradeStatus(projectId);
 
     const folder = await folderDAL.findBySecretPath(projectId, environment, path);
-    if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
+    if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Update secret" });
     const folderId = folder.id;
 
     const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
@@ -742,7 +620,8 @@ export const secretServiceFactory = ({
       inputSecrets,
       folderId,
       isNew: false,
-      blindIndexCfg
+      blindIndexCfg,
+      secretDAL
     });
 
     // now find any secret that needs to update its name
@@ -752,7 +631,8 @@ export const secretServiceFactory = ({
       inputSecrets: nameUpdatedSecrets,
       folderId,
       isNew: true,
-      blindIndexCfg
+      blindIndexCfg,
+      secretDAL
     });
 
     // get all tags
@@ -777,7 +657,11 @@ export const secretServiceFactory = ({
             algorithm: SecretEncryptionAlgo.AES_256_GCM,
             keyEncoding: SecretKeyEncoding.UTF8
           }
-        }))
+        })),
+        secretDAL,
+        secretVersionDAL,
+        secretTagDAL,
+        secretVersionTagDAL
       })
     );
 
@@ -815,7 +699,8 @@ export const secretServiceFactory = ({
       inputSecrets,
       folderId,
       isNew: false,
-      blindIndexCfg
+      blindIndexCfg,
+      secretDAL
     });
 
     const secretsDeleted = await secretDAL.transaction(async (tx) =>

backend/src/services/secret/secret-types.ts

@@ -2,6 +2,14 @@ import { Knex } from "knex";
 
 import { SecretType, TSecretBlindIndexes, TSecrets, TSecretsInsert, TSecretsUpdate } from "@app/db/schemas";
 import { TProjectPermission } from "@app/lib/types";
+import { TProjectDALFactory } from "@app/services/project/project-dal";
+import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
+import { TSecretDALFactory } from "@app/services/secret/secret-dal";
+import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
+import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
+import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
+import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
+import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
 
 type TPartialSecret = Pick<TSecrets, "id" | "secretReminderRepeatDays" | "secretReminderNote">;
 
@@ -181,12 +189,20 @@ export type TFnSecretBulkInsert = {
   folderId: string;
   tx?: Knex;
   inputSecrets: Array<Omit<TSecretsInsert, "folderId"> & { tags?: string[] }>;
+  secretDAL: Pick<TSecretDALFactory, "insertMany">;
+  secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret">;
+  secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
 };
 
 export type TFnSecretBulkUpdate = {
   folderId: string;
   projectId: string;
   inputSecrets: { filter: Partial<TSecrets>; data: TSecretsUpdate & { tags?: string[] } }[];
+  secretDAL: Pick<TSecretDALFactory, "bulkUpdate">;
+  secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret" | "deleteTagsManySecret">;
+  secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
   tx?: Knex;
 };
 
@@ -204,6 +220,7 @@ export type TFnSecretBlindIndexCheck = {
   blindIndexCfg: TSecretBlindIndexes;
   inputSecrets: Array<{ secretName: string; type?: SecretType }>;
   isNew: boolean;
+  secretDAL: Pick<TSecretDALFactory, "findByBlindIndexes">;
 };
 
 // when blind index is already present
@@ -229,3 +246,66 @@ export type TRemoveSecretReminderDTO = {
   secretId: string;
   repeatDays: number;
 };
+
+// ---
+
+export type TCreateManySecretsRawFnFactory = {
+  projectDAL: TProjectDALFactory;
+  projectBotDAL: TProjectBotDALFactory;
+  secretDAL: TSecretDALFactory;
+  secretVersionDAL: TSecretVersionDALFactory;
+  secretBlindIndexDAL: TSecretBlindIndexDALFactory;
+  secretTagDAL: TSecretTagDALFactory;
+  secretVersionTagDAL: TSecretVersionTagDALFactory;
+  folderDAL: TSecretFolderDALFactory;
+};
+
+export type TCreateManySecretsRawFn = {
+  projectId: string;
+  environment: string;
+  path: string;
+  secrets: {
+    secretName: string;
+    secretValue: string;
+    type: SecretType;
+    secretComment?: string;
+    skipMultilineEncoding?: boolean;
+    tags?: string[];
+    metadata?: {
+      source?: string;
+    };
+  }[];
+  userId?: string; // only relevant for personal secret(s)
+};
+
+export type TUpdateManySecretsRawFnFactory = {
+  projectDAL: TProjectDALFactory;
+  projectBotDAL: TProjectBotDALFactory;
+  secretDAL: TSecretDALFactory;
+  secretVersionDAL: TSecretVersionDALFactory;
+  secretBlindIndexDAL: TSecretBlindIndexDALFactory;
+  secretTagDAL: TSecretTagDALFactory;
+  secretVersionTagDAL: TSecretVersionTagDALFactory;
+  folderDAL: TSecretFolderDALFactory;
+};
+
+export type TUpdateManySecretsRawFn = {
+  projectId: string;
+  environment: string;
+  path: string;
+  secrets: {
+    secretName: string;
+    newSecretName?: string;
+    secretValue: string;
+    type: SecretType;
+    secretComment?: string;
+    skipMultilineEncoding?: boolean;
+    secretReminderRepeatDays?: number | null;
+    secretReminderNote?: string | null;
+    tags?: string[];
+    metadata?: {
+      source?: string;
+    };
+  }[];
+  userId?: string;
+};