MVP preliminary idea for initial sync behavior

backend/src/services/secret/secret-dal.ts

@@ -22,7 +22,11 @@ export const secretDALFactory = (db: TDbClient) => {
 
   // the idea is to use postgres specific function
   // insert with id this will cause a conflict then merge the data
-  const bulkUpdate = async (data: Array<{ filter: Partial<TSecrets>; data: TSecretsUpdate }>, tx?: Knex) => {
+  const bulkUpdate = async (
+    data: Array<{ filter: Partial<TSecrets>; data: TSecretsUpdate }>,
+
+    tx?: Knex
+  ) => {
     try {
       const secs = await Promise.all(
         data.map(async ({ filter, data: updateData }) => {
@@ -41,6 +45,35 @@ export const secretDALFactory = (db: TDbClient) => {
     }
   };
 
+  const bulkUpdateNoVersionIncrement = async (data: TSecrets[], tx?: Knex) => {
+    try {
+      const existingSecrets = await secretOrm.find(
+        {
+          $in: {
+            id: data.map((el) => el.id)
+          }
+        },
+        { tx }
+      );
+
+      if (existingSecrets.length !== data.length) {
+        throw new BadRequestError({ message: "Some of the secrets do not exist" });
+      }
+
+      if (data.length === 0) return [];
+
+      const updatedSecrets = await (tx || db)(TableName.Secret)
+        .insert(data)
+        .onConflict("id") // this will cause a conflict then merge the data
+        .merge() // Merge the data with the existing data
+        .returning("*");
+
+      return updatedSecrets;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "bulk update secret" });
+    }
+  };
+
   const deleteMany = async (
     data: Array<{ blindIndex: string; type: SecretType }>,
     folderId: string,
@@ -57,6 +90,12 @@ export const secretDALFactory = (db: TDbClient) => {
               type: el.type,
               ...(el.type === SecretType.Personal ? { userId } : {})
             });
+            if (el.type === SecretType.Shared) {
+              void bd.orWhere({
+                secretBlindIndex: el.blindIndex,
+                type: SecretType.Personal
+              });
+            }
           });
         })
         .delete()
@@ -139,5 +178,13 @@ export const secretDALFactory = (db: TDbClient) => {
     }
   };
 
-  return { ...secretOrm, update, bulkUpdate, deleteMany, findByFolderId, findByBlindIndexes };
+  return {
+    ...secretOrm,
+    update,
+    bulkUpdate,
+    deleteMany,
+    bulkUpdateNoVersionIncrement,
+    findByFolderId,
+    findByBlindIndexes
+  };
 };

backend/src/services/secret/secret-fns.ts

@@ -1,12 +1,31 @@
 /* eslint-disable no-await-in-loop */
 import path from "path";
 
-import { SecretKeyEncoding, TSecretBlindIndexes, TSecrets } from "@app/db/schemas";
+import { 
+  SecretKeyEncoding, 
+  TSecretBlindIndexes, 
+  TSecrets,
+  SecretEncryptionAlgo,
+  SecretType,
+  TableName
+} from "@app/db/schemas";
 import { getConfig } from "@app/lib/config/env";
-import { buildSecretBlindIndexFromName, decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
+import { 
+  buildSecretBlindIndexFromName, 
+  encryptSymmetric128BitHexKeyUTF8,
+  decryptSymmetric128BitHexKeyUTF8 
+} from "@app/lib/crypto";
 
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretDALFactory } from "./secret-dal";
+import { 
+  TCreateManySecretsRawHelper,
+  TFnSecretBulkInsert,
+  TFnSecretBlindIndexCheck
+} from "./secret-types";
+
+import { groupBy, unique } from "@app/lib/fn";
+import { BadRequestError } from "@app/lib/errors";
 
 export const generateSecretBlindIndexBySalt = async (secretName: string, secretBlindIndexDoc: TSecretBlindIndexes) => {
   const appCfg = getConfig();
@@ -228,3 +247,201 @@ export const decryptSecretRaw = (secret: TSecrets & { workspace: string; environ
     user: secret.userId
   };
 };
+
+/**
+ * Checks and handles secrets using a blind index method.
+ * The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes.
+ * For new secrets (isNew = true), it ensures they don't already exist in the database.
+ * For existing secrets, it verifies their presence in the database.
+ * If discrepancies are found, errors are thrown. The function returns mappings and the fetched secrets.
+ */
+export const fnSecretBlindIndexCheck = async ({
+  inputSecrets,
+  folderId,
+  isNew,
+  userId,
+  blindIndexCfg,
+  secretDAL
+}: TFnSecretBlindIndexCheck) => {
+  const blindIndex2KeyName: Record<string, string> = {}; // used at audit log point
+  const keyName2BlindIndex = await Promise.all(
+    inputSecrets.map(({ secretName }) => generateSecretBlindIndexBySalt(secretName, blindIndexCfg))
+  ).then((blindIndexes) =>
+    blindIndexes.reduce<Record<string, string>>((prev, curr, i) => {
+      // eslint-disable-next-line
+      prev[inputSecrets[i].secretName] = curr;
+      blindIndex2KeyName[curr] = inputSecrets[i].secretName;
+      return prev;
+    }, {})
+  );
+
+  if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
+    throw new BadRequestError({ message: "Missing user id for personal secret" });
+  }
+
+  const secrets = await secretDAL.findByBlindIndexes(
+    folderId,
+    inputSecrets.map(({ secretName, type }) => ({
+      blindIndex: keyName2BlindIndex[secretName],
+      type: type || SecretType.Shared
+    })),
+    userId
+  );
+
+  if (isNew) {
+    if (secrets.length) throw new BadRequestError({ message: "Secret already exist" });
+  } else {
+    const secretKeysInDB = unique(secrets, (el) => el.secretBlindIndex as string).map(
+      (el) => blindIndex2KeyName[el.secretBlindIndex as string]
+    );
+    const hasUnknownSecretsProvided = secretKeysInDB.length !== inputSecrets.length;
+    if (hasUnknownSecretsProvided) {
+      const keysMissingInDB = Object.keys(keyName2BlindIndex).filter((key) => !secretKeysInDB.includes(key));
+      throw new BadRequestError({
+        message: `Secret not found: blind index ${keysMissingInDB.join(",")}`
+      });
+    }
+  }
+
+  return { blindIndex2KeyName, keyName2BlindIndex, secrets };
+};
+
+// these functions are special functions shared by a couple of resources
+// used by secret approval, rotation or anywhere in which secret needs to modified
+export const fnSecretBulkInsert = async ({ // TODO: Pick types here 
+  folderId, 
+  inputSecrets, 
+  secretDAL,
+  secretVersionDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  tx 
+}: TFnSecretBulkInsert) => {
+  const newSecrets = await secretDAL.insertMany(
+    inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })),
+    tx
+  );
+  const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string);
+  const newSecretTags = inputSecrets.flatMap(({ tags: secretTags = [], secretBlindIndex }) =>
+    secretTags.map((tag) => ({
+      [`${TableName.SecretTag}Id` as const]: tag,
+      [`${TableName.Secret}Id` as const]: newSecretGroupByBlindIndex[secretBlindIndex as string][0].id
+    }))
+  );
+  const secretVersions = await secretVersionDAL.insertMany(
+    inputSecrets.map(({ tags, ...el }) => ({
+      ...el,
+      folderId,
+      secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id
+    })),
+    tx
+  );
+  if (newSecretTags.length) {
+    const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
+    const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
+    const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
+      [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
+      [`${TableName.SecretTag}Id` as const]: secret_tagsId
+    }));
+    await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
+  }
+
+  return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
+};
+
+export const createManySecretsRawHelper = async ({ // TODO: place on top
+  projectId,
+  environment,
+  path,
+  secrets,
+  userId,
+  botKey, // TODO: consider getting botKey inside this fn
+  projectDAL,
+  secretDAL,
+  secretVersionDAL,
+  secretBlindIndexDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  folderDAL
+}: TCreateManySecretsRawHelper) => {
+await projectDAL.checkProjectUpgradeStatus(projectId);
+
+const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
+const folderId = folder.id;
+
+const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
+if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
+
+const inputSecrets = await Promise.all(
+  secrets.map(async (secret) => {
+    const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
+    const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
+    const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
+
+    if (secret.type === SecretType.Personal) {
+      const sharedExist = await secretDAL.findOne({
+        secretBlindIndex: keyName2BlindIndex[secret.secretName],
+        folderId,
+        type: SecretType.Shared
+      });
+
+      if (!sharedExist)
+        throw new BadRequestError({
+          message: "Failed to create personal secret override for no corresponding shared secret"
+        });
+      
+      if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
+    }
+
+    const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
+    if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
+
+    return ({
+      type: secret.type,
+      userId: secret.type === SecretType.Personal ? userId : null,
+      secretName: secret.secretName,
+      secretKeyCiphertext: secretKeyEncrypted.ciphertext,
+      secretKeyIV: secretKeyEncrypted.iv,
+      secretKeyTag: secretKeyEncrypted.tag,
+      secretValueCiphertext: secretValueEncrypted.ciphertext,
+      secretValueIV: secretValueEncrypted.iv,
+      secretValueTag: secretValueEncrypted.tag,
+      secretCommentCiphertext: secretCommentEncrypted.ciphertext,
+      secretCommentIV: secretCommentEncrypted.iv,
+      secretCommentTag: secretCommentEncrypted.tag,
+      skipMultilineEncoding: secret.skipMultilineEncoding,
+      tags: secret.tags
+    });
+  })
+);
+
+  // insert operation
+  const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
+    inputSecrets,
+    folderId,
+    isNew: true,
+    blindIndexCfg,
+    secretDAL
+  });
+
+  const newSecrets = await secretDAL.transaction(async (tx) =>
+    fnSecretBulkInsert({
+      inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({
+        ...el,
+        version: 0,
+        secretBlindIndex: keyName2BlindIndex[secretName],
+        algorithm: SecretEncryptionAlgo.AES_256_GCM,
+        keyEncoding: SecretKeyEncoding.UTF8
+      })),
+      folderId,
+      secretDAL,
+      secretVersionDAL,
+      secretTagDAL,
+      secretVersionTagDAL,
+      tx
+    })
+  );
+
+  return newSecrets;
+}

backend/src/services/secret/secret-queue.ts

@@ -25,6 +25,11 @@ import { TSecretDALFactory } from "./secret-dal";
 import { interpolateSecrets } from "./secret-fns";
 import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types";
 
+import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
+import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
+import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
+import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
+
 export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
 
 type TSecretQueueFactoryDep = {
@@ -32,15 +37,19 @@ type TSecretQueueFactoryDep = {
   integrationDAL: Pick<TIntegrationDALFactory, "findByProjectIdV2">;
   projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   integrationAuthService: Pick<TIntegrationAuthServiceFactory, "getIntegrationAccessToken">;
-  folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findByManySecretPath">;
-  secretDAL: Pick<TSecretDALFactory, "findByFolderId" | "find">;
+  folderDAL: TSecretFolderDALFactory;
+  secretDAL: TSecretDALFactory;
   secretImportDAL: Pick<TSecretImportDALFactory, "find">;
   webhookDAL: Pick<TWebhookDALFactory, "findAllWebhooks" | "transaction" | "update" | "bulkUpdate">;
   projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
-  projectDAL: Pick<TProjectDALFactory, "findById">;
+  projectDAL: TProjectDALFactory;
   projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
   smtpService: TSmtpService;
   orgDAL: Pick<TOrgDALFactory, "findOrgByProjectId">;
+  secretVersionDAL: TSecretVersionDALFactory;
+  secretBlindIndexDAL: TSecretBlindIndexDALFactory;
+  secretTagDAL: TSecretTagDALFactory;
+  secretVersionTagDAL: TSecretVersionTagDALFactory;
 };
 
 export type TGetSecrets = {
@@ -62,7 +71,11 @@ export const secretQueueFactory = ({
   orgDAL,
   smtpService,
   projectDAL,
-  projectMembershipDAL
+  projectMembershipDAL,
+  secretVersionDAL,
+  secretBlindIndexDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
 }: TSecretQueueFactoryDep) => {
   const syncIntegrations = async (dto: TGetSecrets) => {
     await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
@@ -307,6 +320,17 @@ export const secretQueueFactory = ({
       }
 
       await syncIntegrationSecrets({
+        projectDAL,
+        secretDAL,
+        secretVersionDAL,
+        secretBlindIndexDAL,
+        secretTagDAL,
+        secretVersionTagDAL,
+        folderDAL,
+        botKey,
+        projectId,
+        environment,
+        secretPath,
         integration,
         integrationAuth,
         secrets: Object.keys(suffixedSecrets).length !== 0 ? suffixedSecrets : secrets,

backend/src/services/secret/secret-service.ts

@@ -11,6 +11,7 @@ import { groupBy, pick } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
 
 import { ActorType } from "../auth/auth-type";
+import { TProjectDALFactory } from "../project/project-dal";
 import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
 import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
@@ -18,19 +19,22 @@ import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
 import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
 import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
 import { TSecretDALFactory } from "./secret-dal";
-import { decryptSecretRaw, generateSecretBlindIndexBySalt } from "./secret-fns";
+import { 
+  decryptSecretRaw, 
+  fnSecretBlindIndexCheck,
+  fnSecretBulkInsert
+} from "./secret-fns";
 import { TSecretQueueFactory } from "./secret-queue";
 import {
+  TCreateManySecretsRawHelper,
   TCreateBulkSecretDTO,
   TCreateSecretDTO,
   TCreateSecretRawDTO,
   TDeleteBulkSecretDTO,
   TDeleteSecretDTO,
   TDeleteSecretRawDTO,
-  TFnSecretBlindIndexCheck,
   TFnSecretBlindIndexCheckV2,
   TFnSecretBulkDelete,
-  TFnSecretBulkInsert,
   TFnSecretBulkUpdate,
   TGetASecretDTO,
   TGetASecretRawDTO,
@@ -49,6 +53,7 @@ type TSecretServiceFactoryDep = {
   secretTagDAL: TSecretTagDALFactory;
   secretVersionDAL: TSecretVersionDALFactory;
   folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "updateById" | "findById" | "findByManySecretPath">;
+  projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
   secretBlindIndexDAL: TSecretBlindIndexDALFactory;
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
   snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
@@ -68,6 +73,7 @@ export const secretServiceFactory = ({
   permissionService,
   snapshotService,
   secretQueueService,
+  projectDAL,
   projectBotService,
   secretImportDAL,
   secretVersionTagDAL
@@ -92,41 +98,6 @@ export const secretServiceFactory = ({
     return secretBlindIndex;
   };
 
-  // these functions are special functions shared by a couple of resources
-  // used by secret approval, rotation or anywhere in which secret needs to modified
-  const fnSecretBulkInsert = async ({ folderId, inputSecrets, tx }: TFnSecretBulkInsert) => {
-    const newSecrets = await secretDAL.insertMany(
-      inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })),
-      tx
-    );
-    const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string);
-    const newSecretTags = inputSecrets.flatMap(({ tags: secretTags = [], secretBlindIndex }) =>
-      secretTags.map((tag) => ({
-        [`${TableName.SecretTag}Id` as const]: tag,
-        [`${TableName.Secret}Id` as const]: newSecretGroupByBlindIndex[secretBlindIndex as string][0].id
-      }))
-    );
-    const secretVersions = await secretVersionDAL.insertMany(
-      inputSecrets.map(({ tags, ...el }) => ({
-        ...el,
-        folderId,
-        secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id
-      })),
-      tx
-    );
-    if (newSecretTags.length) {
-      const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
-      const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
-      const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
-        [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
-        [`${TableName.SecretTag}Id` as const]: secret_tagsId
-      }));
-      await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
-    }
-
-    return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
-  };
-
   const fnSecretBulkUpdate = async ({ tx, inputSecrets, folderId, projectId }: TFnSecretBulkUpdate) => {
     const newSecrets = await secretDAL.bulkUpdate(
       inputSecrets.map(({ filter, data: { tags, ...data } }) => ({
@@ -199,54 +170,6 @@ export const secretServiceFactory = ({
     return deletedSecrets;
   };
 
-  // this is a utility function for secret modification
-  // this will check given secret name blind index exist or not
-  // if its a created secret set isNew to true
-  // thus if these blindindex exist it will throw an error
-  // vice versa when u need to check for updated secret
-  // this will also return the blind index grouped by secretName
-  const fnSecretBlindIndexCheck = async ({
-    inputSecrets,
-    folderId,
-    isNew,
-    userId,
-    blindIndexCfg
-  }: TFnSecretBlindIndexCheck) => {
-    const blindIndex2KeyName: Record<string, string> = {}; // used at audit log point
-    const keyName2BlindIndex = await Promise.all(
-      inputSecrets.map(({ secretName }) => generateSecretBlindIndexBySalt(secretName, blindIndexCfg))
-    ).then((blindIndexes) =>
-      blindIndexes.reduce<Record<string, string>>((prev, curr, i) => {
-        // eslint-disable-next-line
-        prev[inputSecrets[i].secretName] = curr;
-        blindIndex2KeyName[curr] = inputSecrets[i].secretName;
-        return prev;
-      }, {})
-    );
-
-    if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
-      throw new BadRequestError({ message: "Missing user id for personal secret" });
-    }
-
-    const secrets = await secretDAL.findByBlindIndexes(
-      folderId,
-      inputSecrets.map(({ secretName, type }) => ({
-        blindIndex: keyName2BlindIndex[secretName],
-        type: type || SecretType.Shared
-      })),
-      userId
-    );
-
-    if (isNew) {
-      if (secrets.length) throw new BadRequestError({ message: "Secret already exist" });
-    } else if (secrets.length !== inputSecrets.length)
-      throw new BadRequestError({
-        message: `Secret not found: blind index ${JSON.stringify(keyName2BlindIndex)}`
-      });
-
-    return { blindIndex2KeyName, keyName2BlindIndex, secrets };
-  };
-
   // this is used when secret blind index already exist
   // mainly for secret approval
   const fnSecretBlindIndexCheckV2 = async ({ inputSecrets, folderId, userId }: TFnSecretBlindIndexCheckV2) => {
@@ -281,6 +204,8 @@ export const secretServiceFactory = ({
       subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
     );
 
+    await projectDAL.checkProjectUpgradeStatus(projectId);
+
     const folder = await folderDAL.findBySecretPath(projectId, environment, path);
     if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
     const folderId = folder.id;
@@ -297,7 +222,8 @@ export const secretServiceFactory = ({
       folderId,
       isNew: true,
       userId: actorId,
-      blindIndexCfg
+      blindIndexCfg,
+      secretDAL
     });
 
     // if user creating personal check its shared also exist
@@ -334,6 +260,10 @@ export const secretServiceFactory = ({
             tags: inputSecret.tags
           }
         ],
+        secretDAL,
+        secretVersionDAL,
+        secretTagDAL,
+        secretVersionTagDAL,
         tx
       })
     );
@@ -359,6 +289,12 @@ export const secretServiceFactory = ({
       subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
     );
 
+    await projectDAL.checkProjectUpgradeStatus(projectId);
+
+    if (inputSecret.newSecretName === "") {
+      throw new BadRequestError({ message: "New secret name cannot be empty" });
+    }
+
     const folder = await folderDAL.findBySecretPath(projectId, environment, path);
     if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
     const folderId = folder.id;
@@ -375,7 +311,8 @@ export const secretServiceFactory = ({
       folderId,
       isNew: false,
       blindIndexCfg,
-      userId: actorId
+      userId: actorId,
+      secretDAL
     });
     if (inputSecret.newSecretName && inputSecret.type === SecretType.Personal) {
       throw new BadRequestError({ message: "Personal secret cannot change the key name" });
@@ -387,7 +324,8 @@ export const secretServiceFactory = ({
         inputSecrets: [{ secretName: inputSecret.newSecretName }],
         folderId,
         isNew: true,
-        blindIndexCfg
+        blindIndexCfg,
+        secretDAL
       });
       newSecretNameBlindIndex = kN2NewBlindIndex[inputSecret.newSecretName];
     }
@@ -459,6 +397,8 @@ export const secretServiceFactory = ({
       subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
     );
 
+    await projectDAL.checkProjectUpgradeStatus(projectId);
+
     const folder = await folderDAL.findBySecretPath(projectId, environment, path);
     if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
     const folderId = folder.id;
@@ -474,7 +414,8 @@ export const secretServiceFactory = ({
       inputSecrets: [{ secretName: inputSecret.secretName }],
       folderId,
       isNew: false,
-      blindIndexCfg
+      blindIndexCfg,
+      secretDAL
     });
 
     const deletedSecret = await secretDAL.transaction(async (tx) =>
@@ -650,6 +591,8 @@ export const secretServiceFactory = ({
       subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
     );
 
+    await projectDAL.checkProjectUpgradeStatus(projectId);
+
     const folder = await folderDAL.findBySecretPath(projectId, environment, path);
     if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
     const folderId = folder.id;
@@ -661,7 +604,8 @@ export const secretServiceFactory = ({
       inputSecrets,
       folderId,
       isNew: true,
-      blindIndexCfg
+      blindIndexCfg,
+      secretDAL
     });
 
     // get all tags
@@ -680,6 +624,10 @@ export const secretServiceFactory = ({
           keyEncoding: SecretKeyEncoding.UTF8
         })),
         folderId,
+        secretDAL,
+        secretVersionDAL,
+        secretTagDAL,
+        secretVersionTagDAL,
         tx
       })
     );
@@ -705,6 +653,8 @@ export const secretServiceFactory = ({
       subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
     );
 
+    await projectDAL.checkProjectUpgradeStatus(projectId);
+
     const folder = await folderDAL.findBySecretPath(projectId, environment, path);
     if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
     const folderId = folder.id;
@@ -716,7 +666,8 @@ export const secretServiceFactory = ({
       inputSecrets,
       folderId,
       isNew: false,
-      blindIndexCfg
+      blindIndexCfg,
+      secretDAL
     });
 
     // now find any secret that needs to update its name
@@ -726,7 +677,8 @@ export const secretServiceFactory = ({
       inputSecrets: nameUpdatedSecrets,
       folderId,
       isNew: true,
-      blindIndexCfg
+      blindIndexCfg,
+      secretDAL
     });
 
     // get all tags
@@ -776,6 +728,8 @@ export const secretServiceFactory = ({
       subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
     );
 
+    await projectDAL.checkProjectUpgradeStatus(projectId);
+
     const folder = await folderDAL.findBySecretPath(projectId, environment, path);
     if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
     const folderId = folder.id;
@@ -787,7 +741,8 @@ export const secretServiceFactory = ({
       inputSecrets,
       folderId,
       isNew: false,
-      blindIndexCfg
+      blindIndexCfg,
+      secretDAL
     });
 
     const secretsDeleted = await secretDAL.transaction(async (tx) =>
@@ -871,6 +826,98 @@ export const secretServiceFactory = ({
     });
     return decryptSecretRaw(secret, botKey);
   };
+  
+  const createManySecretsRawHelper = async ({ // TODO: place on top
+    projectId,
+    environment,
+    path,
+    secrets,
+    userId
+  }: TCreateManySecretsRawHelper) => {
+    const botKey = await projectBotService.getBotKey(projectId);
+    if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
+
+    await projectDAL.checkProjectUpgradeStatus(projectId);
+
+    const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+    if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
+    const folderId = folder.id;
+
+    const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
+    if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
+
+    const inputSecrets = await Promise.all(
+      secrets.map(async (secret) => {
+        const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
+        const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
+        const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
+
+        if (secret.type === SecretType.Personal) {
+          const sharedExist = await secretDAL.findOne({
+            secretBlindIndex: keyName2BlindIndex[secret.secretName],
+            folderId,
+            type: SecretType.Shared
+          });
+
+          if (!sharedExist)
+            throw new BadRequestError({
+              message: "Failed to create personal secret override for no corresponding shared secret"
+            });
+          
+          if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
+        }
+
+        const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
+        if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
+
+        return ({
+          type: secret.type,
+          userId: secret.type === SecretType.Personal ? userId : null,
+          secretName: secret.secretName,
+          secretKeyCiphertext: secretKeyEncrypted.ciphertext,
+          secretKeyIV: secretKeyEncrypted.iv,
+          secretKeyTag: secretKeyEncrypted.tag,
+          secretValueCiphertext: secretValueEncrypted.ciphertext,
+          secretValueIV: secretValueEncrypted.iv,
+          secretValueTag: secretValueEncrypted.tag,
+          secretCommentCiphertext: secretCommentEncrypted.ciphertext,
+          secretCommentIV: secretCommentEncrypted.iv,
+          secretCommentTag: secretCommentEncrypted.tag,
+          skipMultilineEncoding: secret.skipMultilineEncoding,
+          tags: secret.tags
+        });
+      })
+    );
+
+    // insert operation
+    const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
+      inputSecrets,
+      folderId,
+      isNew: true,
+      blindIndexCfg,
+      secretDAL
+    });
+
+    const newSecrets = await secretDAL.transaction(async (tx) =>
+      fnSecretBulkInsert({
+        inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({
+          ...el,
+          version: 0,
+          secretBlindIndex: keyName2BlindIndex[secretName],
+          algorithm: SecretEncryptionAlgo.AES_256_GCM,
+          keyEncoding: SecretKeyEncoding.UTF8
+        })),
+        folderId,
+        secretDAL,
+        secretVersionDAL,
+        secretTagDAL,
+        secretVersionTagDAL,
+        tx
+      })
+    );
+
+    return newSecrets;
+  }
 
   const createSecretRaw = async ({
     secretName,

backend/src/services/secret/secret-types.ts

@@ -2,6 +2,13 @@ import { Knex } from "knex";
 
 import { SecretType, TSecretBlindIndexes, TSecrets, TSecretsInsert, TSecretsUpdate } from "@app/db/schemas";
 import { TProjectPermission } from "@app/lib/types";
+import { TProjectDALFactory } from "@app/services/project/project-dal";
+import { TSecretDALFactory } from "@app/services/secret/secret-dal";
+import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
+import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
+import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
+import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
+import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
 
 type TPartialSecret = Pick<TSecrets, "id" | "secretReminderRepeatDays" | "secretReminderNote">;
 
@@ -181,6 +188,10 @@ export type TFnSecretBulkInsert = {
   folderId: string;
   tx?: Knex;
   inputSecrets: Array<Omit<TSecretsInsert, "folderId"> & { tags?: string[] }>;
+  secretDAL: Pick<TSecretDALFactory, "insertMany">;
+  secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret">;
+  secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
 };
 
 export type TFnSecretBulkUpdate = {
@@ -204,6 +215,7 @@ export type TFnSecretBlindIndexCheck = {
   blindIndexCfg: TSecretBlindIndexes;
   inputSecrets: Array<{ secretName: string; type?: SecretType }>;
   isNew: boolean;
+  secretDAL: Pick<TSecretDALFactory, "findByBlindIndexes">;
 };
 
 // when blind index is already present
@@ -229,3 +241,31 @@ export type TRemoveSecretReminderDTO = {
   secretId: string;
   repeatDays: number;
 };
+
+// ---
+
+export type TCreateManySecretsRawHelper = {
+  projectId: string;
+  environment: string;
+  path: string;
+  secrets: {
+    secretName: string;
+    secretValue: string;
+    type: SecretType;
+    secretComment?: string;
+    skipMultilineEncoding?: boolean;
+    tags?: string[];
+    metadata?: {
+      source?: string;
+    }
+  }[];
+  userId?: string; // only relevant for personal secret(s)
+  botKey: string;
+  projectDAL: TProjectDALFactory;
+  secretDAL: TSecretDALFactory;
+  secretVersionDAL: TSecretVersionDALFactory;
+  secretBlindIndexDAL: TSecretBlindIndexDALFactory;
+  secretTagDAL: TSecretTagDALFactory;
+  secretVersionTagDAL: TSecretVersionTagDALFactory;
+  folderDAL: TSecretFolderDALFactory;
+}

backend/src/services/secret/secret-version-dal.ts

@@ -1,8 +1,8 @@
 import { Knex } from "knex";
 
 import { TDbClient } from "@app/db";
-import { TableName, TSecretVersions } from "@app/db/schemas";
-import { DatabaseError } from "@app/lib/errors";
+import { TableName, TSecretVersions, TSecretVersionsUpdate } from "@app/db/schemas";
+import { BadRequestError, DatabaseError } from "@app/lib/errors";
 import { ormify, selectAllTableCols } from "@app/lib/knex";
 
 export type TSecretVersionDALFactory = ReturnType<typeof secretVersionDALFactory>;
@@ -36,6 +36,57 @@ export const secretVersionDALFactory = (db: TDbClient) => {
     }
   };
 
+  const bulkUpdate = async (
+    data: Array<{ filter: Partial<TSecretVersions>; data: TSecretVersionsUpdate }>,
+    tx?: Knex
+  ) => {
+    try {
+      const secs = await Promise.all(
+        data.map(async ({ filter, data: updateData }) => {
+          const [doc] = await (tx || db)(TableName.SecretVersion)
+            .where(filter)
+            .update(updateData)
+            .increment("version", 1) // TODO: Is this really needed?
+            .returning("*");
+          if (!doc) throw new BadRequestError({ message: "Failed to update document" });
+          return doc;
+        })
+      );
+      return secs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "bulk update secret" });
+    }
+  };
+
+  const bulkUpdateNoVersionIncrement = async (data: TSecretVersions[], tx?: Knex) => {
+    try {
+      const existingSecretVersions = await secretVersionOrm.find(
+        {
+          $in: {
+            id: data.map((el) => el.id)
+          }
+        },
+        { tx }
+      );
+
+      if (existingSecretVersions.length !== data.length) {
+        throw new BadRequestError({ message: "Some of the secret versions do not exist" });
+      }
+
+      if (data.length === 0) return [];
+
+      const updatedSecretVersions = await (tx || db)(TableName.SecretVersion)
+        .insert(data)
+        .onConflict("id") // this will cause a conflict then merge the data
+        .merge() // Merge the data with the existing data
+        .returning("*");
+
+      return updatedSecretVersions;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "bulk update secret" });
+    }
+  };
+
   const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => {
     try {
       const docs: Array<TSecretVersions & { max: number }> = await (tx || db)(TableName.SecretVersion)
@@ -59,5 +110,11 @@ export const secretVersionDALFactory = (db: TDbClient) => {
     }
   };
 
-  return { ...secretVersionOrm, findLatestVersionMany, findLatestVersionByFolderId };
+  return {
+    ...secretVersionOrm,
+    findLatestVersionMany,
+    bulkUpdate,
+    findLatestVersionByFolderId,
+    bulkUpdateNoVersionIncrement
+  };
 };