From 32a84471f2ccfe7a630d0ea425bc30f5da80a63e Mon Sep 17 00:00:00 2001
From: = <akhilmhdh@gmail.com>
Date: Fri, 9 Aug 2024 21:02:26 +0530
Subject: [PATCH] feat: added a new batch insert operation to convert inserts
 into chunks and updated secret migration

---
 backend/src/lib/knex/index.ts                      | 10 ++++++++++
 backend/src/services/secret-tag/secret-tag-dal.ts  |  2 +-
 .../secret-v2-bridge/secret-v2-bridge-dal.ts       |  2 +-
 backend/src/services/secret/secret-queue.ts        | 14 +++++++-------
 4 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/backend/src/lib/knex/index.ts b/backend/src/lib/knex/index.ts
index dcab16218..cbf5db3b0 100644
--- a/backend/src/lib/knex/index.ts
+++ b/backend/src/lib/knex/index.ts
@@ -128,6 +128,16 @@ export const ormify = <DbOps extends object, Tname extends keyof Tables>(db: Kne
       throw new DatabaseError({ error, name: "Create" });
     }
   },
+  // This spilit the insert into multiple chunk
+  batchInsert: async (data: readonly Tables[Tname]["insert"][], tx?: Knex) => {
+    try {
+      if (!data.length) return [];
+      const res = await (tx || db).batchInsert(tableName, data as never).returning("*");
+      return res as Tables[Tname]["base"][];
+    } catch (error) {
+      throw new DatabaseError({ error, name: "batchInsert" });
+    }
+  },
   upsert: async (data: readonly Tables[Tname]["insert"][], onConflictField: keyof Tables[Tname]["base"], tx?: Knex) => {
     try {
       if (!data.length) return [];
diff --git a/backend/src/services/secret-tag/secret-tag-dal.ts b/backend/src/services/secret-tag/secret-tag-dal.ts
index d1f149f35..1df64afa2 100644
--- a/backend/src/services/secret-tag/secret-tag-dal.ts
+++ b/backend/src/services/secret-tag/secret-tag-dal.ts
@@ -51,7 +51,7 @@ export const secretTagDALFactory = (db: TDbClient) => {
     ...secretTagOrm,
     saveTagsToSecret: secretJnTagOrm.insertMany,
     deleteTagsToSecret: secretJnTagOrm.delete,
-    saveTagsToSecretV2: secretV2JnTagOrm.insertMany,
+    saveTagsToSecretV2: secretV2JnTagOrm.batchInsert,
     deleteTagsToSecretV2: secretV2JnTagOrm.delete,
     findSecretTagsByProjectId,
     deleteTagsManySecret,
diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts
index 53ad29b17..fe4d89bac 100644
--- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts
+++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts
@@ -287,7 +287,7 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
           }))
         );
       if (!newSecretReferences.length) return;
-      const secretReferences = await (tx || db)(TableName.SecretReferenceV2).insert(newSecretReferences);
+      const secretReferences = await (tx || db).batchInsert(TableName.SecretReferenceV2, newSecretReferences);
       return secretReferences;
     } catch (error) {
       throw new DatabaseError({ error, name: "UpsertSecretReference" });
diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts
index 28053f4b3..de437a6a5 100644
--- a/backend/src/services/secret/secret-queue.ts
+++ b/backend/src/services/secret/secret-queue.ts
@@ -73,12 +73,12 @@ type TSecretQueueFactoryDep = {
   secretVersionTagDAL: TSecretVersionTagDALFactory;
   kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
   secretV2BridgeDAL: TSecretV2BridgeDALFactory;
-  secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
-  secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
+  secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "batchInsert" | "insertMany" | "findLatestVersionMany">;
+  secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "batchInsert">;
   secretRotationDAL: Pick<TSecretRotationDALFactory, "secretOutputV2InsertMany" | "find">;
   secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "deleteByProjectId">;
   snapshotDAL: Pick<TSnapshotDALFactory, "findNSecretV1SnapshotByFolderId" | "deleteSnapshotsAboveLimit">;
-  snapshotSecretV2BridgeDAL: Pick<TSnapshotSecretV2DALFactory, "insertMany">;
+  snapshotSecretV2BridgeDAL: Pick<TSnapshotSecretV2DALFactory, "insertMany" | "batchInsert">;
 };
 
 export type TGetSecrets = {
@@ -828,7 +828,7 @@ export const secretQueueFactory = ({
             secretId: string;
             references: { environment: string; secretPath: string; secretKey: string }[];
           }[] = [];
-          await secretV2BridgeDAL.insertMany(
+          await secretV2BridgeDAL.batchInsert(
             projectV1Secrets.map((el) => {
               const key = decryptSymmetric128BitHexKeyUTF8({
                 ciphertext: el.secretKeyCiphertext,
@@ -1004,14 +1004,14 @@ export const secretQueueFactory = ({
 
         const projectV3SecretVersions = Object.values(projectV3SecretVersionsGroupById);
         if (projectV3SecretVersions.length) {
-          await secretVersionV2BridgeDAL.insertMany(projectV3SecretVersions, tx);
+          await secretVersionV2BridgeDAL.batchInsert(projectV3SecretVersions, tx);
         }
         if (projectV3SecretVersionTags.length) {
-          await secretVersionTagV2BridgeDAL.insertMany(projectV3SecretVersionTags, tx);
+          await secretVersionTagV2BridgeDAL.batchInsert(projectV3SecretVersionTags, tx);
         }
 
         if (projectV3SnapshotSecrets.length) {
-          await snapshotSecretV2BridgeDAL.insertMany(projectV3SnapshotSecrets, tx);
+          await snapshotSecretV2BridgeDAL.batchInsert(projectV3SnapshotSecrets, tx);
         }
         await snapshotDAL.deleteSnapshotsAboveLimit(folderId, SNAPSHOT_BATCH_SIZE, tx);
       }