awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: added a new batch insert operation to convert inserts into chunks and updated secret migration

Browse Source
This commit is contained in:
=
2024-08-09 21:02:26 +05:30
parent ea14df2cbd
commit 32a84471f2
4 changed files with 19 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/src/lib/knex/index.ts
View File

@@ -128,6 +128,16 @@ export const ormify = <DbOps extends object, Tname extends keyof Tables>(db: Kne
throw new DatabaseError({ error, name: "Create" }); throw new DatabaseError({ error, name: "Create" });
} }
}, },
// This spilit the insert into multiple chunk
batchInsert: async (data: readonly Tables[Tname]["insert"][], tx?: Knex) => {
try {
if (!data.length) return [];
const res = await (tx || db).batchInsert(tableName, data as never).returning("*");
return res as Tables[Tname]["base"][];
} catch (error) {
throw new DatabaseError({ error, name: "batchInsert" });
}
},
upsert: async (data: readonly Tables[Tname]["insert"][], onConflictField: keyof Tables[Tname]["base"], tx?: Knex) => { upsert: async (data: readonly Tables[Tname]["insert"][], onConflictField: keyof Tables[Tname]["base"], tx?: Knex) => {
try { try {
if (!data.length) return []; if (!data.length) return [];

2
backend/src/services/secret-tag/secret-tag-dal.ts
View File

@@ -51,7 +51,7 @@ export const secretTagDALFactory = (db: TDbClient) => {
...secretTagOrm, ...secretTagOrm,
saveTagsToSecret: secretJnTagOrm.insertMany, saveTagsToSecret: secretJnTagOrm.insertMany,
deleteTagsToSecret: secretJnTagOrm.delete, deleteTagsToSecret: secretJnTagOrm.delete,
saveTagsToSecretV2: secretV2JnTagOrm.insertMany, saveTagsToSecretV2: secretV2JnTagOrm.batchInsert,
deleteTagsToSecretV2: secretV2JnTagOrm.delete, deleteTagsToSecretV2: secretV2JnTagOrm.delete,
findSecretTagsByProjectId, findSecretTagsByProjectId,
deleteTagsManySecret, deleteTagsManySecret,

2
backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts
View File

@@ -287,7 +287,7 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
})) }))
); );
if (!newSecretReferences.length) return; if (!newSecretReferences.length) return;
const secretReferences = await (tx || db)(TableName.SecretReferenceV2).insert(newSecretReferences); const secretReferences = await (tx || db).batchInsert(TableName.SecretReferenceV2, newSecretReferences);
return secretReferences; return secretReferences;
} catch (error) { } catch (error) {
throw new DatabaseError({ error, name: "UpsertSecretReference" }); throw new DatabaseError({ error, name: "UpsertSecretReference" });

14
backend/src/services/secret/secret-queue.ts
View File

@@ -73,12 +73,12 @@ type TSecretQueueFactoryDep = {
secretVersionTagDAL: TSecretVersionTagDALFactory; secretVersionTagDAL: TSecretVersionTagDALFactory;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">; kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
secretV2BridgeDAL: TSecretV2BridgeDALFactory; secretV2BridgeDAL: TSecretV2BridgeDALFactory;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">; secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "batchInsert" | "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">; secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "batchInsert">;
secretRotationDAL: Pick<TSecretRotationDALFactory, "secretOutputV2InsertMany" | "find">; secretRotationDAL: Pick<TSecretRotationDALFactory, "secretOutputV2InsertMany" | "find">;
secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "deleteByProjectId">; secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "deleteByProjectId">;
snapshotDAL: Pick<TSnapshotDALFactory, "findNSecretV1SnapshotByFolderId" | "deleteSnapshotsAboveLimit">; snapshotDAL: Pick<TSnapshotDALFactory, "findNSecretV1SnapshotByFolderId" | "deleteSnapshotsAboveLimit">;
snapshotSecretV2BridgeDAL: Pick<TSnapshotSecretV2DALFactory, "insertMany">; snapshotSecretV2BridgeDAL: Pick<TSnapshotSecretV2DALFactory, "insertMany" | "batchInsert">;
}; };
export type TGetSecrets = { export type TGetSecrets = {
@@ -828,7 +828,7 @@ export const secretQueueFactory = ({
secretId: string; secretId: string;
references: { environment: string; secretPath: string; secretKey: string }[]; references: { environment: string; secretPath: string; secretKey: string }[];
}[] = []; }[] = [];
await secretV2BridgeDAL.insertMany( await secretV2BridgeDAL.batchInsert(
projectV1Secrets.map((el) => { projectV1Secrets.map((el) => {
const key = decryptSymmetric128BitHexKeyUTF8({ const key = decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretKeyCiphertext, ciphertext: el.secretKeyCiphertext,
@@ -1004,14 +1004,14 @@ export const secretQueueFactory = ({
const projectV3SecretVersions = Object.values(projectV3SecretVersionsGroupById); const projectV3SecretVersions = Object.values(projectV3SecretVersionsGroupById);
if (projectV3SecretVersions.length) { if (projectV3SecretVersions.length) {
await secretVersionV2BridgeDAL.insertMany(projectV3SecretVersions, tx); await secretVersionV2BridgeDAL.batchInsert(projectV3SecretVersions, tx);
} }
if (projectV3SecretVersionTags.length) { if (projectV3SecretVersionTags.length) {
await secretVersionTagV2BridgeDAL.insertMany(projectV3SecretVersionTags, tx); await secretVersionTagV2BridgeDAL.batchInsert(projectV3SecretVersionTags, tx);
} }
if (projectV3SnapshotSecrets.length) { if (projectV3SnapshotSecrets.length) {
await snapshotSecretV2BridgeDAL.insertMany(projectV3SnapshotSecrets, tx); await snapshotSecretV2BridgeDAL.batchInsert(projectV3SnapshotSecrets, tx);
} }
await snapshotDAL.deleteSnapshotsAboveLimit(folderId, SNAPSHOT_BATCH_SIZE, tx); await snapshotDAL.deleteSnapshotsAboveLimit(folderId, SNAPSHOT_BATCH_SIZE, tx);
} }