awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4603 from Infisical/fix/pitMigrationBatch

Browse Source 
Fix PIT migration batch issue with folders overflowing the batch size
This commit is contained in:
carlosmonastyrski
2025-10-02 22:57:11 -03:00
committed by GitHub
parent e9e9fd1a25 15fe26c7e7
commit 98278aca02
1 changed files with 33 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
backend/src/db/migrations/20250602155452_pit-projects-commits-initialization.ts
View File

@@ -226,7 +226,7 @@ export async function up(knex: Knex): Promise<void> {
// Insert New Commits in batches of 9000
const newCommits = foldersCommitsList.map((folderCommit) => folderCommit.commit);
const commitBatches = chunkArray(newCommits, 9000);
let pendingDeepTreeCommitResources: TFolderCommits[] = [];
let j = 0;
for (const commitBatch of commitBatches) {
j += 1;
@@ -265,12 +265,14 @@ export async function up(knex: Knex): Promise<void> {
});
// Create folder commit changes
const currentBatchFolderIds = new Set(newCommitsInserted.map((commit) => commit.folderId));
// eslint-disable-next-line no-await-in-loop
await knex.batchInsert(
TableName.FolderCommitChanges,
foldersCommitsList
.map((folderCommit) => folderCommit.changes)
.flat()
.filter((change) => currentBatchFolderIds.has(change.folderId))
.map((change) => ({
folderCommitId: newCommitsMap[change.folderId],
changeType: change.changeType,
@@ -289,6 +291,7 @@ export async function up(knex: Knex): Promise<void> {
foldersCommitsList
.map((folderCommit) => folderCommit.changes)
.flat()
.filter((change) => currentBatchFolderIds.has(change.folderId))
.map((change) => ({
folderCheckpointId: newCheckpointsMap[change.folderId],
folderVersionId: change.folderVersionId,
@@ -303,9 +306,11 @@ export async function up(knex: Knex): Promise<void> {
const newTreeCheckpoints = (await knex
.batchInsert(
TableName.FolderTreeCheckpoint,
Object.keys(rootFoldersMap).map((folderId) => ({
folderCommitId: newCommitsMap[folderId]
}))
Object.keys(rootFoldersMap)
.filter((folderId) => currentBatchFolderIds.has(folderId))
.map((folderId) => ({
folderCommitId: newCommitsMap[folderId]
}))
)
.returning("*")) as TFolderTreeCheckpoints[];
@@ -317,18 +322,31 @@ export async function up(knex: Knex): Promise<void> {
});
// Create Folder Tree Checkpoint Resources
// eslint-disable-next-line no-await-in-loop
await knex
.batchInsert(
TableName.FolderTreeCheckpointResources,
newCommitsInserted.map((folderCommit) => ({
folderTreeCheckpointId: newTreeCheckpointsMap[folderCommit.envId],
folderId: folderCommit.folderId,
folderCommitId: folderCommit.id
}))
)
.returning("*");
const commitsToProcess = pendingDeepTreeCommitResources.concat(newCommitsInserted);
const unprocessableCommits: TFolderCommits[] = [];
const processableCommits = commitsToProcess.filter((folderCommit) => {
const isProcessable = newTreeCheckpointsMap[folderCommit.envId];
if (!isProcessable) {
unprocessableCommits.push(folderCommit);
}
return isProcessable;
});
if (processableCommits.length > 0) {
// eslint-disable-next-line no-await-in-loop
await knex
.batchInsert(
TableName.FolderTreeCheckpointResources,
processableCommits.map((folderCommit) => ({
folderTreeCheckpointId: newTreeCheckpointsMap[folderCommit.envId],
folderId: folderCommit.folderId,
folderCommitId: folderCommit.id
}))
)
.returning("*");
}
pendingDeepTreeCommitResources = unprocessableCommits;
logger.info(`Finished inserting folder tree checkpoint resources - batch ${j} of ${commitBatches.length}`);
}
}