From b8e9417466cf47b52b518312639e398b4e97f7f8 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Sat, 1 Jun 2024 03:26:35 +0800 Subject: [PATCH] misc: modified pruning sql logic --- .../services/secret-snapshot/snapshot-dal.ts | 54 ++++++++----------- .../src/services/secret/secret-version-dal.ts | 48 ++++++++--------- 2 files changed, 44 insertions(+), 58 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index a35345c17..44fcadf39 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -325,9 +325,9 @@ export const snapshotDALFactory = (db: TDbClient) => { } }; - const pruneExcessSnapshots = async (tx?: Knex) => { + const pruneExcessSnapshots = async () => { try { - const folders = await (tx || db)(TableName.SecretFolder).select("id"); + const folders = await db(TableName.SecretFolder).select("id"); const folderIds = folders.map((folder) => folder.id); const PRUNE_FOLDER_BATCH_SIZE = 500; @@ -338,35 +338,27 @@ export const snapshotDALFactory = (db: TDbClient) => { } for await (const folderBatch of pruneBatches) { - const rankedSnapshots = (tx || db)(TableName.Snapshot) - .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) - .select( - "folderId", - "id", - (tx || db).raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` - ) - ) - .as("ranked_snapshots"); - - const folderLimits = (tx || db)(TableName.Snapshot) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.Snapshot}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) - .groupBy(`${TableName.Snapshot}.folderId`, `${TableName.Project}.pitVersionLimit`) - .select("folderId", "pitVersionLimit") - .as("folder_limits"); - - const snapshotsToKeep = (tx || db) - .select("id") - .from(rankedSnapshots) - .join(folderLimits, "folder_limits.folderId", "ranked_snapshots.folderId") - .whereRaw(`ranked_snapshots.row_num <= folder_limits."pitVersionLimit"`); - - await (tx || db)(TableName.Snapshot) - .whereIn("folderId", folderBatch) - .whereNotIn("id", snapshotsToKeep) - .delete(); + await secretSnapshotOrm.transaction(async (txn) => { + return txn(TableName.Snapshot) + .with("snapshot_cte", (qb) => { + void qb + .from(TableName.Snapshot) + .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) + .select( + "folderId", + `${TableName.Snapshot}.id as id`, + txn.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Snapshot}.folderId`) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`) + .whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); + }); } } catch (error) { throw new DatabaseError({ error, name: "SnapshotPrune" }); diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 374907c3d..596d0f483 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -110,34 +110,28 @@ export const secretVersionDALFactory = (db: TDbClient) => { } }; - const pruneExcessVersions = async (tx?: Knex) => { + const pruneExcessVersions = async () => { try { - const rankedSecretVersions = (tx || db)(TableName.SecretVersion) - .select( - "id", - "secretId", - "folderId", - (tx || db).raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num` - ) - ) - .as("ranked_secret_versions"); - - const folderLimits = (tx || db)(TableName.SecretVersion) - .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .groupBy(`${TableName.SecretVersion}.folderId`, `${TableName.Project}.pitVersionLimit`) - .select("folderId", "pitVersionLimit") - .as("folder_limits"); - - const versionsToKeep = (tx || db)(rankedSecretVersions) - .select("id") - .from(rankedSecretVersions) - .join(folderLimits, "folder_limits.folderId", "ranked_secret_versions.folderId") - .whereRaw(`ranked_secret_versions.row_num <= folder_limits."pitVersionLimit"`); - - await (tx || db)(TableName.SecretVersion).whereNotIn("id", versionsToKeep).delete(); + await secretVersionOrm.transaction((txn) => { + return txn(TableName.SecretVersion) + .with("version_cte", (qb) => { + void qb + .from(TableName.SecretVersion) + .select( + "id", + "folderId", + txn.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("version_cte", "version_cte.id", `${TableName.SecretVersion}.id`) + .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); + }); } catch (error) { throw new DatabaseError({ error,