awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-06 13:27:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

misc: modified pruning sql logic

Browse Source
This commit is contained in:
Sheen Capadngan
2024-06-01 03:26:35 +08:00
parent 4eb08c64d4
commit b8e9417466
2 changed files with 44 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/ee/services/secret-snapshot/snapshot-dal.ts
+23 -31
View File
@@ -325,9 +325,9 @@ export const snapshotDALFactory = (db: TDbClient) => {
} }
}; };
const pruneExcessSnapshots = async (tx?: Knex) => { const pruneExcessSnapshots = async () => {
try { try {
const folders = await (tx || db)(TableName.SecretFolder).select("id"); const folders = await db(TableName.SecretFolder).select("id");
const folderIds = folders.map((folder) => folder.id); const folderIds = folders.map((folder) => folder.id);
const PRUNE_FOLDER_BATCH_SIZE = 500; const PRUNE_FOLDER_BATCH_SIZE = 500;
@@ -338,35 +338,27 @@ export const snapshotDALFactory = (db: TDbClient) => {
} }
for await (const folderBatch of pruneBatches) { for await (const folderBatch of pruneBatches) {
const rankedSnapshots = (tx || db)(TableName.Snapshot) await secretSnapshotOrm.transaction(async (txn) => {
.whereIn(`${TableName.Snapshot}.folderId`, folderBatch) return txn(TableName.Snapshot)
.select( .with("snapshot_cte", (qb) => {
"folderId", void qb
"id", .from(TableName.Snapshot)
(tx || db).raw( .whereIn(`${TableName.Snapshot}.folderId`, folderBatch)
`ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` .select(
) "folderId",
) `${TableName.Snapshot}.id as id`,
.as("ranked_snapshots"); txn.raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num`
const folderLimits = (tx || db)(TableName.Snapshot) )
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.Snapshot}.envId`) );
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) })
.whereIn(`${TableName.Snapshot}.folderId`, folderBatch) .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Snapshot}.folderId`)
.groupBy(`${TableName.Snapshot}.folderId`, `${TableName.Project}.pitVersionLimit`) .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
.select("folderId", "pitVersionLimit") .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.as("folder_limits"); .join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`)
.whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
const snapshotsToKeep = (tx || db) .delete();
.select("id") });
.from(rankedSnapshots)
.join(folderLimits, "folder_limits.folderId", "ranked_snapshots.folderId")
.whereRaw(`ranked_snapshots.row_num <= folder_limits."pitVersionLimit"`);
await (tx || db)(TableName.Snapshot)
.whereIn("folderId", folderBatch)
.whereNotIn("id", snapshotsToKeep)
.delete();
} }
} catch (error) { } catch (error) {
throw new DatabaseError({ error, name: "SnapshotPrune" }); throw new DatabaseError({ error, name: "SnapshotPrune" });
backend/src/services/secret/secret-version-dal.ts
+21 -27
View File
@@ -110,34 +110,28 @@ export const secretVersionDALFactory = (db: TDbClient) => {
} }
}; };
const pruneExcessVersions = async (tx?: Knex) => { const pruneExcessVersions = async () => {
try { try {
const rankedSecretVersions = (tx || db)(TableName.SecretVersion) await secretVersionOrm.transaction((txn) => {
.select( return txn(TableName.SecretVersion)
"id", .with("version_cte", (qb) => {
"secretId", void qb
"folderId", .from(TableName.SecretVersion)
(tx || db).raw( .select(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num` "id",
) "folderId",
) txn.raw(
.as("ranked_secret_versions"); `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num`
)
const folderLimits = (tx || db)(TableName.SecretVersion) );
.join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`) })
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
.groupBy(`${TableName.SecretVersion}.folderId`, `${TableName.Project}.pitVersionLimit`) .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.select("folderId", "pitVersionLimit") .join("version_cte", "version_cte.id", `${TableName.SecretVersion}.id`)
.as("folder_limits"); .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
.delete();
const versionsToKeep = (tx || db)(rankedSecretVersions) });
.select("id")
.from(rankedSecretVersions)
.join(folderLimits, "folder_limits.folderId", "ranked_secret_versions.folderId")
.whereRaw(`ranked_secret_versions.row_num <= folder_limits."pitVersionLimit"`);
await (tx || db)(TableName.SecretVersion).whereNotIn("id", versionsToKeep).delete();
} catch (error) { } catch (error) {
throw new DatabaseError({ throw new DatabaseError({
error, error,