awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

misc: modified pruning sql logic

Browse Source
This commit is contained in:
Sheen Capadngan
2024-06-01 03:26:35 +08:00
parent 4eb08c64d4
commit b8e9417466
2 changed files with 44 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
backend/src/ee/services/secret-snapshot/snapshot-dal.ts
View File

@@ -325,9 +325,9 @@ export const snapshotDALFactory = (db: TDbClient) => {
}
};
const pruneExcessSnapshots = async (tx?: Knex) => {
const pruneExcessSnapshots = async () => {
try {
const folders = await (tx || db)(TableName.SecretFolder).select("id");
const folders = await db(TableName.SecretFolder).select("id");
const folderIds = folders.map((folder) => folder.id);
const PRUNE_FOLDER_BATCH_SIZE = 500;
@@ -338,35 +338,27 @@ export const snapshotDALFactory = (db: TDbClient) => {
}
for await (const folderBatch of pruneBatches) {
const rankedSnapshots = (tx || db)(TableName.Snapshot)
.whereIn(`${TableName.Snapshot}.folderId`, folderBatch)
.select(
"folderId",
"id",
(tx || db).raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num`
)
)
.as("ranked_snapshots");
const folderLimits = (tx || db)(TableName.Snapshot)
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.Snapshot}.envId`)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.whereIn(`${TableName.Snapshot}.folderId`, folderBatch)
.groupBy(`${TableName.Snapshot}.folderId`, `${TableName.Project}.pitVersionLimit`)
.select("folderId", "pitVersionLimit")
.as("folder_limits");
const snapshotsToKeep = (tx || db)
.select("id")
.from(rankedSnapshots)
.join(folderLimits, "folder_limits.folderId", "ranked_snapshots.folderId")
.whereRaw(`ranked_snapshots.row_num <= folder_limits."pitVersionLimit"`);
await (tx || db)(TableName.Snapshot)
.whereIn("folderId", folderBatch)
.whereNotIn("id", snapshotsToKeep)
.delete();
await secretSnapshotOrm.transaction(async (txn) => {
return txn(TableName.Snapshot)
.with("snapshot_cte", (qb) => {
void qb
.from(TableName.Snapshot)
.whereIn(`${TableName.Snapshot}.folderId`, folderBatch)
.select(
"folderId",
`${TableName.Snapshot}.id as id`,
txn.raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num`
)
);
})
.join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Snapshot}.folderId`)
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`)
.whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
.delete();
});
}
} catch (error) {
throw new DatabaseError({ error, name: "SnapshotPrune" });

48
backend/src/services/secret/secret-version-dal.ts
View File

@@ -110,34 +110,28 @@ export const secretVersionDALFactory = (db: TDbClient) => {
}
};
const pruneExcessVersions = async (tx?: Knex) => {
const pruneExcessVersions = async () => {
try {
const rankedSecretVersions = (tx || db)(TableName.SecretVersion)
.select(
"id",
"secretId",
"folderId",
(tx || db).raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num`
)
)
.as("ranked_secret_versions");
const folderLimits = (tx || db)(TableName.SecretVersion)
.join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`)
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.groupBy(`${TableName.SecretVersion}.folderId`, `${TableName.Project}.pitVersionLimit`)
.select("folderId", "pitVersionLimit")
.as("folder_limits");
const versionsToKeep = (tx || db)(rankedSecretVersions)
.select("id")
.from(rankedSecretVersions)
.join(folderLimits, "folder_limits.folderId", "ranked_secret_versions.folderId")
.whereRaw(`ranked_secret_versions.row_num <= folder_limits."pitVersionLimit"`);
await (tx || db)(TableName.SecretVersion).whereNotIn("id", versionsToKeep).delete();
await secretVersionOrm.transaction((txn) => {
return txn(TableName.SecretVersion)
.with("version_cte", (qb) => {
void qb
.from(TableName.SecretVersion)
.select(
"id",
"folderId",
txn.raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num`
)
);
})
.join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`)
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.join("version_cte", "version_cte.id", `${TableName.SecretVersion}.id`)
.whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
.delete();
});
} catch (error) {
throw new DatabaseError({
error,