awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-03 10:26:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

misc: migrated to using keyset pagnination

Browse Source
This commit is contained in:
Sheen Capadngan
2024-06-03 17:56:00 +08:00
parent ab093dfc85
commit cd6caab508
1 changed files with 29 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/ee/services/secret-snapshot/snapshot-dal.ts
+29 -17
View File
@@ -1,3 +1,4 @@
/* eslint-disable no-await-in-loop */
import { Knex } from "knex"; import { Knex } from "knex";
import { TDbClient } from "@app/db"; import { TDbClient } from "@app/db";
@@ -11,6 +12,7 @@ import {
} from "@app/db/schemas"; } from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors"; import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
export type TSnapshotDALFactory = ReturnType<typeof snapshotDALFactory>; export type TSnapshotDALFactory = ReturnType<typeof snapshotDALFactory>;
@@ -326,28 +328,32 @@ export const snapshotDALFactory = (db: TDbClient) => {
}; };
const pruneExcessSnapshots = async () => { const pruneExcessSnapshots = async () => {
try { const PRUNE_FOLDER_BATCH_SIZE = 10000;
const folders = await db(TableName.SecretFolder).select("id"); let uuidOffset = "00000000-0000-0000-0000-000000000000";
const folderIds = folders.map((folder) => folder.id);
const PRUNE_FOLDER_BATCH_SIZE = 500;
const pruneBatches = []; // eslint-disable-next-line no-constant-condition, no-unreachable-loop
for (let x = 0; x < folderIds.length; x += PRUNE_FOLDER_BATCH_SIZE) { while (true) {
const batch = folderIds.slice(x, x + PRUNE_FOLDER_BATCH_SIZE); const folderBatch = await db(TableName.SecretFolder)
pruneBatches.push(batch); .where("id", ">", uuidOffset)
} .orderBy("id", "asc")
.limit(PRUNE_FOLDER_BATCH_SIZE)
.select("id");
for await (const folderBatch of pruneBatches) { const batchEntries = folderBatch.map((folder) => folder.id);
await secretSnapshotOrm.transaction(async (txn) => { logger.info("UUID offset:", uuidOffset);
return txn(TableName.Snapshot)
if (folderBatch.length) {
try {
logger.info(`Pruning snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}`);
await db(TableName.Snapshot)
.with("snapshot_cte", (qb) => { .with("snapshot_cte", (qb) => {
void qb void qb
.from(TableName.Snapshot) .from(TableName.Snapshot)
.whereIn(`${TableName.Snapshot}.folderId`, folderBatch) .whereIn(`${TableName.Snapshot}.folderId`, batchEntries)
.select( .select(
"folderId", "folderId",
`${TableName.Snapshot}.id as id`, `${TableName.Snapshot}.id as id`,
txn.raw( db.raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num`
) )
); );
@@ -358,10 +364,16 @@ export const snapshotDALFactory = (db: TDbClient) => {
.join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`) .join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`)
.whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`) .whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
.delete(); .delete();
}); } catch (err) {
logger.error(
`Failed to prune snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}`
);
} finally {
uuidOffset = batchEntries[batchEntries.length - 1];
}
} else {
return;
} }
} catch (error) {
throw new DatabaseError({ error, name: "SnapshotPrune" });
} }
}; };