From 30f3dac35ff780671a1987d13497845623230ae7 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Wed, 12 Jun 2024 15:56:47 -0400 Subject: [PATCH] rephrase input and filer for resvered folder --- .../services/secret-snapshot/snapshot-dal.ts | 21 ++++++++++++++++++- .../PointInTimeVersionLimitSection.tsx | 8 +++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index e45098132..92c6b611d 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -327,6 +327,24 @@ export const snapshotDALFactory = (db: TDbClient) => { } }; + /** + * Prunes excess snapshots from the database to ensure only a specified number of recent snapshots are retained for each folder. + * + * This function operates in three main steps: + * 1. Pruning snapshots from root/non-versioned folders. + * 2. Pruning snapshots from versioned folders. + * 3. Removing orphaned snapshots that do not belong to any existing folder or folder version. + * + * The function processes snapshots in batches, determined by the `PRUNE_FOLDER_BATCH_SIZE` constant, + * to manage the large datasets without overwhelming the DB. + * + * Steps: + * - Fetch a batch of folder IDs. + * - For each batch, use a Common Table Expression (CTE) to rank snapshots within each folder by their creation date. + * - Identify and delete snapshots that exceed the project's point-in-time version limit (`pitVersionLimit`). + * - Repeat the process for versioned folders. + * - Finally, delete orphaned snapshots that do not have an associated folder. + */ const pruneExcessSnapshots = async () => { const PRUNE_FOLDER_BATCH_SIZE = 10000; @@ -337,6 +355,7 @@ export const snapshotDALFactory = (db: TDbClient) => { while (true) { const folderBatch = await db(TableName.SecretFolder) .where("id", ">", uuidOffset) + .where("isReserved", false) .orderBy("id", "asc") .limit(PRUNE_FOLDER_BATCH_SIZE) .select("id"); @@ -345,7 +364,7 @@ export const snapshotDALFactory = (db: TDbClient) => { if (folderBatch.length) { try { - logger.info(`Pruning snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}`); + logger.info(`Pruning snapshots in [range=${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}]`); await db(TableName.Snapshot) .with("snapshot_cte", (qb) => { void qb diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx index f3035922b..0ce5c7058 100644 --- a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx @@ -56,12 +56,10 @@ export const PointInTimeVersionLimitSection = () => { return (
-

Point in Time Recovery

+

Version Retention

- This defines the maximum number of folder snapshots, secret versions, and folder versions - that are retained by the system. The cleanup of excess snapshots and versions happens once a - day on midnight of UTC. + This defines the maximum number of recent secret versions to keep per folder. Excess versions will be removed at midnight (UTC) each day.

@@ -73,7 +71,7 @@ export const PointInTimeVersionLimitSection = () => {