Merge pull request #1906 from Infisical/feat/add-version-limits

feat: add limit to the number of retained snapshots and versions

backend/src/services/project/project-service.ts

@@ -39,6 +39,7 @@ import {
   TToggleProjectAutoCapitalizationDTO,
   TUpdateProjectDTO,
   TUpdateProjectNameDTO,
+  TUpdateProjectVersionLimitDTO,
   TUpgradeProjectDTO
 } from "./project-types";
 
@@ -133,7 +134,8 @@ export const projectServiceFactory = ({
           name: workspaceName,
           orgId: organization.id,
           slug: projectSlug || slugify(`${workspaceName}-${alphaNumericNanoId(4)}`),
-          version: ProjectVersion.V2
+          version: ProjectVersion.V2,
+          pitVersionLimit: 10
         },
         tx
       );
@@ -406,6 +408,35 @@ export const projectServiceFactory = ({
     return updatedProject;
   };
 
+  const updateVersionLimit = async ({
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod,
+    pitVersionLimit,
+    workspaceSlug
+  }: TUpdateProjectVersionLimitDTO) => {
+    const project = await projectDAL.findProjectBySlug(workspaceSlug, actorOrgId);
+    if (!project) {
+      throw new BadRequestError({
+        message: "Project not found"
+      });
+    }
+
+    const { hasRole } = await permissionService.getProjectPermission(
+      actor,
+      actorId,
+      project.id,
+      actorAuthMethod,
+      actorOrgId
+    );
+
+    if (!hasRole(ProjectMembershipRole.Admin))
+      throw new BadRequestError({ message: "Only admins are allowed to take this action" });
+
+    return projectDAL.updateById(project.id, { pitVersionLimit });
+  };
+
   const updateName = async ({
     projectId,
     actor,
@@ -501,6 +532,7 @@ export const projectServiceFactory = ({
     getAProject,
     toggleAutoCapitalization,
     updateName,
-    upgradeProject
+    upgradeProject,
+    updateVersionLimit
   };
 };

backend/src/services/project/project-types.ts

@@ -43,6 +43,11 @@ export type TToggleProjectAutoCapitalizationDTO = {
   autoCapitalization: boolean;
 } & TProjectPermission;
 
+export type TUpdateProjectVersionLimitDTO = {
+  pitVersionLimit: number;
+  workspaceSlug: string;
+} & Omit<TProjectPermission, "projectId">;
+
 export type TUpdateProjectNameDTO = {
   name: string;
 } & TProjectPermission;

backend/src/services/resource-cleanup/resource-cleanup-queue.ts

@@ -1,13 +1,19 @@
 import { TAuditLogDALFactory } from "@app/ee/services/audit-log/audit-log-dal";
+import { TSnapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal";
 import { logger } from "@app/lib/logger";
 import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
 
 import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal";
+import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
+import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal";
 import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal";
 
 type TDailyResourceCleanUpQueueServiceFactoryDep = {
   auditLogDAL: Pick<TAuditLogDALFactory, "pruneAuditLog">;
   identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "removeExpiredTokens">;
+  secretVersionDAL: Pick<TSecretVersionDALFactory, "pruneExcessVersions">;
+  secretFolderVersionDAL: Pick<TSecretFolderVersionDALFactory, "pruneExcessVersions">;
+  snapshotDAL: Pick<TSnapshotDALFactory, "pruneExcessSnapshots">;
   secretSharingDAL: Pick<TSecretSharingDALFactory, "pruneExpiredSharedSecrets">;
   queueService: TQueueServiceFactory;
 };
@@ -17,6 +23,9 @@ export type TDailyResourceCleanUpQueueServiceFactory = ReturnType<typeof dailyRe
 export const dailyResourceCleanUpQueueServiceFactory = ({
   auditLogDAL,
   queueService,
+  snapshotDAL,
+  secretVersionDAL,
+  secretFolderVersionDAL,
   identityAccessTokenDAL,
   secretSharingDAL
 }: TDailyResourceCleanUpQueueServiceFactoryDep) => {
@@ -25,6 +34,9 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
     await auditLogDAL.pruneAuditLog();
     await identityAccessTokenDAL.removeExpiredTokens();
     await secretSharingDAL.pruneExpiredSharedSecrets();
+    await snapshotDAL.pruneExcessSnapshots();
+    await secretVersionDAL.pruneExcessVersions();
+    await secretFolderVersionDAL.pruneExcessVersions();
     logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`);
   });
 

backend/src/services/secret-folder/secret-folder-version-dal.ts

@@ -62,5 +62,32 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
     }
   };
 
-  return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId };
+  const pruneExcessVersions = async () => {
+    try {
+      await db(TableName.SecretFolderVersion)
+        .with("folder_cte", (qb) => {
+          void qb
+            .from(TableName.SecretFolderVersion)
+            .select(
+              "id",
+              "folderId",
+              db.raw(
+                `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretFolderVersion}."folderId" ORDER BY ${TableName.SecretFolderVersion}."createdAt" DESC) AS row_num`
+              )
+            );
+        })
+        .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`)
+        .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
+        .join("folder_cte", "folder_cte.id", `${TableName.SecretFolderVersion}.id`)
+        .whereRaw(`folder_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
+        .delete();
+    } catch (error) {
+      throw new DatabaseError({
+        error,
+        name: "Secret Folder Version Prune"
+      });
+    }
+  };
+
+  return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId, pruneExcessVersions };
 };

backend/src/services/secret/secret-version-dal.ts

@@ -111,8 +111,37 @@ export const secretVersionDALFactory = (db: TDbClient) => {
     }
   };
 
+  const pruneExcessVersions = async () => {
+    try {
+      await db(TableName.SecretVersion)
+        .with("version_cte", (qb) => {
+          void qb
+            .from(TableName.SecretVersion)
+            .select(
+              "id",
+              "folderId",
+              db.raw(
+                `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num`
+              )
+            );
+        })
+        .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`)
+        .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
+        .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
+        .join("version_cte", "version_cte.id", `${TableName.SecretVersion}.id`)
+        .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
+        .delete();
+    } catch (error) {
+      throw new DatabaseError({
+        error,
+        name: "Secret Version Prune"
+      });
+    }
+  };
+
   return {
     ...secretVersionOrm,
+    pruneExcessVersions,
     findLatestVersionMany,
     bulkUpdate,
     findLatestVersionByFolderId,