Merge pull request #3555 from Infisical/feat/point-in-time-revamp-2710

Feat/point in time revamp 2710

backend/src/services/secret-folder/secret-folder-dal.ts

@@ -488,6 +488,73 @@ export const secretFolderDALFactory = (db: TDbClient) => {
     }
   };
 
+  const findFoldersByRootAndIds = async ({ rootId, folderIds }: { rootId: string; folderIds: string[] }, tx?: Knex) => {
+    try {
+      // First, get all descendant folders of rootId
+      const descendants = await (tx || db.replicaNode())
+        .withRecursive("descendants", (qb) =>
+          qb
+            .select(
+              selectAllTableCols(TableName.SecretFolder),
+              db.raw("0 as depth"),
+              db.raw(`'/' as path`),
+              db.ref(`${TableName.Environment}.slug`).as("environment")
+            )
+            .from(TableName.SecretFolder)
+            .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
+            .where(`${TableName.SecretFolder}.id`, rootId)
+            .union((un) => {
+              void un
+                .select(
+                  selectAllTableCols(TableName.SecretFolder),
+                  db.raw("descendants.depth + 1 as depth"),
+                  db.raw(
+                    `CONCAT(
+                      CASE WHEN descendants.path = '/' THEN '' ELSE descendants.path END,
+                      CASE WHEN ${TableName.SecretFolder}."parentId" is NULL THEN '' ELSE CONCAT('/', secret_folders.name) END
+                    )`
+                  ),
+                  db.ref("descendants.environment")
+                )
+                .from(TableName.SecretFolder)
+                .where(`${TableName.SecretFolder}.isReserved`, false)
+                .join("descendants", `${TableName.SecretFolder}.parentId`, "descendants.id");
+            })
+        )
+        .select<(TSecretFolders & { path: string; depth: number; environment: string })[]>("*")
+        .from("descendants")
+        .whereIn(`id`, folderIds)
+        .orderBy("depth")
+        .orderBy(`name`);
+
+      return descendants;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindFoldersByRootAndIds" });
+    }
+  };
+
+  const findByParentId = async (parentId: string, tx?: Knex) => {
+    try {
+      const folders = await (tx || db.replicaNode())(TableName.SecretFolder)
+        .where({ parentId })
+        .select(selectAllTableCols(TableName.SecretFolder));
+      return folders;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "findByParentId" });
+    }
+  };
+
+  const findByEnvId = async (envId: string, tx?: Knex) => {
+    try {
+      const folders = await (tx || db.replicaNode())(TableName.SecretFolder)
+        .where({ envId })
+        .select(selectAllTableCols(TableName.SecretFolder));
+      return folders;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "findByEnvId" });
+    }
+  };
+
   return {
     ...secretFolderOrm,
     update,
@@ -499,6 +566,9 @@ export const secretFolderDALFactory = (db: TDbClient) => {
     findClosestFolder,
     findByProjectId,
     findByMultiEnv,
-    findByEnvsDeep
+    findByEnvsDeep,
+    findByParentId,
+    findByEnvId,
+    findFoldersByRootAndIds
   };
 };

backend/src/services/secret-folder/secret-folder-service.ts

@@ -10,6 +10,7 @@ import { BadRequestError, NotFoundError } from "@app/lib/errors";
 import { OrderByDirection, OrgServiceActor } from "@app/lib/types";
 import { buildFolderPath } from "@app/services/secret-folder/secret-folder-fns";
 
+import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
 import { TProjectDALFactory } from "../project/project-dal";
 import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
 import { TSecretFolderDALFactory } from "./secret-folder-dal";
@@ -29,7 +30,8 @@ type TSecretFolderServiceFactoryDep = {
   snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
   folderDAL: TSecretFolderDALFactory;
   projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne" | "findBySlugs" | "find">;
-  folderVersionDAL: TSecretFolderVersionDALFactory;
+  folderVersionDAL: Pick<TSecretFolderVersionDALFactory, "findLatestFolderVersions" | "create" | "insertMany" | "find">;
+  folderCommitService: Pick<TFolderCommitServiceFactory, "createCommit">;
   projectDAL: Pick<TProjectDALFactory, "findProjectBySlug">;
 };
 
@@ -41,6 +43,7 @@ export const secretFolderServiceFactory = ({
   permissionService,
   projectEnvDAL,
   folderVersionDAL,
+  folderCommitService,
   projectDAL
 }: TSecretFolderServiceFactoryDep) => {
   const createFolder = async ({
@@ -111,7 +114,7 @@ export const secretFolderServiceFactory = ({
           });
           parentFolderId = newFolders.at(-1)?.id as string;
           const docs = await folderDAL.insertMany(newFolders, tx);
-          await folderVersionDAL.insertMany(
+          const folderVersions = await folderVersionDAL.insertMany(
             docs.map((doc) => ({
               name: doc.name,
               envId: doc.envId,
@@ -120,6 +123,23 @@ export const secretFolderServiceFactory = ({
             })),
             tx
           );
+          await folderCommitService.createCommit(
+            {
+              actor: {
+                type: actor,
+                metadata: {
+                  id: actorId
+                }
+              },
+              message: "Folder created",
+              folderId: parentFolderId,
+              changes: folderVersions.map((fv) => ({
+                type: "add",
+                folderVersionId: fv.id
+              }))
+            },
+            tx
+          );
         }
       }
 
@@ -127,7 +147,7 @@ export const secretFolderServiceFactory = ({
         { name, envId: env.id, version: 1, parentId: parentFolderId, description },
         tx
       );
-      await folderVersionDAL.create(
+      const folderVersion = await folderVersionDAL.create(
         {
           name: doc.name,
           envId: doc.envId,
@@ -136,6 +156,25 @@ export const secretFolderServiceFactory = ({
         },
         tx
       );
+      await folderCommitService.createCommit(
+        {
+          actor: {
+            type: actor,
+            metadata: {
+              id: actorId
+            }
+          },
+          message: "Folder created",
+          folderId: parentFolderId,
+          changes: [
+            {
+              type: "add",
+              folderVersionId: folderVersion.id
+            }
+          ]
+        },
+        tx
+      );
       return doc;
     });
 
@@ -225,7 +264,7 @@ export const secretFolderServiceFactory = ({
             { name, description },
             tx
           );
-          await folderVersionDAL.create(
+          const folderVersion = await folderVersionDAL.create(
             {
               name: doc.name,
               envId: doc.envId,
@@ -234,6 +273,26 @@ export const secretFolderServiceFactory = ({
             },
             tx
           );
+          await folderCommitService.createCommit(
+            {
+              actor: {
+                type: actor,
+                metadata: {
+                  id: actorId
+                }
+              },
+              message: "Folder updated",
+              folderId: parentFolder.id,
+              changes: [
+                {
+                  type: "add",
+                  isUpdate: true,
+                  folderVersionId: folderVersion.id
+                }
+              ]
+            },
+            tx
+          );
           if (!doc) {
             throw new NotFoundError({
               message: `Failed to update folder with id '${id}', not found`,
@@ -321,7 +380,7 @@ export const secretFolderServiceFactory = ({
         { name, description },
         tx
       );
-      await folderVersionDAL.create(
+      const folderVersion = await folderVersionDAL.create(
         {
           name: doc.name,
           envId: doc.envId,
@@ -330,6 +389,26 @@ export const secretFolderServiceFactory = ({
         },
         tx
       );
+      await folderCommitService.createCommit(
+        {
+          actor: {
+            type: actor,
+            metadata: {
+              id: actorId
+            }
+          },
+          message: "Folder updated",
+          folderId: parentFolder.id,
+          changes: [
+            {
+              type: "add",
+              isUpdate: true,
+              folderVersionId: folderVersion.id
+            }
+          ]
+        },
+        tx
+      );
       if (!doc) throw new NotFoundError({ message: `Failed to update folder with ID '${id}'`, name: "UpdateFolder" });
       return doc;
     });
@@ -381,7 +460,30 @@ export const secretFolderServiceFactory = ({
         },
         tx
       );
+
       if (!doc) throw new NotFoundError({ message: `Failed to delete folder with ID '${idOrName}', not found` });
+
+      const folderVersions = await folderVersionDAL.findLatestFolderVersions([doc.id]);
+
+      await folderCommitService.createCommit(
+        {
+          actor: {
+            type: actor,
+            metadata: {
+              id: actorId
+            }
+          },
+          message: "Folder deleted",
+          folderId: parentFolder.id,
+          changes: [
+            {
+              type: "delete",
+              folderVersionId: folderVersions[doc.id].id
+            }
+          ]
+        },
+        tx
+      );
       return doc;
     });
 
@@ -665,6 +767,22 @@ export const secretFolderServiceFactory = ({
     return environmentFolders;
   };
 
+  const getFolderVersionsByIds = async ({
+    folderId,
+    folderVersions
+  }: {
+    folderId: string;
+    folderVersions: string[];
+  }) => {
+    const versions = await folderVersionDAL.find({
+      folderId,
+      $in: {
+        version: folderVersions.map((v) => Number.parseInt(v, 10))
+      }
+    });
+    return versions;
+  };
+
   return {
     createFolder,
     updateFolder,
@@ -675,6 +793,7 @@ export const secretFolderServiceFactory = ({
     getProjectFolderCount,
     getFoldersMultiEnv,
     getFoldersDeepByEnvs,
-    getProjectEnvironmentsFolders
+    getProjectEnvironmentsFolders,
+    getFolderVersionsByIds
   };
 };

backend/src/services/secret-folder/secret-folder-version-dal.ts

@@ -43,7 +43,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
       const docs: Array<TSecretFolderVersions & { max: number }> = await (tx || db.replicaNode())(
         TableName.SecretFolderVersion
       )
-        .whereIn("folderId", folderIds)
+        .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds)
         .join(
           (tx || db)(TableName.SecretFolderVersion)
             .groupBy("folderId")
@@ -97,5 +97,80 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
     logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret folder versions completed`);
   };
 
-  return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId, pruneExcessVersions };
+  const findByIdsWithLatestVersion = async (folderIds: string[], versionIds?: string[], tx?: Knex) => {
+    try {
+      if (!folderIds.length && (!versionIds || !versionIds.length)) return {};
+
+      const knexInstance = tx || db.replicaNode();
+      let allDocs: Array<TSecretFolderVersions & { max?: number }> = [];
+
+      // Process latest versions for folderIds
+      if (folderIds.length) {
+        const latestVersions: Array<TSecretFolderVersions & { max: number }> = await knexInstance(
+          TableName.SecretFolderVersion
+        )
+          .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds)
+          .join(
+            knexInstance(TableName.SecretFolderVersion)
+              .groupBy("folderId")
+              .max("version")
+              .select("folderId")
+              .as("latestVersion"),
+            (bd) => {
+              bd.on(`${TableName.SecretFolderVersion}.folderId`, "latestVersion.folderId").andOn(
+                `${TableName.SecretFolderVersion}.version`,
+                "latestVersion.max"
+              );
+            }
+          );
+
+        allDocs = [...allDocs, ...latestVersions];
+      }
+
+      // Process specific versions by versionIds
+      if (versionIds && versionIds.length) {
+        // Get the specific versions
+        const specificVersions = await knexInstance(TableName.SecretFolderVersion).whereIn("id", versionIds);
+
+        // Get the folderIds from these versions
+        const specificFolderIds = [...new Set(specificVersions.map((v) => v.folderId).filter(Boolean))];
+
+        // Get max versions for these folderIds
+        const maxVersionsQuery = (await knexInstance(TableName.SecretFolderVersion)
+          .whereIn("folderId", specificFolderIds)
+          .groupBy("folderId")
+          .select("folderId")
+          .max("version as max")) as Array<{ folderId: string; max: number }>;
+
+        // Create a lookup map for max versions
+        const maxVersionMap = maxVersionsQuery.reduce<Record<string, number>>((acc, item) => {
+          acc[item.folderId] = item.max;
+          return acc;
+        }, {});
+
+        // Add max version to each specific version
+        const specificVersionsWithMax = specificVersions.map((version) => ({
+          ...version,
+          max: maxVersionMap[version.folderId]
+        }));
+
+        allDocs = [...allDocs, ...specificVersionsWithMax];
+      }
+
+      return allDocs.reduce<Record<string, TSecretFolderVersions & { max?: number }>>(
+        (prev, curr) => ({ ...prev, [curr.folderId || ""]: curr }),
+        {}
+      );
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindByIdsWithLatestVersion" });
+    }
+  };
+
+  return {
+    ...secretFolderVerOrm,
+    findLatestFolderVersions,
+    findLatestVersionByFolderId,
+    pruneExcessVersions,
+    findByIdsWithLatestVersion
+  };
 };