PIT: Add folder reconstruction functions

backend/src/ee/routes/v1/index.ts

@@ -18,6 +18,7 @@ import { registerLdapRouter } from "./ldap-router";
 import { registerLicenseRouter } from "./license-router";
 import { registerOidcRouter } from "./oidc-router";
 import { registerOrgRoleRouter } from "./org-role-router";
+import { registerPITRouter } from "./pit-router";
 import { registerProjectRoleRouter } from "./project-role-router";
 import { registerProjectRouter } from "./project-router";
 import { registerRateLimitRouter } from "./rate-limit-router";
@@ -53,6 +54,7 @@ export const registerV1EERoutes = async (server: FastifyZodProvider) => {
     { prefix: "/workspace" }
   );
   await server.register(registerSnapshotRouter, { prefix: "/secret-snapshot" });
+  await server.register(registerPITRouter, { prefix: "/pit" });
   await server.register(registerSecretApprovalPolicyRouter, { prefix: "/secret-approvals" });
   await server.register(registerSecretApprovalRequestRouter, {
     prefix: "/secret-approval-requests"

backend/src/ee/routes/v1/pit-router.ts

@@ -0,0 +1,27 @@
+import { z } from "zod";
+
+import { readLimit } from "@app/server/config/rateLimiter";
+
+export const registerPITRouter = async (server: FastifyZodProvider) => {
+  server.route({
+    method: "GET",
+    url: "/diff",
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      querystring: z.object({
+        fromCommit: z.string().trim(),
+        toCommit: z.string().trim()
+      }),
+      response: {
+        200: z.any()
+      }
+    },
+    handler: async (req) => {
+      const backup = await server.services.folderCommit.compareFolderStates(req.query.fromCommit, req.query.toCommit);
+
+      return backup;
+    }
+  });
+};

backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts

@@ -1,24 +1,52 @@
 import { Knex } from "knex";
 
 import { TDbClient } from "@app/db";
-import { TableName, TFolderCheckpointResources, TFolderCheckpoints } from "@app/db/schemas";
+import {
+  TableName,
+  TFolderCheckpointResources,
+  TFolderCheckpoints,
+  TSecretFolderVersions,
+  TSecretVersionsV2
+} from "@app/db/schemas";
 import { DatabaseError } from "@app/lib/errors";
 import { ormify, selectAllTableCols } from "@app/lib/knex";
 
 export type TFolderCheckpointResourcesDALFactory = ReturnType<typeof folderCheckpointResourcesDALFactory>;
 
-type ResourceWithCheckpointInfo = TFolderCheckpointResources & {
+export type ResourceWithCheckpointInfo = TFolderCheckpointResources & {
   folderCommitId: string;
 };
 
 export const folderCheckpointResourcesDALFactory = (db: TDbClient) => {
   const folderCheckpointResourcesOrm = ormify(db, TableName.FolderCheckpointResources);
 
-  const findByCheckpointId = async (folderCheckpointId: string, tx?: Knex): Promise<TFolderCheckpointResources[]> => {
+  const findByCheckpointId = async (
+    folderCheckpointId: string,
+    tx?: Knex
+  ): Promise<
+    (TFolderCheckpointResources & {
+      referencedSecretId?: string;
+      referencedFolderId?: string;
+    })[]
+  > => {
     try {
       const docs = await (tx || db.replicaNode())<TFolderCheckpointResources>(TableName.FolderCheckpointResources)
         .where({ folderCheckpointId })
-        .select(selectAllTableCols(TableName.FolderCheckpointResources));
+        .leftJoin<TSecretVersionsV2>(
+          TableName.SecretVersionV2,
+          `${TableName.FolderCheckpointResources}.secretVersionId`,
+          `${TableName.SecretVersionV2}.id`
+        )
+        .leftJoin<TSecretFolderVersions>(
+          TableName.SecretFolderVersion,
+          `${TableName.FolderCheckpointResources}.folderVersionId`,
+          `${TableName.SecretFolderVersion}.id`
+        )
+        .select(selectAllTableCols(TableName.FolderCheckpointResources))
+        .select(
+          db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"),
+          db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId")
+        );
       return docs;
     } catch (error) {
       throw new DatabaseError({ error, name: "FindByCheckpointId" });

backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts

@@ -1,7 +1,7 @@
 import { Knex } from "knex";
 
 import { TDbClient } from "@app/db";
-import { TableName, TFolderCheckpoints } from "@app/db/schemas";
+import { TableName, TFolderCheckpoints, TFolderCommits } from "@app/db/schemas";
 import { DatabaseError } from "@app/lib/errors";
 import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex";
 
@@ -33,7 +33,11 @@ export const folderCheckpointDALFactory = (db: TDbClient) => {
   const findByFolderId = async (folderId: string, limit?: number, tx?: Knex): Promise<CheckpointWithCommitInfo[]> => {
     try {
       let query = (tx || db.replicaNode())(TableName.FolderCheckpoint)
-        .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`)
+        .join<TFolderCommits>(
+          TableName.FolderCommit,
+          `${TableName.FolderCheckpoint}.folderCommitId`,
+          `${TableName.FolderCommit}.id`
+        )
         // eslint-disable-next-line @typescript-eslint/no-misused-promises
         .where(buildFindFilter({ folderId }, TableName.FolderCommit))
         .select(selectAllTableCols(TableName.FolderCheckpoint))
@@ -50,8 +54,7 @@ export const folderCheckpointDALFactory = (db: TDbClient) => {
         query = query.limit(limit);
       }
 
-      const docs = await query;
+      return await query;
-      return docs;
     } catch (error) {
       throw new DatabaseError({ error, name: "FindByFolderId" });
     }
@@ -60,7 +63,11 @@ export const folderCheckpointDALFactory = (db: TDbClient) => {
   const findLatestByFolderId = async (folderId: string, tx?: Knex): Promise<CheckpointWithCommitInfo | undefined> => {
     try {
       const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint)
-        .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`)
+        .join<TFolderCommits>(
+          TableName.FolderCommit,
+          `${TableName.FolderCheckpoint}.folderCommitId`,
+          `${TableName.FolderCommit}.id`
+        )
         // eslint-disable-next-line @typescript-eslint/no-misused-promises
         .where(buildFindFilter({ folderId }, TableName.FolderCommit))
         .select(selectAllTableCols(TableName.FolderCheckpoint))
@@ -79,10 +86,53 @@ export const folderCheckpointDALFactory = (db: TDbClient) => {
     }
   };
 
+  const findNearestCheckpoint = async (
+    folderCommitId: string,
+    tx?: Knex
+  ): Promise<(CheckpointWithCommitInfo & { commitId: number }) | undefined> => {
+    try {
+      // First, get the commit info to find the folder ID and commit sequence number
+      const commit = await (tx || db.replicaNode())(TableName.FolderCommit)
+        .where({ id: folderCommitId })
+        .select("commitId", "folderId")
+        .first();
+
+      if (!commit) {
+        return undefined;
+      }
+
+      // Get the checkpoint with the highest commitId that's still less than or equal to our commit
+      const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderCheckpoint)
+        .join<TFolderCommits>(
+          TableName.FolderCommit,
+          `${TableName.FolderCheckpoint}.folderCommitId`,
+          `${TableName.FolderCommit}.id`
+        )
+        .where(`${TableName.FolderCommit}.folderId`, commit.folderId)
+        .where(`${TableName.FolderCommit}.commitId`, "<=", commit.commitId.toString())
+        .select(selectAllTableCols(TableName.FolderCheckpoint))
+        .select(
+          db.ref("actorMetadata").withSchema(TableName.FolderCommit),
+          db.ref("actorType").withSchema(TableName.FolderCommit),
+          db.ref("message").withSchema(TableName.FolderCommit),
+          db.ref("commitId").withSchema(TableName.FolderCommit),
+          db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"),
+          db.ref("folderId").withSchema(TableName.FolderCommit)
+        )
+        .orderBy(`${TableName.FolderCommit}.commitId`, "desc")
+        .first();
+
+      return nearestCheckpoint;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindNearestCheckpoint" });
+    }
+  };
+
   return {
     ...folderCheckpointOrm,
     findByCommitId,
     findByFolderId,
-    findLatestByFolderId
+    findLatestByFolderId,
+    findNearestCheckpoint
   };
 };

backend/src/services/folder-commit/folder-commit-dal.ts

@@ -1,9 +1,15 @@
 import { Knex } from "knex";
 
 import { TDbClient } from "@app/db";
-import { TableName, TFolderCommits } from "@app/db/schemas";
+import {
+  TableName,
+  TFolderCommitChanges,
+  TFolderCommits,
+  TSecretFolderVersions,
+  TSecretVersionsV2
+} from "@app/db/schemas";
 import { DatabaseError } from "@app/lib/errors";
-import { ormify, selectAllTableCols } from "@app/lib/knex";
+import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex";
 
 export type TFolderCommitDALFactory = ReturnType<typeof folderCommitDALFactory>;
 
@@ -56,10 +62,80 @@ export const folderCommitDALFactory = (db: TDbClient) => {
     }
   };
 
+  const findCommitsToRecreate = async (
+    folderId: string,
+    targetCommitNumber: number,
+    checkpointCommitNumber: number,
+    tx?: Knex
+  ): Promise<
+    (TFolderCommits & {
+      changes: (TFolderCommitChanges & { referencedSecretId?: string; referencedFolderId?: string })[];
+    })[]
+  > => {
+    try {
+      // First get all the commits in the range
+      const commits = await (tx || db.replicaNode())(TableName.FolderCommit)
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
+        .where(buildFindFilter({ folderId }, TableName.FolderCommit))
+        .andWhere(`${TableName.FolderCommit}.commitId`, ">", checkpointCommitNumber)
+        .andWhere(`${TableName.FolderCommit}.commitId`, "<=", targetCommitNumber)
+        .select(selectAllTableCols(TableName.FolderCommit))
+        .orderBy(`${TableName.FolderCommit}.commitId`, "asc");
+
+      // If no commits found, return empty array
+      if (!commits.length) {
+        return [];
+      }
+
+      // Get all the commit IDs
+      const commitIds = commits.map((commit) => commit.id);
+
+      // Get all changes for these commits in a single query
+      const allChanges = await (tx || db.replicaNode())(TableName.FolderCommitChanges)
+        .whereIn("folderCommitId", commitIds)
+        .leftJoin<TSecretVersionsV2>(
+          TableName.SecretVersionV2,
+          `${TableName.FolderCommitChanges}.secretVersionId`,
+          `${TableName.SecretVersionV2}.id`
+        )
+        .leftJoin<TSecretFolderVersions>(
+          TableName.SecretFolderVersion,
+          `${TableName.FolderCommitChanges}.folderVersionId`,
+          `${TableName.SecretFolderVersion}.id`
+        )
+        .select(selectAllTableCols(TableName.FolderCommitChanges))
+        .select(
+          db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"),
+          db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId")
+        );
+
+      // Organize changes by commit ID
+      const changesByCommitId = allChanges.reduce(
+        (acc, change) => {
+          if (!acc[change.folderCommitId]) {
+            acc[change.folderCommitId] = [];
+          }
+          acc[change.folderCommitId].push(change);
+          return acc;
+        },
+        {} as Record<string, TFolderCommitChanges[]>
+      );
+
+      // Attach changes to each commit
+      return commits.map((commit) => ({
+        ...commit,
+        changes: changesByCommitId[commit.id] || []
+      }));
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindCommitsToRecreate" });
+    }
+  };
+
   return {
     ...restOfOrm,
     findByFolderId,
     findLatestCommit,
-    getNumberOfCommitsSince
+    getNumberOfCommitsSince,
+    findCommitsToRecreate
   };
 };

backend/src/services/folder-commit/folder-commit-service.ts

@@ -1,3 +1,4 @@
+/* eslint-disable no-await-in-loop */
 import { Knex } from "knex";
 
 import { TSecretFolders } from "@app/db/schemas";
@@ -20,17 +21,32 @@ import { TFolderCommitDALFactory } from "./folder-commit-dal";
 type TFolderCommitServiceFactoryDep = {
   folderCommitDAL: Pick<
     TFolderCommitDALFactory,
-    "create" | "findById" | "findByFolderId" | "findLatestCommit" | "transaction" | "getNumberOfCommitsSince"
+    | "create"
+    | "findById"
+    | "findByFolderId"
+    | "findLatestCommit"
+    | "transaction"
+    | "getNumberOfCommitsSince"
+    | "findCommitsToRecreate"
   >;
   folderCommitChangesDAL: Pick<TFolderCommitChangesDALFactory, "create" | "findByCommitId" | "insertMany">;
-  folderCheckpointDAL: Pick<TFolderCheckpointDALFactory, "create" | "findByFolderId" | "findLatestByFolderId">;
+  folderCheckpointDAL: Pick<
-  folderCheckpointResourcesDAL: Pick<TFolderCheckpointResourcesDALFactory, "insertMany">;
+    TFolderCheckpointDALFactory,
+    "create" | "findByFolderId" | "findLatestByFolderId" | "findNearestCheckpoint"
+  >;
+  folderCheckpointResourcesDAL: Pick<TFolderCheckpointResourcesDALFactory, "insertMany" | "findByCheckpointId">;
   folderTreeCheckpointDAL: Pick<TFolderTreeCheckpointDALFactory, "findByProjectId" | "findLatestByProjectId">;
   userDAL: Pick<TUserDALFactory, "findById">;
   identityDAL: Pick<TIdentityDALFactory, "findById">;
   folderDAL: Pick<TSecretFolderDALFactory, "findByParentId" | "findByProjectId">;
-  folderVersionDAL: Pick<TSecretFolderVersionDALFactory, "findLatestFolderVersions">;
+  folderVersionDAL: Pick<
-  secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "findLatestVersionByFolderId">;
+    TSecretFolderVersionDALFactory,
+    "findLatestFolderVersions" | "findById" | "deleteById" | "create" | "updateById"
+  >;
+  secretVersionV2BridgeDAL: Pick<
+    TSecretVersionV2DALFactory,
+    "findLatestVersionByFolderId" | "findById" | "deleteById" | "create" | "updateById"
+  >;
   projectDAL: Pick<TProjectDALFactory, "findById">;
 };
 
@@ -100,15 +116,20 @@ export const folderCommitServiceFactory = ({
     tx?: Knex;
   }) => {
     let latestCommitId = folderCommitId;
+    const latestCheckpoint = await folderCheckpointDAL.findLatestByFolderId(folderId, tx);
     if (!latestCommitId) {
-      latestCommitId = (await folderCheckpointDAL.findLatestByFolderId(folderId, tx))?.folderCommitId;
+      latestCommitId = (await folderCommitDAL.findLatestCommit(folderId, tx))?.id;
     }
     if (!latestCommitId) {
       throw new BadRequestError({ message: "Latest commit ID not found" });
       return;
     }
-    if (!force) {
+    if (!force && latestCheckpoint) {
-      const commitsSinceLastCheckpoint = await folderCommitDAL.getNumberOfCommitsSince(folderId, latestCommitId, tx);
+      const commitsSinceLastCheckpoint = await folderCommitDAL.getNumberOfCommitsSince(
+        folderId,
+        latestCheckpoint.folderCommitId,
+        tx
+      );
       if (commitsSinceLastCheckpoint < Number(appCfg.CHECKPOINT_WINDOW)) {
         return;
       }
@@ -129,6 +150,175 @@ export const folderCommitServiceFactory = ({
     }
   };
 
+  const reconstructFolderState = async (
+    folderCommitId: string,
+    tx?: Knex
+  ): Promise<{ type: string; id: string; versionId: string }[]> => {
+    const targetCommit = await folderCommitDAL.findById(folderCommitId, tx);
+    if (!targetCommit) {
+      throw new NotFoundError({ message: `Commit with ID ${folderCommitId} not found` });
+    }
+
+    const nearestCheckpoint = await folderCheckpointDAL.findNearestCheckpoint(folderCommitId, tx);
+    if (!nearestCheckpoint) {
+      throw new NotFoundError({ message: `Nearest checkpoint not found for commit ${folderCommitId}` });
+    }
+
+    const checkpointResources = await folderCheckpointResourcesDAL.findByCheckpointId(nearestCheckpoint.id, tx);
+
+    const folderState: Record<string, { type: string; id: string; versionId: string }> = {};
+
+    // Add all checkpoint resources to initial state
+    checkpointResources.forEach((resource) => {
+      if (resource.secretVersionId && resource.referencedSecretId) {
+        folderState[`secret-${resource.referencedSecretId}`] = {
+          type: "secret",
+          id: resource.referencedSecretId,
+          versionId: resource.secretVersionId
+        };
+      } else if (resource.folderVersionId && resource.referencedFolderId) {
+        folderState[`folder-${resource.referencedFolderId}`] = {
+          type: "folder",
+          id: resource.referencedFolderId,
+          versionId: resource.folderVersionId
+        };
+      }
+    });
+
+    const commitsToRecreate = await folderCommitDAL.findCommitsToRecreate(
+      targetCommit.folderId,
+      targetCommit.commitId,
+      nearestCheckpoint.commitId,
+      tx
+    );
+
+    // Process commits to recreate final state
+    for (const commit of commitsToRecreate) {
+      // eslint-disable-next-line no-continue
+      if (!commit.changes) continue;
+
+      for (const change of commit.changes) {
+        if (change.secretVersionId && change.referencedSecretId) {
+          const key = `secret-${change.referencedSecretId}`;
+
+          if (change.changeType.toLowerCase() === "add") {
+            folderState[key] = {
+              type: "secret",
+              id: change.referencedSecretId,
+              versionId: change.secretVersionId
+            };
+          } else if (change.changeType.toLowerCase() === "delete") {
+            delete folderState[key];
+          }
+        } else if (change.folderVersionId && change.referencedFolderId) {
+          const key = `folder-${change.referencedFolderId}`;
+
+          if (change.changeType.toLowerCase() === "add") {
+            folderState[key] = {
+              type: "folder",
+              id: change.referencedFolderId,
+              versionId: change.folderVersionId
+            };
+          } else if (change.changeType.toLowerCase() === "delete") {
+            delete folderState[key];
+          }
+        }
+      }
+    }
+    return Object.values(folderState);
+  };
+
+  const compareFolderStates = async (currentCommitId: string, targetCommitId: string, tx?: Knex) => {
+    // Reconstruct state for both commits
+    const currentState = await reconstructFolderState(currentCommitId, tx);
+    const targetState = await reconstructFolderState(targetCommitId, tx);
+
+    // Create lookup maps for easier comparison
+    const currentMap: Record<string, { type: string; id: string; versionId: string }> = {};
+    const targetMap: Record<string, { type: string; id: string; versionId: string }> = {};
+
+    // Build lookup map for current state
+    currentState.forEach((resource) => {
+      const key = `${resource.type}-${resource.id}`;
+      currentMap[key] = resource;
+    });
+
+    // Build lookup map for target state
+    targetState.forEach((resource) => {
+      const key = `${resource.type}-${resource.id}`;
+      targetMap[key] = resource;
+    });
+
+    // Track differences
+    const differences: {
+      type: string;
+      id: string;
+      versionId: string;
+      changeType: "create" | "update" | "delete";
+    }[] = [];
+
+    // Find deletes and updates (resources in current but not in target, or with different versions)
+    Object.keys(currentMap).forEach((key) => {
+      const currentResource = currentMap[key];
+      const targetResource = targetMap[key];
+
+      if (!targetResource) {
+        // Resource exists in current but not in target - it's a delete
+        differences.push({
+          type: currentResource.type,
+          id: currentResource.id,
+          versionId: currentResource.versionId,
+          changeType: "delete"
+        });
+      } else if (currentResource.versionId !== targetResource.versionId) {
+        // Resource exists in both but with different versions - it's an update
+        differences.push({
+          type: targetResource.type,
+          id: targetResource.id,
+          versionId: targetResource.versionId,
+          changeType: "update"
+        });
+      }
+      // If versions are the same, it's unchanged - exclude from result
+    });
+
+    // Find creates (resources in target but not in current)
+    Object.keys(targetMap).forEach((key) => {
+      if (!currentMap[key]) {
+        const targetResource = targetMap[key];
+        differences.push({
+          type: targetResource.type,
+          id: targetResource.id,
+          versionId: targetResource.versionId,
+          changeType: "create"
+        });
+      }
+    });
+
+    return differences;
+  };
+
+  // Add a change to an existing commit
+  const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => {
+    try {
+      if (!data.secretVersionId && !data.folderVersionId) {
+        throw new BadRequestError({ message: "Either secretVersionId or folderVersionId must be provided" });
+      }
+
+      const commit = await folderCommitDAL.findById(data.folderCommitId, tx);
+      if (!commit) {
+        throw new NotFoundError({ message: `Commit with ID ${data.folderCommitId} not found` });
+      }
+
+      return await folderCommitChangesDAL.create(data, tx);
+    } catch (error) {
+      if (error instanceof NotFoundError || error instanceof BadRequestError) {
+        throw error;
+      }
+      throw new DatabaseError({ error, name: "AddCommitChange" });
+    }
+  };
+
   const createCommit = async (data: TCreateCommitDTO, tx?: Knex) => {
     const metadata = data.actor.metadata || {};
     try {
@@ -159,34 +349,13 @@ export const folderCommitServiceFactory = ({
         tx
       );
 
-      await createFolderCheckpoint({ folderId: data.folderId, folderCommitId: newCommit.id, tx });
+      await createFolderCheckpoint({ folderId: data.folderId, tx });
       return newCommit;
     } catch (error) {
       throw new DatabaseError({ error, name: "CreateCommit" });
     }
   };
 
-  // Add a change to an existing commit
-  const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => {
-    try {
-      if (!data.secretVersionId && !data.folderVersionId) {
-        throw new BadRequestError({ message: "Either secretVersionId or folderVersionId must be provided" });
-      }
-
-      const commit = await folderCommitDAL.findById(data.folderCommitId, tx);
-      if (!commit) {
-        throw new NotFoundError({ message: `Commit with ID ${data.folderCommitId} not found` });
-      }
-
-      return await folderCommitChangesDAL.create(data, tx);
-    } catch (error) {
-      if (error instanceof NotFoundError || error instanceof BadRequestError) {
-        throw error;
-      }
-      throw new DatabaseError({ error, name: "AddCommitChange" });
-    }
-  };
-
   // Retrieve a commit by ID
   const getCommitById = async (id: string, tx?: Knex) => {
     return folderCommitDAL.findById(id, tx);
@@ -297,7 +466,8 @@ export const folderCommitServiceFactory = ({
     getLatestTreeCheckpoint,
     initializeFolder,
     initializeProject,
-    createFolderCheckpoint
+    createFolderCheckpoint,
+    compareFolderStates
   };
 };