From 78d5bc823d364fd38e7549b60a49d4b6315590b0 Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Fri, 9 May 2025 09:20:17 -0300 Subject: [PATCH] PIT: Add folder reconstruction functions --- backend/src/ee/routes/v1/index.ts | 2 + backend/src/ee/routes/v1/pit-router.ts | 27 ++ .../folder-checkpoint-resources-dal.ts | 36 ++- .../folder-checkpoint-dal.ts | 62 ++++- .../folder-commit/folder-commit-dal.ts | 82 ++++++- .../folder-commit/folder-commit-service.ts | 232 +++++++++++++++--- 6 files changed, 397 insertions(+), 44 deletions(-) create mode 100644 backend/src/ee/routes/v1/pit-router.ts diff --git a/backend/src/ee/routes/v1/index.ts b/backend/src/ee/routes/v1/index.ts index 0b8c78586..9ba7f734e 100644 --- a/backend/src/ee/routes/v1/index.ts +++ b/backend/src/ee/routes/v1/index.ts @@ -18,6 +18,7 @@ import { registerLdapRouter } from "./ldap-router"; import { registerLicenseRouter } from "./license-router"; import { registerOidcRouter } from "./oidc-router"; import { registerOrgRoleRouter } from "./org-role-router"; +import { registerPITRouter } from "./pit-router"; import { registerProjectRoleRouter } from "./project-role-router"; import { registerProjectRouter } from "./project-router"; import { registerRateLimitRouter } from "./rate-limit-router"; @@ -53,6 +54,7 @@ export const registerV1EERoutes = async (server: FastifyZodProvider) => { { prefix: "/workspace" } ); await server.register(registerSnapshotRouter, { prefix: "/secret-snapshot" }); + await server.register(registerPITRouter, { prefix: "/pit" }); await server.register(registerSecretApprovalPolicyRouter, { prefix: "/secret-approvals" }); await server.register(registerSecretApprovalRequestRouter, { prefix: "/secret-approval-requests" diff --git a/backend/src/ee/routes/v1/pit-router.ts b/backend/src/ee/routes/v1/pit-router.ts new file mode 100644 index 000000000..0045a7efe --- /dev/null +++ b/backend/src/ee/routes/v1/pit-router.ts @@ -0,0 +1,27 @@ +import { z } from "zod"; + +import { readLimit } from "@app/server/config/rateLimiter"; + +export const registerPITRouter = async (server: FastifyZodProvider) => { + server.route({ + method: "GET", + url: "/diff", + config: { + rateLimit: readLimit + }, + schema: { + querystring: z.object({ + fromCommit: z.string().trim(), + toCommit: z.string().trim() + }), + response: { + 200: z.any() + } + }, + handler: async (req) => { + const backup = await server.services.folderCommit.compareFolderStates(req.query.fromCommit, req.query.toCommit); + + return backup; + } + }); +}; diff --git a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts index 2457abe14..7a36ec641 100644 --- a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts +++ b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts @@ -1,24 +1,52 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName, TFolderCheckpointResources, TFolderCheckpoints } from "@app/db/schemas"; +import { + TableName, + TFolderCheckpointResources, + TFolderCheckpoints, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderCheckpointResourcesDALFactory = ReturnType; -type ResourceWithCheckpointInfo = TFolderCheckpointResources & { +export type ResourceWithCheckpointInfo = TFolderCheckpointResources & { folderCommitId: string; }; export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { const folderCheckpointResourcesOrm = ormify(db, TableName.FolderCheckpointResources); - const findByCheckpointId = async (folderCheckpointId: string, tx?: Knex): Promise => { + const findByCheckpointId = async ( + folderCheckpointId: string, + tx?: Knex + ): Promise< + (TFolderCheckpointResources & { + referencedSecretId?: string; + referencedFolderId?: string; + })[] + > => { try { const docs = await (tx || db.replicaNode())(TableName.FolderCheckpointResources) .where({ folderCheckpointId }) - .select(selectAllTableCols(TableName.FolderCheckpointResources)); + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCheckpointResources}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCheckpointResources}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .select( + db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId") + ); return docs; } catch (error) { throw new DatabaseError({ error, name: "FindByCheckpointId" }); diff --git a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts index c598a8ea0..5ebb39f5b 100644 --- a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts +++ b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts @@ -1,7 +1,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName, TFolderCheckpoints } from "@app/db/schemas"; +import { TableName, TFolderCheckpoints, TFolderCommits } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; @@ -33,7 +33,11 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { const findByFolderId = async (folderId: string, limit?: number, tx?: Knex): Promise => { try { let query = (tx || db.replicaNode())(TableName.FolderCheckpoint) - .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) // eslint-disable-next-line @typescript-eslint/no-misused-promises .where(buildFindFilter({ folderId }, TableName.FolderCommit)) .select(selectAllTableCols(TableName.FolderCheckpoint)) @@ -50,8 +54,7 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { query = query.limit(limit); } - const docs = await query; - return docs; + return await query; } catch (error) { throw new DatabaseError({ error, name: "FindByFolderId" }); } @@ -60,7 +63,11 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { const findLatestByFolderId = async (folderId: string, tx?: Knex): Promise => { try { const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) - .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) // eslint-disable-next-line @typescript-eslint/no-misused-promises .where(buildFindFilter({ folderId }, TableName.FolderCommit)) .select(selectAllTableCols(TableName.FolderCheckpoint)) @@ -79,10 +86,53 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { } }; + const findNearestCheckpoint = async ( + folderCommitId: string, + tx?: Knex + ): Promise<(CheckpointWithCommitInfo & { commitId: number }) | undefined> => { + try { + // First, get the commit info to find the folder ID and commit sequence number + const commit = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id: folderCommitId }) + .select("commitId", "folderId") + .first(); + + if (!commit) { + return undefined; + } + + // Get the checkpoint with the highest commitId that's still less than or equal to our commit + const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .where(`${TableName.FolderCommit}.folderId`, commit.folderId) + .where(`${TableName.FolderCommit}.commitId`, "<=", commit.commitId.toString()) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("commitId").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCommit}.commitId`, "desc") + .first(); + + return nearestCheckpoint; + } catch (error) { + throw new DatabaseError({ error, name: "FindNearestCheckpoint" }); + } + }; + return { ...folderCheckpointOrm, findByCommitId, findByFolderId, - findLatestByFolderId + findLatestByFolderId, + findNearestCheckpoint }; }; diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts index 68394ddb3..f52da95b5 100644 --- a/backend/src/services/folder-commit/folder-commit-dal.ts +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -1,9 +1,15 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName, TFolderCommits } from "@app/db/schemas"; +import { + TableName, + TFolderCommitChanges, + TFolderCommits, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; -import { ormify, selectAllTableCols } from "@app/lib/knex"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderCommitDALFactory = ReturnType; @@ -56,10 +62,80 @@ export const folderCommitDALFactory = (db: TDbClient) => { } }; + const findCommitsToRecreate = async ( + folderId: string, + targetCommitNumber: number, + checkpointCommitNumber: number, + tx?: Knex + ): Promise< + (TFolderCommits & { + changes: (TFolderCommitChanges & { referencedSecretId?: string; referencedFolderId?: string })[]; + })[] + > => { + try { + // First get all the commits in the range + const commits = await (tx || db.replicaNode())(TableName.FolderCommit) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) + .andWhere(`${TableName.FolderCommit}.commitId`, ">", checkpointCommitNumber) + .andWhere(`${TableName.FolderCommit}.commitId`, "<=", targetCommitNumber) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy(`${TableName.FolderCommit}.commitId`, "asc"); + + // If no commits found, return empty array + if (!commits.length) { + return []; + } + + // Get all the commit IDs + const commitIds = commits.map((commit) => commit.id); + + // Get all changes for these commits in a single query + const allChanges = await (tx || db.replicaNode())(TableName.FolderCommitChanges) + .whereIn("folderCommitId", commitIds) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCommitChanges}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCommitChanges}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .select( + db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId") + ); + + // Organize changes by commit ID + const changesByCommitId = allChanges.reduce( + (acc, change) => { + if (!acc[change.folderCommitId]) { + acc[change.folderCommitId] = []; + } + acc[change.folderCommitId].push(change); + return acc; + }, + {} as Record + ); + + // Attach changes to each commit + return commits.map((commit) => ({ + ...commit, + changes: changesByCommitId[commit.id] || [] + })); + } catch (error) { + throw new DatabaseError({ error, name: "FindCommitsToRecreate" }); + } + }; + return { ...restOfOrm, findByFolderId, findLatestCommit, - getNumberOfCommitsSince + getNumberOfCommitsSince, + findCommitsToRecreate }; }; diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts index df834b88e..c39aaa7e7 100644 --- a/backend/src/services/folder-commit/folder-commit-service.ts +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-await-in-loop */ import { Knex } from "knex"; import { TSecretFolders } from "@app/db/schemas"; @@ -20,17 +21,32 @@ import { TFolderCommitDALFactory } from "./folder-commit-dal"; type TFolderCommitServiceFactoryDep = { folderCommitDAL: Pick< TFolderCommitDALFactory, - "create" | "findById" | "findByFolderId" | "findLatestCommit" | "transaction" | "getNumberOfCommitsSince" + | "create" + | "findById" + | "findByFolderId" + | "findLatestCommit" + | "transaction" + | "getNumberOfCommitsSince" + | "findCommitsToRecreate" >; folderCommitChangesDAL: Pick; - folderCheckpointDAL: Pick; - folderCheckpointResourcesDAL: Pick; + folderCheckpointDAL: Pick< + TFolderCheckpointDALFactory, + "create" | "findByFolderId" | "findLatestByFolderId" | "findNearestCheckpoint" + >; + folderCheckpointResourcesDAL: Pick; folderTreeCheckpointDAL: Pick; userDAL: Pick; identityDAL: Pick; folderDAL: Pick; - folderVersionDAL: Pick; - secretVersionV2BridgeDAL: Pick; + folderVersionDAL: Pick< + TSecretFolderVersionDALFactory, + "findLatestFolderVersions" | "findById" | "deleteById" | "create" | "updateById" + >; + secretVersionV2BridgeDAL: Pick< + TSecretVersionV2DALFactory, + "findLatestVersionByFolderId" | "findById" | "deleteById" | "create" | "updateById" + >; projectDAL: Pick; }; @@ -100,15 +116,20 @@ export const folderCommitServiceFactory = ({ tx?: Knex; }) => { let latestCommitId = folderCommitId; + const latestCheckpoint = await folderCheckpointDAL.findLatestByFolderId(folderId, tx); if (!latestCommitId) { - latestCommitId = (await folderCheckpointDAL.findLatestByFolderId(folderId, tx))?.folderCommitId; + latestCommitId = (await folderCommitDAL.findLatestCommit(folderId, tx))?.id; } if (!latestCommitId) { throw new BadRequestError({ message: "Latest commit ID not found" }); return; } - if (!force) { - const commitsSinceLastCheckpoint = await folderCommitDAL.getNumberOfCommitsSince(folderId, latestCommitId, tx); + if (!force && latestCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getNumberOfCommitsSince( + folderId, + latestCheckpoint.folderCommitId, + tx + ); if (commitsSinceLastCheckpoint < Number(appCfg.CHECKPOINT_WINDOW)) { return; } @@ -129,6 +150,175 @@ export const folderCommitServiceFactory = ({ } }; + const reconstructFolderState = async ( + folderCommitId: string, + tx?: Knex + ): Promise<{ type: string; id: string; versionId: string }[]> => { + const targetCommit = await folderCommitDAL.findById(folderCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `Commit with ID ${folderCommitId} not found` }); + } + + const nearestCheckpoint = await folderCheckpointDAL.findNearestCheckpoint(folderCommitId, tx); + if (!nearestCheckpoint) { + throw new NotFoundError({ message: `Nearest checkpoint not found for commit ${folderCommitId}` }); + } + + const checkpointResources = await folderCheckpointResourcesDAL.findByCheckpointId(nearestCheckpoint.id, tx); + + const folderState: Record = {}; + + // Add all checkpoint resources to initial state + checkpointResources.forEach((resource) => { + if (resource.secretVersionId && resource.referencedSecretId) { + folderState[`secret-${resource.referencedSecretId}`] = { + type: "secret", + id: resource.referencedSecretId, + versionId: resource.secretVersionId + }; + } else if (resource.folderVersionId && resource.referencedFolderId) { + folderState[`folder-${resource.referencedFolderId}`] = { + type: "folder", + id: resource.referencedFolderId, + versionId: resource.folderVersionId + }; + } + }); + + const commitsToRecreate = await folderCommitDAL.findCommitsToRecreate( + targetCommit.folderId, + targetCommit.commitId, + nearestCheckpoint.commitId, + tx + ); + + // Process commits to recreate final state + for (const commit of commitsToRecreate) { + // eslint-disable-next-line no-continue + if (!commit.changes) continue; + + for (const change of commit.changes) { + if (change.secretVersionId && change.referencedSecretId) { + const key = `secret-${change.referencedSecretId}`; + + if (change.changeType.toLowerCase() === "add") { + folderState[key] = { + type: "secret", + id: change.referencedSecretId, + versionId: change.secretVersionId + }; + } else if (change.changeType.toLowerCase() === "delete") { + delete folderState[key]; + } + } else if (change.folderVersionId && change.referencedFolderId) { + const key = `folder-${change.referencedFolderId}`; + + if (change.changeType.toLowerCase() === "add") { + folderState[key] = { + type: "folder", + id: change.referencedFolderId, + versionId: change.folderVersionId + }; + } else if (change.changeType.toLowerCase() === "delete") { + delete folderState[key]; + } + } + } + } + return Object.values(folderState); + }; + + const compareFolderStates = async (currentCommitId: string, targetCommitId: string, tx?: Knex) => { + // Reconstruct state for both commits + const currentState = await reconstructFolderState(currentCommitId, tx); + const targetState = await reconstructFolderState(targetCommitId, tx); + + // Create lookup maps for easier comparison + const currentMap: Record = {}; + const targetMap: Record = {}; + + // Build lookup map for current state + currentState.forEach((resource) => { + const key = `${resource.type}-${resource.id}`; + currentMap[key] = resource; + }); + + // Build lookup map for target state + targetState.forEach((resource) => { + const key = `${resource.type}-${resource.id}`; + targetMap[key] = resource; + }); + + // Track differences + const differences: { + type: string; + id: string; + versionId: string; + changeType: "create" | "update" | "delete"; + }[] = []; + + // Find deletes and updates (resources in current but not in target, or with different versions) + Object.keys(currentMap).forEach((key) => { + const currentResource = currentMap[key]; + const targetResource = targetMap[key]; + + if (!targetResource) { + // Resource exists in current but not in target - it's a delete + differences.push({ + type: currentResource.type, + id: currentResource.id, + versionId: currentResource.versionId, + changeType: "delete" + }); + } else if (currentResource.versionId !== targetResource.versionId) { + // Resource exists in both but with different versions - it's an update + differences.push({ + type: targetResource.type, + id: targetResource.id, + versionId: targetResource.versionId, + changeType: "update" + }); + } + // If versions are the same, it's unchanged - exclude from result + }); + + // Find creates (resources in target but not in current) + Object.keys(targetMap).forEach((key) => { + if (!currentMap[key]) { + const targetResource = targetMap[key]; + differences.push({ + type: targetResource.type, + id: targetResource.id, + versionId: targetResource.versionId, + changeType: "create" + }); + } + }); + + return differences; + }; + + // Add a change to an existing commit + const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => { + try { + if (!data.secretVersionId && !data.folderVersionId) { + throw new BadRequestError({ message: "Either secretVersionId or folderVersionId must be provided" }); + } + + const commit = await folderCommitDAL.findById(data.folderCommitId, tx); + if (!commit) { + throw new NotFoundError({ message: `Commit with ID ${data.folderCommitId} not found` }); + } + + return await folderCommitChangesDAL.create(data, tx); + } catch (error) { + if (error instanceof NotFoundError || error instanceof BadRequestError) { + throw error; + } + throw new DatabaseError({ error, name: "AddCommitChange" }); + } + }; + const createCommit = async (data: TCreateCommitDTO, tx?: Knex) => { const metadata = data.actor.metadata || {}; try { @@ -159,34 +349,13 @@ export const folderCommitServiceFactory = ({ tx ); - await createFolderCheckpoint({ folderId: data.folderId, folderCommitId: newCommit.id, tx }); + await createFolderCheckpoint({ folderId: data.folderId, tx }); return newCommit; } catch (error) { throw new DatabaseError({ error, name: "CreateCommit" }); } }; - // Add a change to an existing commit - const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => { - try { - if (!data.secretVersionId && !data.folderVersionId) { - throw new BadRequestError({ message: "Either secretVersionId or folderVersionId must be provided" }); - } - - const commit = await folderCommitDAL.findById(data.folderCommitId, tx); - if (!commit) { - throw new NotFoundError({ message: `Commit with ID ${data.folderCommitId} not found` }); - } - - return await folderCommitChangesDAL.create(data, tx); - } catch (error) { - if (error instanceof NotFoundError || error instanceof BadRequestError) { - throw error; - } - throw new DatabaseError({ error, name: "AddCommitChange" }); - } - }; - // Retrieve a commit by ID const getCommitById = async (id: string, tx?: Knex) => { return folderCommitDAL.findById(id, tx); @@ -297,7 +466,8 @@ export const folderCommitServiceFactory = ({ getLatestTreeCheckpoint, initializeFolder, initializeProject, - createFolderCheckpoint + createFolderCheckpoint, + compareFolderStates }; };