diff --git a/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts b/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts index 7ad865b10..5c254afcd 100644 --- a/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts +++ b/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts @@ -1,12 +1,15 @@ import { Knex } from "knex"; +import { inMemoryKeyStore } from "@app/keystore/memory"; + import { ProjectType, TableName } from "../schemas"; import { getMigrationPITServices } from "./utils/services"; export async function up(knex: Knex): Promise { const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); if (hasFolderCommitTable) { - const { folderCommitService } = await getMigrationPITServices({ db: knex }); + const keyStore = inMemoryKeyStore(); + const { folderCommitService } = await getMigrationPITServices({ db: knex, keyStore }); const projects = await knex(TableName.Project).where({ version: 3, type: ProjectType.SecretManager }).select("id"); await knex.transaction(async (tx) => { for (const project of projects) { diff --git a/backend/src/db/migrations/utils/services.ts b/backend/src/db/migrations/utils/services.ts index 610112618..1ee81536c 100644 --- a/backend/src/db/migrations/utils/services.ts +++ b/backend/src/db/migrations/utils/services.ts @@ -18,6 +18,7 @@ import { orgDALFactory } from "@app/services/org/org-dal"; import { projectDALFactory } from "@app/services/project/project-dal"; import { secretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal"; +import { secretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; import { secretVersionV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; import { userDALFactory } from "@app/services/user/user-dal"; @@ -62,7 +63,7 @@ export const getMigrationEncryptionServices = async ({ envConfig, db, keyStore } return { kmsService }; }; -export const getMigrationPITServices = async ({ db }: { db: Knex }) => { +export const getMigrationPITServices = async ({ db, keyStore }: { db: Knex; keyStore: TKeyStoreFactory }) => { const projectDAL = projectDALFactory(db); const folderCommitDAL = folderCommitDALFactory(db); const folderCommitChangesDAL = folderCommitChangesDALFactory(db); @@ -74,6 +75,7 @@ export const getMigrationPITServices = async ({ db }: { db: Knex }) => { const folderVersionDAL = secretFolderVersionDALFactory(db); const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db); const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); + const secretV2BridgeDAL = secretV2BridgeDALFactory({ db, keyStore }); const folderCommitService = folderCommitServiceFactory({ folderCommitDAL, @@ -86,7 +88,8 @@ export const getMigrationPITServices = async ({ db }: { db: Knex }) => { folderVersionDAL, secretVersionV2BridgeDAL, projectDAL, - folderCheckpointResourcesDAL + folderCheckpointResourcesDAL, + secretV2BridgeDAL }); return { folderCommitService }; diff --git a/backend/src/ee/routes/v1/pit-router.ts b/backend/src/ee/routes/v1/pit-router.ts index 0045a7efe..9abef5382 100644 --- a/backend/src/ee/routes/v1/pit-router.ts +++ b/backend/src/ee/routes/v1/pit-router.ts @@ -24,4 +24,38 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { return backup; } }); + + server.route({ + method: "POST", + url: "/rollback", + config: { + rateLimit: readLimit + }, + schema: { + body: z.object({ + fromCommit: z.string().trim(), + toCommit: z.string().trim(), + folderId: z.string().trim(), + projectId: z.string().trim() + }), + response: { + 200: z.any() + } + }, + handler: async (req) => { + const diff = await server.services.folderCommit.compareFolderStates(req.body.fromCommit, req.body.toCommit); + const response = await server.services.folderCommit.applyFolderStateDifferences( + diff, + { + actorType: req.permission?.type || "PLATFORM", + actorId: req.permission?.id, + message: "Rollback to previous commit" + }, + req.body.folderId, + req.body.projectId + ); + + return response; + } + }); }; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 4367f69c5..38c5f5dd0 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -574,7 +574,8 @@ export const registerRoutes = async ( folderVersionDAL, secretVersionV2BridgeDAL, projectDAL, - folderCheckpointResourcesDAL + folderCheckpointResourcesDAL, + secretV2BridgeDAL }); const scimService = scimServiceFactory({ licenseService, diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts index c39aaa7e7..e40847d06 100644 --- a/backend/src/services/folder-commit/folder-commit-service.ts +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -14,6 +14,7 @@ import { TIdentityDALFactory } from "../identity/identity-dal"; import { TProjectDALFactory } from "../project/project-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; import { TUserDALFactory } from "../user/user-dal"; import { TFolderCommitDALFactory } from "./folder-commit-dal"; @@ -38,14 +39,33 @@ type TFolderCommitServiceFactoryDep = { folderTreeCheckpointDAL: Pick; userDAL: Pick; identityDAL: Pick; - folderDAL: Pick; + folderDAL: Pick< + TSecretFolderDALFactory, + "findByParentId" | "findByProjectId" | "deleteById" | "create" | "updateById" | "update" + >; folderVersionDAL: Pick< TSecretFolderVersionDALFactory, - "findLatestFolderVersions" | "findById" | "deleteById" | "create" | "updateById" + | "findLatestFolderVersions" + | "findById" + | "deleteById" + | "create" + | "updateById" + | "find" + | "findByIdsWithLatestVersion" >; secretVersionV2BridgeDAL: Pick< TSecretVersionV2DALFactory, - "findLatestVersionByFolderId" | "findById" | "deleteById" | "create" | "updateById" + | "findLatestVersionByFolderId" + | "findById" + | "deleteById" + | "create" + | "updateById" + | "find" + | "findByIdsWithLatestVersion" + >; + secretV2BridgeDAL: Pick< + TSecretV2BridgeDALFactory, + "deleteById" | "create" | "updateById" | "update" | "insertMany" | "invalidateSecretCacheByProjectId" >; projectDAL: Pick; }; @@ -85,7 +105,8 @@ export const folderCommitServiceFactory = ({ folderDAL, folderVersionDAL, secretVersionV2BridgeDAL, - projectDAL + projectDAL, + secretV2BridgeDAL }: TFolderCommitServiceFactoryDep) => { const appCfg = getConfig(); @@ -356,6 +377,261 @@ export const folderCommitServiceFactory = ({ } }; + const applyFolderStateDifferences = async ( + differences: Array<{ + type: string; + id: string; + versionId: string; + oldVersionId?: string; + changeType: "create" | "update" | "delete"; + }>, + actorInfo: { + actorType: string; + actorId?: string; + message?: string; + }, + folderId: string, + projectId: string + ) => { + let result = {}; + await folderCommitDAL.transaction(async (tx) => { + // Group differences by type for more efficient processing + const secretChanges = differences.filter((diff) => diff.type === "secret"); + const folderChanges = differences.filter((diff) => diff.type === "folder"); + + const secretVersions = await secretVersionV2BridgeDAL.findByIdsWithLatestVersion( + folderId, + secretChanges.map((diff) => diff.id), + secretChanges.map((diff) => diff.versionId) + ); + const folderVersions = await folderVersionDAL.findByIdsWithLatestVersion( + folderChanges.map((diff) => diff.id), + folderChanges.map((diff) => diff.versionId) + ); + + // Track all changes for commit recording + const commitChanges = []; + + // Process secret changes + for (const change of secretChanges) { + const secretVersion = secretVersions[change.id]; + switch (change.changeType) { + case "create": + if (secretVersion) { + const newSecret = [ + { + id: change.id, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + version: secretVersion.version + 1, + type: secretVersion.type, + key: secretVersion.key, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + encryptedValue: secretVersion.encryptedValue, + encryptedComment: secretVersion.encryptedComment, + userId: secretVersion.userId, + metadata: secretVersion.metadata, + folderId + } + ]; + await secretV2BridgeDAL.insertMany(newSecret, tx); + + const newVersion = await secretVersionV2BridgeDAL.create( + { + folderId, + secretId: secretVersion.secretId, + version: secretVersion.version + 1, + encryptedValue: secretVersion.encryptedValue, + key: secretVersion.key, + encryptedComment: secretVersion.encryptedComment, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + userId: secretVersion.userId, + metadata: secretVersion.metadata, + actorType: actorInfo.actorType, + envId: secretVersion.envId, + ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), + ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) + }, + tx + ); + + commitChanges.push({ + type: "add", + secretVersionId: newVersion.id + }); + } + break; + + case "update": + // Update secret to specific version + if (secretVersion) { + await secretV2BridgeDAL.updateById( + change.id, + { + skipMultilineEncoding: secretVersion?.skipMultilineEncoding, + version: secretVersion?.version, + type: secretVersion?.type, + key: secretVersion?.key, + reminderNote: secretVersion?.reminderNote, + reminderRepeatDays: secretVersion?.reminderRepeatDays, + encryptedValue: secretVersion?.encryptedValue, + encryptedComment: secretVersion?.encryptedComment, + userId: secretVersion?.userId, + metadata: secretVersion?.metadata + }, + tx + ); + + const newVersion = await secretVersionV2BridgeDAL.create( + { + version: secretVersion.version + 1, + encryptedValue: secretVersion.encryptedValue, + key: secretVersion.key, + encryptedComment: secretVersion.encryptedComment, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + userId: secretVersion.userId, + metadata: secretVersion.metadata, + actorType: actorInfo.actorType, + envId: secretVersion.envId, + folderId, + secretId: secretVersion.secretId, + ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), + ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) + }, + tx + ); + + commitChanges.push({ + type: "add", + secretVersionId: newVersion.id + }); + } + break; + + case "delete": + await secretV2BridgeDAL.deleteById(change.id, tx); + + commitChanges.push({ + type: "delete", + secretVersionId: change.versionId + }); + break; + + default: + throw new BadRequestError({ message: `Unknown change type: ${change.changeType as string}` }); + } + } + + // process folder changes + for (const change of folderChanges) { + const folderVersion = folderVersions[change.id]; + switch (change.changeType) { + case "create": + // Add new folder + if (folderVersion) { + const newFolder = { + id: change.id, + parentId: folderId, + envId: folderVersion.envId, + version: (folderVersion.version || 1) + 1, + name: folderVersion.name + }; + await folderDAL.create(newFolder, tx); + + const newFolderVersion = await folderVersionDAL.create( + { + folderId: change.id, + version: (folderVersion.version || 1) + 1, + name: folderVersion.name, + envId: folderVersion.envId + }, + tx + ); + + commitChanges.push({ + type: "add", + folderVersionId: newFolderVersion.id + }); + } + break; + + case "update": + // Update folder to specific version + if (change.versionId) { + await folderVersionDAL.findById(change.versionId, tx).then(async (versionDetails) => { + if (versionDetails) { + await folderDAL.updateById( + change.id, + { + parentId: folderId, + envId: versionDetails.envId, + version: (versionDetails.version || 1) + 1, + name: versionDetails.name + }, + tx + ); + + const newFolderVersion = await folderVersionDAL.create( + { + folderId: change.id, + version: (versionDetails.version || 1) + 1, + name: versionDetails.name, + envId: versionDetails.envId + }, + tx + ); + + commitChanges.push({ + type: "add", + folderVersionId: newFolderVersion.id + }); + } + }); + } + break; + + case "delete": + await folderDAL.deleteById(change.id, tx); + + commitChanges.push({ + type: "delete", + folderVersionId: change.versionId + }); + break; + + default: + throw new BadRequestError({ message: `Unknown change type: ${change.changeType as string}` }); + } + } + + await createCommit( + { + actor: { + type: actorInfo.actorType, + metadata: { id: actorInfo.actorId } + }, + message: actorInfo.message || "Rolled back folder state", + folderId, + changes: commitChanges + }, + tx + ); + + result = { + secretChangesCount: secretChanges.length, + folderChangesCount: folderChanges.length, + totalChanges: differences.length + }; + await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId); + }); + + return result; + }; + // Retrieve a commit by ID const getCommitById = async (id: string, tx?: Knex) => { return folderCommitDAL.findById(id, tx); @@ -467,7 +743,8 @@ export const folderCommitServiceFactory = ({ initializeFolder, initializeProject, createFolderCheckpoint, - compareFolderStates + compareFolderStates, + applyFolderStateDifferences }; }; diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index 437b1e1eb..56ac4f1f7 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -96,5 +96,80 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret folder versions completed`); }; - return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId, pruneExcessVersions }; + const findByIdsWithLatestVersion = async (folderIds: string[], versionIds?: string[], tx?: Knex) => { + try { + if (!folderIds.length && (!versionIds || !versionIds.length)) return {}; + + const knexInstance = tx || db.replicaNode(); + let allDocs: Array = []; + + // Process latest versions for folderIds + if (folderIds.length) { + const latestVersions: Array = await knexInstance( + TableName.SecretFolderVersion + ) + .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds) + .join( + knexInstance(TableName.SecretFolderVersion) + .groupBy("folderId") + .max("version") + .select("folderId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretFolderVersion}.folderId`, "latestVersion.folderId").andOn( + `${TableName.SecretFolderVersion}.version`, + "latestVersion.max" + ); + } + ); + + allDocs = [...allDocs, ...latestVersions]; + } + + // Process specific versions by versionIds + if (versionIds && versionIds.length) { + // Get the specific versions + const specificVersions = await knexInstance(TableName.SecretFolderVersion).whereIn("id", versionIds); + + // Get the folderIds from these versions + const specificFolderIds = [...new Set(specificVersions.map((v) => v.folderId).filter(Boolean))]; + + // Get max versions for these folderIds + const maxVersionsQuery = (await knexInstance(TableName.SecretFolderVersion) + .whereIn("folderId", specificFolderIds) + .groupBy("folderId") + .select("folderId") + .max("version as max")) as Array<{ folderId: string; max: number }>; + + // Create a lookup map for max versions + const maxVersionMap = maxVersionsQuery.reduce>((acc, item) => { + acc[item.folderId] = item.max; + return acc; + }, {}); + + // Add max version to each specific version + const specificVersionsWithMax = specificVersions.map((version) => ({ + ...version, + max: maxVersionMap[version.folderId] + })); + + allDocs = [...allDocs, ...specificVersionsWithMax]; + } + + return allDocs.reduce>( + (prev, curr) => ({ ...prev, [curr.folderId || ""]: curr }), + {} + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByIdsWithLatestVersion" }); + } + }; + + return { + ...secretFolderVerOrm, + findLatestFolderVersions, + findLatestVersionByFolderId, + pruneExcessVersions, + findByIdsWithLatestVersion + }; }; diff --git a/backend/src/services/secret-v2-bridge/secret-version-dal.ts b/backend/src/services/secret-v2-bridge/secret-version-dal.ts index 2f9574ac9..291fec22f 100644 --- a/backend/src/services/secret-v2-bridge/secret-version-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-version-dal.ts @@ -138,7 +138,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { {} ); } catch (error) { - throw new DatabaseError({ error, name: "FindLatestVersinMany" }); + throw new DatabaseError({ error, name: "FindLatestVersionMany" }); } }; @@ -261,6 +261,84 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { } }; + const findByIdsWithLatestVersion = async ( + folderId: string, + secretIds: string[], + versionIds?: string[], + tx?: Knex + ) => { + try { + if (!secretIds.length && (!versionIds || !versionIds.length)) return {}; + + const knexInstance = tx || db.replicaNode(); + let allDocs: Array = []; + + // Process latest versions for secretIds + if (secretIds.length) { + const latestVersions: Array = await knexInstance(TableName.SecretVersionV2) + .where("folderId", folderId) + .whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds) + .join( + knexInstance(TableName.SecretVersionV2) + .groupBy("secretId") + .max("version") + .select("secretId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretVersionV2}.secretId`, "latestVersion.secretId").andOn( + `${TableName.SecretVersionV2}.version`, + "latestVersion.max" + ); + } + ); + + allDocs = [...allDocs, ...latestVersions]; + } + + // Process specific versions by versionIds + if (versionIds && versionIds.length) { + // Get the specific versions + const specificVersions = await knexInstance(TableName.SecretVersionV2) + .where("folderId", folderId) + .whereIn("id", versionIds); + + // Get the secretIds from these versions + const specificSecretIds = [...new Set(specificVersions.map((v) => v.secretId).filter(Boolean))]; + + // Get max versions for these secretIds + const maxVersionsQuery = (await knexInstance(TableName.SecretVersionV2) + .whereIn("secretId", specificSecretIds) + .groupBy("secretId") + .select("secretId") + .max("version as max")) as Array<{ secretId: string; max: number }>; + + // Create a lookup map for max versions + const maxVersionMap = maxVersionsQuery.reduce( + (acc, item) => { + acc[item.secretId] = item.max; + return acc; + }, + {} as Record + ); + + // Add max version to each specific version + const specificVersionsWithMax = specificVersions.map((version) => ({ + ...version, + max: maxVersionMap[version.secretId] + })); + + allDocs = [...allDocs, ...specificVersionsWithMax]; + } + + return allDocs.reduce>( + (prev, curr) => ({ ...prev, [curr.secretId || ""]: curr }), + {} + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByIdsWithLatestVersion" }); + } + }; + return { ...secretVersionV2Orm, pruneExcessVersions, @@ -268,6 +346,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { bulkUpdate, findLatestVersionByFolderId, findVersionsBySecretIdWithActors, - findBySecretId + findBySecretId, + findByIdsWithLatestVersion }; };