From d36d0784ca96b0ff67bc24663aa7b00d231ae730 Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Mon, 26 May 2025 21:51:43 -0300 Subject: [PATCH] PIT: Add delete commit for cascade deletion --- .../secret-snapshot-service.ts | 4 +- .../folder-commit/folder-commit-service.ts | 112 +++++++++++++++++- .../secret-folder/secret-folder-dal.ts | 1 + .../secret-folder/secret-folder-service.ts | 3 +- .../secret-folder-version-dal.ts | 15 ++- .../ProjectRoleModifySection.utils.tsx | 2 +- 6 files changed, 130 insertions(+), 7 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index d245578c5..6321f6054 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -415,6 +415,7 @@ export const secretSnapshotServiceFactory = ({ secretVersionId?: string; folderVersionId?: string; isUpdate?: boolean; + folderId?: string; }[] = []; // this will remove all secrets in current folder except rotated secrets which we ignore @@ -625,7 +626,8 @@ export const secretSnapshotServiceFactory = ({ // Folder was only deleted commitChanges.push({ type: CommitType.DELETE, - folderVersionId: deletedInfo.versionId + folderVersionId: deletedInfo.versionId, + folderId: deletedInfo.id }); } }); diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts index 42eeab31b..4aa98a250 100644 --- a/backend/src/services/folder-commit/folder-commit-service.ts +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -30,7 +30,7 @@ import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metad import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; -import * as secretV2BridgeDal from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; import { TUserDALFactory } from "../user/user-dal"; import { TFolderCommitDALFactory } from "./folder-commit-dal"; @@ -68,6 +68,7 @@ type TCreateCommitDTO = { secretVersionId?: string; folderVersionId?: string; isUpdate?: boolean; + folderId?: string; }[]; }; @@ -143,7 +144,7 @@ type TFolderCommitServiceFactoryDep = { folderDAL: TSecretFolderDALFactory; folderVersionDAL: TSecretFolderVersionDALFactory; secretVersionV2BridgeDAL: TSecretVersionV2DALFactory; - secretV2BridgeDAL: secretV2BridgeDal.TSecretV2BridgeDALFactory; + secretV2BridgeDAL: TSecretV2BridgeDALFactory; projectDAL: Pick; folderCommitQueueService?: Pick< TFolderCommitQueueServiceFactory, @@ -676,6 +677,95 @@ export const folderCommitServiceFactory = ({ } }; + const createDeleteCommitForNestedFolders = async ({ + folderId, + actorMetadata, + actorType, + envId, + parentFolderName, + step = 1, + tx + }: { + folderId: string; + actorMetadata: Record; + actorType: string; + envId: string; + parentFolderName: string; + step?: number; + tx?: Knex; + }) => { + if (step > 20) { + logger.info(`createDeleteCommitForNestedFolders - Max step reached for folder ${folderId}`); + return; + } + logger.info(`Creating delete commit for nested folders ${folderId}`); + const folderVersion = await folderVersionDAL.findLatestVersion(folderId, tx); + if (!folderVersion) { + logger.info(`No folder version found for ${folderId}`); + return; + } + const lastFolderCommit = await folderCommitDAL.findLatestCommit(folderId, undefined, tx); + if (!lastFolderCommit) { + logger.info(`No commit found for folder ${folderId}`); + return; + } + const folderState = await reconstructFolderState(lastFolderCommit.id, tx); + const changes = folderState.map((resource) => ({ + type: ChangeType.DELETE, + folderId: resource.id, + folderName: resource.folderName, + secretVersionId: resource.type === ResourceType.SECRET ? resource.versionId : undefined, + folderVersionId: resource.type === ResourceType.FOLDER ? resource.versionId : undefined, + secretKey: resource.secretKey + })); + logger.info(`Found ${changes.length} changes for ${folderId}`); + + await Promise.all( + changes + .filter((change) => change.type === ChangeType.DELETE && change.folderVersionId) + .map(async (change) => { + await createDeleteCommitForNestedFolders({ + folderId: change.folderId, + actorMetadata, + actorType, + envId, + parentFolderName: folderVersion.name, + step: step + 1, + tx + }); + }) + ); + + const newCommit = await folderCommitDAL.create( + { + actorMetadata, + actorType, + message: `Parent folder ${parentFolderName} deleted`, + folderId, + envId + }, + tx + ); + + const batchSize = 500; + const chunks = chunkArray(changes, batchSize); + + await Promise.all( + chunks.map(async (chunk) => { + await folderCommitChangesDAL.insertMany( + chunk.map((change) => ({ + folderCommitId: newCommit.id, + changeType: CommitType.DELETE, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId, + isUpdate: false + })), + tx + ); + }) + ); + }; + /** * Creates a new commit with the provided changes */ @@ -698,6 +788,21 @@ export const folderCommitServiceFactory = ({ throw new NotFoundError({ message: `Folder with ID ${data.folderId} not found` }); } + await Promise.all( + data.changes.map(async (change) => { + if (change.type === ChangeType.DELETE && change.folderId) { + await createDeleteCommitForNestedFolders({ + folderId: change.folderId, + actorMetadata: metadata, + actorType: data.actor.type, + envId: folder.envId, + parentFolderName: folder.name, + tx + }); + } + }) + ); + const newCommit = await folderCommitDAL.create( { actorMetadata: metadata, @@ -1102,7 +1207,8 @@ export const folderCommitServiceFactory = ({ commitChanges.push({ type: ChangeType.DELETE, - folderVersionId: change.versionId + folderVersionId: change.versionId, + folderId: change.id }); break; diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index c03324ab9..7dfeaddcf 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -537,6 +537,7 @@ export const secretFolderDALFactory = (db: TDbClient) => { try { const folders = await (tx || db.replicaNode())(TableName.SecretFolder) .where({ parentId }) + .andWhere({ isReserved: false }) .select(selectAllTableCols(TableName.SecretFolder)); return folders; } catch (error) { diff --git a/backend/src/services/secret-folder/secret-folder-service.ts b/backend/src/services/secret-folder/secret-folder-service.ts index 354f7e7da..a93e4717f 100644 --- a/backend/src/services/secret-folder/secret-folder-service.ts +++ b/backend/src/services/secret-folder/secret-folder-service.ts @@ -478,7 +478,8 @@ export const secretFolderServiceFactory = ({ changes: [ { type: CommitType.DELETE, - folderVersionId: folderVersions[doc.id].id + folderVersionId: folderVersions[doc.id].id, + folderId: doc.id } ] }, diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index b2c169a5a..46ff49692 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -180,11 +180,24 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { } }; + const findLatestVersion = async (folderId: string, tx?: Knex) => { + try { + const doc = await (tx || db.replicaNode())(TableName.SecretFolderVersion) + .where(`${TableName.SecretFolderVersion}.folderId`, folderId) + .select(selectAllTableCols(TableName.SecretFolderVersion)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "findLatestVersion" }); + } + }; + return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId, pruneExcessVersions, - findByIdsWithLatestVersion + findByIdsWithLatestVersion, + findLatestVersion }; }; diff --git a/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx b/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx index 2ee383b2c..59bb28714 100644 --- a/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx +++ b/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx @@ -12,8 +12,8 @@ import { } from "@app/context"; import { PermissionConditionOperators, - ProjectPermissionCommitsActions, ProjectPermissionApprovalActions, + ProjectPermissionCommitsActions, ProjectPermissionDynamicSecretActions, ProjectPermissionGroupActions, ProjectPermissionIdentityActions,