From 549d388f592b67265ea0e344739f5be2cf2766c6 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 26 Jul 2024 15:08:23 +0530 Subject: [PATCH] feat: improved migration wizard to info user prerequisite check list --- .../secret-approval-request-dal.ts | 17 +- .../secret-approval-request-service.ts | 2 +- .../services/secret-snapshot/snapshot-dal.ts | 18 +- backend/src/server/routes/index.ts | 4 +- backend/src/services/secret/secret-queue.ts | 154 +++++++++--------- backend/src/services/secret/secret-service.ts | 6 +- .../src/components/v2/Checkbox/Checkbox.tsx | 7 +- .../views/SecretMainPage/SecretMainPage.tsx | 2 + .../SecretV2MigrationSection.tsx | 137 +++++++++++++++- 9 files changed, 254 insertions(+), 93 deletions(-) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts index ea5241188..e5b2ea8a4 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts @@ -489,11 +489,26 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => { } }; + const deleteByProjectId = async (projectId: string, tx?: Knex) => { + try { + const query = await (tx || db.replicaNode())(TableName.SecretApprovalRequest) + .join(TableName.SecretFolder, `${TableName.SecretApprovalRequest}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where({ projectId }) + .delete(); + + return query; + } catch (error) { + throw new DatabaseError({ error, name: "DeleteByProjectId" }); + } + }; + return { ...secretApprovalRequestOrm, findById, findProjectRequestCount, findByProjectId, - findByProjectIdBridgeSecretV2 + findByProjectIdBridgeSecretV2, + deleteByProjectId }; }; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 5157c78c0..6a8317dfe 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -1274,7 +1274,7 @@ export const secretApprovalRequestServiceFactory = ({ const commitsGroupByKey = groupBy(approvalCommits, (i) => i.key); if (tagIds.length) { - await secretApprovalRequestSecretDAL.insertApprovalSecretTags( + await secretApprovalRequestSecretDAL.insertApprovalSecretV2Tags( Object.keys(commitTagIds).flatMap((blindIndex) => commitTagIds[blindIndex] ? commitTagIds[blindIndex].map((tagId) => ({ diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index b0dec99fb..e59497368 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -743,7 +743,7 @@ export const snapshotDALFactory = (db: TDbClient) => { db.ref("envId").withSchema(TableName.SnapshotSecret).as("snapshotEnvId"), db.ref("id").withSchema(TableName.SecretVersionTag).as("secretVersionTagId"), db.ref("secret_versionsId").withSchema(TableName.SecretVersionTag).as("secretVersionTagSecretId"), - db.ref("secret_versionsId").withSchema(TableName.SecretVersionTag).as("secretVersionTagSecretTagId"), + db.ref("secret_tagsId").withSchema(TableName.SecretVersionTag).as("secretVersionTagSecretTagId"), db.raw( `DENSE_RANK() OVER (partition by ${TableName.Snapshot}."id" ORDER BY ${TableName.SecretVersion}."createdAt") as rank` ) @@ -789,6 +789,19 @@ export const snapshotDALFactory = (db: TDbClient) => { } }; + const deleteSnapshotsAboveLimit = async (folderId: string, n = 15, tx?: Knex) => { + try { + const query = await (tx || db.replicaNode())(TableName.Snapshot) + .orderBy(`${TableName.Snapshot}.createdAt`, "desc") + .where(`${TableName.Snapshot}.folderId`, folderId) + .offset(n) + .delete(); + return query; + } catch (error) { + throw new DatabaseError({ error, name: "DeleteSnapshotsAboveLimit" }); + } + }; + return { ...secretSnapshotOrm, findById, @@ -799,6 +812,7 @@ export const snapshotDALFactory = (db: TDbClient) => { findSecretSnapshotDataById, findSecretSnapshotV2DataById, pruneExcessSnapshots, - findNSecretV1SnapshotByFolderId + findNSecretV1SnapshotByFolderId, + deleteSnapshotsAboveLimit }; }; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index fded5cb57..023e13000 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -723,8 +723,8 @@ export const registerRoutes = async ( secretRotationDAL, integrationAuthDAL, snapshotDAL, - secretApprovalRequestSecretDAL, - snapshotSecretV2BridgeDAL + snapshotSecretV2BridgeDAL, + secretApprovalRequestDAL }); const secretImportService = secretImportServiceFactory({ licenseService, diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 87a24fdf1..f04055dc4 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -2,7 +2,7 @@ import { AxiosError } from "axios"; import { ProjectUpgradeStatus, ProjectVersion, TSecretSnapshotSecretsV2, TSecretVersionsV2 } from "@app/db/schemas"; -import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; +import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; import { TSecretRotationDALFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal"; import { TSnapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal"; import { TSnapshotSecretV2DALFactory } from "@app/ee/services/secret-snapshot/snapshot-secret-v2-dal"; @@ -76,11 +76,8 @@ type TSecretQueueFactoryDep = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; secretRotationDAL: Pick; - secretApprovalRequestSecretDAL: Pick< - TSecretApprovalRequestSecretDALFactory, - "findByProjectId" | "insertV2Bridge" | "insertApprovalSecretV2Tags" - >; - snapshotDAL: Pick; + secretApprovalRequestDAL: Pick; + snapshotDAL: Pick; snapshotSecretV2BridgeDAL: Pick; }; @@ -123,9 +120,9 @@ export const secretQueueFactory = ({ kmsService, secretVersionTagV2BridgeDAL, secretRotationDAL, - secretApprovalRequestSecretDAL, snapshotDAL, - snapshotSecretV2BridgeDAL + snapshotSecretV2BridgeDAL, + secretApprovalRequestDAL }: TSecretQueueFactoryDep) => { const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { const appCfg = getConfig(); @@ -792,11 +789,6 @@ export const secretQueueFactory = ({ QueueJobs.ProjectV3Migration, { projectId }, { - attempts: 2, - backoff: { - type: "exponential", - delay: 3000 - }, removeOnComplete: true, removeOnFail: true } @@ -805,12 +797,17 @@ export const secretQueueFactory = ({ queueService.start(QueueName.ProjectV3Migration, async (job) => { const { projectId } = job.data; - const { botKey, shouldUseSecretV2Bridge: isProjectUpgradedToV3 } = await projectBotService.getBotKey(projectId); - if (isProjectUpgradedToV3) { + const { + botKey, + shouldUseSecretV2Bridge: isProjectUpgradedToV3, + project + } = await projectBotService.getBotKey(projectId); + if (isProjectUpgradedToV3 || project.upgradeStatus === ProjectUpgradeStatus.InProgress) { return; } if (!botKey) throw new BadRequestError({ message: "Bot not found" }); await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.InProgress }); + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ projectId, type: KmsDataKey.SecretManager @@ -884,7 +881,8 @@ export const secretQueueFactory = ({ await secretV2BridgeDAL.upsertSecretReferences(secretReferences, tx); } - const snapshots = await snapshotDAL.findNSecretV1SnapshotByFolderId(folderId, 10, tx); + const SNAPSHOT_BATCH_SIZE = 15; + const snapshots = await snapshotDAL.findNSecretV1SnapshotByFolderId(folderId, SNAPSHOT_BATCH_SIZE, tx); const projectV3SecretVersionsGroupById: Record = {}; const projectV3SecretVersionTags: { secret_versions_v2Id: string; secret_tagsId: string }[] = []; const projectV3SnapshotSecrets: Omit[] = []; @@ -959,6 +957,7 @@ export const secretQueueFactory = ({ if (projectV3SnapshotSecrets.length) { await snapshotSecretV2BridgeDAL.insertMany(projectV3SnapshotSecrets, tx); } + await snapshotDAL.deleteSnapshotsAboveLimit(folderId, SNAPSHOT_BATCH_SIZE, tx); } /* * Secret Tag Migration @@ -1056,67 +1055,70 @@ export const secretQueueFactory = ({ ); /* - * approvals + * approvals: we will delete all approvals this is because some secret versions may not be added yet + * Thus doesn't make sense for rest to be there * */ - const projectV1ApprovalSecrets = await secretApprovalRequestSecretDAL.findByProjectId(projectId); - if (projectV1ApprovalSecrets.length) { - await secretApprovalRequestSecretDAL.insertV2Bridge( - projectV1ApprovalSecrets.map((el) => { - const key = decryptSymmetric128BitHexKeyUTF8({ - ciphertext: el.secretKeyCiphertext, - iv: el.secretKeyIV, - tag: el.secretKeyTag, - key: botKey - }); - const value = decryptSymmetric128BitHexKeyUTF8({ - ciphertext: el.secretValueCiphertext, - iv: el.secretValueIV, - tag: el.secretValueTag, - key: botKey - }); - const comment = - el.secretCommentCiphertext && el.secretCommentTag && el.secretCommentIV - ? decryptSymmetric128BitHexKeyUTF8({ - ciphertext: el.secretCommentCiphertext, - iv: el.secretCommentIV, - tag: el.secretCommentTag, - key: botKey - }) - : ""; - const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob; - const encryptedComment = comment - ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob - : null; - return { - id: el.id, - createdAt: el.createdAt, - updatedAt: el.updatedAt, - skipMultilineEncoding: el.skipMultilineEncoding, - encryptedComment, - encryptedValue, - key, - version: el.version, - metadata: el.metadata, - reminderNote: el.secretReminderNote, - reminderRepeatDays: el.secretReminderRepeatDays, - requestId: el.requestId, - op: el.op, - secretId: el.secretId, - secretVersion: el.secretVersion - }; - }), - tx - ); - } - const projectV1SecretApprovalSecretTags = projectV1ApprovalSecrets.flatMap((el) => - el.tags.map((tag) => ({ - secretId: tag.secretApprovalTagSecretId, - tagId: tag.secretApprovalTagId - })) - ); - if (projectV1SecretApprovalSecretTags.length) { - await secretApprovalRequestSecretDAL.insertApprovalSecretV2Tags(projectV1SecretApprovalSecretTags, tx); - } + await secretApprovalRequestDAL.deleteByProjectId(projectId, tx); + // const projectV1ApprovalSecrets = await secretApprovalRequestSecretDAL.findByProjectId(projectId); + // if (projectV1ApprovalSecrets.length) { + // await secretApprovalRequestSecretDAL.insertV2Bridge( + // projectV1ApprovalSecrets.map((el) => { + // const key = decryptSymmetric128BitHexKeyUTF8({ + // ciphertext: el.secretKeyCiphertext, + // iv: el.secretKeyIV, + // tag: el.secretKeyTag, + // key: botKey + // }); + // const value = decryptSymmetric128BitHexKeyUTF8({ + // ciphertext: el.secretValueCiphertext, + // iv: el.secretValueIV, + // tag: el.secretValueTag, + // key: botKey + // }); + // const comment = + // el.secretCommentCiphertext && el.secretCommentTag && el.secretCommentIV + // ? decryptSymmetric128BitHexKeyUTF8({ + // ciphertext: el.secretCommentCiphertext, + // iv: el.secretCommentIV, + // tag: el.secretCommentTag, + // key: botKey + // }) + // : ""; + // const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob; + // const encryptedComment = comment + // ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob + // : null; + // return { + // id: el.id, + // createdAt: el.createdAt, + // updatedAt: el.updatedAt, + // skipMultilineEncoding: el.skipMultilineEncoding, + // encryptedComment, + // encryptedValue, + // key, + // version: el.version, + // metadata: el.metadata, + // reminderNote: el.secretReminderNote, + // reminderRepeatDays: el.secretReminderRepeatDays, + // requestId: el.requestId, + // op: el.op, + // secretId: el.secretId, + // secretVersion: el.secretVersion + // }; + // }), + // tx + // ); + // } + // const projectV1SecretApprovalSecretTags = projectV1ApprovalSecrets.flatMap((el) => + // el.tags.map((tag) => ({ + // secretId: tag.secretApprovalTagSecretId, + // tagId: tag.secretApprovalTagId + // })) + // ); + // if (projectV1SecretApprovalSecretTags.length) { + // await secretApprovalRequestSecretDAL.insertApprovalSecretV2Tags(projectV1SecretApprovalSecretTags, tx); + // } + await projectDAL.updateById(projectId, { upgradeStatus: null, version: ProjectVersion.V3 }, tx); }); }); diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 36e8db64a..3a14f0e30 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -4,6 +4,7 @@ import { ForbiddenError, subject } from "@casl/ability"; import { ProjectMembershipRole, + ProjectUpgradeStatus, SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, @@ -2666,8 +2667,11 @@ export const secretServiceFactory = ({ if (!hasRole(ProjectMembershipRole.Admin)) throw new BadRequestError({ message: "Only admins are allowed to take this action" }); - const { shouldUseSecretV2Bridge: isProjectV3 } = await projectBotService.getBotKey(projectId); + const { shouldUseSecretV2Bridge: isProjectV3, project } = await projectBotService.getBotKey(projectId); if (isProjectV3) throw new BadRequestError({ message: "project is already in v3" }); + if (project.upgradeStatus === ProjectUpgradeStatus.InProgress) + throw new BadRequestError({ message: "project is upgrading" }); + await secretQueueService.startSecretV2Migration(projectId); return { message: "Migrating project to new KMS architecture" }; }; diff --git a/frontend/src/components/v2/Checkbox/Checkbox.tsx b/frontend/src/components/v2/Checkbox/Checkbox.tsx index 0e4581372..8c7472b26 100644 --- a/frontend/src/components/v2/Checkbox/Checkbox.tsx +++ b/frontend/src/components/v2/Checkbox/Checkbox.tsx @@ -14,6 +14,7 @@ export type CheckboxProps = Omit< isChecked?: boolean; isRequired?: boolean; checkIndicatorBg?: string | undefined; + isError?: boolean; }; export const Checkbox = ({ @@ -24,6 +25,7 @@ export const Checkbox = ({ isDisabled, isRequired, checkIndicatorBg, + isError, ...props }: CheckboxProps): JSX.Element => { return ( @@ -46,7 +48,10 @@ export const Checkbox = ({ -