From ab88e6c4144b6d62bed9b855535037f8563c8af5 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 24 Jul 2024 20:56:27 +0530 Subject: [PATCH] checkpoint --- backend/src/ee/routes/v1/project-router.ts | 31 +++++ backend/src/queue/queue-service.ts | 10 +- .../secret-folder/secret-folder-dal.ts | 24 +++- backend/src/services/secret/secret-queue.ts | 106 ++++++++++++++++- backend/src/services/secret/secret-service.ts | 33 +++++- backend/src/services/secret/secret-types.ts | 2 + frontend/src/hooks/api/workspace/index.tsx | 3 +- .../src/hooks/api/workspace/mutations.tsx | 12 ++ frontend/src/hooks/api/workspace/queries.tsx | 8 +- frontend/src/hooks/api/workspace/types.ts | 3 +- frontend/src/layouts/AppLayout/AppLayout.tsx | 2 - .../SecretOverviewPage/SecretOverviewPage.tsx | 6 +- .../ProjectIndexSecretsSection.tsx | 109 ------------------ .../ProjectIndexSecretsSection/index.tsx | 1 - .../SecretV2MigrationSection.tsx | 90 +++++++++++++++ .../SecretV2MigrationSection/index.tsx | 1 + 16 files changed, 315 insertions(+), 126 deletions(-) delete mode 100644 frontend/src/views/SecretOverviewPage/components/ProjectIndexSecretsSection/ProjectIndexSecretsSection.tsx delete mode 100644 frontend/src/views/SecretOverviewPage/components/ProjectIndexSecretsSection/index.tsx create mode 100644 frontend/src/views/SecretOverviewPage/components/SecretV2MigrationSection/SecretV2MigrationSection.tsx create mode 100644 frontend/src/views/SecretOverviewPage/components/SecretV2MigrationSection/index.tsx diff --git a/backend/src/ee/routes/v1/project-router.ts b/backend/src/ee/routes/v1/project-router.ts index 7d0d5f3b3..aa32fcb5f 100644 --- a/backend/src/ee/routes/v1/project-router.ts +++ b/backend/src/ee/routes/v1/project-router.ts @@ -349,4 +349,35 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { return backup; } }); + + server.route({ + method: "POST", + url: "/:workspaceId/migrate-v3", + config: { + rateLimit: writeLimit + }, + schema: { + params: z.object({ + workspaceId: z.string().trim() + }), + + response: { + 200: z.object({ + message: z.string() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const migration = await server.services.secret.startSecretV2Migration({ + actor: req.permission.type, + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + projectId: req.params.workspaceId + }); + + return migration; + } + }); }; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index d51a8e683..1f58dfec3 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -25,7 +25,8 @@ export enum QueueName { DynamicSecretRevocation = "dynamic-secret-revocation", CaCrlRotation = "ca-crl-rotation", SecretReplication = "secret-replication", - SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication + SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication + ProjectV3Migration = "project-v3-migration" } export enum QueueJobs { @@ -44,7 +45,8 @@ export enum QueueJobs { DynamicSecretPruning = "dynamic-secret-pruning", CaCrlRotation = "ca-crl-rotation-job", SecretReplication = "secret-replication", - SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication + SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication + ProjectV3Migration = "project-v3-migration" } export type TQueueJobTypes = { @@ -136,6 +138,10 @@ export type TQueueJobTypes = { name: QueueJobs.SecretSync; payload: TSyncSecretsDTO; }; + [QueueName.ProjectV3Migration]: { + name: QueueJobs.ProjectV3Migration; + payload: { projectId: string }; + }; }; export type TQueueServiceFactory = ReturnType; diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index f9768ecc2..a2a01be68 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -331,6 +331,27 @@ export const secretFolderDALFactory = (db: TDbClient) => { } }; + // special query for project migration + const findByProjectId = async (projectId: string, tx?: Knex) => { + try { + const folders = await (tx || db.replicaNode())(TableName.SecretFolder) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .join(TableName.Project, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) + .select(selectAllTableCols(TableName.SecretFolder)) + .where({ projectId }) + .select( + db.ref("id").withSchema(TableName.Environment).as("envId"), + db.ref("slug").withSchema(TableName.Environment).as("envSlug"), + db.ref("name").withSchema(TableName.Environment).as("envName"), + db.ref("projectId").withSchema(TableName.Environment), + db.ref("version").withSchema(TableName.Project).as("projectVersion") + ); + return folders; + } catch (error) { + throw new DatabaseError({ error, name: "Find by id" }); + } + }; + return { ...secretFolderOrm, update, @@ -338,6 +359,7 @@ export const secretFolderDALFactory = (db: TDbClient) => { findById, findByManySecretPath, findSecretPathByFolderIds, - findClosestFolder + findClosestFolder, + findByProjectId }; }; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index e1c13496b..674455ba1 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -41,6 +41,7 @@ import { TRemoveSecretReminderDTO, TSyncSecretsDTO } from "./secret-types"; +import { ProjectUpgradeStatus, ProjectVersion } from "@app/db/schemas"; export type TSecretQueueFactory = ReturnType; type TSecretQueueFactoryDep = { @@ -52,7 +53,7 @@ type TSecretQueueFactoryDep = { secretDAL: TSecretDALFactory; secretImportDAL: Pick; webhookDAL: Pick; - projectEnvDAL: Pick; + projectEnvDAL: Pick; projectDAL: TProjectDALFactory; projectBotDAL: TProjectBotDALFactory; projectMembershipDAL: Pick; @@ -761,6 +762,108 @@ export const secretQueueFactory = ({ }); }); + const startSecretV2Migration = async (projectId: string) => { + await queueService.queue( + QueueName.ProjectV3Migration, + QueueJobs.ProjectV3Migration, + { projectId }, + { + attempts: 2, + backoff: { + type: "exponential", + delay: 3000 + }, + removeOnComplete: true, + removeOnFail: true + } + ); + }; + + const MIGRATION_BATCH_SIZE = 10000; + queueService.start(QueueName.ProjectV3Migration, async (job) => { + const { projectId } = job.data; + const { botKey, shouldUseSecretV2Bridge: isProjectUpgradedToV3 } = await projectBotService.getBotKey(projectId); + if (isProjectUpgradedToV3) { + return; + } + if (!botKey) throw new BadRequestError({ message: "Bot not found" }); + await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.InProgress }); + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + projectId, + type: KmsDataKey.SecretManager + }); + + const folders = await folderDAL.findByProjectId(projectId); + // except secret version and snapshot migrate rest of everything first in a transaction + await secretDAL.transaction(async (tx) => { + for (const folder of folders) { + const folderId = folder.id; + let projectV1Secrets; + do { + // eslint-disable-next-line no-await-in-loop + projectV1Secrets = await secretDAL.find({ folderId }, { limit: MIGRATION_BATCH_SIZE, tx }); + await secretV2BridgeDAL.insertMany( + projectV1Secrets.map((el) => { + const key = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretKeyCiphertext, + iv: el.secretKeyIV, + tag: el.secretKeyTag, + key: botKey + }); + const value = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag, + key: botKey + }); + const comment = + el.secretCommentCiphertext && el.secretCommentTag && el.secretCommentIV + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretCommentCiphertext, + iv: el.secretCommentIV, + tag: el.secretCommentTag, + key: botKey + }) + : ""; + const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob; + const encryptedComment = comment + ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob + : null; + return { + id: el.id, + createdAt: el.createdAt, + updatedAt: el.updatedAt, + skipMultilineEncoding: el.skipMultilineEncoding, + encryptedComment, + encryptedValue, + key, + version: el.version, + type: el.type, + userId: el.userId, + folderId: el.folderId, + metadata: el.metadata, + reminderNote: el.secretReminderNote, + reminderRepeatDays: el.secretReminderRepeatDays + }; + }), + tx + ); + projectV1Secrets = await secretDAL.delete({ folderId, $in: { id: projectV1Secrets.map((el) => el.id) } }, tx); + } while (projectV1Secrets.length > 0); + } + await projectDAL.updateById(projectId, { upgradeStatus: null, version: ProjectVersion.V3 }, tx); + }); + }); + + // eslint-disable-next-line + queueService.listen(QueueName.ProjectV3Migration, "failed", async (job, err) => { + if (job?.data) { + const { projectId } = job.data; + await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.Failed }); + logger.error(err, `Failed to migrate project to v3: ${projectId}`); + } + }); + queueService.listen(QueueName.IntegrationSync, "failed", (job, err) => { logger.error(err, "Failed to sync integration %s", job?.id); }); @@ -772,6 +875,7 @@ export const secretQueueFactory = ({ return { // depth is internal only field thus no need to make it available outside syncSecrets, + startSecretV2Migration, syncIntegrations, addSecretReminder, removeSecretReminder, diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 2ce7ed898..18fd7ff8d 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -68,6 +68,7 @@ import { TGetSecretsRawDTO, TGetSecretVersionsDTO, TMoveSecretsDTO, + TStartSecretsV2MigrationDTO, TUpdateBulkSecretDTO, TUpdateManySecretRawDTO, TUpdateSecretDTO, @@ -90,7 +91,10 @@ type TSecretServiceFactoryDep = { secretBlindIndexDAL: TSecretBlindIndexDALFactory; permissionService: Pick; snapshotService: Pick; - secretQueueService: Pick; + secretQueueService: Pick< + TSecretQueueFactory, + "syncSecrets" | "handleSecretReminder" | "removeSecretReminder" | "startSecretV2Migration" + >; projectBotService: Pick; secretImportDAL: Pick; secretVersionTagDAL: Pick; @@ -2638,6 +2642,30 @@ export const secretServiceFactory = ({ }; }; + const startSecretV2Migration = async ({ + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod + }: TStartSecretsV2MigrationDTO) => { + const { hasRole } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + if (!hasRole(ProjectMembershipRole.Admin)) + throw new BadRequestError({ message: "Only admins are allowed to take this action" }); + + const { shouldUseSecretV2Bridge: isProjectV3 } = await projectBotService.getBotKey(projectId); + if (isProjectV3) throw new BadRequestError({ message: "project is already in v3" }); + await secretQueueService.startSecretV2Migration(projectId); + return { message: "Migrating project to new KMS architecture" }; + }; + return { attachTags, detachTags, @@ -2659,6 +2687,7 @@ export const secretServiceFactory = ({ deleteManySecretsRaw, getSecretVersions, backfillSecretReferences, - moveSecrets + moveSecrets, + startSecretV2Migration }; }; diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index e6dcf19e2..99bde05f2 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -449,3 +449,5 @@ export enum SecretProtectionType { Approval = "approval", Direct = "direct" } + +export type TStartSecretsV2MigrationDTO = TProjectPermission; diff --git a/frontend/src/hooks/api/workspace/index.tsx b/frontend/src/hooks/api/workspace/index.tsx index f5e5855d4..f9f7f9d78 100644 --- a/frontend/src/hooks/api/workspace/index.tsx +++ b/frontend/src/hooks/api/workspace/index.tsx @@ -2,7 +2,8 @@ export { useAddGroupToWorkspace, useDeleteGroupFromWorkspace, useLeaveProject, - useUpdateGroupWorkspaceRole + useUpdateGroupWorkspaceRole, + useMigrateProjectToV3 } from "./mutations"; export { useAddIdentityToWorkspace, diff --git a/frontend/src/hooks/api/workspace/mutations.tsx b/frontend/src/hooks/api/workspace/mutations.tsx index 5aba02098..60948c9a7 100644 --- a/frontend/src/hooks/api/workspace/mutations.tsx +++ b/frontend/src/hooks/api/workspace/mutations.tsx @@ -74,3 +74,15 @@ export const useLeaveProject = () => { } }); }; + +export const useMigrateProjectToV3 = () => { + const queryClient = useQueryClient(); + return useMutation<{}, {}, { workspaceId: string }>({ + mutationFn: ({ workspaceId }) => { + return apiRequest.delete(`/api/v1/workspace/${workspaceId}/migrate-v3`); + }, + onSuccess: () => { + queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace); + } + }); +}; diff --git a/frontend/src/hooks/api/workspace/queries.tsx b/frontend/src/hooks/api/workspace/queries.tsx index 679f1bc7d..f3ffaf86e 100644 --- a/frontend/src/hooks/api/workspace/queries.tsx +++ b/frontend/src/hooks/api/workspace/queries.tsx @@ -150,11 +150,15 @@ export const useGetWorkspaceSecrets = (workspaceId: string) => { }); }; -export const useGetWorkspaceById = (workspaceId: string) => { +export const useGetWorkspaceById = ( + workspaceId: string, + dto?: { refetchInterval?: number | false } +) => { return useQuery({ queryKey: workspaceKeys.getWorkspaceById(workspaceId), queryFn: () => fetchWorkspaceById(workspaceId), - enabled: Boolean(workspaceId) + enabled: Boolean(workspaceId), + refetchInterval: dto?.refetchInterval }); }; diff --git a/frontend/src/hooks/api/workspace/types.ts b/frontend/src/hooks/api/workspace/types.ts index e2a31725d..783b15f1e 100644 --- a/frontend/src/hooks/api/workspace/types.ts +++ b/frontend/src/hooks/api/workspace/types.ts @@ -1,6 +1,7 @@ export enum ProjectVersion { V1 = 1, - V2 = 2 + V2 = 2, + V3 = 3 } export enum ProjectUserMembershipTemporaryMode { diff --git a/frontend/src/layouts/AppLayout/AppLayout.tsx b/frontend/src/layouts/AppLayout/AppLayout.tsx index bfcd585a9..f5f8b5449 100644 --- a/frontend/src/layouts/AppLayout/AppLayout.tsx +++ b/frontend/src/layouts/AppLayout/AppLayout.tsx @@ -55,7 +55,6 @@ import { SelectItem, UpgradePlanModal } from "@app/components/v2"; -import { UpgradeOverlay } from "@app/components/v2/UpgradeOverlay"; import { OrgPermissionActions, OrgPermissionSubjects, @@ -335,7 +334,6 @@ export const AppLayout = ({ children }: LayoutProps) => {