From a53d0b23349b84617b857cc2c30d70c61135b9d1 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 24 Jul 2024 23:54:18 +0530 Subject: [PATCH] feat: added first version of migrator to secret v2 --- .../secret-rotation/secret-rotation-dal.ts | 2 + backend/src/lib/knex/index.ts | 13 ++ backend/src/server/routes/index.ts | 4 +- .../src/services/secret-tag/secret-tag-dal.ts | 15 +- backend/src/services/secret/secret-queue.ts | 129 ++++++++++++++++-- frontend/src/hooks/api/users/types.ts | 1 - frontend/src/hooks/api/workspace/index.tsx | 5 +- .../src/hooks/api/workspace/mutations.tsx | 2 +- .../SecretOverviewPage/SecretOverviewPage.tsx | 2 +- .../SecretV2MigrationSection.tsx | 42 ++---- 10 files changed, 170 insertions(+), 45 deletions(-) diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-dal.ts b/backend/src/ee/services/secret-rotation/secret-rotation-dal.ts index 8927928d5..7f885e4f9 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-dal.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-dal.ts @@ -144,6 +144,8 @@ export const secretRotationDALFactory = (db: TDbClient) => { const findRotationOutputsV2ByRotationId = async (rotationId: string) => secretRotationOutputV2Orm.find({ rotationId }); + // special query + return { ...secretRotationOrm, find, diff --git a/backend/src/lib/knex/index.ts b/backend/src/lib/knex/index.ts index bd103af9d..c01d146ec 100644 --- a/backend/src/lib/knex/index.ts +++ b/backend/src/lib/knex/index.ts @@ -104,6 +104,19 @@ export const ormify = (db: Kne throw new DatabaseError({ error, name: "Create" }); } }, + upsert: async (data: readonly Tables[Tname]["insert"][], onConflictField: keyof Tables[Tname]["base"], tx?: Knex) => { + try { + if (!data.length) return []; + const res = await (tx || db)(tableName) + .insert(data as never) + .onConflict(onConflictField as never) + .merge() + .returning("*"); + return res; + } catch (error) { + throw new DatabaseError({ error, name: "Create" }); + } + }, updateById: async ( id: string, { diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 2d096221c..01aebc1eb 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -719,7 +719,9 @@ export const registerRoutes = async ( kmsService, secretVersionV2BridgeDAL, secretV2BridgeDAL, - secretVersionTagV2BridgeDAL + secretVersionTagV2BridgeDAL, + secretRotationDAL, + integrationAuthDAL }); const secretImportService = secretImportServiceFactory({ licenseService, diff --git a/backend/src/services/secret-tag/secret-tag-dal.ts b/backend/src/services/secret-tag/secret-tag-dal.ts index b1dd1a00b..d1f149f35 100644 --- a/backend/src/services/secret-tag/secret-tag-dal.ts +++ b/backend/src/services/secret-tag/secret-tag-dal.ts @@ -3,7 +3,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { TableName } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; -import { ormify } from "@app/lib/knex"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TSecretTagDALFactory = ReturnType; @@ -35,12 +35,25 @@ export const secretTagDALFactory = (db: TDbClient) => { } }; + // special query for migration + const findSecretTagsByProjectId = async (projectId: string, tx?: Knex) => { + try { + const tags = await (tx || db.replicaNode())(TableName.JnSecretTag) + .join(TableName.SecretTag, `${TableName.JnSecretTag}.secret_tagsId`, `${TableName.SecretTag}.id`) + .where({ projectId }) + .select(selectAllTableCols(TableName.JnSecretTag)); + return tags; + } catch (error) { + throw new DatabaseError({ error, name: "Find all by ids" }); + } + }; return { ...secretTagOrm, saveTagsToSecret: secretJnTagOrm.insertMany, deleteTagsToSecret: secretJnTagOrm.delete, saveTagsToSecretV2: secretV2JnTagOrm.insertMany, deleteTagsToSecretV2: secretV2JnTagOrm.delete, + findSecretTagsByProjectId, deleteTagsManySecret, findManyTagsById }; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 674455ba1..49cd227cf 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -1,6 +1,8 @@ /* eslint-disable no-await-in-loop */ import { AxiosError } from "axios"; +import { ProjectUpgradeStatus, ProjectVersion } from "@app/db/schemas"; +import { TSecretRotationDALFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal"; import { getConfig } from "@app/lib/config/env"; import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { daysToMillisecond, secondsToMillis } from "@app/lib/dates"; @@ -16,6 +18,7 @@ import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/se import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { TIntegrationDALFactory } from "../integration/integration-dal"; +import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal"; import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service"; import { syncIntegrationSecrets } from "../integration-auth/integration-sync-secret"; import { TKmsServiceFactory } from "../kms/kms-service"; @@ -28,6 +31,7 @@ import { TProjectMembershipDALFactory } from "../project-membership/project-memb import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { getAllNestedSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns"; import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service"; @@ -41,12 +45,12 @@ import { TRemoveSecretReminderDTO, TSyncSecretsDTO } from "./secret-types"; -import { ProjectUpgradeStatus, ProjectVersion } from "@app/db/schemas"; export type TSecretQueueFactory = ReturnType; type TSecretQueueFactoryDep = { queueService: TQueueServiceFactory; integrationDAL: Pick; + integrationAuthDAL: Pick; projectBotService: Pick; integrationAuthService: Pick; folderDAL: TSecretFolderDALFactory; @@ -67,6 +71,7 @@ type TSecretQueueFactoryDep = { secretV2BridgeDAL: TSecretV2BridgeDALFactory; secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; + secretRotationDAL: Pick; }; export type TGetSecrets = { @@ -86,6 +91,7 @@ type TIntegrationSecret = Record< export const secretQueueFactory = ({ queueService, integrationDAL, + integrationAuthDAL, projectBotService, integrationAuthService, secretDAL, @@ -105,7 +111,8 @@ export const secretQueueFactory = ({ secretV2BridgeDAL, secretVersionV2BridgeDAL, kmsService, - secretVersionTagV2BridgeDAL + secretVersionTagV2BridgeDAL, + secretRotationDAL }: TSecretQueueFactoryDep) => { const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { const appCfg = getConfig(); @@ -779,7 +786,6 @@ export const secretQueueFactory = ({ ); }; - const MIGRATION_BATCH_SIZE = 10000; queueService.start(QueueName.ProjectV3Migration, async (job) => { const { projectId } = job.data; const { botKey, shouldUseSecretV2Bridge: isProjectUpgradedToV3 } = await projectBotService.getBotKey(projectId); @@ -798,10 +804,16 @@ export const secretQueueFactory = ({ await secretDAL.transaction(async (tx) => { for (const folder of folders) { const folderId = folder.id; - let projectV1Secrets; - do { - // eslint-disable-next-line no-await-in-loop - projectV1Secrets = await secretDAL.find({ folderId }, { limit: MIGRATION_BATCH_SIZE, tx }); + /* + * Secrets Migration + * */ + // eslint-disable-next-line no-await-in-loop + const projectV1Secrets = await secretDAL.find({ folderId }, { tx }); + if (projectV1Secrets.length) { + const secretReferences: { + secretId: string; + references: { environment: string; secretPath: string; secretKey: string }[]; + }[] = []; await secretV2BridgeDAL.insertMany( projectV1Secrets.map((el) => { const key = decryptSymmetric128BitHexKeyUTF8({ @@ -826,6 +838,10 @@ export const secretQueueFactory = ({ }) : ""; const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob; + // create references + const references = getAllNestedSecretReferences(value); + secretReferences.push({ secretId: el.id, references }); + const encryptedComment = comment ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob : null; @@ -848,9 +864,104 @@ export const secretQueueFactory = ({ }), tx ); - projectV1Secrets = await secretDAL.delete({ folderId, $in: { id: projectV1Secrets.map((el) => el.id) } }, tx); - } while (projectV1Secrets.length > 0); + await secretV2BridgeDAL.upsertSecretReferences(secretReferences); + } } + /* + * Secret Tag Migration + * */ + // eslint-disable-next-line no-await-in-loop + const projectV1SecretTags = await secretTagDAL.findSecretTagsByProjectId(projectId, tx); + if (projectV1SecretTags.length) { + await secretTagDAL.saveTagsToSecretV2( + projectV1SecretTags.map((el) => ({ + secrets_v2Id: el.secretsId, + secret_tagsId: el.secret_tagsId + })), + tx + ); + } + + /* + * Integration Auth Migration + * Saving the new encrypted colum + * */ + // eslint-disable-next-line no-await-in-loop + const projectV1IntegrationAuths = await integrationAuthDAL.find({ projectId }, { tx }); + await integrationAuthDAL.upsert( + projectV1IntegrationAuths.map((el) => { + const accessToken = + el.accessIV && el.accessTag && el.accessCiphertext + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.accessCiphertext, + iv: el.accessIV, + tag: el.accessTag, + key: botKey + }) + : undefined; + const accessId = + el.accessIdIV && el.accessIdTag && el.accessIdCiphertext + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.accessIdCiphertext, + iv: el.accessIdIV, + tag: el.accessIdTag, + key: botKey + }) + : undefined; + const refreshToken = + el.refreshIV && el.refreshTag && el.refreshCiphertext + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.refreshCiphertext, + iv: el.refreshIV, + tag: el.refreshTag, + key: botKey + }) + : undefined; + const awsAssumeRoleArn = + el.awsAssumeIamRoleArnCipherText && el.awsAssumeIamRoleArnIV && el.awsAssumeIamRoleArnTag + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.awsAssumeIamRoleArnCipherText, + iv: el.awsAssumeIamRoleArnIV, + tag: el.awsAssumeIamRoleArnTag, + key: botKey + }) + : undefined; + + const encryptedAccess = accessToken + ? secretManagerEncryptor({ plainText: Buffer.from(accessToken) }).cipherTextBlob + : null; + const encryptedAccessId = accessId + ? secretManagerEncryptor({ plainText: Buffer.from(accessId) }).cipherTextBlob + : null; + const encryptedRefresh = refreshToken + ? secretManagerEncryptor({ plainText: Buffer.from(refreshToken) }).cipherTextBlob + : null; + const encryptedAwsAssumeIamRoleArn = awsAssumeRoleArn + ? secretManagerEncryptor({ plainText: Buffer.from(awsAssumeRoleArn) }).cipherTextBlob + : null; + return { + ...el, + encryptedAccess, + encryptedRefresh, + encryptedAccessId, + encryptedAwsAssumeIamRoleArn + }; + }), + "id", + tx + ); + /* + * Secret Rotation Secret Migration + * Saving the new encrypted colum + * */ + + const projectV1SecretRotations = await secretRotationDAL.find({ projectId }, tx); + await secretRotationDAL.secretOutputV2InsertMany( + projectV1SecretRotations.flatMap((el) => + el.outputs.map((output) => ({ rotationId: el.id, key: output.key, secretId: output.secret.id })) + ), + tx + ); await projectDAL.updateById(projectId, { upgradeStatus: null, version: ProjectVersion.V3 }, tx); }); }); diff --git a/frontend/src/hooks/api/users/types.ts b/frontend/src/hooks/api/users/types.ts index 4e2d7d3a2..1e5661d1a 100644 --- a/frontend/src/hooks/api/users/types.ts +++ b/frontend/src/hooks/api/users/types.ts @@ -66,7 +66,6 @@ export type OrgUser = { export type TProjectMembership = { id: string; - role: string; createdAt: string; updatedAt: string; projectId: string; diff --git a/frontend/src/hooks/api/workspace/index.tsx b/frontend/src/hooks/api/workspace/index.tsx index f9f7f9d78..7b528cfbd 100644 --- a/frontend/src/hooks/api/workspace/index.tsx +++ b/frontend/src/hooks/api/workspace/index.tsx @@ -2,9 +2,8 @@ export { useAddGroupToWorkspace, useDeleteGroupFromWorkspace, useLeaveProject, - useUpdateGroupWorkspaceRole, - useMigrateProjectToV3 -} from "./mutations"; + useMigrateProjectToV3, + useUpdateGroupWorkspaceRole} from "./mutations"; export { useAddIdentityToWorkspace, useCreateWorkspace, diff --git a/frontend/src/hooks/api/workspace/mutations.tsx b/frontend/src/hooks/api/workspace/mutations.tsx index 60948c9a7..3b12c9210 100644 --- a/frontend/src/hooks/api/workspace/mutations.tsx +++ b/frontend/src/hooks/api/workspace/mutations.tsx @@ -79,7 +79,7 @@ export const useMigrateProjectToV3 = () => { const queryClient = useQueryClient(); return useMutation<{}, {}, { workspaceId: string }>({ mutationFn: ({ workspaceId }) => { - return apiRequest.delete(`/api/v1/workspace/${workspaceId}/migrate-v3`); + return apiRequest.post(`/api/v1/workspace/${workspaceId}/migrate-v3`); }, onSuccess: () => { queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace); diff --git a/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx b/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx index 42e050f21..f9f279fe7 100644 --- a/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx +++ b/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx @@ -69,10 +69,10 @@ import { ProjectVersion } from "@app/hooks/api/workspace/types"; import { FolderForm } from "../SecretMainPage/components/ActionBar/FolderForm"; import { CreateSecretForm } from "./components/CreateSecretForm"; import { FolderBreadCrumbs } from "./components/FolderBreadCrumbs"; -import { SecretV2MigrationSection } from "./components/SecretV2MigrationSection"; import { SecretOverviewDynamicSecretRow } from "./components/SecretOverviewDynamicSecretRow"; import { SecretOverviewFolderRow } from "./components/SecretOverviewFolderRow"; import { SecretOverviewTableRow } from "./components/SecretOverviewTableRow"; +import { SecretV2MigrationSection } from "./components/SecretV2MigrationSection"; import { SelectionPanel } from "./components/SelectionPanel/SelectionPanel"; export enum EntryType { diff --git a/frontend/src/views/SecretOverviewPage/components/SecretV2MigrationSection/SecretV2MigrationSection.tsx b/frontend/src/views/SecretOverviewPage/components/SecretV2MigrationSection/SecretV2MigrationSection.tsx index dd6bb144d..182b25385 100644 --- a/frontend/src/views/SecretOverviewPage/components/SecretV2MigrationSection/SecretV2MigrationSection.tsx +++ b/frontend/src/views/SecretOverviewPage/components/SecretV2MigrationSection/SecretV2MigrationSection.tsx @@ -1,13 +1,6 @@ import { createNotification } from "@app/components/notifications"; -import { ProjectPermissionCan } from "@app/components/permissions"; - import { Button, Spinner } from "@app/components/v2"; -import { - ProjectPermissionActions, - ProjectPermissionSub, - useProjectPermission, - useWorkspace -} from "@app/context"; +import { useProjectPermission, useWorkspace } from "@app/context"; import { useGetWorkspaceById, useMigrateProjectToV3 } from "@app/hooks/api"; import { ProjectMembershipRole } from "@app/hooks/api/roles/types"; import { ProjectVersion } from "@app/hooks/api/workspace/types"; @@ -50,8 +43,8 @@ export const SecretV2MigrationSection = () => { }); } }; - // for non admin this would throw an error - // so no need to render + + const isAdmin = membership?.roles.includes(ProjectMembershipRole.Admin); return (
{isUpgrading && ( @@ -66,25 +59,18 @@ export const SecretV2MigrationSection = () => {

Action Required

There is a new update for your project. Introducing Infisical KMS. - - {membership.role !== ProjectMembershipRole.Admin && "This is an admin only operation."} - + {!isAdmin && "This is an admin only operation."}

- - {(isAllowed) => ( - - )} - + +
); };