From 643d13b0eccc1e85e43e9218752d1a2a1c09a79b Mon Sep 17 00:00:00 2001 From: = Date: Sun, 21 Jul 2024 22:09:24 +0530 Subject: [PATCH] checkpoint --- .../secret-approval-request-service.ts | 2 +- .../secret-rotation-queue.ts | 163 ++++++++++++------ .../integration-sync-secret.ts | 18 +- .../secret-v2-bridge-service.ts | 22 ++- backend/src/services/secret/secret-fns.ts | 162 +++++++++++++++-- backend/src/services/secret/secret-queue.ts | 150 +++++++++++++--- backend/src/services/secret/secret-types.ts | 18 ++ 7 files changed, 432 insertions(+), 103 deletions(-) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 11e24f104..db805df8d 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -730,7 +730,7 @@ export const secretApprovalRequestServiceFactory = ({ }); } - await snapshotService.performSnapshot(folderId); + await snapshotService.performSnapshot(folderId, shouldUseSecretV2Bridge); const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]); if (!folder) throw new BadRequestError({ message: "Folder not found" }); await secretQueueService.syncSecrets({ diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index e9eedaee8..b10873e5b 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -39,6 +39,10 @@ import { secretRotationPreSetFn } from "./secret-rotation-queue-fn"; import { TSecretRotationData, TSecretRotationDbFn, TSecretRotationEncData } from "./secret-rotation-queue-types"; +import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; +import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; +import { TKmsServiceFactory } from "@app/services/kms/kms-service"; +import { KmsDataKey } from "@app/services/kms/kms-types"; export type TSecretRotationQueueFactory = ReturnType; @@ -47,8 +51,11 @@ type TSecretRotationQueueFactoryDep = { secretRotationDAL: TSecretRotationDALFactory; projectBotService: Pick; secretDAL: Pick; + secretV2BridgeDAL: Pick; secretVersionDAL: Pick; + secretVersionV2BridgeDAL: Pick; telemetryService: Pick; + kmsService: Pick; }; // These error should stop the repeatable job and ask user to reconfigure rotation @@ -70,7 +77,10 @@ export const secretRotationQueueFactory = ({ projectBotService, secretDAL, secretVersionDAL, - telemetryService + telemetryService, + secretV2BridgeDAL, + secretVersionV2BridgeDAL, + kmsService }: TSecretRotationQueueFactoryDep) => { const addToQueue = async (rotationId: string, interval: number) => { const appCfg = getConfig(); @@ -267,62 +277,113 @@ export const secretRotationQueueFactory = ({ internal: newCredential.internal }); const encVarData = infisicalSymmetricEncypt(JSON.stringify(variables)); - const key = await projectBotService.getBotKey(secretRotation.projectId); - const encryptedSecrets = rotationOutputs.map(({ key: outputKey, secretId }) => ({ - secretId, - value: encryptSymmetric128BitHexKeyUTF8( - typeof newCredential.outputs[outputKey] === "object" - ? JSON.stringify(newCredential.outputs[outputKey]) - : String(newCredential.outputs[outputKey]), - key - ) - })); - // map the final values to output keys in the board - await secretRotationDAL.transaction(async (tx) => { - await secretRotationDAL.updateById( - rotationId, - { - encryptedData: encVarData.ciphertext, - encryptedDataIV: encVarData.iv, - encryptedDataTag: encVarData.tag, - keyEncoding: encVarData.encoding, - algorithm: encVarData.algorithm, - lastRotatedAt: new Date(), - statusMessage: "Rotated successfull", - status: "success" - }, - tx - ); - const updatedSecrets = await secretDAL.bulkUpdate( - encryptedSecrets.map(({ secretId, value }) => ({ - // this secret id is validated when user is inserted - filter: { id: secretId, type: SecretType.Shared }, - data: { - secretValueCiphertext: value.ciphertext, - secretValueIV: value.iv, - secretValueTag: value.tag - } - })), - tx - ); - await secretVersionDAL.insertMany( - updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => { - if (!el.secretBlindIndex) throw new BadRequestError({ message: "Missing blind index" }); - return { - ...el, - secretId: id, - secretBlindIndex: el.secretBlindIndex - }; - }), - tx - ); + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(secretRotation.projectId); + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId: secretRotation.projectId }); + const numberOfSecretsRotated = rotationOutputs.length; + if (shouldUseSecretV2Bridge) { + const encryptedSecrets = rotationOutputs.map(({ key: outputKey, secretId }) => ({ + secretId, + value: + typeof newCredential.outputs[outputKey] === "object" + ? JSON.stringify(newCredential.outputs[outputKey]) + : String(newCredential.outputs[outputKey]) + })); + // map the final values to output keys in the board + await secretRotationDAL.transaction(async (tx) => { + await secretRotationDAL.updateById( + rotationId, + { + encryptedData: encVarData.ciphertext, + encryptedDataIV: encVarData.iv, + encryptedDataTag: encVarData.tag, + keyEncoding: encVarData.encoding, + algorithm: encVarData.algorithm, + lastRotatedAt: new Date(), + statusMessage: "Rotated successfull", + status: "success" + }, + tx + ); + const updatedSecrets = await secretV2BridgeDAL.bulkUpdate( + encryptedSecrets.map(({ secretId, value }) => ({ + // this secret id is validated when user is inserted + filter: { id: secretId, type: SecretType.Shared }, + data: { + encryptedValue: secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob + } + })), + tx + ); + await secretVersionV2BridgeDAL.insertMany( + updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => ({ + ...el, + secretId: id + })), + tx + ); + }); + } else { + if (!botKey) throw new BadRequestError({ message: "Bot not found" }); + const encryptedSecrets = rotationOutputs.map(({ key: outputKey, secretId }) => ({ + secretId, + value: encryptSymmetric128BitHexKeyUTF8( + typeof newCredential.outputs[outputKey] === "object" + ? JSON.stringify(newCredential.outputs[outputKey]) + : String(newCredential.outputs[outputKey]), + botKey + ) + })); + // map the final values to output keys in the board + await secretRotationDAL.transaction(async (tx) => { + await secretRotationDAL.updateById( + rotationId, + { + encryptedData: encVarData.ciphertext, + encryptedDataIV: encVarData.iv, + encryptedDataTag: encVarData.tag, + keyEncoding: encVarData.encoding, + algorithm: encVarData.algorithm, + lastRotatedAt: new Date(), + statusMessage: "Rotated successfull", + status: "success" + }, + tx + ); + const updatedSecrets = await secretDAL.bulkUpdate( + encryptedSecrets.map(({ secretId, value }) => ({ + // this secret id is validated when user is inserted + filter: { id: secretId, type: SecretType.Shared }, + data: { + secretValueCiphertext: value.ciphertext, + secretValueIV: value.iv, + secretValueTag: value.tag + } + })), + tx + ); + await secretVersionDAL.insertMany( + updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => { + if (!el.secretBlindIndex) throw new BadRequestError({ message: "Missing blind index" }); + return { + ...el, + secretId: id, + secretBlindIndex: el.secretBlindIndex + }; + }), + tx + ); + }); + } + await telemetryService.sendPostHogEvents({ event: PostHogEventTypes.SecretRotated, distinctId: "", properties: { - numberOfSecrets: encryptedSecrets.length, + numberOfSecrets: numberOfSecretsRotated, environment: secretRotation.environment.slug, secretPath: secretRotation.secretPath, workspaceId: secretRotation.projectId diff --git a/backend/src/services/integration-auth/integration-sync-secret.ts b/backend/src/services/integration-auth/integration-sync-secret.ts index 4d32f819a..4fc69b76c 100644 --- a/backend/src/services/integration-auth/integration-sync-secret.ts +++ b/backend/src/services/integration-auth/integration-sync-secret.ts @@ -26,7 +26,7 @@ import sodium from "libsodium-wrappers"; import isEqual from "lodash.isequal"; import { z } from "zod"; -import { SecretType, TIntegrationAuths, TIntegrations, TSecrets } from "@app/db/schemas"; +import { SecretType, TIntegrationAuths, TIntegrations } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; import { request } from "@app/lib/config/request"; import { BadRequestError } from "@app/lib/errors"; @@ -275,8 +275,8 @@ const syncSecretsAzureKeyVault = async ({ }; secrets: Record; accessToken: string; - createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; - updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; + createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; + updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; }) => { interface GetAzureKeyVaultSecret { id: string; // secret URI @@ -903,8 +903,8 @@ const syncSecretsHeroku = async ({ secrets, accessToken }: { - createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; - updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; + createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; + updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; integration: TIntegrations & { projectId: string; environment: { @@ -2464,8 +2464,8 @@ const syncSecretsTerraformCloud = async ({ accessToken, integrationDAL }: { - createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; - updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; + createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; + updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; integration: TIntegrations & { projectId: string; environment: { @@ -3612,8 +3612,8 @@ export const syncIntegrationSecrets = async ({ appendices, projectId }: { - createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; - updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; + createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; + updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; integrationDAL: Pick; integration: TIntegrations & { projectId: string; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 2a1976ca8..0753a4f50 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -179,7 +179,7 @@ export const secretV2BridgeServiceFactory = ({ }) ); - await snapshotService.performSnapshot(folderId); + await snapshotService.performSnapshot(folderId, true); await secretQueueService.syncSecrets({ secretPath, actorId, @@ -319,7 +319,7 @@ export const secretV2BridgeServiceFactory = ({ projectId }); - await snapshotService.performSnapshot(folderId); + await snapshotService.performSnapshot(folderId, true); await secretQueueService.syncSecrets({ actor, actorId, @@ -385,7 +385,7 @@ export const secretV2BridgeServiceFactory = ({ }) ); - await snapshotService.performSnapshot(folderId); + await snapshotService.performSnapshot(folderId, true); await secretQueueService.syncSecrets({ actor, actorId, @@ -717,7 +717,7 @@ export const secretV2BridgeServiceFactory = ({ }) ); - await snapshotService.performSnapshot(folderId); + await snapshotService.performSnapshot(folderId, true); await secretQueueService.syncSecrets({ actor, actorId, @@ -775,6 +775,7 @@ export const secretV2BridgeServiceFactory = ({ ); if (secretsToUpdate.length !== inputSecrets.length) throw new BadRequestError({ message: `Secret not exist: ${secretsToUpdate.map((el) => el.key).join(",")}` }); + const secretsToUpdateInDBGroupedByKey = groupBy(secretsToUpdate, (i) => i.key); // now find any secret that needs to update its name // same process as above @@ -788,7 +789,9 @@ export const secretV2BridgeServiceFactory = ({ })) ); if (secrets.length) - throw new BadRequestError({ message: `Secret not exist: ${secretsToUpdate.map((el) => el.key).join(",")}` }); + throw new BadRequestError({ + message: `Secret with new name exist: ${secretsWithNewName.map((el) => el.newSecretName).join(",")}` + }); } // get all tags @@ -804,6 +807,7 @@ export const secretV2BridgeServiceFactory = ({ folderId, tx, inputSecrets: inputSecrets.map((el) => { + const originalSecret = secretsToUpdateInDBGroupedByKey[el.secretKey][0]; const encryptedValue = typeof el.secretValue !== "undefined" ? { @@ -812,7 +816,7 @@ export const secretV2BridgeServiceFactory = ({ } : {}; return { - filter: { key: el.secretKey, type: SecretType.Shared }, + filter: { id: originalSecret.id, type: SecretType.Shared }, data: { reminderRepeatDays: el.secretReminderRepeatDays, encryptedComment: secretEncryptionHelper.encryptValue(secretManagerEncryptor, el.secretComment), @@ -830,7 +834,7 @@ export const secretV2BridgeServiceFactory = ({ secretVersionTagDAL }) ); - await snapshotService.performSnapshot(folderId); + await snapshotService.performSnapshot(folderId, true); await secretQueueService.syncSecrets({ actor, actorId, @@ -1309,7 +1313,7 @@ export const secretV2BridgeServiceFactory = ({ }); if (isDestinationUpdated) { - await snapshotService.performSnapshot(destinationFolder.id); + await snapshotService.performSnapshot(destinationFolder.id, true); await secretQueueService.syncSecrets({ projectId, secretPath: destinationFolder.path, @@ -1320,7 +1324,7 @@ export const secretV2BridgeServiceFactory = ({ } if (isSourceUpdated) { - await snapshotService.performSnapshot(sourceFolder.id); + await snapshotService.performSnapshot(sourceFolder.id, true); await secretQueueService.syncSecrets({ projectId, secretPath: sourceFolder.path, diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 5f6f9bb90..f4c66841c 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -39,6 +39,12 @@ import { TUpdateManySecretsRawFn, TUpdateManySecretsRawFnFactory } from "./secret-types"; +import { KmsDataKey } from "../kms/kms-types"; +import { + fnSecretBulkInsert as fnSecretV2BridgeBulkInsert, + fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate, + getAllNestedSecretReferences as getAllNestedSecretReferencesV2Bridge +} from "@app/services/secret-v2-bridge/secret-v2-bridge-fns"; export const generateSecretBlindIndexBySalt = async (secretName: string, secretBlindIndexDoc: TSecretBlindIndexes) => { const appCfg = getConfig(); @@ -679,7 +685,11 @@ export const createManySecretsRawFnFactory = ({ secretBlindIndexDAL, secretTagDAL, secretVersionTagDAL, - folderDAL + folderDAL, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + secretVersionTagV2BridgeDAL, + kmsService }: TCreateManySecretsRawFnFactory) => { const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL); const createManySecretsRawFn = async ({ @@ -689,10 +699,7 @@ export const createManySecretsRawFnFactory = ({ secrets, userId }: TCreateManySecretsRawFn) => { - const { botKey } = await getBotKeyFn(projectId); - if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); - - await projectDAL.checkProjectUpgradeStatus(projectId); + const { botKey, shouldUseSecretV2Bridge } = await getBotKeyFn(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) @@ -701,6 +708,62 @@ export const createManySecretsRawFnFactory = ({ name: "Create secret" }); const folderId = folder.id; + if (shouldUseSecretV2Bridge) { + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + + const secretsStoredInDB = await secretV2BridgeDAL.findBySecretKeys( + folderId, + secrets.map((el) => ({ + key: el.secretName, + type: SecretType.Shared + })) + ); + if (secretsStoredInDB.length) + throw new BadRequestError({ + message: `Secret already exist: ${secretsStoredInDB.map((el) => el.key).join(",")}` + }); + + const inputSecrets = secrets.map((secret) => { + return { + type: secret.type, + userId: secret.type === SecretType.Personal ? userId : null, + key: secret.secretName, + encryptedValue: secretManagerEncryptor({ plainText: Buffer.from(secret.secretValue) }).cipherTextBlob, + encryptedComent: secret.secretComment + ? secretManagerEncryptor({ plainText: Buffer.from(secret.secretComment) }).cipherTextBlob + : null, + skipMultilineEncoding: secret.skipMultilineEncoding, + tags: secret.tags, + references: getAllNestedSecretReferencesV2Bridge(secret.secretValue) + }; + }); + + // get all tags + const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags); + const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; + if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" }); + + const newSecrets = await secretDAL.transaction(async (tx) => + fnSecretV2BridgeBulkInsert({ + inputSecrets: inputSecrets.map((el) => ({ + ...el, + version: 1, + tagIds: el.tags + })), + folderId, + secretDAL: secretV2BridgeDAL, + secretVersionDAL: secretVersionV2BridgeDAL, + secretTagDAL, + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + tx + }) + ); + + return newSecrets; + } const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Create secret" }); @@ -715,6 +778,7 @@ export const createManySecretsRawFnFactory = ({ secretDAL }); + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); const inputSecrets = secrets.map((secret) => { const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); @@ -777,7 +841,11 @@ export const updateManySecretsRawFnFactory = ({ secretBlindIndexDAL, secretTagDAL, secretVersionTagDAL, - folderDAL + folderDAL, + secretVersionTagV2BridgeDAL, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + kmsService }: TUpdateManySecretsRawFnFactory) => { const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL); const updateManySecretsRawFn = async ({ @@ -786,11 +854,8 @@ export const updateManySecretsRawFnFactory = ({ path: secretPath, secrets, // consider accepting instead ciphertext secrets userId - }: TUpdateManySecretsRawFn): Promise> => { - const { botKey } = await getBotKeyFn(projectId); - if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); - - await projectDAL.checkProjectUpgradeStatus(projectId); + }: TUpdateManySecretsRawFn): Promise> => { + const { botKey, shouldUseSecretV2Bridge } = await getBotKeyFn(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) @@ -799,7 +864,82 @@ export const updateManySecretsRawFnFactory = ({ name: "Update secret" }); const folderId = folder.id; + if (shouldUseSecretV2Bridge) { + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const secretsToUpdate = await secretV2BridgeDAL.findBySecretKeys( + folderId, + secrets.map((el) => ({ + key: el.secretName, + type: SecretType.Shared + })) + ); + if (secretsToUpdate.length !== secrets.length) + throw new BadRequestError({ message: `Secret not exist: ${secretsToUpdate.map((el) => el.key).join(",")}` }); + + // now find any secret that needs to update its name + // same process as above + const secretsWithNewName = secrets.filter(({ newSecretName }) => Boolean(newSecretName)); + if (secretsWithNewName.length) { + const secretsWithNewNameInDB = await secretV2BridgeDAL.findBySecretKeys( + folderId, + secrets.map((el) => ({ + key: el.secretName, + type: SecretType.Shared + })) + ); + if (secretsWithNewNameInDB.length) + throw new BadRequestError({ + message: `Secret not exist: ${secretsWithNewName.map((el) => el.newSecretName).join(",")}` + }); + } + + const secretsToUpdateInDBGroupedByKey = groupBy(secretsToUpdate, (i) => i.key); + const inputSecrets = secrets.map((secret) => { + if (secret.newSecretName === "") { + throw new BadRequestError({ message: "New secret name cannot be empty" }); + } + + return { + type: secret.type, + userId: secret.type === SecretType.Personal ? userId : null, + key: secret.newSecretName || secret.secretName, + encryptedValue: secretManagerEncryptor({ plainText: Buffer.from(secret.secretValue) }).cipherTextBlob, + encryptedComent: secret.secretComment + ? secretManagerEncryptor({ plainText: Buffer.from(secret.secretComment) }).cipherTextBlob + : null, + skipMultilineEncoding: secret.skipMultilineEncoding, + tags: secret.tags, + references: getAllNestedSecretReferencesV2Bridge(secret.secretValue) + }; + }); + + const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags); + const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; + if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); + + const updatedSecrets = await secretDAL.transaction(async (tx) => + fnSecretV2BridgeBulkUpdate({ + folderId, + tx, + inputSecrets: inputSecrets.map((el) => ({ + filter: { id: secretsToUpdateInDBGroupedByKey[el.key][0].id, type: SecretType.Shared }, + data: el + })), + secretDAL: secretV2BridgeDAL, + secretVersionDAL: secretVersionV2BridgeDAL, + secretTagDAL, + secretVersionTagDAL: secretVersionTagV2BridgeDAL + }) + ); + + return updatedSecrets; + } + + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" }); diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 0f0645f58..337eba8f5 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -36,6 +36,10 @@ import { TRemoveSecretReminderDTO, TSyncSecretsDTO } from "./secret-types"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; export type TSecretQueueFactory = ReturnType; type TSecretQueueFactoryDep = { @@ -57,6 +61,10 @@ type TSecretQueueFactoryDep = { secretBlindIndexDAL: TSecretBlindIndexDALFactory; secretTagDAL: TSecretTagDALFactory; secretVersionTagDAL: TSecretVersionTagDALFactory; + kmsService: Pick; + secretV2BridgeDAL: TSecretV2BridgeDALFactory; + secretVersionV2BridgeDAL: Pick; + secretVersionTagV2BridgeDAL: Pick; }; export type TGetSecrets = { @@ -91,7 +99,11 @@ export const secretQueueFactory = ({ secretVersionDAL, secretBlindIndexDAL, secretTagDAL, - secretVersionTagDAL + secretVersionTagDAL, + secretV2BridgeDAL, + secretVersionV2BridgeDAL, + kmsService, + secretVersionTagV2BridgeDAL }: TSecretQueueFactoryDep) => { const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { const appCfg = getConfig(); @@ -195,7 +207,11 @@ export const secretQueueFactory = ({ secretBlindIndexDAL, secretTagDAL, secretVersionTagDAL, - folderDAL + folderDAL, + kmsService, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + secretVersionTagV2BridgeDAL }); const updateManySecretsRawFn = updateManySecretsRawFnFactory({ @@ -206,9 +222,82 @@ export const secretQueueFactory = ({ secretBlindIndexDAL, secretTagDAL, secretVersionTagDAL, - folderDAL + folderDAL, + kmsService, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + secretVersionTagV2BridgeDAL }); + /** + * Return the secrets in a given [folderId] including secrets from + * nested imported folders recursively. + */ + const getIntegrationSecretsV2 = async (dto: { + projectId: string; + environment: string; + folderId: string; + depth: number; + decryptor: (value: Buffer | null | undefined) => string; + }) => { + let content: TIntegrationSecret = {}; + if (dto.depth > MAX_SYNC_SECRET_DEPTH) { + logger.info( + `getIntegrationSecrets: secret depth exceeded for [projectId=${dto.projectId}] [folderId=${dto.folderId}] [depth=${dto.depth}]` + ); + return content; + } + + // process secrets in current folder + const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId); + secrets.forEach((secret) => { + const secretKey = secret.key; + const secretValue = dto.decryptor(secret.encryptedValue); + content[secretKey] = { value: secretValue }; + + if (secret.encryptedComment) { + const commentValue = dto.decryptor(secret.encryptedComment); + content[secretKey].comment = commentValue; + } + + content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding); + }); + + // TODO(akhilmhdh-sev2): change this to v2 expand secrets + + // check if current folder has any imports from other folders + const secretImport = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false }); + + // if no imports then return secrets in the current folder + if (!secretImport) return content; + + const importedFolders = await folderDAL.findByManySecretPath( + secretImport.map(({ importEnv, importPath }) => ({ + envId: importEnv.id, + secretPath: importPath + })) + ); + + for await (const folder of importedFolders) { + if (folder) { + // get secrets contained in each imported folder by recursively calling + // this function against the imported folder + const importedSecrets = await getIntegrationSecretsV2({ + environment: dto.environment, + projectId: dto.projectId, + folderId: folder.id, + depth: dto.depth + 1, + decryptor: dto.decryptor + }); + + // add the imported secrets to the current folder secrets + content = { ...importedSecrets, ...content }; + } + } + + return content; + }; + /** * Return the secrets in a given [folderId] including secrets from * nested imported folders recursively. @@ -467,24 +556,35 @@ export const secretQueueFactory = ({ ); } - const secretReferences = await secretDAL.findReferencedSecretReferences( - projectId, - folder.environment.slug, - secretPath - ); - if (secretReferences.length) { - const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(projectId); + let referencedFolderIds; + if (shouldUseSecretV2Bridge) { + const secretReferences = await secretV2BridgeDAL.findReferencedSecretReferences( + projectId, + folder.environment.slug, + secretPath + ); + referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId); + } else { + const secretReferences = await secretDAL.findReferencedSecretReferences( + projectId, + folder.environment.slug, + secretPath + ); + referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId); + } + if (referencedFolderIds.length) { const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds); const referencedFoldersGroupedById = groupBy(referencedFolders.filter(Boolean), (i) => i?.id as string); logger.info( `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` ); await Promise.all( - secretReferences - .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0]?.path)) + referencedFolderIds + .filter((folderId) => Boolean(referencedFoldersGroupedById[folderId][0]?.path)) // filter out already synced ones .filter( - ({ folderId }) => + (folderId) => !deDupeQueue[ uniqueSecretQueueKey( referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, @@ -492,7 +592,7 @@ export const secretQueueFactory = ({ ) ] ) - .map(({ folderId }) => + .map((folderId) => syncSecrets({ projectId, secretPath: referencedFoldersGroupedById[folderId][0]?.path as string, @@ -523,7 +623,6 @@ export const secretQueueFactory = ({ projectId: integration.projectId }; - const botKey = await projectBotService.getBotKey(projectId); const { accessToken, accessId } = await integrationAuthService.getIntegrationAccessToken(integrationAuth, botKey); const awsAssumeRoleArn = integrationAuth.awsAssumeIamRoleArnTag && @@ -537,13 +636,20 @@ export const secretQueueFactory = ({ }) : null; - const secrets = await getIntegrationSecrets({ - environment, - projectId, - folderId: folder.id, - key: botKey, - depth: 1 - }); + const secrets = shouldUseSecretV2Bridge + ? await getIntegrationSecretsV2({ + environment, + projectId, + folderId: folder.id, + depth: 1 + }) + : await getIntegrationSecrets({ + environment, + projectId, + folderId: folder.id, + key: botKey, + depth: 1 + }); const suffixedSecrets: typeof secrets = {}; const metadata = integration.metadata as Record; if (metadata) { diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 16bcf6172..cac5adef2 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -12,6 +12,10 @@ import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-fold import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; +import { TKmsServiceFactory } from "../kms/kms-service"; type TPartialSecret = Pick; @@ -341,6 +345,13 @@ export type TCreateManySecretsRawFnFactory = { secretTagDAL: TSecretTagDALFactory; secretVersionTagDAL: TSecretVersionTagDALFactory; folderDAL: TSecretFolderDALFactory; + kmsService: Pick; + secretV2BridgeDAL: Pick< + TSecretV2BridgeDALFactory, + "insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "bulkUpdate" | "deleteMany" + >; + secretVersionV2BridgeDAL: Pick; + secretVersionTagV2BridgeDAL: Pick; }; export type TCreateManySecretsRawFn = { @@ -370,6 +381,13 @@ export type TUpdateManySecretsRawFnFactory = { secretTagDAL: TSecretTagDALFactory; secretVersionTagDAL: TSecretVersionTagDALFactory; folderDAL: TSecretFolderDALFactory; + kmsService: Pick; + secretV2BridgeDAL: Pick< + TSecretV2BridgeDALFactory, + "insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "bulkUpdate" | "deleteMany" + >; + secretVersionV2BridgeDAL: Pick; + secretVersionTagV2BridgeDAL: Pick; }; export type TUpdateManySecretsRawFn = {