From 1847491cb3696fde91f053e02e87636ce8e75fd1 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 21 Aug 2024 00:54:13 +0530 Subject: [PATCH] feat: implemented new secret reference strategy --- .../integration-delete-secret.ts | 2 + .../secret-import/secret-import-fns.ts | 44 ++-- .../secret-v2-bridge/secret-v2-bridge-fns.ts | 180 ++++++-------- .../secret-v2-bridge-service.ts | 49 ++-- backend/src/services/secret/secret-fns.ts | 223 ++++++++---------- backend/src/services/secret/secret-queue.ts | 123 +++++----- backend/src/services/secret/secret-service.ts | 144 ++++------- 7 files changed, 336 insertions(+), 429 deletions(-) diff --git a/backend/src/services/integration-auth/integration-delete-secret.ts b/backend/src/services/integration-auth/integration-delete-secret.ts index 5da48fb6e..bd9619ad4 100644 --- a/backend/src/services/integration-auth/integration-delete-secret.ts +++ b/backend/src/services/integration-auth/integration-delete-secret.ts @@ -30,6 +30,7 @@ const getIntegrationSecretsV2 = async ( environment: string; folderId: string; depth: number; + secretPath: string; decryptor: (value: Buffer | null | undefined) => string; }, secretV2BridgeDAL: Pick, @@ -306,6 +307,7 @@ export const deleteIntegrationSecrets = async ({ ? await getIntegrationSecretsV2( { environment: integration.environment.id, + secretPath: integration.secretPath, projectId: integration.projectId, folderId: folder.id, depth: 1, diff --git a/backend/src/services/secret-import/secret-import-fns.ts b/backend/src/services/secret-import/secret-import-fns.ts index f7c31ac61..981a1bf62 100644 --- a/backend/src/services/secret-import/secret-import-fns.ts +++ b/backend/src/services/secret-import/secret-import-fns.ts @@ -158,9 +158,12 @@ export const fnSecretsV2FromImports = async ({ depth?: number; cyclicDetector?: Set; decryptor: (value?: Buffer | null) => string; - expandSecretReferences?: ( - secrets: Record - ) => Promise>; + expandSecretReferences?: (inputSecret: { + value?: string; + skipMultilineEncoding?: boolean | null; + secretPath: string; + environment: string; + }) => Promise; }) => { // avoid going more than a depth if (depth >= LEVEL_BREAK) return []; @@ -244,26 +247,21 @@ export const fnSecretsV2FromImports = async ({ }); if (expandSecretReferences) { - await Promise.all( - processedImports.map(async (processedImport) => { - const secretsGroupByKey = processedImport.secrets.reduce( - (acc, item) => { - acc[item.secretKey] = { - value: item.secretValue, - comment: item.secretComment, - skipMultilineEncoding: item.skipMultilineEncoding - }; - return acc; - }, - {} as Record - ); - // eslint-disable-next-line - await expandSecretReferences(secretsGroupByKey); - processedImport.secrets.forEach((decryptedSecret) => { - // eslint-disable-next-line no-param-reassign - decryptedSecret.secretValue = secretsGroupByKey[decryptedSecret.secretKey].value; - }); - }) + await Promise.allSettled( + processedImports.map((processedImport) => + Promise.allSettled( + processedImport.secrets.map(async (decryptedSecret, index) => { + const expandedSecretValue = await expandSecretReferences({ + value: decryptedSecret.secretValue, + secretPath: processedImport.secretPath, + environment: processedImport.environment, + skipMultilineEncoding: decryptedSecret.skipMultilineEncoding + }); + // eslint-disable-next-line no-param-reassign + processedImport.secrets[index].secretValue = expandedSecretValue || ""; + }) + ) + ) ); } diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index e58960a4b..7bd073d52 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -377,150 +377,118 @@ type TInterpolateSecretArg = { folderDAL: Pick; }; +const MAX_SECRET_REFERENCE_DEPTH = 10; export const expandSecretReferencesFactory = ({ projectId, decryptSecretValue: decryptSecret, secretDAL, folderDAL }: TInterpolateSecretArg) => { - const fetchSecretFactory = () => { - const secretCache: Record> = {}; + const secretCache: Record> = {}; + const getCacheUniqueKey = (environment: string, secretPath: string) => `${environment}-${secretPath}`; - return async (secRefEnv: string, secRefPath: string[], secRefKey: string) => { - const referredSecretPathURL = path.join("/", ...secRefPath); - const uniqueKey = `${secRefEnv}-${referredSecretPathURL}`; + const fetchSecret = async (environment: string, secretPath: string, secretKey: string) => { + const cacheKey = getCacheUniqueKey(environment, secretPath); - if (secretCache?.[uniqueKey]) { - return secretCache[uniqueKey][secRefKey]; - } + if (secretCache?.[cacheKey]) { + return secretCache[cacheKey][secretKey] || ""; + } - const folder = await folderDAL.findBySecretPath(projectId, secRefEnv, referredSecretPathURL); - if (!folder) return ""; - const secrets = await secretDAL.findByFolderId(folder.id); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + if (!folder) return ""; + const secrets = await secretDAL.findByFolderId(folder.id); - const decryptedSecret = secrets.reduce>((prev, secret) => { - // eslint-disable-next-line - prev[secret.key] = decryptSecret(secret.encryptedValue) || ""; - return prev; - }, {}); + const decryptedSecret = secrets.reduce>((prev, secret) => { + // eslint-disable-next-line + prev[secret.key] = decryptSecret(secret.encryptedValue) || ""; + return prev; + }, {}); - secretCache[uniqueKey] = decryptedSecret; + secretCache[cacheKey] = decryptedSecret; - return secretCache[uniqueKey][secRefKey]; - }; + return secretCache[cacheKey][secretKey] || ""; }; - const recursivelyExpandSecret = async ( - expandedSec: Record, - interpolatedSec: Record, - fetchSecret: (env: string, secPath: string[], secKey: string) => Promise, - recursionChainBreaker: Record, - key: string - ): Promise => { - if (expandedSec?.[key] !== undefined) { - return expandedSec[key]; - } - if (recursionChainBreaker?.[key]) { - return ""; - } - // eslint-disable-next-line - recursionChainBreaker[key] = true; + const recursivelyExpandSecret = async ({ + value, + secretPath, + environment, + depth = 1 + }: { + value?: string; + secretPath: string; + environment: string; + depth?: number; + }) => { + if (!value) return ""; + if (depth > MAX_SECRET_REFERENCE_DEPTH) return ""; - let interpolatedValue = interpolatedSec[key]; - if (!interpolatedValue) { - // eslint-disable-next-line no-console - console.error(`Couldn't find referenced value - ${key}`); - return ""; - } - - const refs = interpolatedValue.match(INTERPOLATION_SYNTAX_REG); + const refs = value.match(INTERPOLATION_SYNTAX_REG); + let expandedValue = value; if (refs) { for (const interpolationSyntax of refs) { const interpolationKey = interpolationSyntax.slice(2, interpolationSyntax.length - 1); const entities = interpolationKey.trim().split("."); if (entities.length === 1) { + const [secretKey] = entities; // eslint-disable-next-line - const val = await recursivelyExpandSecret( - expandedSec, - interpolatedSec, - fetchSecret, - recursionChainBreaker, - interpolationKey - ); - if (val) { - interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val); + let referenceValue = await fetchSecret(environment, secretPath, secretKey); + if (INTERPOLATION_SYNTAX_REG.test(referenceValue)) { + // eslint-disable-next-line + referenceValue = await recursivelyExpandSecret({ + environment, + secretPath, + value: referenceValue, + depth: depth + 1 + }); } - // eslint-disable-next-line - continue; + const cacheKey = getCacheUniqueKey(environment, secretPath); + secretCache[cacheKey][secretKey] = referenceValue; + expandedValue = expandedValue.replaceAll(interpolationSyntax, referenceValue); } if (entities.length > 1) { - const secRefEnv = entities[0]; - const secRefPath = entities.slice(1, entities.length - 1); - const secRefKey = entities[entities.length - 1]; + const secretReferenceEnvironment = entities[0]; + const secretReferencePath = path.join("/", ...entities.slice(1, entities.length - 1)); + const secretReferenceKey = entities[entities.length - 1]; // eslint-disable-next-line - const val = await fetchSecret(secRefEnv, secRefPath, secRefKey); - if (val) { - interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val); + let referenceValue = await fetchSecret(secretReferenceEnvironment, secretReferencePath, secretReferenceKey); + if (INTERPOLATION_SYNTAX_REG.test(referenceValue)) { + // eslint-disable-next-line + referenceValue = await recursivelyExpandSecret({ + environment: secretReferenceEnvironment, + secretPath: secretReferencePath, + value: referenceValue, + depth: depth + 1 + }); } + const cacheKey = getCacheUniqueKey(secretReferenceEnvironment, secretReferencePath); + secretCache[cacheKey][secretReferenceKey] = referenceValue; + expandedValue = expandedValue.replaceAll(interpolationSyntax, referenceValue); } } } - // eslint-disable-next-line - expandedSec[key] = interpolatedValue; - return interpolatedValue; + return expandedValue; }; - const fetchSecret = fetchSecretFactory(); - const expandSecrets = async ( - inputSecrets: Record - ) => { - const expandedSecrets: Record = {}; - const toBeExpandedSecrets: Record = {}; + const expandSecret = async (inputSecret: { + value?: string; + skipMultilineEncoding?: boolean | null; + secretPath: string; + environment: string; + }) => { + if (!inputSecret.value) return inputSecret.value; - Object.keys(inputSecrets).forEach((key) => { - if (inputSecrets[key].value?.match(INTERPOLATION_SYNTAX_REG)) { - toBeExpandedSecrets[key] = inputSecrets[key].value; - } else { - expandedSecrets[key] = inputSecrets[key].value; - } - }); + const shouldExpand = Boolean(inputSecret.value?.match(INTERPOLATION_SYNTAX_REG)); + if (!shouldExpand) return inputSecret.value; - for (const key of Object.keys(inputSecrets)) { - if (expandedSecrets?.[key]) { - // should not do multi line encoding if user has set it to skip - // eslint-disable-next-line - inputSecrets[key].value = inputSecrets[key].skipMultilineEncoding - ? formatMultiValueEnv(expandedSecrets[key]) - : expandedSecrets[key]; - // eslint-disable-next-line - continue; - } - - // this is to avoid recursion loop. So the graph should be direct graph rather than cyclic - // so for any recursion building if there is an entity two times same key meaning it will be looped - const recursionChainBreaker: Record = {}; - // eslint-disable-next-line - const expandedVal = await recursivelyExpandSecret( - expandedSecrets, - toBeExpandedSecrets, - fetchSecret, - recursionChainBreaker, - key - ); - - // eslint-disable-next-line - inputSecrets[key].value = inputSecrets[key].skipMultilineEncoding - ? formatMultiValueEnv(expandedVal) - : expandedVal; - } - - return inputSecrets; + const expandedSecretValue = await recursivelyExpandSecret(inputSecret); + return inputSecret.skipMultilineEncoding ? formatMultiValueEnv(expandedSecretValue) : expandedSecretValue; }; - return expandSecrets; + return expandSecret; }; export const reshapeBridgeSecret = ( diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index b594f98ca..ed9cd3a7d 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -521,27 +521,22 @@ export const secretV2BridgeServiceFactory = ({ if (shouldExpandSecretReferences) { const secretsGroupByPath = groupBy(filteredSecrets, (i) => i.secretPath); - for (const secretPathKey in secretsGroupByPath) { - if (Object.hasOwn(secretsGroupByPath, secretPathKey)) { - const secretsGroupByKey = secretsGroupByPath[secretPathKey].reduce( - (acc, item) => { - acc[item.secretKey] = { - value: item.secretValue, - comment: item.secretComment, - skipMultilineEncoding: item.skipMultilineEncoding - }; - return acc; - }, - {} as Record - ); - // eslint-disable-next-line - await expandSecretReferences(secretsGroupByKey); - secretsGroupByPath[secretPathKey].forEach((decryptedSecret) => { - // eslint-disable-next-line no-param-reassign - decryptedSecret.secretValue = secretsGroupByKey[decryptedSecret.secretKey].value || ""; - }); - } - } + await Promise.allSettled( + Object.keys(secretsGroupByPath).map((groupedPath) => + Promise.allSettled( + secretsGroupByPath[groupedPath].map(async (decryptedSecret, index) => { + const expandedSecretValue = await expandSecretReferences({ + value: decryptedSecret.secretValue, + secretPath: groupedPath, + environment, + skipMultilineEncoding: decryptedSecret.skipMultilineEncoding + }); + // eslint-disable-next-line no-param-reassign + secretsGroupByPath[groupedPath][index].secretValue = expandedSecretValue || ""; + }) + ) + ) + ); } if (!includeImports) { @@ -693,12 +688,14 @@ export const secretV2BridgeServiceFactory = ({ ? secretManagerDecryptor({ cipherTextBlob: secret.encryptedValue }).toString() : ""; if (shouldExpandSecretReferences && secretValue) { - const secretReferenceExpandedRecord = { - [secret.key]: { value: secretValue } - }; // eslint-disable-next-line - await expandSecretReferences(secretReferenceExpandedRecord); - secretValue = secretReferenceExpandedRecord[secret.key].value; + const expandedSecretValue = await expandSecretReferences({ + environment, + secretPath: path, + value: secretValue, + skipMultilineEncoding: secret.skipMultilineEncoding + }); + secretValue = expandedSecretValue || ""; } return reshapeBridgeSecret(projectId, environment, path, { diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index d41770846..7837b716b 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -196,6 +196,13 @@ export const recursivelyGetSecretPaths = ({ return getPaths; }; +// used to convert multi line ones to quotes ones with \n +const formatMultiValueEnv = (val?: string) => { + if (!val) return ""; + if (!val.match("\n")) return val; + return `"${val.replace(/\n/g, "\\n")}"`; +}; + type TInterpolateSecretArg = { projectId: string; secretEncKey: string; @@ -203,162 +210,128 @@ type TInterpolateSecretArg = { folderDAL: Pick; }; +const MAX_SECRET_REFERENCE_DEPTH = 5; const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g; export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderDAL }: TInterpolateSecretArg) => { - const fetchSecretsCrossEnv = () => { - const fetchCache: Record> = {}; + const secretCache: Record> = {}; + const getCacheUniqueKey = (environment: string, secretPath: string) => `${environment}-${secretPath}`; - return async (secRefEnv: string, secRefPath: string[], secRefKey: string) => { - const secRefPathUrl = path.join("/", ...secRefPath); - const uniqKey = `${secRefEnv}-${secRefPathUrl}`; + const fetchSecret = async (environment: string, secretPath: string, secretKey: string) => { + const cacheKey = getCacheUniqueKey(environment, secretPath); + const uniqKey = `${environment}-${cacheKey}`; - if (fetchCache?.[uniqKey]) { - return fetchCache[uniqKey][secRefKey]; - } + if (secretCache?.[uniqKey]) { + return secretCache[uniqKey][secretKey] || ""; + } - const folder = await folderDAL.findBySecretPath(projectId, secRefEnv, secRefPathUrl); - if (!folder) return ""; - const secrets = await secretDAL.findByFolderId(folder.id); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + if (!folder) return ""; + const secrets = await secretDAL.findByFolderId(folder.id); - const decryptedSec = secrets.reduce>((prev, secret) => { - const secretKey = decryptSymmetric128BitHexKeyUTF8({ - ciphertext: secret.secretKeyCiphertext, - iv: secret.secretKeyIV, - tag: secret.secretKeyTag, - key: secretEncKey - }); - const secretValue = decryptSymmetric128BitHexKeyUTF8({ - ciphertext: secret.secretValueCiphertext, - iv: secret.secretValueIV, - tag: secret.secretValueTag, - key: secretEncKey - }); + const decryptedSec = secrets.reduce>((prev, secret) => { + const decryptedSecretKey = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretKeyCiphertext, + iv: secret.secretKeyIV, + tag: secret.secretKeyTag, + key: secretEncKey + }); + const decryptedSecretValue = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretValueCiphertext, + iv: secret.secretValueIV, + tag: secret.secretValueTag, + key: secretEncKey + }); - // eslint-disable-next-line - prev[secretKey] = secretValue; - return prev; - }, {}); + // eslint-disable-next-line + prev[decryptedSecretKey] = decryptedSecretValue; + return prev; + }, {}); - fetchCache[uniqKey] = decryptedSec; + secretCache[uniqKey] = decryptedSec; - return fetchCache[uniqKey][secRefKey]; - }; + return secretCache[uniqKey][secretKey] || ""; }; - const recursivelyExpandSecret = async ( - expandedSec: Record, - interpolatedSec: Record, - fetchCrossEnv: (env: string, secPath: string[], secKey: string) => Promise, - recursionChainBreaker: Record, - key: string - ) => { - if (expandedSec?.[key] !== undefined) { - return expandedSec[key]; - } - if (recursionChainBreaker?.[key]) { - return ""; - } - // eslint-disable-next-line - recursionChainBreaker[key] = true; + const recursivelyExpandSecret = async ({ + value, + secretPath, + environment, + depth = 0 + }: { + value?: string; + secretPath: string; + environment: string; + depth?: number; + }) => { + if (!value) return ""; + if (depth > MAX_SECRET_REFERENCE_DEPTH) return ""; - let interpolatedValue = interpolatedSec[key]; - if (!interpolatedValue) { - // eslint-disable-next-line no-console - console.error(`Couldn't find referenced value - ${key}`); - return ""; - } - - const refs = interpolatedValue.match(INTERPOLATION_SYNTAX_REG); + const refs = value.match(INTERPOLATION_SYNTAX_REG); + let expandedValue = value; if (refs) { for (const interpolationSyntax of refs) { const interpolationKey = interpolationSyntax.slice(2, interpolationSyntax.length - 1); const entities = interpolationKey.trim().split("."); if (entities.length === 1) { - const val = await recursivelyExpandSecret( - expandedSec, - interpolatedSec, - fetchCrossEnv, - recursionChainBreaker, - interpolationKey - ); - if (val) { - interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val); - } + const [secretKey] = entities; // eslint-disable-next-line - continue; + let referenceValue = await fetchSecret(environment, secretPath, secretKey); + if (INTERPOLATION_SYNTAX_REG.test(referenceValue)) { + // eslint-disable-next-line + referenceValue = await recursivelyExpandSecret({ + environment, + secretPath, + value: referenceValue, + depth: depth + 1 + }); + } + const cacheKey = getCacheUniqueKey(environment, secretPath); + secretCache[cacheKey][secretKey] = referenceValue; + expandedValue = expandedValue.replaceAll(interpolationSyntax, referenceValue); } if (entities.length > 1) { - const secRefEnv = entities[0]; - const secRefPath = entities.slice(1, entities.length - 1); - const secRefKey = entities[entities.length - 1]; + const secretReferenceEnvironment = entities[0]; + const secretReferencePath = path.join("/", ...entities.slice(1, entities.length - 1)); + const secretReferenceKey = entities[entities.length - 1]; - const val = await fetchCrossEnv(secRefEnv, secRefPath, secRefKey); - if (val) { - interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val); + // eslint-disable-next-line + let referenceValue = await fetchSecret(secretReferenceEnvironment, secretReferencePath, secretReferenceKey); + if (INTERPOLATION_SYNTAX_REG.test(referenceValue)) { + // eslint-disable-next-line + referenceValue = await recursivelyExpandSecret({ + environment: secretReferenceEnvironment, + secretPath: secretReferencePath, + value: referenceValue, + depth: depth + 1 + }); } + const cacheKey = getCacheUniqueKey(secretReferenceEnvironment, secretReferencePath); + secretCache[cacheKey][secretReferenceKey] = referenceValue; + expandedValue = expandedValue.replaceAll(interpolationSyntax, referenceValue); } } } - // eslint-disable-next-line - expandedSec[key] = interpolatedValue; - return interpolatedValue; + return expandedValue; }; - // used to convert multi line ones to quotes ones with \n - const formatMultiValueEnv = (val?: string) => { - if (!val) return ""; - if (!val.match("\n")) return val; - return `"${val.replace(/\n/g, "\\n")}"`; + const expandSecret = async (inputSecret: { + value?: string; + skipMultilineEncoding?: boolean | null; + secretPath: string; + environment: string; + }) => { + if (!inputSecret.value) return inputSecret.value; + + const shouldExpand = Boolean(inputSecret.value?.match(INTERPOLATION_SYNTAX_REG)); + if (!shouldExpand) return inputSecret.value; + + const expandedSecretValue = await recursivelyExpandSecret(inputSecret); + return inputSecret.skipMultilineEncoding ? formatMultiValueEnv(expandedSecretValue) : expandedSecretValue; }; - - const expandSecrets = async ( - secrets: Record - ) => { - const expandedSec: Record = {}; - const interpolatedSec: Record = {}; - - const crossSecEnvFetch = fetchSecretsCrossEnv(); - - Object.keys(secrets).forEach((key) => { - if (secrets[key].value.match(INTERPOLATION_SYNTAX_REG)) { - interpolatedSec[key] = secrets[key].value; - } else { - expandedSec[key] = secrets[key].value; - } - }); - - for (const key of Object.keys(secrets)) { - if (expandedSec?.[key]) { - // should not do multi line encoding if user has set it to skip - // eslint-disable-next-line - secrets[key].value = secrets[key].skipMultilineEncoding - ? formatMultiValueEnv(expandedSec[key]) - : expandedSec[key]; - // eslint-disable-next-line - continue; - } - - // this is to avoid recursion loop. So the graph should be direct graph rather than cyclic - // so for any recursion building if there is an entity two times same key meaning it will be looped - const recursionChainBreaker: Record = {}; - const expandedVal = await recursivelyExpandSecret( - expandedSec, - interpolatedSec, - crossSecEnvFetch, - recursionChainBreaker, - key - ); - - // eslint-disable-next-line - secrets[key].value = secrets[key].skipMultilineEncoding ? formatMultiValueEnv(expandedVal) : expandedVal; - } - - return secrets; - }; - return expandSecrets; + return expandSecret; }; export const decryptSecretRaw = ( diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 36edcf195..0c63eb147 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -258,6 +258,7 @@ export const secretQueueFactory = ({ const getIntegrationSecretsV2 = async (dto: { projectId: string; environment: string; + secretPath: string; folderId: string; depth: number; decryptor: (value: Buffer | null | undefined) => string; @@ -269,30 +270,36 @@ export const secretQueueFactory = ({ ); return content; } - - // process secrets in current folder - const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId); - secrets.forEach((secret) => { - const secretKey = secret.key; - const secretValue = dto.decryptor(secret.encryptedValue); - content[secretKey] = { value: secretValue }; - - if (secret.encryptedComment) { - const commentValue = dto.decryptor(secret.encryptedComment); - content[secretKey].comment = commentValue; - } - - content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding); - }); - const expandSecretReferences = expandSecretReferencesFactory({ decryptSecretValue: dto.decryptor, secretDAL: secretV2BridgeDAL, folderDAL, projectId: dto.projectId }); + // process secrets in current folder + const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId); + + await Promise.allSettled( + secrets.map(async (secret) => { + const secretKey = secret.key; + const secretValue = dto.decryptor(secret.encryptedValue); + const expandedSecretValue = await expandSecretReferences({ + environment: dto.environment, + secretPath: dto.secretPath, + skipMultilineEncoding: secret.skipMultilineEncoding, + value: secretValue + }); + content[secretKey] = { value: expandedSecretValue || "" }; + + if (secret.encryptedComment) { + const commentValue = dto.decryptor(secret.encryptedComment); + content[secretKey].comment = commentValue; + } + + content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding); + }) + ); - await expandSecretReferences(content); // check if current folder has any imports from other folders const secretImports = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false }); @@ -329,6 +336,7 @@ export const secretQueueFactory = ({ const getIntegrationSecrets = async (dto: { projectId: string; environment: string; + secretPath: string; folderId: string; key: string; depth: number; @@ -341,46 +349,52 @@ export const secretQueueFactory = ({ return content; } - // process secrets in current folder - const secrets = await secretDAL.findByFolderId(dto.folderId); - secrets.forEach((secret) => { - const secretKey = decryptSymmetric128BitHexKeyUTF8({ - ciphertext: secret.secretKeyCiphertext, - iv: secret.secretKeyIV, - tag: secret.secretKeyTag, - key: dto.key - }); - - const secretValue = decryptSymmetric128BitHexKeyUTF8({ - ciphertext: secret.secretValueCiphertext, - iv: secret.secretValueIV, - tag: secret.secretValueTag, - key: dto.key - }); - - content[secretKey] = { value: secretValue }; - - if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) { - const commentValue = decryptSymmetric128BitHexKeyUTF8({ - ciphertext: secret.secretCommentCiphertext, - iv: secret.secretCommentIV, - tag: secret.secretCommentTag, - key: dto.key - }); - content[secretKey].comment = commentValue; - } - - content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding); - }); - - const expandSecrets = interpolateSecrets({ + const expandSecretReferences = interpolateSecrets({ projectId: dto.projectId, secretEncKey: dto.key, folderDAL, secretDAL }); - await expandSecrets(content); + // process secrets in current folder + const secrets = await secretDAL.findByFolderId(dto.folderId); + await Promise.allSettled( + secrets.map(async (secret) => { + const secretKey = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretKeyCiphertext, + iv: secret.secretKeyIV, + tag: secret.secretKeyTag, + key: dto.key + }); + + const secretValue = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretValueCiphertext, + iv: secret.secretValueIV, + tag: secret.secretValueTag, + key: dto.key + }); + const expandedSecretValue = await expandSecretReferences({ + environment: dto.environment, + secretPath: dto.secretPath, + skipMultilineEncoding: secret.skipMultilineEncoding, + value: secretValue + }); + + content[secretKey] = { value: expandedSecretValue || "" }; + + if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) { + const commentValue = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretCommentCiphertext, + iv: secret.secretCommentIV, + tag: secret.secretCommentTag, + key: dto.key + }); + content[secretKey].comment = commentValue; + } + + content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding); + }) + ); // check if current folder has any imports from other folders const secretImport = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false }); @@ -404,7 +418,8 @@ export const secretQueueFactory = ({ projectId: dto.projectId, folderId: folder.id, key: dto.key, - depth: dto.depth + 1 + depth: dto.depth + 1, + secretPath: dto.secretPath }); // add the imported secrets to the current folder secrets @@ -686,6 +701,7 @@ export const secretQueueFactory = ({ projectId, folderId: folder.id, depth: 1, + secretPath, decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : "") }) : await getIntegrationSecrets({ @@ -693,7 +709,8 @@ export const secretQueueFactory = ({ projectId, folderId: folder.id, key: botKey as string, - depth: 1 + depth: 1, + secretPath }); for (const integration of toBeSyncedIntegrations) { diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index c62a73f69..b7a9f0fc6 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -1047,74 +1047,47 @@ export const secretServiceFactory = ({ }; }); + const expandSecret = interpolateSecrets({ + folderDAL, + projectId, + secretDAL, + secretEncKey: botKey + }); + if (expandSecretReferences) { - const expandSecrets = interpolateSecrets({ - folderDAL, - projectId, - secretDAL, - secretEncKey: botKey - }); - - const batchSecretsExpand = async ( - secretBatch: { - secretKey: string; - secretValue: string; - secretComment?: string; - secretPath: string; - skipMultilineEncoding: boolean | null | undefined; - }[] - ) => { - // Group secrets by secretPath - const secretsByPath: Record< - string, - { - secretKey: string; - secretValue: string; - secretComment?: string; - skipMultilineEncoding: boolean | null | undefined; - }[] - > = {}; - - secretBatch.forEach((secret) => { - if (!secretsByPath[secret.secretPath]) { - secretsByPath[secret.secretPath] = []; - } - secretsByPath[secret.secretPath].push(secret); - }); - - // Expand secrets for each group - for (const secPath in secretsByPath) { - if (!Object.hasOwn(secretsByPath, path)) { - // eslint-disable-next-line no-continue - continue; - } - - const secretRecord: Record< - string, - { value: string; comment?: string; skipMultilineEncoding: boolean | null | undefined } - > = {}; - secretsByPath[secPath].forEach((decryptedSecret) => { - secretRecord[decryptedSecret.secretKey] = { - value: decryptedSecret.secretValue, - comment: decryptedSecret.secretComment, - skipMultilineEncoding: decryptedSecret.skipMultilineEncoding - }; - }); - - await expandSecrets(secretRecord); - - secretsByPath[secPath].forEach((decryptedSecret) => { - // eslint-disable-next-line no-param-reassign - decryptedSecret.secretValue = secretRecord[decryptedSecret.secretKey].value; - }); - } - }; - - // expand secrets - await batchSecretsExpand(filteredSecrets); - - // expand imports by batch - await Promise.all(processedImports.map((processedImport) => batchSecretsExpand(processedImport.secrets))); + const secretsGroupByPath = groupBy(filteredSecrets, (i) => i.secretPath); + await Promise.allSettled( + Object.keys(secretsGroupByPath).map((groupedPath) => + Promise.allSettled( + secretsGroupByPath[groupedPath].map(async (decryptedSecret, index) => { + const expandedSecretValue = await expandSecret({ + value: decryptedSecret.secretValue, + secretPath: groupedPath, + environment, + skipMultilineEncoding: decryptedSecret.skipMultilineEncoding + }); + // eslint-disable-next-line no-param-reassign + secretsGroupByPath[groupedPath][index].secretValue = expandedSecretValue || ""; + }) + ) + ) + ); + await Promise.allSettled( + processedImports.map((processedImport) => + Promise.allSettled( + processedImport.secrets.map(async (decryptedSecret, index) => { + const expandedSecretValue = await expandSecret({ + value: decryptedSecret.secretValue, + secretPath: path, + environment, + skipMultilineEncoding: decryptedSecret.skipMultilineEncoding + }); + // eslint-disable-next-line no-param-reassign + processedImport.secrets[index].secretValue = expandedSecretValue || ""; + }) + ) + ) + ); } return { @@ -1177,40 +1150,19 @@ export const secretServiceFactory = ({ const decryptedSecret = decryptSecretRaw(encryptedSecret, botKey); if (expandSecretReferences) { - const expandSecrets = interpolateSecrets({ + const expandSecret = interpolateSecrets({ folderDAL, projectId, secretDAL, secretEncKey: botKey }); - - const expandSingleSecret = async (secret: { - secretKey: string; - secretValue: string; - secretComment?: string; - secretPath: string; - skipMultilineEncoding: boolean | null | undefined; - }) => { - const secretRecord: Record< - string, - { value: string; comment?: string; skipMultilineEncoding: boolean | null | undefined } - > = { - [secret.secretKey]: { - value: secret.secretValue, - comment: secret.secretComment, - skipMultilineEncoding: secret.skipMultilineEncoding - } - }; - - await expandSecrets(secretRecord); - - // Update the secret with the expanded value - // eslint-disable-next-line no-param-reassign - secret.secretValue = secretRecord[secret.secretKey].value; - }; - - // Expand the secret - await expandSingleSecret(decryptedSecret); + const expandedSecretValue = await expandSecret({ + environment, + secretPath: path, + value: decryptedSecret.secretValue, + skipMultilineEncoding: decryptedSecret.skipMultilineEncoding + }); + decryptedSecret.secretValue = expandedSecretValue || ""; } return decryptedSecret;