feat: implemented new secret reference strategy

backend/src/services/integration-auth/integration-delete-secret.ts

@@ -30,6 +30,7 @@ const getIntegrationSecretsV2 = async (
     environment: string;
     folderId: string;
     depth: number;
+    secretPath: string;
     decryptor: (value: Buffer | null | undefined) => string;
   },
   secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "findByFolderId">,
@@ -306,6 +307,7 @@ export const deleteIntegrationSecrets = async ({
     ? await getIntegrationSecretsV2(
         {
           environment: integration.environment.id,
+          secretPath: integration.secretPath,
           projectId: integration.projectId,
           folderId: folder.id,
           depth: 1,

backend/src/services/secret-import/secret-import-fns.ts

@@ -158,9 +158,12 @@ export const fnSecretsV2FromImports = async ({
   depth?: number;
   cyclicDetector?: Set<string>;
   decryptor: (value?: Buffer | null) => string;
-  expandSecretReferences?: (
+  expandSecretReferences?: (inputSecret: {
-    secrets: Record<string, { value?: string; comment?: string; skipMultilineEncoding?: boolean | null }>
+    value?: string;
-  ) => Promise<Record<string, { value?: string; comment?: string; skipMultilineEncoding?: boolean | null }>>;
+    skipMultilineEncoding?: boolean | null;
+    secretPath: string;
+    environment: string;
+  }) => Promise<string | undefined>;
 }) => {
   // avoid going more than a depth
   if (depth >= LEVEL_BREAK) return [];
@@ -244,26 +247,21 @@ export const fnSecretsV2FromImports = async ({
   });
 
   if (expandSecretReferences) {
-    await Promise.all(
+    await Promise.allSettled(
-      processedImports.map(async (processedImport) => {
+      processedImports.map((processedImport) =>
-        const secretsGroupByKey = processedImport.secrets.reduce(
+        Promise.allSettled(
-          (acc, item) => {
+          processedImport.secrets.map(async (decryptedSecret, index) => {
-            acc[item.secretKey] = {
+            const expandedSecretValue = await expandSecretReferences({
-              value: item.secretValue,
+              value: decryptedSecret.secretValue,
-              comment: item.secretComment,
+              secretPath: processedImport.secretPath,
-              skipMultilineEncoding: item.skipMultilineEncoding
+              environment: processedImport.environment,
-            };
+              skipMultilineEncoding: decryptedSecret.skipMultilineEncoding
-            return acc;
+            });
-          },
+            // eslint-disable-next-line no-param-reassign
-          {} as Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean | null }>
+            processedImport.secrets[index].secretValue = expandedSecretValue || "";
-        );
+          })
-        // eslint-disable-next-line
+        )
-        await expandSecretReferences(secretsGroupByKey);
+      )
-        processedImport.secrets.forEach((decryptedSecret) => {
-          // eslint-disable-next-line no-param-reassign
-          decryptedSecret.secretValue = secretsGroupByKey[decryptedSecret.secretKey].value;
-        });
-      })
     );
   }
 

backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts

@@ -377,150 +377,118 @@ type TInterpolateSecretArg = {
   folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath">;
 };
 
+const MAX_SECRET_REFERENCE_DEPTH = 10;
 export const expandSecretReferencesFactory = ({
   projectId,
   decryptSecretValue: decryptSecret,
   secretDAL,
   folderDAL
 }: TInterpolateSecretArg) => {
-  const fetchSecretFactory = () => {
+  const secretCache: Record<string, Record<string, string>> = {};
-    const secretCache: Record<string, Record<string, string>> = {};
+  const getCacheUniqueKey = (environment: string, secretPath: string) => `${environment}-${secretPath}`;
 
-    return async (secRefEnv: string, secRefPath: string[], secRefKey: string) => {
+  const fetchSecret = async (environment: string, secretPath: string, secretKey: string) => {
-      const referredSecretPathURL = path.join("/", ...secRefPath);
+    const cacheKey = getCacheUniqueKey(environment, secretPath);
-      const uniqueKey = `${secRefEnv}-${referredSecretPathURL}`;
 
-      if (secretCache?.[uniqueKey]) {
+    if (secretCache?.[cacheKey]) {
-        return secretCache[uniqueKey][secRefKey];
+      return secretCache[cacheKey][secretKey] || "";
-      }
+    }
 
-      const folder = await folderDAL.findBySecretPath(projectId, secRefEnv, referredSecretPathURL);
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
-      if (!folder) return "";
+    if (!folder) return "";
-      const secrets = await secretDAL.findByFolderId(folder.id);
+    const secrets = await secretDAL.findByFolderId(folder.id);
 
-      const decryptedSecret = secrets.reduce<Record<string, string>>((prev, secret) => {
+    const decryptedSecret = secrets.reduce<Record<string, string>>((prev, secret) => {
-        // eslint-disable-next-line
+      // eslint-disable-next-line
-        prev[secret.key] = decryptSecret(secret.encryptedValue) || "";
+      prev[secret.key] = decryptSecret(secret.encryptedValue) || "";
-        return prev;
+      return prev;
-      }, {});
+    }, {});
 
-      secretCache[uniqueKey] = decryptedSecret;
+    secretCache[cacheKey] = decryptedSecret;
 
-      return secretCache[uniqueKey][secRefKey];
+    return secretCache[cacheKey][secretKey] || "";
-    };
   };
 
-  const recursivelyExpandSecret = async (
+  const recursivelyExpandSecret = async ({
-    expandedSec: Record<string, string | undefined>,
+    value,
-    interpolatedSec: Record<string, string | undefined>,
+    secretPath,
-    fetchSecret: (env: string, secPath: string[], secKey: string) => Promise<string>,
+    environment,
-    recursionChainBreaker: Record<string, boolean>,
+    depth = 1
-    key: string
+  }: {
-  ): Promise<string | undefined> => {
+    value?: string;
-    if (expandedSec?.[key] !== undefined) {
+    secretPath: string;
-      return expandedSec[key];
+    environment: string;
-    }
+    depth?: number;
-    if (recursionChainBreaker?.[key]) {
+  }) => {
-      return "";
+    if (!value) return "";
-    }
+    if (depth > MAX_SECRET_REFERENCE_DEPTH) return "";
-    // eslint-disable-next-line
-    recursionChainBreaker[key] = true;
 
-    let interpolatedValue = interpolatedSec[key];
+    const refs = value.match(INTERPOLATION_SYNTAX_REG);
-    if (!interpolatedValue) {
+    let expandedValue = value;
-      // eslint-disable-next-line no-console
-      console.error(`Couldn't find referenced value - ${key}`);
-      return "";
-    }
-
-    const refs = interpolatedValue.match(INTERPOLATION_SYNTAX_REG);
     if (refs) {
       for (const interpolationSyntax of refs) {
         const interpolationKey = interpolationSyntax.slice(2, interpolationSyntax.length - 1);
         const entities = interpolationKey.trim().split(".");
 
         if (entities.length === 1) {
+          const [secretKey] = entities;
           // eslint-disable-next-line
-          const val = await recursivelyExpandSecret(
+          let referenceValue = await fetchSecret(environment, secretPath, secretKey);
-            expandedSec,
+          if (INTERPOLATION_SYNTAX_REG.test(referenceValue)) {
-            interpolatedSec,
+            // eslint-disable-next-line
-            fetchSecret,
+            referenceValue = await recursivelyExpandSecret({
-            recursionChainBreaker,
+              environment,
-            interpolationKey
+              secretPath,
-          );
+              value: referenceValue,
-          if (val) {
+              depth: depth + 1
-            interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val);
+            });
           }
-          // eslint-disable-next-line
+          const cacheKey = getCacheUniqueKey(environment, secretPath);
-          continue;
+          secretCache[cacheKey][secretKey] = referenceValue;
+          expandedValue = expandedValue.replaceAll(interpolationSyntax, referenceValue);
         }
 
         if (entities.length > 1) {
-          const secRefEnv = entities[0];
+          const secretReferenceEnvironment = entities[0];
-          const secRefPath = entities.slice(1, entities.length - 1);
+          const secretReferencePath = path.join("/", ...entities.slice(1, entities.length - 1));
-          const secRefKey = entities[entities.length - 1];
+          const secretReferenceKey = entities[entities.length - 1];
 
           // eslint-disable-next-line
-          const val = await fetchSecret(secRefEnv, secRefPath, secRefKey);
+          let referenceValue = await fetchSecret(secretReferenceEnvironment, secretReferencePath, secretReferenceKey);
-          if (val) {
+          if (INTERPOLATION_SYNTAX_REG.test(referenceValue)) {
-            interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val);
+            // eslint-disable-next-line
+            referenceValue = await recursivelyExpandSecret({
+              environment: secretReferenceEnvironment,
+              secretPath: secretReferencePath,
+              value: referenceValue,
+              depth: depth + 1
+            });
           }
+          const cacheKey = getCacheUniqueKey(secretReferenceEnvironment, secretReferencePath);
+          secretCache[cacheKey][secretReferenceKey] = referenceValue;
+          expandedValue = expandedValue.replaceAll(interpolationSyntax, referenceValue);
         }
       }
     }
 
-    // eslint-disable-next-line
+    return expandedValue;
-    expandedSec[key] = interpolatedValue;
-    return interpolatedValue;
   };
 
-  const fetchSecret = fetchSecretFactory();
+  const expandSecret = async (inputSecret: {
-  const expandSecrets = async (
+    value?: string;
-    inputSecrets: Record<string, { value?: string; comment?: string; skipMultilineEncoding?: boolean | null }>
+    skipMultilineEncoding?: boolean | null;
-  ) => {
+    secretPath: string;
-    const expandedSecrets: Record<string, string | undefined> = {};
+    environment: string;
-    const toBeExpandedSecrets: Record<string, string | undefined> = {};
+  }) => {
+    if (!inputSecret.value) return inputSecret.value;
 
-    Object.keys(inputSecrets).forEach((key) => {
+    const shouldExpand = Boolean(inputSecret.value?.match(INTERPOLATION_SYNTAX_REG));
-      if (inputSecrets[key].value?.match(INTERPOLATION_SYNTAX_REG)) {
+    if (!shouldExpand) return inputSecret.value;
-        toBeExpandedSecrets[key] = inputSecrets[key].value;
-      } else {
-        expandedSecrets[key] = inputSecrets[key].value;
-      }
-    });
 
-    for (const key of Object.keys(inputSecrets)) {
+    const expandedSecretValue = await recursivelyExpandSecret(inputSecret);
-      if (expandedSecrets?.[key]) {
+    return inputSecret.skipMultilineEncoding ? formatMultiValueEnv(expandedSecretValue) : expandedSecretValue;
-        // should not do multi line encoding if user has set it to skip
-        // eslint-disable-next-line
-        inputSecrets[key].value = inputSecrets[key].skipMultilineEncoding
-          ? formatMultiValueEnv(expandedSecrets[key])
-          : expandedSecrets[key];
-        // eslint-disable-next-line
-        continue;
-      }
-
-      // this is to avoid recursion loop. So the graph should be direct graph rather than cyclic
-      // so for any recursion building if there is an entity two times same key meaning it will be looped
-      const recursionChainBreaker: Record<string, boolean> = {};
-      // eslint-disable-next-line
-      const expandedVal = await recursivelyExpandSecret(
-        expandedSecrets,
-        toBeExpandedSecrets,
-        fetchSecret,
-        recursionChainBreaker,
-        key
-      );
-
-      // eslint-disable-next-line
-      inputSecrets[key].value = inputSecrets[key].skipMultilineEncoding
-        ? formatMultiValueEnv(expandedVal)
-        : expandedVal;
-    }
-
-    return inputSecrets;
   };
-  return expandSecrets;
+  return expandSecret;
 };
 
 export const reshapeBridgeSecret = (

backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts

@@ -521,27 +521,22 @@ export const secretV2BridgeServiceFactory = ({
 
     if (shouldExpandSecretReferences) {
       const secretsGroupByPath = groupBy(filteredSecrets, (i) => i.secretPath);
-      for (const secretPathKey in secretsGroupByPath) {
+      await Promise.allSettled(
-        if (Object.hasOwn(secretsGroupByPath, secretPathKey)) {
+        Object.keys(secretsGroupByPath).map((groupedPath) =>
-          const secretsGroupByKey = secretsGroupByPath[secretPathKey].reduce(
+          Promise.allSettled(
-            (acc, item) => {
+            secretsGroupByPath[groupedPath].map(async (decryptedSecret, index) => {
-              acc[item.secretKey] = {
+              const expandedSecretValue = await expandSecretReferences({
-                value: item.secretValue,
+                value: decryptedSecret.secretValue,
-                comment: item.secretComment,
+                secretPath: groupedPath,
-                skipMultilineEncoding: item.skipMultilineEncoding
+                environment,
-              };
+                skipMultilineEncoding: decryptedSecret.skipMultilineEncoding
-              return acc;
+              });
-            },
+              // eslint-disable-next-line no-param-reassign
-            {} as Record<string, { value?: string; comment?: string; skipMultilineEncoding?: boolean | null }>
+              secretsGroupByPath[groupedPath][index].secretValue = expandedSecretValue || "";
-          );
+            })
-          // eslint-disable-next-line
+          )
-          await expandSecretReferences(secretsGroupByKey);
+        )
-          secretsGroupByPath[secretPathKey].forEach((decryptedSecret) => {
+      );
-            // eslint-disable-next-line no-param-reassign
-            decryptedSecret.secretValue = secretsGroupByKey[decryptedSecret.secretKey].value || "";
-          });
-        }
-      }
     }
 
     if (!includeImports) {
@@ -693,12 +688,14 @@ export const secretV2BridgeServiceFactory = ({
       ? secretManagerDecryptor({ cipherTextBlob: secret.encryptedValue }).toString()
       : "";
     if (shouldExpandSecretReferences && secretValue) {
-      const secretReferenceExpandedRecord = {
-        [secret.key]: { value: secretValue }
-      };
       // eslint-disable-next-line
-      await expandSecretReferences(secretReferenceExpandedRecord);
+      const expandedSecretValue = await expandSecretReferences({
-      secretValue = secretReferenceExpandedRecord[secret.key].value;
+        environment,
+        secretPath: path,
+        value: secretValue,
+        skipMultilineEncoding: secret.skipMultilineEncoding
+      });
+      secretValue = expandedSecretValue || "";
     }
 
     return reshapeBridgeSecret(projectId, environment, path, {

backend/src/services/secret/secret-fns.ts

@@ -196,6 +196,13 @@ export const recursivelyGetSecretPaths = ({
   return getPaths;
 };
 
+// used to convert multi line ones to quotes ones with \n
+const formatMultiValueEnv = (val?: string) => {
+  if (!val) return "";
+  if (!val.match("\n")) return val;
+  return `"${val.replace(/\n/g, "\\n")}"`;
+};
+
 type TInterpolateSecretArg = {
   projectId: string;
   secretEncKey: string;
@@ -203,162 +210,128 @@ type TInterpolateSecretArg = {
   folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath">;
 };
 
+const MAX_SECRET_REFERENCE_DEPTH = 5;
 const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g;
 export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderDAL }: TInterpolateSecretArg) => {
-  const fetchSecretsCrossEnv = () => {
+  const secretCache: Record<string, Record<string, string>> = {};
-    const fetchCache: Record<string, Record<string, string>> = {};
+  const getCacheUniqueKey = (environment: string, secretPath: string) => `${environment}-${secretPath}`;
 
-    return async (secRefEnv: string, secRefPath: string[], secRefKey: string) => {
+  const fetchSecret = async (environment: string, secretPath: string, secretKey: string) => {
-      const secRefPathUrl = path.join("/", ...secRefPath);
+    const cacheKey = getCacheUniqueKey(environment, secretPath);
-      const uniqKey = `${secRefEnv}-${secRefPathUrl}`;
+    const uniqKey = `${environment}-${cacheKey}`;
 
-      if (fetchCache?.[uniqKey]) {
+    if (secretCache?.[uniqKey]) {
-        return fetchCache[uniqKey][secRefKey];
+      return secretCache[uniqKey][secretKey] || "";
-      }
+    }
 
-      const folder = await folderDAL.findBySecretPath(projectId, secRefEnv, secRefPathUrl);
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
-      if (!folder) return "";
+    if (!folder) return "";
-      const secrets = await secretDAL.findByFolderId(folder.id);
+    const secrets = await secretDAL.findByFolderId(folder.id);
 
-      const decryptedSec = secrets.reduce<Record<string, string>>((prev, secret) => {
+    const decryptedSec = secrets.reduce<Record<string, string>>((prev, secret) => {
-        const secretKey = decryptSymmetric128BitHexKeyUTF8({
+      const decryptedSecretKey = decryptSymmetric128BitHexKeyUTF8({
-          ciphertext: secret.secretKeyCiphertext,
+        ciphertext: secret.secretKeyCiphertext,
-          iv: secret.secretKeyIV,
+        iv: secret.secretKeyIV,
-          tag: secret.secretKeyTag,
+        tag: secret.secretKeyTag,
-          key: secretEncKey
+        key: secretEncKey
-        });
+      });
-        const secretValue = decryptSymmetric128BitHexKeyUTF8({
+      const decryptedSecretValue = decryptSymmetric128BitHexKeyUTF8({
-          ciphertext: secret.secretValueCiphertext,
+        ciphertext: secret.secretValueCiphertext,
-          iv: secret.secretValueIV,
+        iv: secret.secretValueIV,
-          tag: secret.secretValueTag,
+        tag: secret.secretValueTag,
-          key: secretEncKey
+        key: secretEncKey
-        });
+      });
 
-        // eslint-disable-next-line
+      // eslint-disable-next-line
-        prev[secretKey] = secretValue;
+      prev[decryptedSecretKey] = decryptedSecretValue;
-        return prev;
+      return prev;
-      }, {});
+    }, {});
 
-      fetchCache[uniqKey] = decryptedSec;
+    secretCache[uniqKey] = decryptedSec;
 
-      return fetchCache[uniqKey][secRefKey];
+    return secretCache[uniqKey][secretKey] || "";
-    };
   };
 
-  const recursivelyExpandSecret = async (
+  const recursivelyExpandSecret = async ({
-    expandedSec: Record<string, string>,
+    value,
-    interpolatedSec: Record<string, string>,
+    secretPath,
-    fetchCrossEnv: (env: string, secPath: string[], secKey: string) => Promise<string>,
+    environment,
-    recursionChainBreaker: Record<string, boolean>,
+    depth = 0
-    key: string
+  }: {
-  ) => {
+    value?: string;
-    if (expandedSec?.[key] !== undefined) {
+    secretPath: string;
-      return expandedSec[key];
+    environment: string;
-    }
+    depth?: number;
-    if (recursionChainBreaker?.[key]) {
+  }) => {
-      return "";
+    if (!value) return "";
-    }
+    if (depth > MAX_SECRET_REFERENCE_DEPTH) return "";
-    // eslint-disable-next-line
-    recursionChainBreaker[key] = true;
 
-    let interpolatedValue = interpolatedSec[key];
+    const refs = value.match(INTERPOLATION_SYNTAX_REG);
-    if (!interpolatedValue) {
+    let expandedValue = value;
-      // eslint-disable-next-line no-console
-      console.error(`Couldn't find referenced value - ${key}`);
-      return "";
-    }
-
-    const refs = interpolatedValue.match(INTERPOLATION_SYNTAX_REG);
     if (refs) {
       for (const interpolationSyntax of refs) {
         const interpolationKey = interpolationSyntax.slice(2, interpolationSyntax.length - 1);
         const entities = interpolationKey.trim().split(".");
 
         if (entities.length === 1) {
-          const val = await recursivelyExpandSecret(
+          const [secretKey] = entities;
-            expandedSec,
-            interpolatedSec,
-            fetchCrossEnv,
-            recursionChainBreaker,
-            interpolationKey
-          );
-          if (val) {
-            interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val);
-          }
           // eslint-disable-next-line
-          continue;
+          let referenceValue = await fetchSecret(environment, secretPath, secretKey);
+          if (INTERPOLATION_SYNTAX_REG.test(referenceValue)) {
+            // eslint-disable-next-line
+            referenceValue = await recursivelyExpandSecret({
+              environment,
+              secretPath,
+              value: referenceValue,
+              depth: depth + 1
+            });
+          }
+          const cacheKey = getCacheUniqueKey(environment, secretPath);
+          secretCache[cacheKey][secretKey] = referenceValue;
+          expandedValue = expandedValue.replaceAll(interpolationSyntax, referenceValue);
         }
 
         if (entities.length > 1) {
-          const secRefEnv = entities[0];
+          const secretReferenceEnvironment = entities[0];
-          const secRefPath = entities.slice(1, entities.length - 1);
+          const secretReferencePath = path.join("/", ...entities.slice(1, entities.length - 1));
-          const secRefKey = entities[entities.length - 1];
+          const secretReferenceKey = entities[entities.length - 1];
 
-          const val = await fetchCrossEnv(secRefEnv, secRefPath, secRefKey);
+          // eslint-disable-next-line
-          if (val) {
+          let referenceValue = await fetchSecret(secretReferenceEnvironment, secretReferencePath, secretReferenceKey);
-            interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val);
+          if (INTERPOLATION_SYNTAX_REG.test(referenceValue)) {
+            // eslint-disable-next-line
+            referenceValue = await recursivelyExpandSecret({
+              environment: secretReferenceEnvironment,
+              secretPath: secretReferencePath,
+              value: referenceValue,
+              depth: depth + 1
+            });
           }
+          const cacheKey = getCacheUniqueKey(secretReferenceEnvironment, secretReferencePath);
+          secretCache[cacheKey][secretReferenceKey] = referenceValue;
+          expandedValue = expandedValue.replaceAll(interpolationSyntax, referenceValue);
         }
       }
     }
 
-    // eslint-disable-next-line
+    return expandedValue;
-    expandedSec[key] = interpolatedValue;
-    return interpolatedValue;
   };
 
-  // used to convert multi line ones to quotes ones with \n
+  const expandSecret = async (inputSecret: {
-  const formatMultiValueEnv = (val?: string) => {
+    value?: string;
-    if (!val) return "";
+    skipMultilineEncoding?: boolean | null;
-    if (!val.match("\n")) return val;
+    secretPath: string;
-    return `"${val.replace(/\n/g, "\\n")}"`;
+    environment: string;
+  }) => {
+    if (!inputSecret.value) return inputSecret.value;
+
+    const shouldExpand = Boolean(inputSecret.value?.match(INTERPOLATION_SYNTAX_REG));
+    if (!shouldExpand) return inputSecret.value;
+
+    const expandedSecretValue = await recursivelyExpandSecret(inputSecret);
+    return inputSecret.skipMultilineEncoding ? formatMultiValueEnv(expandedSecretValue) : expandedSecretValue;
   };
-
+  return expandSecret;
-  const expandSecrets = async (
-    secrets: Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean | null }>
-  ) => {
-    const expandedSec: Record<string, string> = {};
-    const interpolatedSec: Record<string, string> = {};
-
-    const crossSecEnvFetch = fetchSecretsCrossEnv();
-
-    Object.keys(secrets).forEach((key) => {
-      if (secrets[key].value.match(INTERPOLATION_SYNTAX_REG)) {
-        interpolatedSec[key] = secrets[key].value;
-      } else {
-        expandedSec[key] = secrets[key].value;
-      }
-    });
-
-    for (const key of Object.keys(secrets)) {
-      if (expandedSec?.[key]) {
-        // should not do multi line encoding if user has set it to skip
-        // eslint-disable-next-line
-        secrets[key].value = secrets[key].skipMultilineEncoding
-          ? formatMultiValueEnv(expandedSec[key])
-          : expandedSec[key];
-        // eslint-disable-next-line
-        continue;
-      }
-
-      // this is to avoid recursion loop. So the graph should be direct graph rather than cyclic
-      // so for any recursion building if there is an entity two times same key meaning it will be looped
-      const recursionChainBreaker: Record<string, boolean> = {};
-      const expandedVal = await recursivelyExpandSecret(
-        expandedSec,
-        interpolatedSec,
-        crossSecEnvFetch,
-        recursionChainBreaker,
-        key
-      );
-
-      // eslint-disable-next-line
-      secrets[key].value = secrets[key].skipMultilineEncoding ? formatMultiValueEnv(expandedVal) : expandedVal;
-    }
-
-    return secrets;
-  };
-  return expandSecrets;
 };
 
 export const decryptSecretRaw = (

backend/src/services/secret/secret-queue.ts

@@ -258,6 +258,7 @@ export const secretQueueFactory = ({
   const getIntegrationSecretsV2 = async (dto: {
     projectId: string;
     environment: string;
+    secretPath: string;
     folderId: string;
     depth: number;
     decryptor: (value: Buffer | null | undefined) => string;
@@ -269,30 +270,36 @@ export const secretQueueFactory = ({
       );
       return content;
     }
-
-    // process secrets in current folder
-    const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId);
-    secrets.forEach((secret) => {
-      const secretKey = secret.key;
-      const secretValue = dto.decryptor(secret.encryptedValue);
-      content[secretKey] = { value: secretValue };
-
-      if (secret.encryptedComment) {
-        const commentValue = dto.decryptor(secret.encryptedComment);
-        content[secretKey].comment = commentValue;
-      }
-
-      content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
-    });
-
     const expandSecretReferences = expandSecretReferencesFactory({
       decryptSecretValue: dto.decryptor,
       secretDAL: secretV2BridgeDAL,
       folderDAL,
       projectId: dto.projectId
     });
+    // process secrets in current folder
+    const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId);
+
+    await Promise.allSettled(
+      secrets.map(async (secret) => {
+        const secretKey = secret.key;
+        const secretValue = dto.decryptor(secret.encryptedValue);
+        const expandedSecretValue = await expandSecretReferences({
+          environment: dto.environment,
+          secretPath: dto.secretPath,
+          skipMultilineEncoding: secret.skipMultilineEncoding,
+          value: secretValue
+        });
+        content[secretKey] = { value: expandedSecretValue || "" };
+
+        if (secret.encryptedComment) {
+          const commentValue = dto.decryptor(secret.encryptedComment);
+          content[secretKey].comment = commentValue;
+        }
+
+        content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
+      })
+    );
 
-    await expandSecretReferences(content);
     // check if current folder has any imports from other folders
     const secretImports = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false });
 
@@ -329,6 +336,7 @@ export const secretQueueFactory = ({
   const getIntegrationSecrets = async (dto: {
     projectId: string;
     environment: string;
+    secretPath: string;
     folderId: string;
     key: string;
     depth: number;
@@ -341,46 +349,52 @@ export const secretQueueFactory = ({
       return content;
     }
 
-    // process secrets in current folder
+    const expandSecretReferences = interpolateSecrets({
-    const secrets = await secretDAL.findByFolderId(dto.folderId);
-    secrets.forEach((secret) => {
-      const secretKey = decryptSymmetric128BitHexKeyUTF8({
-        ciphertext: secret.secretKeyCiphertext,
-        iv: secret.secretKeyIV,
-        tag: secret.secretKeyTag,
-        key: dto.key
-      });
-
-      const secretValue = decryptSymmetric128BitHexKeyUTF8({
-        ciphertext: secret.secretValueCiphertext,
-        iv: secret.secretValueIV,
-        tag: secret.secretValueTag,
-        key: dto.key
-      });
-
-      content[secretKey] = { value: secretValue };
-
-      if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) {
-        const commentValue = decryptSymmetric128BitHexKeyUTF8({
-          ciphertext: secret.secretCommentCiphertext,
-          iv: secret.secretCommentIV,
-          tag: secret.secretCommentTag,
-          key: dto.key
-        });
-        content[secretKey].comment = commentValue;
-      }
-
-      content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
-    });
-
-    const expandSecrets = interpolateSecrets({
       projectId: dto.projectId,
       secretEncKey: dto.key,
       folderDAL,
       secretDAL
     });
 
-    await expandSecrets(content);
+    // process secrets in current folder
+    const secrets = await secretDAL.findByFolderId(dto.folderId);
+    await Promise.allSettled(
+      secrets.map(async (secret) => {
+        const secretKey = decryptSymmetric128BitHexKeyUTF8({
+          ciphertext: secret.secretKeyCiphertext,
+          iv: secret.secretKeyIV,
+          tag: secret.secretKeyTag,
+          key: dto.key
+        });
+
+        const secretValue = decryptSymmetric128BitHexKeyUTF8({
+          ciphertext: secret.secretValueCiphertext,
+          iv: secret.secretValueIV,
+          tag: secret.secretValueTag,
+          key: dto.key
+        });
+        const expandedSecretValue = await expandSecretReferences({
+          environment: dto.environment,
+          secretPath: dto.secretPath,
+          skipMultilineEncoding: secret.skipMultilineEncoding,
+          value: secretValue
+        });
+
+        content[secretKey] = { value: expandedSecretValue || "" };
+
+        if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) {
+          const commentValue = decryptSymmetric128BitHexKeyUTF8({
+            ciphertext: secret.secretCommentCiphertext,
+            iv: secret.secretCommentIV,
+            tag: secret.secretCommentTag,
+            key: dto.key
+          });
+          content[secretKey].comment = commentValue;
+        }
+
+        content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
+      })
+    );
 
     // check if current folder has any imports from other folders
     const secretImport = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false });
@@ -404,7 +418,8 @@ export const secretQueueFactory = ({
           projectId: dto.projectId,
           folderId: folder.id,
           key: dto.key,
-          depth: dto.depth + 1
+          depth: dto.depth + 1,
+          secretPath: dto.secretPath
         });
 
         // add the imported secrets to the current folder secrets
@@ -686,6 +701,7 @@ export const secretQueueFactory = ({
             projectId,
             folderId: folder.id,
             depth: 1,
+            secretPath,
             decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : "")
           })
         : await getIntegrationSecrets({
@@ -693,7 +709,8 @@ export const secretQueueFactory = ({
             projectId,
             folderId: folder.id,
             key: botKey as string,
-            depth: 1
+            depth: 1,
+            secretPath
           });
 
       for (const integration of toBeSyncedIntegrations) {

backend/src/services/secret/secret-service.ts

@@ -1047,74 +1047,47 @@ export const secretServiceFactory = ({
       };
     });
 
+    const expandSecret = interpolateSecrets({
+      folderDAL,
+      projectId,
+      secretDAL,
+      secretEncKey: botKey
+    });
+
     if (expandSecretReferences) {
-      const expandSecrets = interpolateSecrets({
+      const secretsGroupByPath = groupBy(filteredSecrets, (i) => i.secretPath);
-        folderDAL,
+      await Promise.allSettled(
-        projectId,
+        Object.keys(secretsGroupByPath).map((groupedPath) =>
-        secretDAL,
+          Promise.allSettled(
-        secretEncKey: botKey
+            secretsGroupByPath[groupedPath].map(async (decryptedSecret, index) => {
-      });
+              const expandedSecretValue = await expandSecret({
-
+                value: decryptedSecret.secretValue,
-      const batchSecretsExpand = async (
+                secretPath: groupedPath,
-        secretBatch: {
+                environment,
-          secretKey: string;
+                skipMultilineEncoding: decryptedSecret.skipMultilineEncoding
-          secretValue: string;
+              });
-          secretComment?: string;
+              // eslint-disable-next-line no-param-reassign
-          secretPath: string;
+              secretsGroupByPath[groupedPath][index].secretValue = expandedSecretValue || "";
-          skipMultilineEncoding: boolean | null | undefined;
+            })
-        }[]
+          )
-      ) => {
+        )
-        // Group secrets by secretPath
+      );
-        const secretsByPath: Record<
+      await Promise.allSettled(
-          string,
+        processedImports.map((processedImport) =>
-          {
+          Promise.allSettled(
-            secretKey: string;
+            processedImport.secrets.map(async (decryptedSecret, index) => {
-            secretValue: string;
+              const expandedSecretValue = await expandSecret({
-            secretComment?: string;
+                value: decryptedSecret.secretValue,
-            skipMultilineEncoding: boolean | null | undefined;
+                secretPath: path,
-          }[]
+                environment,
-        > = {};
+                skipMultilineEncoding: decryptedSecret.skipMultilineEncoding
-
+              });
-        secretBatch.forEach((secret) => {
+              // eslint-disable-next-line no-param-reassign
-          if (!secretsByPath[secret.secretPath]) {
+              processedImport.secrets[index].secretValue = expandedSecretValue || "";
-            secretsByPath[secret.secretPath] = [];
+            })
-          }
+          )
-          secretsByPath[secret.secretPath].push(secret);
+        )
-        });
+      );
-
-        // Expand secrets for each group
-        for (const secPath in secretsByPath) {
-          if (!Object.hasOwn(secretsByPath, path)) {
-            // eslint-disable-next-line no-continue
-            continue;
-          }
-
-          const secretRecord: Record<
-            string,
-            { value: string; comment?: string; skipMultilineEncoding: boolean | null | undefined }
-          > = {};
-          secretsByPath[secPath].forEach((decryptedSecret) => {
-            secretRecord[decryptedSecret.secretKey] = {
-              value: decryptedSecret.secretValue,
-              comment: decryptedSecret.secretComment,
-              skipMultilineEncoding: decryptedSecret.skipMultilineEncoding
-            };
-          });
-
-          await expandSecrets(secretRecord);
-
-          secretsByPath[secPath].forEach((decryptedSecret) => {
-            // eslint-disable-next-line no-param-reassign
-            decryptedSecret.secretValue = secretRecord[decryptedSecret.secretKey].value;
-          });
-        }
-      };
-
-      // expand secrets
-      await batchSecretsExpand(filteredSecrets);
-
-      // expand imports by batch
-      await Promise.all(processedImports.map((processedImport) => batchSecretsExpand(processedImport.secrets)));
     }
 
     return {
@@ -1177,40 +1150,19 @@ export const secretServiceFactory = ({
     const decryptedSecret = decryptSecretRaw(encryptedSecret, botKey);
 
     if (expandSecretReferences) {
-      const expandSecrets = interpolateSecrets({
+      const expandSecret = interpolateSecrets({
         folderDAL,
         projectId,
         secretDAL,
         secretEncKey: botKey
       });
-
+      const expandedSecretValue = await expandSecret({
-      const expandSingleSecret = async (secret: {
+        environment,
-        secretKey: string;
+        secretPath: path,
-        secretValue: string;
+        value: decryptedSecret.secretValue,
-        secretComment?: string;
+        skipMultilineEncoding: decryptedSecret.skipMultilineEncoding
-        secretPath: string;
+      });
-        skipMultilineEncoding: boolean | null | undefined;
+      decryptedSecret.secretValue = expandedSecretValue || "";
-      }) => {
-        const secretRecord: Record<
-          string,
-          { value: string; comment?: string; skipMultilineEncoding: boolean | null | undefined }
-        > = {
-          [secret.secretKey]: {
-            value: secret.secretValue,
-            comment: secret.secretComment,
-            skipMultilineEncoding: secret.skipMultilineEncoding
-          }
-        };
-
-        await expandSecrets(secretRecord);
-
-        // Update the secret with the expanded value
-        // eslint-disable-next-line no-param-reassign
-        secret.secretValue = secretRecord[secret.secretKey].value;
-      };
-
-      // Expand the secret
-      await expandSingleSecret(decryptedSecret);
     }
 
     return decryptedSecret;