Merge pull request #813 from akhilmhdh/feat/sec-exp-ingtegration

Secret expansion and import in integrations

backend/src/helpers/bot.ts

@@ -4,18 +4,20 @@ import {
   decryptAsymmetric,
   decryptSymmetric128BitHexKeyUTF8,
   encryptSymmetric128BitHexKeyUTF8,
-  generateKeyPair,
+  generateKeyPair
 } from "../utils/crypto";
 import {
   ALGORITHM_AES_256_GCM,
   ENCODING_SCHEME_BASE64,
   ENCODING_SCHEME_UTF8,
-  SECRET_SHARED,
+  SECRET_SHARED
 } from "../variables";
 import { client, getEncryptionKey, getRootEncryptionKey } from "../config";
 import { InternalServerError } from "../utils/errors";
 import Folder from "../models/folder";
 import { getFolderByPath } from "../services/FolderService";
+import { getAllImportedSecrets } from "../services/SecretImportService";
+import { expandSecrets } from "./secrets";
 
 /**
  * Create an inactive bot with name [name] for workspace with id [workspaceId]
@@ -25,7 +27,7 @@ import { getFolderByPath } from "../services/FolderService";
  */
 export const createBot = async ({
   name,
-  workspaceId,
+  workspaceId
 }: {
   name: string;
   workspaceId: Types.ObjectId;
@@ -36,10 +38,7 @@ export const createBot = async ({
   const { publicKey, privateKey } = generateKeyPair();
 
   if (rootEncryptionKey) {
-    const { ciphertext, iv, tag } = client.encryptSymmetric(
+    const { ciphertext, iv, tag } = client.encryptSymmetric(privateKey, rootEncryptionKey);
-      privateKey,
-      rootEncryptionKey
-    );
 
     return await new Bot({
       name,
@@ -50,12 +49,12 @@ export const createBot = async ({
       iv,
       tag,
       algorithm: ALGORITHM_AES_256_GCM,
-      keyEncoding: ENCODING_SCHEME_BASE64,
+      keyEncoding: ENCODING_SCHEME_BASE64
     }).save();
   } else if (encryptionKey) {
     const { ciphertext, iv, tag } = encryptSymmetric128BitHexKeyUTF8({
       plaintext: privateKey,
-      key: await getEncryptionKey(),
+      key: await getEncryptionKey()
     });
 
     return await new Bot({
@@ -67,12 +66,12 @@ export const createBot = async ({
       iv,
       tag,
       algorithm: ALGORITHM_AES_256_GCM,
-      keyEncoding: ENCODING_SCHEME_UTF8,
+      keyEncoding: ENCODING_SCHEME_UTF8
     }).save();
   }
 
   throw InternalServerError({
-    message: "Failed to create new bot due to missing encryption key",
+    message: "Failed to create new bot due to missing encryption key"
   });
 };
 
@@ -82,7 +81,7 @@ export const createBot = async ({
  */
 export const getIsWorkspaceE2EEHelper = async (workspaceId: Types.ObjectId) => {
   const botKey = await BotKey.exists({
-    workspace: workspaceId,
+    workspace: workspaceId
   });
 
   return botKey ? false : true;
@@ -98,19 +97,19 @@ export const getIsWorkspaceE2EEHelper = async (workspaceId: Types.ObjectId) => {
 export const getSecretsBotHelper = async ({
   workspaceId,
   environment,
-  secretPath,
+  secretPath
 }: {
   workspaceId: Types.ObjectId;
   environment: string;
   secretPath: string;
 }) => {
-  const content = {} as any;
+  const content: Record<string, { value: string; comment?: string }> = {};
   const key = await getKey({ workspaceId: workspaceId });
 
   let folderId = "root";
   const folders = await Folder.findOne({
     workspace: workspaceId,
-    environment,
+    environment
   });
 
   if (!folders && secretPath !== "/") {
@@ -129,7 +128,43 @@ export const getSecretsBotHelper = async ({
     workspace: workspaceId,
     environment,
     type: SECRET_SHARED,
-    folder: folderId,
+    folder: folderId
+  });
+
+  const importedSecrets = await getAllImportedSecrets(
+    workspaceId.toString(),
+    environment,
+    folderId
+  );
+
+  importedSecrets.forEach(({ secrets }) => {
+    secrets.forEach((secret) => {
+      const secretKey = decryptSymmetric128BitHexKeyUTF8({
+        ciphertext: secret.secretKeyCiphertext,
+        iv: secret.secretKeyIV,
+        tag: secret.secretKeyTag,
+        key
+      });
+
+      const secretValue = decryptSymmetric128BitHexKeyUTF8({
+        ciphertext: secret.secretValueCiphertext,
+        iv: secret.secretValueIV,
+        tag: secret.secretValueTag,
+        key
+      });
+
+      content[secretKey] = { value: secretValue };
+
+      if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) {
+        const commentValue = decryptSymmetric128BitHexKeyUTF8({
+          ciphertext: secret.secretCommentCiphertext,
+          iv: secret.secretCommentIV,
+          tag: secret.secretCommentTag,
+          key
+        });
+        content[secretKey].comment = commentValue;
+      }
+    });
   });
 
   secrets.forEach((secret: ISecret) => {
@@ -137,19 +172,31 @@ export const getSecretsBotHelper = async ({
       ciphertext: secret.secretKeyCiphertext,
       iv: secret.secretKeyIV,
       tag: secret.secretKeyTag,
-      key,
+      key
     });
 
     const secretValue = decryptSymmetric128BitHexKeyUTF8({
       ciphertext: secret.secretValueCiphertext,
       iv: secret.secretValueIV,
       tag: secret.secretValueTag,
-      key,
+      key
     });
 
-    content[secretKey] = secretValue;
+    content[secretKey] = { value: secretValue };
+
+    if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) {
+      const commentValue = decryptSymmetric128BitHexKeyUTF8({
+        ciphertext: secret.secretCommentCiphertext,
+        iv: secret.secretCommentIV,
+        tag: secret.secretCommentTag,
+        key
+      });
+      content[secretKey].comment = commentValue;
+    }
   });
 
+  await expandSecrets(workspaceId.toString(), key, content);
+
   return content;
 };
 
@@ -160,22 +207,18 @@ export const getSecretsBotHelper = async ({
  * @param {String} obj.workspaceId - id of workspace
  * @returns {String} key - decrypted workspace key
  */
-export const getKey = async ({
+export const getKey = async ({ workspaceId }: { workspaceId: Types.ObjectId }) => {
-  workspaceId,
-}: {
-  workspaceId: Types.ObjectId;
-}) => {
   const encryptionKey = await getEncryptionKey();
   const rootEncryptionKey = await getRootEncryptionKey();
 
   const botKey = await BotKey.findOne({
-    workspace: workspaceId,
+    workspace: workspaceId
   }).populate<{ sender: IUser }>("sender", "publicKey");
 
   if (!botKey) throw new Error("Failed to find bot key");
 
   const bot = await Bot.findOne({
-    workspace: workspaceId,
+    workspace: workspaceId
   }).select("+encryptedPrivateKey +iv +tag +algorithm +keyEncoding");
 
   if (!bot) throw new Error("Failed to find bot");
@@ -194,7 +237,7 @@ export const getKey = async ({
       ciphertext: botKey.encryptedKey,
       nonce: botKey.nonce,
       publicKey: botKey.sender.publicKey as string,
-      privateKey: privateKeyBot,
+      privateKey: privateKeyBot
     });
   } else if (encryptionKey && bot.keyEncoding === ENCODING_SCHEME_UTF8) {
     // case: encoding scheme is utf8
@@ -202,20 +245,19 @@ export const getKey = async ({
       ciphertext: bot.encryptedPrivateKey,
       iv: bot.iv,
       tag: bot.tag,
-      key: encryptionKey,
+      key: encryptionKey
     });
 
     return decryptAsymmetric({
       ciphertext: botKey.encryptedKey,
       nonce: botKey.nonce,
       publicKey: botKey.sender.publicKey as string,
-      privateKey: privateKeyBot,
+      privateKey: privateKeyBot
     });
   }
 
   throw InternalServerError({
-    message:
+    message: "Failed to obtain bot's copy of workspace key needed for bot operations"
-      "Failed to obtain bot's copy of workspace key needed for bot operations",
   });
 };
 
@@ -228,7 +270,7 @@ export const getKey = async ({
  */
 export const encryptSymmetricHelper = async ({
   workspaceId,
-  plaintext,
+  plaintext
 }: {
   workspaceId: Types.ObjectId;
   plaintext: string;
@@ -236,13 +278,13 @@ export const encryptSymmetricHelper = async ({
   const key = await getKey({ workspaceId: workspaceId });
   const { ciphertext, iv, tag } = encryptSymmetric128BitHexKeyUTF8({
     plaintext,
-    key,
+    key
   });
 
   return {
     ciphertext,
     iv,
-    tag,
+    tag
   };
 };
 /**
@@ -258,7 +300,7 @@ export const decryptSymmetricHelper = async ({
   workspaceId,
   ciphertext,
   iv,
-  tag,
+  tag
 }: {
   workspaceId: Types.ObjectId;
   ciphertext: string;
@@ -270,7 +312,7 @@ export const decryptSymmetricHelper = async ({
     ciphertext,
     iv,
     tag,
-    key,
+    key
   });
 
   return plaintext;
@@ -281,24 +323,24 @@ export const decryptSymmetricHelper = async ({
  * and [envionment] using bot
  * @param {Object} obj
  * @param {String} obj.workspaceId - id of workspace
- * @param {String} obj.environment - environment 
+ * @param {String} obj.environment - environment
  */
 export const getSecretsCommentBotHelper = async ({
   workspaceId,
   environment,
   secretPath
-} : {
+}: {
   workspaceId: Types.ObjectId;
   environment: string;
   secretPath: string;
 }) => {
   const content = {} as any;
   const key = await getKey({ workspaceId: workspaceId });
-  
+
   let folderId = "root";
   const folders = await Folder.findOne({
     workspace: workspaceId,
-    environment,
+    environment
   });
 
   if (!folders && secretPath !== "/") {
@@ -317,23 +359,23 @@ export const getSecretsCommentBotHelper = async ({
     workspace: workspaceId,
     environment,
     type: SECRET_SHARED,
-    folder: folderId,
+    folder: folderId
   });
 
   secrets.forEach((secret: ISecret) => {
-    if(secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) {
+    if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) {
       const secretKey = decryptSymmetric128BitHexKeyUTF8({
         ciphertext: secret.secretKeyCiphertext,
         iv: secret.secretKeyIV,
         tag: secret.secretKeyTag,
-        key,
+        key
       });
-  
+
       const commentValue = decryptSymmetric128BitHexKeyUTF8({
         ciphertext: secret.secretCommentCiphertext,
         iv: secret.secretCommentIV,
         tag: secret.secretCommentTag,
-        key,
+        key
       });
 
       content[secretKey] = commentValue;
@@ -341,4 +383,4 @@ export const getSecretsCommentBotHelper = async ({
   });
 
   return content;
-}
+};

backend/src/helpers/integration.ts

@@ -6,7 +6,7 @@ import {
   ALGORITHM_AES_256_GCM,
   ENCODING_SCHEME_UTF8,
   INTEGRATION_NETLIFY,
-  INTEGRATION_VERCEL,
+  INTEGRATION_VERCEL
 } from "../variables";
 import { UnauthorizedRequestError } from "../utils/errors";
 import * as Sentry from "@sentry/node";
@@ -34,7 +34,7 @@ export const handleOAuthExchangeHelper = async ({
   workspaceId,
   integration,
   code,
-  environment,
+  environment
 }: {
   workspaceId: string;
   integration: string;
@@ -43,21 +43,20 @@ export const handleOAuthExchangeHelper = async ({
 }) => {
   const bot = await Bot.findOne({
     workspace: workspaceId,
-    isActive: true,
+    isActive: true
   });
 
-  if (!bot)
+  if (!bot) throw new Error("Bot must be enabled for OAuth2 code-token exchange");
-    throw new Error("Bot must be enabled for OAuth2 code-token exchange");
 
   // exchange code for access and refresh tokens
   const res = await exchangeCode({
     integration,
-    code,
+    code
   });
 
   const update: Update = {
     workspace: workspaceId,
-    integration,
+    integration
   };
 
   switch (integration) {
@@ -72,12 +71,12 @@ export const handleOAuthExchangeHelper = async ({
   const integrationAuth = await IntegrationAuth.findOneAndUpdate(
     {
       workspace: workspaceId,
-      integration,
+      integration
     },
     update,
     {
       new: true,
-      upsert: true,
+      upsert: true
     }
   );
 
@@ -86,7 +85,7 @@ export const handleOAuthExchangeHelper = async ({
     // set integration auth refresh token
     await setIntegrationAuthRefreshHelper({
       integrationAuthId: integrationAuth._id.toString(),
-      refreshToken: res.refreshToken,
+      refreshToken: res.refreshToken
     });
   }
 
@@ -97,7 +96,7 @@ export const handleOAuthExchangeHelper = async ({
       integrationAuthId: integrationAuth._id.toString(),
       accessId: null,
       accessToken: res.accessToken,
-      accessExpiresAt: res.accessExpiresAt,
+      accessExpiresAt: res.accessExpiresAt
     });
   }
 
@@ -111,7 +110,7 @@ export const handleOAuthExchangeHelper = async ({
  */
 export const syncIntegrationsHelper = async ({
   workspaceId,
-  environment,
+  environment
 }: {
   workspaceId: Types.ObjectId;
   environment?: string;
@@ -121,11 +120,11 @@ export const syncIntegrationsHelper = async ({
       workspace: workspaceId,
       ...(environment
         ? {
-          environment,
+            environment
-        }
+          }
-      : {}),
+        : {}),
       isActive: true,
-      app: { $ne: null },
+      app: { $ne: null }
     });
 
     // for each workspace integration, sync/push secrets
@@ -135,25 +134,16 @@ export const syncIntegrationsHelper = async ({
       const secrets = await BotService.getSecrets({
         workspaceId: integration.workspace,
         environment: integration.environment,
-        secretPath: integration.secretPath,
+        secretPath: integration.secretPath
       });
 
-      // get workspace, environment (shared) secrets comments
+      const integrationAuth = await IntegrationAuth.findById(integration.integrationAuth);
-      const secretComments = await BotService.getSecretComments({
-        workspaceId: integration.workspace,
-        environment: integration.environment,
-        secretPath: integration.secretPath,
-      })
-
-      const integrationAuth = await IntegrationAuth.findById(
-        integration.integrationAuth
-      );
 
       if (!integrationAuth) throw new Error("Failed to find integration auth");
-      
+
       // get integration auth access token
       const access = await getIntegrationAuthAccessHelper({
-        integrationAuthId: integration.integrationAuth,
+        integrationAuthId: integration.integrationAuth
       });
 
       // sync secrets to integration
@@ -162,14 +152,17 @@ export const syncIntegrationsHelper = async ({
         integrationAuth,
         secrets,
         accessId: access.accessId === undefined ? null : access.accessId,
-        accessToken: access.accessToken,
+        accessToken: access.accessToken
-        secretComments
       });
     }
   } catch (err) {
     Sentry.captureException(err);
-    console.log(`syncIntegrationsHelper: failed with [workspaceId=${workspaceId}] [environment=${environment}]`, err) // eslint-disable-line no-use-before-define
+    // eslint-disable-next-line
-    throw err
+    console.log(
+      `syncIntegrationsHelper: failed with [workspaceId=${workspaceId}] [environment=${environment}]`,
+      err
+    ); // eslint-disable-line no-use-before-define
+    throw err;
   }
 };
 
@@ -182,24 +175,24 @@ export const syncIntegrationsHelper = async ({
  * @param {String} refreshToken - decrypted refresh token
  */
 export const getIntegrationAuthRefreshHelper = async ({
-  integrationAuthId,
+  integrationAuthId
 }: {
   integrationAuthId: Types.ObjectId;
 }) => {
-  const integrationAuth = await IntegrationAuth.findById(
+  const integrationAuth = await IntegrationAuth.findById(integrationAuthId).select(
-    integrationAuthId
+    "+refreshCiphertext +refreshIV +refreshTag"
-  ).select("+refreshCiphertext +refreshIV +refreshTag");
+  );
 
   if (!integrationAuth)
     throw UnauthorizedRequestError({
-      message: "Failed to locate Integration Authentication credentials",
+      message: "Failed to locate Integration Authentication credentials"
     });
 
   const refreshToken = await BotService.decryptSymmetric({
     workspaceId: integrationAuth.workspace,
     ciphertext: integrationAuth.refreshCiphertext as string,
     iv: integrationAuth.refreshIV as string,
-    tag: integrationAuth.refreshTag as string,
+    tag: integrationAuth.refreshTag as string
   });
 
   return refreshToken;
@@ -214,28 +207,26 @@ export const getIntegrationAuthRefreshHelper = async ({
  * @returns {String} accessToken - decrypted access token
  */
 export const getIntegrationAuthAccessHelper = async ({
-  integrationAuthId,
+  integrationAuthId
 }: {
   integrationAuthId: Types.ObjectId;
 }) => {
   let accessId;
   let accessToken;
-  const integrationAuth = await IntegrationAuth.findById(
+  const integrationAuth = await IntegrationAuth.findById(integrationAuthId).select(
-    integrationAuthId
-  ).select(
     "workspace integration +accessCiphertext +accessIV +accessTag +accessExpiresAt + refreshCiphertext +accessIdCiphertext +accessIdIV +accessIdTag"
   );
 
   if (!integrationAuth)
     throw UnauthorizedRequestError({
-      message: "Failed to locate Integration Authentication credentials",
+      message: "Failed to locate Integration Authentication credentials"
     });
 
   accessToken = await BotService.decryptSymmetric({
     workspaceId: integrationAuth.workspace,
     ciphertext: integrationAuth.accessCiphertext as string,
     iv: integrationAuth.accessIV as string,
-    tag: integrationAuth.accessTag as string,
+    tag: integrationAuth.accessTag as string
   });
 
   if (integrationAuth?.accessExpiresAt && integrationAuth?.refreshCiphertext) {
@@ -245,11 +236,11 @@ export const getIntegrationAuthAccessHelper = async ({
     if (integrationAuth.accessExpiresAt < new Date()) {
       // access token is expired
       const refreshToken = await getIntegrationAuthRefreshHelper({
-        integrationAuthId,
+        integrationAuthId
       });
       accessToken = await exchangeRefresh({
         integrationAuth,
-        refreshToken,
+        refreshToken
       });
     }
   }
@@ -263,13 +254,13 @@ export const getIntegrationAuthAccessHelper = async ({
       workspaceId: integrationAuth.workspace,
       ciphertext: integrationAuth.accessIdCiphertext as string,
       iv: integrationAuth.accessIdIV as string,
-      tag: integrationAuth.accessIdTag as string,
+      tag: integrationAuth.accessIdTag as string
     });
   }
 
   return {
     accessId,
-    accessToken,
+    accessToken
   };
 };
 
@@ -283,7 +274,7 @@ export const getIntegrationAuthAccessHelper = async ({
  */
 export const setIntegrationAuthRefreshHelper = async ({
   integrationAuthId,
-  refreshToken,
+  refreshToken
 }: {
   integrationAuthId: string;
   refreshToken: string;
@@ -294,22 +285,22 @@ export const setIntegrationAuthRefreshHelper = async ({
 
   const obj = await BotService.encryptSymmetric({
     workspaceId: integrationAuth.workspace,
-    plaintext: refreshToken,
+    plaintext: refreshToken
   });
 
   integrationAuth = await IntegrationAuth.findOneAndUpdate(
     {
-      _id: integrationAuthId,
+      _id: integrationAuthId
     },
     {
       refreshCiphertext: obj.ciphertext,
       refreshIV: obj.iv,
       refreshTag: obj.tag,
       algorithm: ALGORITHM_AES_256_GCM,
-      keyEncoding: ENCODING_SCHEME_UTF8,
+      keyEncoding: ENCODING_SCHEME_UTF8
     },
     {
-      new: true,
+      new: true
     }
   );
 
@@ -329,7 +320,7 @@ export const setIntegrationAuthAccessHelper = async ({
   integrationAuthId,
   accessId,
   accessToken,
-  accessExpiresAt,
+  accessExpiresAt
 }: {
   integrationAuthId: string;
   accessId: string | null;
@@ -342,20 +333,20 @@ export const setIntegrationAuthAccessHelper = async ({
 
   const encryptedAccessTokenObj = await BotService.encryptSymmetric({
     workspaceId: integrationAuth.workspace,
-    plaintext: accessToken,
+    plaintext: accessToken
   });
 
   let encryptedAccessIdObj;
   if (accessId) {
     encryptedAccessIdObj = await BotService.encryptSymmetric({
       workspaceId: integrationAuth.workspace,
-      plaintext: accessId,
+      plaintext: accessId
     });
   }
 
   integrationAuth = await IntegrationAuth.findOneAndUpdate(
     {
-      _id: integrationAuthId,
+      _id: integrationAuthId
     },
     {
       accessIdCiphertext: encryptedAccessIdObj?.ciphertext ?? undefined,
@@ -366,10 +357,10 @@ export const setIntegrationAuthAccessHelper = async ({
       accessTag: encryptedAccessTokenObj.tag,
       accessExpiresAt,
       algorithm: ALGORITHM_AES_256_GCM,
-      keyEncoding: ENCODING_SCHEME_UTF8,
+      keyEncoding: ENCODING_SCHEME_UTF8
     },
     {
-      new: true,
+      new: true
     }
   );
 

backend/src/helpers/secrets.ts

@@ -42,9 +42,10 @@ import { TelemetryService } from "../services";
 import { client, getEncryptionKey, getRootEncryptionKey } from "../config";
 import { EELogService, EESecretService } from "../ee/services";
 import { getAuthDataPayloadIdObj, getAuthDataPayloadUserObj } from "../utils/auth";
-import { getFolderIdFromServiceToken } from "../services/FolderService";
+import { getFolderByPath, getFolderIdFromServiceToken } from "../services/FolderService";
 import picomatch from "picomatch";
 import path from "path";
+import Folder, { TFolderRootSchema } from "../models/folder";
 
 export const isValidScope = (
   authPayload: IServiceTokenData,
@@ -64,10 +65,9 @@ export const isValidScope = (
 export function containsGlobPatterns(secretPath: string) {
   const globChars = ["*", "?", "[", "]", "{", "}", "**"];
   const normalizedPath = path.normalize(secretPath);
-  return globChars.some(char => normalizedPath.includes(char));
+  return globChars.some((char) => normalizedPath.includes(char));
 }
 
-
 /**
  * Returns an object containing secret [secret] but with its value, key, comment decrypted.
  *
@@ -929,3 +929,164 @@ export const deleteSecretHelper = async ({
     secret
   };
 };
+
+const fetchSecretsCrossEnv = (workspaceId: string, folders: TFolderRootSchema[], key: string) => {
+  const fetchCache: Record<string, Record<string, string>> = {};
+
+  return async (secRefEnv: string, secRefPath: string[], secRefKey: string) => {
+    const secRefPathUrl = path.join("/", ...secRefPath);
+    const uniqKey = `${secRefEnv}-${secRefPathUrl}`;
+
+    if (fetchCache?.[uniqKey]) {
+      return fetchCache[uniqKey][secRefKey];
+    }
+
+    let folderId = "root";
+    const folder = folders.find(({ environment }) => environment === secRefEnv);
+    if (!folder && secRefPathUrl !== "/") {
+      throw BadRequestError({ message: "Folder not found" });
+    }
+
+    if (folder) {
+      const selectedFolder = getFolderByPath(folder.nodes, secRefPathUrl);
+      if (!selectedFolder) {
+        throw BadRequestError({ message: "Folder not found" });
+      }
+      folderId = selectedFolder.id;
+    }
+
+    const secrets = await Secret.find({
+      workspace: workspaceId,
+      environment: secRefEnv,
+      type: SECRET_SHARED,
+      folder: folderId
+    });
+
+    const decryptedSec = secrets.reduce<Record<string, string>>((prev, secret) => {
+      const secretKey = decryptSymmetric128BitHexKeyUTF8({
+        ciphertext: secret.secretKeyCiphertext,
+        iv: secret.secretKeyIV,
+        tag: secret.secretKeyTag,
+        key
+      });
+      const secretValue = decryptSymmetric128BitHexKeyUTF8({
+        ciphertext: secret.secretValueCiphertext,
+        iv: secret.secretValueIV,
+        tag: secret.secretValueTag,
+        key
+      });
+
+      prev[secretKey] = secretValue;
+      return prev;
+    }, {});
+
+    fetchCache[uniqKey] = decryptedSec;
+
+    return fetchCache[uniqKey][secRefKey];
+  };
+};
+
+const INTERPOLATION_SYNTAX_REG = new RegExp(/\${([^}]+)}/g);
+const recursivelyExpandSecret = async (
+  expandedSec: Record<string, string>,
+  interpolatedSec: Record<string, string>,
+  fetchCrossEnv: (env: string, secPath: string[], secKey: string) => Promise<string>,
+  recursionChainBreaker: Record<string, boolean>,
+  key: string
+) => {
+  if (expandedSec?.[key]) {
+    return expandedSec[key];
+  }
+  if (recursionChainBreaker?.[key]) {
+    return "";
+  }
+  recursionChainBreaker[key] = true;
+
+  let interpolatedValue = interpolatedSec[key];
+  if (!interpolatedValue) {
+    throw new Error(`Couldn't find referenced value - ${key}`);
+  }
+
+  const refs = interpolatedValue.match(INTERPOLATION_SYNTAX_REG);
+  if (refs) {
+    for (const interpolationSyntax of refs) {
+      const interpolationKey = interpolationSyntax.slice(2, interpolationSyntax.length - 1);
+      const entities = interpolationKey.trim().split(".");
+
+      if (entities.length === 1) {
+        const val = await recursivelyExpandSecret(
+          expandedSec,
+          interpolatedSec,
+          fetchCrossEnv,
+          recursionChainBreaker,
+          interpolationKey
+        );
+        if (val) {
+          interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val);
+        }
+        continue;
+      }
+
+      if (entities.length > 1) {
+        const secRefEnv = entities[0];
+        const secRefPath = entities.slice(1, entities.length - 1);
+        const secRefKey = entities[entities.length - 1];
+
+        const val = await fetchCrossEnv(secRefEnv, secRefPath, secRefKey);
+        interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val);
+      }
+    }
+  }
+
+  expandedSec[key] = interpolatedValue;
+  return interpolatedValue;
+};
+
+// used to convert multi line ones to quotes ones with \n
+const formatMultiValueEnv = (val?: string) => {
+  if (!val) return "";
+  if (!val.match("\n")) return val;
+  return `"${val.replace(/\n/g, "\\n")}"`;
+};
+
+export const expandSecrets = async (
+  workspaceId: string,
+  rootEncKey: string,
+  secrets: Record<string, { value: string; comment?: string }>
+) => {
+  const expandedSec: Record<string, string> = {};
+  const interpolatedSec: Record<string, string> = {};
+
+  const folders = await Folder.find({ workspace: workspaceId });
+  const crossSecEnvFetch = fetchSecretsCrossEnv(workspaceId, folders, rootEncKey);
+
+  Object.keys(secrets).forEach((key) => {
+    if (secrets[key].value.match(INTERPOLATION_SYNTAX_REG)) {
+      interpolatedSec[key] = secrets[key].value;
+    } else {
+      expandedSec[key] = secrets[key].value;
+    }
+  });
+
+  for (const key of Object.keys(secrets)) {
+    if (expandedSec?.[key]) {
+      secrets[key].value = formatMultiValueEnv(expandedSec[key]);
+      continue;
+    }
+
+    // this is to avoid recursion loop. So the graph should be direct graph rather than cyclic
+    // so for any recursion building if there is an entity two times same key meaning it will be looped
+    const recursionChainBreaker: Record<string, boolean> = {};
+    const expandedVal = await recursivelyExpandSecret(
+      expandedSec,
+      interpolatedSec,
+      crossSecEnvFetch,
+      recursionChainBreaker,
+      key
+    );
+
+    secrets[key].value = formatMultiValueEnv(expandedVal);
+  }
+
+  return secrets;
+};

backend/tsconfig.json

@@ -1,9 +1,7 @@
 {
   "compilerOptions": {
     "target": "es2016",
-    "lib": [
+    "lib": ["es6", "es2021"],
-      "es6"
-    ],
     "module": "commonjs",
     "rootDir": "src",
     "resolveJsonModule": true,
@@ -15,15 +13,8 @@
     "strict": true,
     "noImplicitAny": true,
     "skipLibCheck": true,
-    "typeRoots": [
+    "typeRoots": ["./src/types", "./node_modules/@types"]
-      "./src/types",
-      "./node_modules/@types"
-    ]
   },
-  "include": [
+  "include": ["src/**/*"],
-    "src/**/*"
+  "exclude": ["node_modules"]
-  ],
+}
-  "exclude": [
-    "node_modules"
-  ]
-}