Revert "Revert "feat: secret reference graph for understanding how its pulled""

backend/src/lib/api-docs/constants.ts

@@ -669,6 +669,12 @@ export const RAW_SECRETS = {
     type: "The type of the secret to delete.",
     projectSlug: "The slug of the project to delete the secret in.",
     workspaceId: "The ID of the project where the secret is located."
+  },
+  GET_REFERENCE_TREE: {
+    secretName: "The name of the secret to get the reference tree for.",
+    workspaceId: "The ID of the project where the secret is located.",
+    environment: "The slug of the environment where the the secret is located.",
+    secretPath: "The folder path where the secret is located."
   }
 } as const;
 

backend/src/server/routes/v3/secret-router.ts

@@ -23,6 +23,18 @@ import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
 
 import { secretRawSchema } from "../sanitizedSchemas";
 
+const SecretReferenceNode = z.object({
+  key: z.string(),
+  value: z.string().optional(),
+  environment: z.string(),
+  secretPath: z.string()
+});
+type TSecretReferenceNode = z.infer<typeof SecretReferenceNode> & { children: TSecretReferenceNode[] };
+
+const SecretReferenceNodeTree: z.ZodType<TSecretReferenceNode> = SecretReferenceNode.extend({
+  children: z.lazy(() => SecretReferenceNodeTree.array())
+});
+
 export const registerSecretRouter = async (server: FastifyZodProvider) => {
   server.route({
     method: "POST",
@@ -2102,6 +2114,58 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
     }
   });
 
+  server.route({
+    method: "GET",
+    url: "/raw/:secretName/secret-reference-tree",
+    config: {
+      rateLimit: secretsLimit
+    },
+    schema: {
+      description: "Get secret reference tree",
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
+      params: z.object({
+        secretName: z.string().trim().describe(RAW_SECRETS.GET_REFERENCE_TREE.secretName)
+      }),
+      querystring: z.object({
+        workspaceId: z.string().trim().describe(RAW_SECRETS.GET_REFERENCE_TREE.workspaceId),
+        environment: z.string().trim().describe(RAW_SECRETS.GET_REFERENCE_TREE.environment),
+        secretPath: z
+          .string()
+          .trim()
+          .default("/")
+          .transform(removeTrailingSlash)
+          .describe(RAW_SECRETS.GET_REFERENCE_TREE.secretPath)
+      }),
+      response: {
+        200: z.object({
+          tree: SecretReferenceNodeTree,
+          value: z.string().optional()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { secretName } = req.params;
+      const { secretPath, environment, workspaceId } = req.query;
+      const { tree, value } = await server.services.secret.getSecretReferenceTree({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        projectId: workspaceId,
+        secretName,
+        secretPath,
+        environment
+      });
+
+      return { tree, value };
+    }
+  });
+
   server.route({
     method: "POST",
     url: "/backfill-secret-references",

backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts

@@ -11,6 +11,8 @@ import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal";
 import { TFnSecretBulkDelete, TFnSecretBulkInsert, TFnSecretBulkUpdate } from "./secret-v2-bridge-types";
 
 const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g;
+// akhilmhdh: JS regex with global save state in .test
+const INTERPOLATION_SYNTAX_REG_NON_GLOBAL = /\${([^}]+)}/;
 
 export const shouldUseSecretV2Bridge = (version: number) => version === 3;
 
@@ -376,6 +378,13 @@ const formatMultiValueEnv = (val?: string) => {
   return `"${val.replace(/\n/g, "\\n")}"`;
 };
 
+type TSecretReferenceTraceNode = {
+  key: string;
+  value?: string;
+  environment: string;
+  secretPath: string;
+  children: TSecretReferenceTraceNode[];
+};
 type TInterpolateSecretArg = {
   projectId: string;
   decryptSecretValue: (encryptedValue?: Buffer | null) => string | undefined;
@@ -417,14 +426,21 @@ export const expandSecretReferencesFactory = ({
     return secretCache[cacheKey][secretKey] || { value: "", tags: [] };
   };
 
-  const recursivelyExpandSecret = async (dto: { value?: string; secretPath: string; environment: string }) => {
-    if (!dto.value) return "";
+  const recursivelyExpandSecret = async (dto: {
+    value?: string;
+    secretPath: string;
+    environment: string;
+    shouldStackTrace?: boolean;
+  }) => {
+    const stackTrace = { ...dto, key: "root", children: [] } as TSecretReferenceTraceNode;
 
-    const stack = [{ ...dto, depth: 0 }];
+    if (!dto.value) return { expandedValue: "", stackTrace };
+    const stack = [{ ...dto, depth: 0, trace: stackTrace }];
     let expandedValue = dto.value;
 
     while (stack.length) {
-      const { value, secretPath, environment, depth } = stack.pop()!;
+      const { value, secretPath, environment, depth, trace } = stack.pop()!;
+
       // eslint-disable-next-line no-continue
       if (depth > MAX_SECRET_REFERENCE_DEPTH) continue;
       const refs = value?.match(INTERPOLATION_SYNTAX_REG);
@@ -437,6 +453,11 @@ export const expandSecretReferencesFactory = ({
           // eslint-disable-next-line no-continue
           if (!entities.length) continue;
 
+          let referencedSecretPath = "";
+          let referencedSecretKey = "";
+          let referencedSecretEnvironmentSlug = "";
+          let referencedSecretValue = "";
+
           if (entities.length === 1) {
             const [secretKey] = entities;
 
@@ -449,17 +470,11 @@ export const expandSecretReferencesFactory = ({
 
             const cacheKey = getCacheUniqueKey(environment, secretPath);
             secretCache[cacheKey][secretKey] = referredValue;
-            if (INTERPOLATION_SYNTAX_REG.test(referredValue.value)) {
-              stack.push({
-                value: referredValue.value,
-                secretPath,
-                environment,
-                depth: depth + 1
-              });
-            }
-            if (referredValue) {
-              expandedValue = expandedValue.replaceAll(interpolationSyntax, referredValue.value);
-            }
+
+            referencedSecretValue = referredValue.value;
+            referencedSecretKey = secretKey;
+            referencedSecretPath = secretPath;
+            referencedSecretEnvironmentSlug = environment;
           } else {
             const secretReferenceEnvironment = entities[0];
             const secretReferencePath = path.join("/", ...entities.slice(1, entities.length - 1));
@@ -474,24 +489,42 @@ export const expandSecretReferencesFactory = ({
 
             const cacheKey = getCacheUniqueKey(secretReferenceEnvironment, secretReferencePath);
             secretCache[cacheKey][secretReferenceKey] = referedValue;
-            if (INTERPOLATION_SYNTAX_REG.test(referedValue.value)) {
-              stack.push({
-                value: referedValue.value,
-                secretPath: secretReferencePath,
-                environment: secretReferenceEnvironment,
-                depth: depth + 1
-              });
-            }
 
-            if (referedValue) {
-              expandedValue = expandedValue.replaceAll(interpolationSyntax, referedValue.value);
+            referencedSecretValue = referedValue.value;
+            referencedSecretKey = secretReferenceKey;
+            referencedSecretPath = secretReferencePath;
+            referencedSecretEnvironmentSlug = secretReferenceEnvironment;
+          }
+
+          const node = {
+            value: referencedSecretValue,
+            secretPath: referencedSecretPath,
+            environment: referencedSecretEnvironmentSlug,
+            depth: depth + 1,
+            trace
+          };
+
+          const shouldExpandMore = INTERPOLATION_SYNTAX_REG_NON_GLOBAL.test(referencedSecretValue);
+          if (dto.shouldStackTrace) {
+            const stackTraceNode = { ...node, children: [], key: referencedSecretKey, trace: null };
+            trace?.children.push(stackTraceNode);
+            // if stack trace this would be child node
+            if (shouldExpandMore) {
+              stack.push({ ...node, trace: stackTraceNode });
             }
+          } else if (shouldExpandMore) {
+            // if no stack trace is needed we just keep going with root node
+            stack.push(node);
+          }
+
+          if (referencedSecretValue) {
+            expandedValue = expandedValue.replaceAll(interpolationSyntax, referencedSecretValue);
           }
         }
       }
     }
 
-    return expandedValue;
+    return { expandedValue, stackTrace };
   };
 
   const expandSecret = async (inputSecret: {
@@ -505,10 +538,21 @@ export const expandSecretReferencesFactory = ({
     const shouldExpand = Boolean(inputSecret.value?.match(INTERPOLATION_SYNTAX_REG));
     if (!shouldExpand) return inputSecret.value;
 
-    const expandedSecretValue = await recursivelyExpandSecret(inputSecret);
-    return inputSecret.skipMultilineEncoding ? formatMultiValueEnv(expandedSecretValue) : expandedSecretValue;
+    const { expandedValue } = await recursivelyExpandSecret(inputSecret);
+
+    return inputSecret.skipMultilineEncoding ? formatMultiValueEnv(expandedValue) : expandedValue;
   };
-  return expandSecret;
+
+  const getExpandedSecretStackTrace = async (inputSecret: {
+    value?: string;
+    secretPath: string;
+    environment: string;
+  }) => {
+    const { stackTrace, expandedValue } = await recursivelyExpandSecret({ ...inputSecret, shouldStackTrace: true });
+    return { stackTrace, expandedValue };
+  };
+
+  return { expandSecretReferences: expandSecret, getExpandedSecretStackTrace };
 };
 
 export const reshapeBridgeSecret = (

backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts

@@ -41,6 +41,7 @@ import {
   TDeleteManySecretDTO,
   TDeleteSecretDTO,
   TGetASecretDTO,
+  TGetSecretReferencesTreeDTO,
   TGetSecretsDTO,
   TGetSecretVersionsDTO,
   TMoveSecretsDTO,
@@ -815,7 +816,7 @@ export const secretV2BridgeServiceFactory = ({
         })
       );
 
-    const expandSecretReferences = expandSecretReferencesFactory({
+    const { expandSecretReferences } = expandSecretReferencesFactory({
       projectId,
       folderDAL,
       secretDAL,
@@ -965,7 +966,7 @@ export const secretV2BridgeServiceFactory = ({
       })
     );
 
-    const expandSecretReferences = expandSecretReferencesFactory({
+    const { expandSecretReferences } = expandSecretReferencesFactory({
       projectId,
       folderDAL,
       secretDAL,
@@ -1032,6 +1033,7 @@ export const secretV2BridgeServiceFactory = ({
         value: secretValue,
         skipMultilineEncoding: secret.skipMultilineEncoding
       });
+
       secretValue = expandedSecretValue || "";
     }
 
@@ -1928,6 +1930,88 @@ export const secretV2BridgeServiceFactory = ({
     };
   };
 
+  const getSecretReferenceTree = async ({
+    environment,
+    secretPath,
+    projectId,
+    actor,
+    actorId,
+    actorOrgId,
+    secretName,
+    actorAuthMethod
+  }: TGetSecretReferencesTreeDTO) => {
+    const { permission } = await permissionService.getProjectPermission(
+      actor,
+      actorId,
+      projectId,
+      actorAuthMethod,
+      actorOrgId
+    );
+
+    ForbiddenError.from(permission).throwUnlessCan(
+      ProjectPermissionActions.Read,
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
+    );
+
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
+    if (!folder)
+      throw new NotFoundError({
+        message: "Folder not found for the given environment slug & secret path",
+        name: "Create secret"
+      });
+    const folderId = folder.id;
+
+    const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
+      type: KmsDataKey.SecretManager,
+      projectId
+    });
+
+    const secret = await secretDAL.findOne({
+      folderId,
+      key: secretName,
+      type: SecretType.Shared
+    });
+
+    ForbiddenError.from(permission).throwUnlessCan(
+      ProjectPermissionActions.Read,
+      subject(ProjectPermissionSub.Secrets, {
+        environment,
+        secretPath,
+        secretName,
+        secretTags: (secret?.tags || []).map((el) => el.slug)
+      })
+    );
+
+    const secretValue = secret.encryptedValue
+      ? secretManagerDecryptor({ cipherTextBlob: secret.encryptedValue }).toString()
+      : "";
+
+    const { getExpandedSecretStackTrace } = expandSecretReferencesFactory({
+      projectId,
+      folderDAL,
+      secretDAL,
+      decryptSecretValue: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : undefined),
+      canExpandValue: (expandEnvironment, expandSecretPath, expandSecretName, expandSecretTags) =>
+        permission.can(
+          ProjectPermissionActions.Read,
+          subject(ProjectPermissionSub.Secrets, {
+            environment: expandEnvironment,
+            secretPath: expandSecretPath,
+            secretName: expandSecretName,
+            secretTags: expandSecretTags
+          })
+        )
+    });
+
+    const { expandedValue, stackTrace } = await getExpandedSecretStackTrace({
+      environment,
+      secretPath,
+      value: secretValue
+    });
+
+    return { tree: stackTrace, value: expandedValue };
+  };
+
   return {
     createSecret,
     deleteSecret,
@@ -1942,6 +2026,7 @@ export const secretV2BridgeServiceFactory = ({
     moveSecrets,
     getSecretsCount,
     getSecretsCountMultiEnv,
-    getSecretsMultiEnv
+    getSecretsMultiEnv,
+    getSecretReferenceTree
   };
 };

backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts

@@ -278,3 +278,10 @@ export type TAttachSecretTagsDTO = {
   secretPath: string;
   type: SecretType;
 } & Omit<TProjectPermission, "projectId">;
+
+export type TGetSecretReferencesTreeDTO = {
+  projectId: string;
+  secretName: string;
+  environment: string;
+  secretPath: string;
+} & Omit<TProjectPermission, "projectId">;

backend/src/services/secret/secret-queue.ts

@@ -299,7 +299,7 @@ export const secretQueueFactory = ({
       );
       return content;
     }
-    const expandSecretReferences = expandSecretReferencesFactory({
+    const { expandSecretReferences } = expandSecretReferencesFactory({
       decryptSecretValue: dto.decryptor,
       secretDAL: secretV2BridgeDAL,
       folderDAL,

backend/src/services/secret/secret-service.ts

@@ -38,6 +38,7 @@ import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
 import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
 import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
 import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service";
+import { TGetSecretReferencesTreeDTO } from "../secret-v2-bridge/secret-v2-bridge-types";
 import { TSecretDALFactory } from "./secret-dal";
 import {
   decryptSecretRaw,
@@ -1099,6 +1100,18 @@ export const secretServiceFactory = ({
     return secrets;
   };
 
+  const getSecretReferenceTree = async (dto: TGetSecretReferencesTreeDTO) => {
+    const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(dto.projectId);
+
+    if (!shouldUseSecretV2Bridge)
+      throw new BadRequestError({
+        message: "Project version does not support secret reference tree",
+        name: "SecretReferenceTreeNotSupported"
+      });
+
+    return secretV2BridgeService.getSecretReferenceTree(dto);
+  };
+
   const getSecretsRaw = async ({
     projectId,
     path,
@@ -2857,6 +2870,7 @@ export const secretServiceFactory = ({
     startSecretV2Migration,
     getSecretsCount,
     getSecretsCountMultiEnv,
-    getSecretsRawMultiEnv
+    getSecretsRawMultiEnv,
+    getSecretReferenceTree
   };
 };