Feat: Recursively get secrets from all nested secret paths

backend/src/lib/api-docs/constants.ts

@@ -215,6 +215,7 @@ export const SECRETS = {
 
 export const RAW_SECRETS = {
   LIST: {
+    recursive: "Whether or not to fetch all secrets from the specified base path, and all of its subdirectories.",
     workspaceId: "The ID of the project to list secrets from.",
     workspaceSlug: "The slug of the project to list secrets from. This parameter is only usable by machine identities.",
     environment: "The slug of the environment to list secrets from.",

backend/src/server/routes/v3/secret-router.ts

@@ -157,6 +157,11 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
         workspaceSlug: z.string().trim().optional().describe(RAW_SECRETS.LIST.workspaceSlug),
         environment: z.string().trim().optional().describe(RAW_SECRETS.LIST.environment),
         secretPath: z.string().trim().default("/").transform(removeTrailingSlash).describe(RAW_SECRETS.LIST.secretPath),
+        recursive: z
+          .enum(["true", "false"])
+          .default("false")
+          .transform((value) => value === "true")
+          .describe(RAW_SECRETS.LIST.recursive),
         include_imports: z
           .enum(["true", "false"])
           .default("false")
@@ -165,7 +170,13 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
       }),
       response: {
         200: z.object({
-          secrets: secretRawSchema.array(),
+          secrets: secretRawSchema
+            .merge(
+              z.object({
+                secretPath: z.string().optional()
+              })
+            )
+            .array(),
           imports: z
             .object({
               secretPath: z.string(),
@@ -218,7 +229,8 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
         actorAuthMethod: req.permission.authMethod,
         projectId: workspaceId,
         path: secretPath,
-        includeImports: req.query.include_imports
+        includeImports: req.query.include_imports,
+        recursive: req.query.recursive
       });
 
       await server.services.auditLog.createAuditLog({

backend/src/services/secret/secret-fns.ts

@@ -9,6 +9,7 @@ import {
   TSecretBlindIndexes,
   TSecrets
 } from "@app/db/schemas";
+import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import { getConfig } from "@app/lib/config/env";
 import {
   buildSecretBlindIndexFromName,
@@ -17,8 +18,10 @@ import {
 } from "@app/lib/crypto";
 import { BadRequestError } from "@app/lib/errors";
 import { groupBy, unique } from "@app/lib/fn";
+import { logger } from "@app/lib/logger";
 
 import { getBotKeyFnFactory } from "../project-bot/project-bot-fns";
+import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretDALFactory } from "./secret-dal";
 import {
@@ -45,6 +48,68 @@ export const generateSecretBlindIndexBySalt = async (secretName: string, secretB
   return secretBlindIndex;
 };
 
+type TRecursivelyFetchSecretsFromFoldersArg = {
+  permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
+  projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
+  folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "find">;
+};
+
+export const recursivelyGetSecretPaths = ({ folderDAL, projectEnvDAL }: TRecursivelyFetchSecretsFromFoldersArg) => {
+  const getPaths = async (projectId: string, environment: string, currentPath: string) => {
+    let secretPaths: string[] = [];
+
+    // Get secrets in the current folder.
+    try {
+      const folder = await folderDAL.findBySecretPath(projectId, environment, currentPath);
+
+      if (!folder) {
+        throw new Error(`Base directory '${currentPath}' not found.`);
+      }
+
+      secretPaths.push(currentPath);
+    } catch (error) {
+      logger.error(error, "Error fetching secrets from base directory");
+      throw error;
+    }
+
+    // List all subfolders in the current folder.
+    try {
+      const env = await projectEnvDAL.findOne({ projectId, slug: environment });
+      const parentFolder = await folderDAL.findBySecretPath(projectId, environment, currentPath);
+
+      if (!env) {
+        throw new Error(`Environment with not found`);
+      }
+
+      if (!parentFolder) {
+        throw new Error(`Parent folder not found`);
+      }
+
+      const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id });
+
+      // Use Promise.all to handle recursive calls concurrently for efficiency.
+      const secretsFromSubFolders = await Promise.all(
+        folders.map(async (folder) => {
+          // Ensure the path is correctly formatted for the next level.
+          const subFolderPath = `${currentPath}${currentPath !== "/" ? "/" : ""}${folder.name}`;
+
+          return getPaths(projectId, environment, subFolderPath);
+        })
+      );
+
+      // Flatten the array of arrays and concatenate with the current secrets array.
+      secretPaths = secretPaths.concat(...secretsFromSubFolders);
+    } catch (error) {
+      logger.error(error, "Error fetching secrets from subdirectories");
+      throw error;
+    }
+
+    return secretPaths;
+  };
+
+  return getPaths;
+};
+
 type TInterpolateSecretArg = {
   projectId: string;
   secretEncKey: string;
@@ -202,9 +267,7 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD
       );
 
       // eslint-disable-next-line
-      secrets[key].value = secrets[key].skipMultilineEncoding
-        ? expandedVal
-        : formatMultiValueEnv(expandedVal);
+      secrets[key].value = secrets[key].skipMultilineEncoding ? expandedVal : formatMultiValueEnv(expandedVal);
     }
 
     return secrets;

backend/src/services/secret/secret-service.ts

@@ -13,13 +13,20 @@ import { logger } from "@app/lib/logger";
 import { ActorType } from "../auth/auth-type";
 import { TProjectDALFactory } from "../project/project-dal";
 import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
+import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
 import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
 import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
 import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
 import { TSecretDALFactory } from "./secret-dal";
-import { decryptSecretRaw, fnSecretBlindIndexCheck, fnSecretBulkInsert, fnSecretBulkUpdate } from "./secret-fns";
+import {
+  decryptSecretRaw,
+  fnSecretBlindIndexCheck,
+  fnSecretBulkInsert,
+  fnSecretBulkUpdate,
+  recursivelyGetSecretPaths
+} from "./secret-fns";
 import { TSecretQueueFactory } from "./secret-queue";
 import {
   TAttachSecretTagsDTO,
@@ -47,8 +54,12 @@ type TSecretServiceFactoryDep = {
   secretDAL: TSecretDALFactory;
   secretTagDAL: TSecretTagDALFactory;
   secretVersionDAL: TSecretVersionDALFactory;
-  folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "updateById" | "findById" | "findByManySecretPath">;
   projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus" | "findProjectBySlug">;
+  projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
+  folderDAL: Pick<
+    TSecretFolderDALFactory,
+    "findBySecretPath" | "updateById" | "findById" | "findByManySecretPath" | "find"
+  >;
   secretBlindIndexDAL: TSecretBlindIndexDALFactory;
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
   snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
@@ -61,6 +72,7 @@ type TSecretServiceFactoryDep = {
 export type TSecretServiceFactory = ReturnType<typeof secretServiceFactory>;
 export const secretServiceFactory = ({
   secretDAL,
+  projectEnvDAL,
   secretTagDAL,
   secretVersionDAL,
   folderDAL,
@@ -789,21 +801,64 @@ export const secretServiceFactory = ({
     actorOrgId,
     actorAuthMethod,
     environment,
-    includeImports
+    includeImports,
+    recursive
   }: TGetSecretsRawDTO) => {
     const botKey = await projectBotService.getBotKey(projectId);
     if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
 
-    const { secrets, imports } = await getSecrets({
-      actorId,
-      projectId,
-      environment,
-      actor,
-      actorOrgId,
-      actorAuthMethod,
-      path,
-      includeImports
-    });
+    let secrets: Awaited<ReturnType<typeof getSecrets>>["secrets"];
+    let imports: Awaited<ReturnType<typeof getSecrets>>["imports"];
+
+    if (recursive) {
+      const getPaths = recursivelyGetSecretPaths({
+        permissionService,
+        folderDAL,
+        projectEnvDAL
+      });
+
+      const paths = await getPaths(projectId, environment, path);
+
+      const result = await Promise.all(
+        paths.map(async (currentPath) => {
+          const secs = await getSecrets({
+            actorId,
+            projectId,
+            environment,
+            actor,
+            actorOrgId,
+            actorAuthMethod,
+            path: currentPath,
+            includeImports
+          });
+
+          return {
+            secrets: {
+              ...secs.secrets,
+              secretPath: currentPath
+            },
+            imports: secs.imports
+          };
+        })
+      );
+
+      secrets = result.flatMap((el) => el.secrets);
+      imports = result.flatMap((el) => el.imports || []);
+    } else {
+      const result = await getSecrets({
+        actorId,
+        projectId,
+        environment,
+        actor,
+        actorOrgId,
+        actorAuthMethod,
+        path,
+        includeImports
+      });
+
+      secrets = result.secrets;
+      imports = result.imports;
+    }
 
     return {
       secrets: secrets.map((el) => decryptSecretRaw(el, botKey)),

backend/src/services/secret/secret-types.ts

@@ -140,6 +140,7 @@ export type TGetSecretsRawDTO = {
   path: string;
   environment: string;
   includeImports?: boolean;
+  recursive?: boolean;
 } & TProjectPermission;
 
 export type TGetASecretRawDTO = {