Fix: Refactor to in-memory approach

backend/src/services/secret/secret-fns.ts

@@ -8,6 +8,7 @@ import {
   SecretType,
   TableName,
   TSecretBlindIndexes,
+  TSecretFolders,
   TSecrets
 } from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
@@ -20,7 +21,6 @@ import {
 } from "@app/lib/crypto";
 import { BadRequestError } from "@app/lib/errors";
 import { groupBy, unique } from "@app/lib/fn";
-import { logger } from "@app/lib/logger";
 
 import { ActorAuthMethod, ActorType } from "../auth/auth-type";
 import { getBotKeyFnFactory } from "../project-bot/project-bot-fns";
@@ -53,8 +53,8 @@ export const generateSecretBlindIndexBySalt = async (secretName: string, secretB
 
 type TRecursivelyFetchSecretsFromFoldersArg = {
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
-  projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
   folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "find">;
+  projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
 };
 
 type TGetPathsDTO = {
@@ -70,65 +70,73 @@ type TGetPathsDTO = {
   };
 };
 
+// Introduce a new interface for mapping parent IDs to their children
+interface FolderMap {
+  [parentId: string]: TSecretFolders[];
+}
+
 export const recursivelyGetSecretPaths = ({
   folderDAL,
   projectEnvDAL,
   permissionService
 }: TRecursivelyFetchSecretsFromFoldersArg) => {
-  const getPaths = async ({ projectId, environment, currentPath }: Omit<TGetPathsDTO, "auth">) => {
+  const buildHierarchy = (folders: TSecretFolders[]): FolderMap => {
-    let secretPaths: string[] = [];
+    const map: FolderMap = {};
+    map.null = []; // Initialize mapping for root directory
 
-    // Get secrets in the current folder.
+    folders.forEach((folder) => {
-    try {
+      const parentId = folder.parentId || "null";
-      const folder = await folderDAL.findBySecretPath(projectId, environment, currentPath);
+      if (!map[parentId]) {
-
+        map[parentId] = [];
-      if (!folder) {
-        throw new Error(`Base directory '${currentPath}' not found.`);
       }
+      map[parentId].push(folder);
+    });
 
-      secretPaths.push(currentPath);
+    return map;
-    } catch (error) {
+  };
-      logger.error(error, "Error fetching secrets from base directory");
-      throw error;
-    }
 
-    // List all subfolders in the current folder.
+  const generatePaths = (map: FolderMap, parentId: string = "null", basePath: string = ""): string[] => {
-    try {
+    const children = map[parentId || "null"] || [];
-      const env = await projectEnvDAL.findOne({ projectId, slug: environment });
+    let paths: string[] = [];
-      const parentFolder = await folderDAL.findBySecretPath(projectId, environment, currentPath);
 
-      if (!env) {
+    children.forEach((child) => {
-        throw new Error(`Environment with not found`);
+      // Determine if this is the root folder of the environment. If no parentId is present and the name is root, it's the root folder
-      }
+      const isRootFolder = child.name === "root" && !child.parentId;
 
-      if (!parentFolder) {
+      // Form the current path based on the base path and the current child
-        throw new Error(`Parent folder not found`);
+      // eslint-disable-next-line no-nested-ternary
-      }
+      const currPath = basePath === "" ? (isRootFolder ? "/" : `/${child.name}`) : `${basePath}/${child.name}`;
 
-      const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id });
+      paths.push(currPath); // Add the current path
 
-      // Use Promise.all to handle recursive calls concurrently for efficiency.
+      // Recursively generate paths for children, passing down the formatted pathh
-      const secretsFromSubFolders = await Promise.all(
+      const childPaths = generatePaths(map, child.id, currPath);
-        folders.map(async (folder) => {
+      paths = paths.concat(childPaths);
-          // Ensure the path is correctly formatted for the next level.
+    });
-          const subFolderPath = `${currentPath}${currentPath !== "/" ? "/" : ""}${folder.name}`;
 
-          return getPaths({ projectId, environment, currentPath: subFolderPath });
+    return paths;
-        })
-      );
-
-      // Flatten the array of arrays and concatenate with the current secrets array.
-      secretPaths = secretPaths.concat(...secretsFromSubFolders);
-    } catch (error) {
-      logger.error(error, "Error fetching secrets from subdirectories");
-      throw error;
-    }
-
-    return secretPaths;
   };
 
   return async ({ projectId, environment, currentPath, auth }: TGetPathsDTO) => {
-    const paths = await getPaths({ projectId, environment, currentPath });
+    const env = await projectEnvDAL.findOne({
+      projectId,
+      slug: environment
+    });
+
+    if (!env) {
+      throw new Error(`'${environment}' environment not found in project with ID ${projectId}`);
+    }
+
+    // Fetch all folders in env once with a single query
+    const folders = await folderDAL.find({
+      envId: env.id
+    });
+
+    // Build the folder hierarchy map
+    const folderMap = buildHierarchy(folders);
+
+    // Generate the paths paths and normalize the root path toe /
+    const paths = generatePaths(folderMap).map((p) => (p === "/" ? p : p.substring(1)));
 
     const { permission } = await permissionService.getProjectPermission(
       auth.actor,
@@ -138,17 +146,16 @@ export const recursivelyGetSecretPaths = ({
       auth.actorOrgId
     );
 
-    const allowedPaths = paths.filter((p) =>
+    // Filter out paths that the user does not have permission to access, and paths that are not in the current path
-      // if its service token allow full access over imported one
+    const allowedPaths = paths.filter(
-      auth.actor === ActorType.SERVICE
+      (p) =>
-        ? true
+        permission.can(
-        : permission.can(
+          ProjectPermissionActions.Read,
-            ProjectPermissionActions.Read,
+          subject(ProjectPermissionSub.Secrets, {
-            subject(ProjectPermissionSub.Secrets, {
+            environment,
-              environment,
+            secretPath: p
-              secretPath: p
+          })
-            })
+        ) && p.startsWith(currentPath === "/" ? "" : currentPath)
-          )
     );
 
     return allowedPaths;