feature: server-side pagination/filtering/sorting for secrets overview and main pages

backend/src/services/secret-folder/secret-folder-dal.ts

@@ -5,6 +5,8 @@ import { TableName, TProjectEnvironments, TSecretFolders, TSecretFoldersUpdate }
 import { BadRequestError, DatabaseError } from "@app/lib/errors";
 import { groupBy, removeTrailingSlash } from "@app/lib/fn";
 import { ormify, selectAllTableCols } from "@app/lib/knex";
+import { OrderByDirection } from "@app/lib/types";
+import { SecretsOrderBy } from "@app/services/secret/secret-types";
 
 export const validateFolderName = (folderName: string) => {
   const validNameRegex = /^[a-zA-Z0-9-_]+$/;
@@ -83,7 +85,7 @@ const sqlFindMultipleFolderByEnvPathQuery = (db: Knex, query: Array<{ envId: str
     .from<TSecretFolders & { depth: number; path: string }>("parent");
 };
 
-const sqlFindFolderByPathQuery = (db: Knex, projectId: string, environment: string, secretPath: string) => {
+const sqlFindFolderByPathQuery = (db: Knex, projectId: string, environments: string[], secretPath: string) => {
   // this is removing an trailing slash like /folder1/folder2/ -> /folder1/folder2
   const formatedPath = secretPath.at(-1) === "/" && secretPath.length > 1 ? secretPath.slice(0, -1) : secretPath;
   // next goal to sanitize saw the raw sql query is safe
@@ -111,7 +113,7 @@ const sqlFindFolderByPathQuery = (db: Knex, projectId: string, environment: stri
           projectId,
           parentId: null
         })
-        .where(`${TableName.Environment}.slug`, environment)
+        .whereIn(`${TableName.Environment}.slug`, environments)
         .select(selectAllTableCols(TableName.SecretFolder))
         .union(
           (qb) =>
@@ -139,14 +141,14 @@ const sqlFindFolderByPathQuery = (db: Knex, projectId: string, environment: stri
     .from<TSecretFolders & { depth: number; path: string }>("parent")
     .leftJoin<TProjectEnvironments>(TableName.Environment, `${TableName.Environment}.id`, "parent.envId")
     .select<
-      TSecretFolders & {
+      (TSecretFolders & {
         depth: number;
         path: string;
         envId: string;
         envSlug: string;
         envName: string;
         projectId: string;
-      }
+      })[]
     >(
       selectAllTableCols("parent" as TableName.SecretFolder),
       db.ref("id").withSchema(TableName.Environment).as("envId"),
@@ -214,7 +216,7 @@ export const secretFolderDALFactory = (db: TDbClient) => {
       const folder = await sqlFindFolderByPathQuery(
         tx || db.replicaNode(),
         projectId,
-        environment,
+        [environment],
         removeTrailingSlash(path)
       )
         .orderBy("depth", "desc")
@@ -230,6 +232,35 @@ export const secretFolderDALFactory = (db: TDbClient) => {
     }
   };
 
+  // finds folders by path for multiple envs
+  const findBySecretPathMultiEnv = async (projectId: string, environments: string[], path: string, tx?: Knex) => {
+    try {
+      const pathDepth = removeTrailingSlash(path).split("/").filter(Boolean).length + 1;
+
+      const folders = await sqlFindFolderByPathQuery(
+        tx || db.replicaNode(),
+        projectId,
+        environments,
+        removeTrailingSlash(path)
+      )
+        .orderBy("depth", "desc")
+        .where("depth", pathDepth);
+
+      const firstFolder = folders[0];
+
+      if (firstFolder && firstFolder.path !== removeTrailingSlash(path)) {
+        return [];
+      }
+
+      return folders.map((folder) => {
+        const { envId: id, envName: name, envSlug: slug, ...el } = folder;
+        return { ...el, envId: id, environment: { id, name, slug } };
+      });
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find folders by secret path multi env" });
+    }
+  };
+
   // used in folder creation
   // even if its the original given /path1/path2
   // it will stop automatically at /path2
@@ -238,7 +269,7 @@ export const secretFolderDALFactory = (db: TDbClient) => {
       const folder = await sqlFindFolderByPathQuery(
         tx || db.replicaNode(),
         projectId,
-        environment,
+        [environment],
         removeTrailingSlash(path)
       )
         .orderBy("depth", "desc")
@@ -352,14 +383,71 @@ export const secretFolderDALFactory = (db: TDbClient) => {
     }
   };
 
+  // find project folders for multiple envs
+  const findByMultiEnv = async (
+    {
+      environmentIds,
+      parentIds,
+      search,
+      limit,
+      offset = 0,
+      orderBy = SecretsOrderBy.Name,
+      orderDirection = OrderByDirection.ASC
+    }: {
+      environmentIds: string[];
+      parentIds: string[];
+      search?: string;
+      limit?: number;
+      offset?: number;
+      orderBy?: SecretsOrderBy;
+      orderDirection?: OrderByDirection;
+    },
+    tx?: Knex
+  ) => {
+    try {
+      const query = (tx || db.replicaNode())(TableName.SecretFolder)
+        .whereIn("parentId", parentIds)
+        .whereIn("envId", environmentIds)
+        .where("isReserved", false)
+        .where((bd) => {
+          if (search) {
+            void bd.whereILike("name", `%${search}%`);
+          }
+        })
+        .select(
+          selectAllTableCols(TableName.SecretFolder),
+          db.raw(`DENSE_RANK() OVER (ORDER BY "name" ${orderDirection ?? OrderByDirection.ASC}) as rank`)
+        )
+        .orderBy(orderBy, orderDirection);
+
+      if (limit) {
+        const rankOffset = offset + 1; // ranks start from 1
+        return await (tx || db)
+          .with("w", query)
+          .select("*")
+          .from<Awaited<typeof query>[number]>("w")
+          .where("w.rank", ">=", rankOffset)
+          .andWhere("w.rank", "<", rankOffset + limit);
+      }
+
+      const folders = await query;
+
+      return folders;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find folders multi env" });
+    }
+  };
+
   return {
     ...secretFolderOrm,
     update,
     findBySecretPath,
+    findBySecretPathMultiEnv,
     findById,
     findByManySecretPath,
     findSecretPathByFolderIds,
     findClosestFolder,
-    findByProjectId
+    findByProjectId,
+    findByMultiEnv
   };
 };

backend/src/services/secret-folder/secret-folder-service.ts

@@ -2,11 +2,12 @@ import { ForbiddenError, subject } from "@casl/ability";
 import path from "path";
 import { v4 as uuidv4, validate as uuidValidate } from "uuid";
 
-import { TSecretFoldersInsert } from "@app/db/schemas";
+import { TSecretFolders, TSecretFoldersInsert } from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
 import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
 import { BadRequestError, NotFoundError } from "@app/lib/errors";
+import { OrderByDirection } from "@app/lib/types";
 
 import { TProjectDALFactory } from "../project/project-dal";
 import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
@@ -26,7 +27,7 @@ type TSecretFolderServiceFactoryDep = {
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
   snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
   folderDAL: TSecretFolderDALFactory;
-  projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
+  projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne" | "findBySlugs">;
   folderVersionDAL: TSecretFolderVersionDALFactory;
   projectDAL: Pick<TProjectDALFactory, "findProjectBySlug">;
 };
@@ -396,7 +397,12 @@ export const secretFolderServiceFactory = ({
     actorOrgId,
     actorAuthMethod,
     environment,
-    path: secretPath
+    path: secretPath,
+    search,
+    orderBy,
+    orderDirection,
+    limit,
+    offset
   }: TGetFolderDTO) => {
     // folder list is allowed to be read by anyone
     // permission to check does user has access
@@ -408,11 +414,100 @@ export const secretFolderServiceFactory = ({
     const parentFolder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!parentFolder) return [];
 
-    const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id, isReserved: false });
-
+    const folders = await folderDAL.find(
+      {
+        envId: env.id,
+        parentId: parentFolder.id,
+        isReserved: false,
+        $search: search ? { name: `%${search}%` } : undefined
+      },
+      {
+        sort: orderBy ? [[orderBy, orderDirection ?? OrderByDirection.ASC]] : undefined,
+        limit,
+        offset
+      }
+    );
     return folders;
   };
 
+  // get folders for multiple envs
+  const getFoldersMultiEnv = async ({
+    projectId,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod,
+    environments,
+    path: secretPath,
+    ...params
+  }: Omit<TGetFolderDTO, "environment"> & { environments: string[] }) => {
+    // folder list is allowed to be read by anyone
+    // permission to check does user has access
+    await permissionService.getProjectPermission(actor, actorId, projectId, actorAuthMethod, actorOrgId);
+
+    const envs = await projectEnvDAL.findBySlugs(projectId, environments);
+    const data: { [key: string]: TSecretFolders[] } = {};
+
+    if (!envs.length)
+      throw new BadRequestError({ message: "Environment(s) not found", name: "get project folder count" });
+
+    const parentFolders = await folderDAL.findBySecretPathMultiEnv(projectId, environments, secretPath);
+    if (!parentFolders.length) return data;
+
+    const folders = await folderDAL.findByMultiEnv({
+      environmentIds: envs.map((env) => env.id),
+      parentIds: parentFolders.map((folder) => folder.id),
+      ...params
+    });
+
+    // reorganize folders into respective envs
+    const envMap: Map<string, string> = new Map(envs.map((env) => [env.id, env.slug]));
+    folders.forEach((folder) => {
+      const slug = envMap.get(folder.envId);
+      data[slug!] = [...(data[slug!] ?? []), folder];
+    });
+
+    return data;
+  };
+
+  // get the unique count of folders within a project path
+  const getProjectFolderCount = async ({
+    projectId,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod,
+    environments,
+    path: secretPath,
+    search
+  }: Omit<TGetFolderDTO, "environment"> & { environments: string[] }) => {
+    // folder list is allowed to be read by anyone
+    // permission to check does user has access
+    await permissionService.getProjectPermission(actor, actorId, projectId, actorAuthMethod, actorOrgId);
+
+    const envs = await projectEnvDAL.findBySlugs(projectId, environments);
+
+    if (!envs.length)
+      throw new BadRequestError({ message: "Environment(s) not found", name: "get project folder count" });
+
+    const parentFolders = await folderDAL.findBySecretPathMultiEnv(projectId, environments, secretPath);
+    if (!parentFolders.length) return 0;
+
+    const folders = await folderDAL.find(
+      {
+        $in: {
+          envId: envs.map((env) => env.id),
+          parentId: parentFolders.map((folder) => folder.id)
+        },
+        isReserved: false,
+        $search: search ? { name: `%${search}%` } : undefined
+      },
+      { countDistinct: "name" }
+    );
+
+    return Number(folders[0]?.count ?? 0);
+  };
+
   const getFolderById = async ({ actor, actorId, actorOrgId, actorAuthMethod, id }: TGetFolderByIdDTO) => {
     const folder = await folderDAL.findById(id);
     if (!folder) throw new NotFoundError({ message: "folder not found" });
@@ -429,6 +524,8 @@ export const secretFolderServiceFactory = ({
     updateManyFolders,
     deleteFolder,
     getFolders,
-    getFolderById
+    getFolderById,
+    getProjectFolderCount,
+    getFoldersMultiEnv
   };
 };

backend/src/services/secret-folder/secret-folder-types.ts

@@ -1,4 +1,5 @@
-import { TProjectPermission } from "@app/lib/types";
+import { OrderByDirection, TProjectPermission } from "@app/lib/types";
+import { SecretsOrderBy } from "@app/services/secret/secret-types";
 
 export enum ReservedFolders {
   SecretReplication = "__reserve_replication_"
@@ -36,6 +37,11 @@ export type TDeleteFolderDTO = {
 export type TGetFolderDTO = {
   environment: string;
   path: string;
+  search?: string;
+  orderBy?: SecretsOrderBy;
+  orderDirection?: OrderByDirection;
+  limit?: number;
+  offset?: number;
 } & TProjectPermission;
 
 export type TGetFolderByIdDTO = {