pam: accounts table server-side filter, search, pagination

backend/src/ee/routes/v1/pam-account-routers/pam-account-router.ts

@@ -2,11 +2,14 @@ import { z } from "zod";
 
 import { PamFoldersSchema } from "@app/db/schemas";
 import { EventType } from "@app/ee/services/audit-log/audit-log-types";
+import { PamAccountOrderBy, PamAccountView } from "@app/ee/services/pam-account/pam-account-types";
 import { SanitizedMySQLAccountWithResourceSchema } from "@app/ee/services/pam-resource/mysql/mysql-resource-schemas";
 import { PamResource } from "@app/ee/services/pam-resource/pam-resource-enums";
 import { SanitizedPostgresAccountWithResourceSchema } from "@app/ee/services/pam-resource/postgres/postgres-resource-schemas";
 import { BadRequestError } from "@app/lib/errors";
+import { removeTrailingSlash } from "@app/lib/fn";
 import { ms } from "@app/lib/ms";
+import { OrderByDirection } from "@app/lib/types";
 import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
@@ -26,33 +29,58 @@ export const registerPamAccountRouter = async (server: FastifyZodProvider) => {
     schema: {
       description: "List PAM accounts",
       querystring: z.object({
-        projectId: z.string().uuid()
+        projectId: z.string().uuid(),
+        accountPath: z.string().trim().default("/").transform(removeTrailingSlash),
+        accountView: z.nativeEnum(PamAccountView).default(PamAccountView.Flat),
+        offset: z.coerce.number().min(0).default(0),
+        limit: z.coerce.number().min(1).max(100).default(100),
+        orderBy: z.nativeEnum(PamAccountOrderBy).default(PamAccountOrderBy.Name),
+        orderDirection: z.nativeEnum(OrderByDirection).default(OrderByDirection.ASC),
+        search: z.string().trim().optional()
       }),
       response: {
         200: z.object({
           accounts: SanitizedAccountSchema.array(),
-          folders: PamFoldersSchema.array()
+          folders: PamFoldersSchema.array(),
+          totalCount: z.number().default(0),
+          folderId: z.string().optional(),
+          folderPaths: z.record(z.string(), z.string())
         })
       }
     },
     onRequest: verifyAuth([AuthMode.JWT]),
     handler: async (req) => {
-      const response = await server.services.pamAccount.list(req.query.projectId, req.permission);
+      const { projectId, accountPath, accountView, limit, offset, search, orderBy, orderDirection } = req.query;
+
+      const { accounts, folders, totalCount, folderId, folderPaths } = await server.services.pamAccount.list({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        projectId,
+        accountPath,
+        accountView,
+        limit,
+        offset,
+        search,
+        orderBy,
+        orderDirection
+      });
 
       await server.services.auditLog.createAuditLog({
         ...req.auditLogInfo,
         orgId: req.permission.orgId,
-        projectId: req.query.projectId,
+        projectId,
         event: {
           type: EventType.PAM_ACCOUNT_LIST,
           metadata: {
-            accountCount: response.accounts.length,
-            folderCount: response.folders.length
+            accountCount: accounts.length,
+            folderCount: folders.length
           }
         }
       });
 
-      return response;
+      return { accounts, folders, totalCount, folderId, folderPaths };
     }
   });
 

backend/src/ee/services/pam-account/pam-account-dal.ts

@@ -1,46 +1,100 @@
 import { Knex } from "knex";
 
 import { TDbClient } from "@app/db";
-import { TableName, TPamAccounts } from "@app/db/schemas";
-import { buildFindFilter, ormify, prependTableNameToFindFilter, selectAllTableCols } from "@app/lib/knex";
+import { TableName } from "@app/db/schemas";
+import { DatabaseError } from "@app/lib/errors";
+import { ormify, selectAllTableCols } from "@app/lib/knex";
+import { OrderByDirection } from "@app/lib/types";
+
+import { PamAccountOrderBy, PamAccountView } from "./pam-account-types";
 
 export type TPamAccountDALFactory = ReturnType<typeof pamAccountDALFactory>;
 
-type PamAccountFindFilter = Parameters<typeof buildFindFilter<TPamAccounts>>[0];
-
 export const pamAccountDALFactory = (db: TDbClient) => {
   const orm = ormify(db, TableName.PamAccount);
 
-  const findWithResourceDetails = async (filter: PamAccountFindFilter, tx?: Knex) => {
-    const query = (tx || db.replicaNode())(TableName.PamAccount)
-      .leftJoin(TableName.PamResource, `${TableName.PamAccount}.resourceId`, `${TableName.PamResource}.id`)
-      .select(selectAllTableCols(TableName.PamAccount))
-      .select(
+  const findByProjectIdWithResourceDetails = async (
+    {
+      projectId,
+      folderId,
+      accountView = PamAccountView.Nested,
+      search,
+      limit,
+      offset = 0,
+      orderBy = PamAccountOrderBy.Name,
+      orderDirection = OrderByDirection.ASC
+    }: {
+      projectId: string;
+      folderId?: string | null;
+      accountView?: PamAccountView;
+      search?: string;
+      limit?: number;
+      offset?: number;
+      orderBy?: PamAccountOrderBy;
+      orderDirection?: OrderByDirection;
+    },
+    tx?: Knex
+  ) => {
+    try {
+      const dbInstance = tx || db.replicaNode();
+      const query = dbInstance(TableName.PamAccount)
+        .leftJoin(TableName.PamResource, `${TableName.PamAccount}.resourceId`, `${TableName.PamResource}.id`)
+        .where(`${TableName.PamAccount}.projectId`, projectId);
+
+      if (accountView === PamAccountView.Nested) {
+        if (folderId) {
+          void query.where(`${TableName.PamAccount}.folderId`, folderId);
+        } else {
+          void query.whereNull(`${TableName.PamAccount}.folderId`);
+        }
+      }
+
+      if (search) {
+        void query.where((q) => {
+          void q
+            .whereILike(`${TableName.PamAccount}.name`, `%${search}%`)
+            .orWhereILike(`${TableName.PamResource}.name`, `%${search}%`)
+            .orWhereILike(`${TableName.PamAccount}.description`, `%${search}%`);
+        });
+      }
+
+      const countQuery = query.clone().count("*", { as: "count" }).first();
+
+      void query.select(selectAllTableCols(TableName.PamAccount)).select(
         // resource
         db.ref("name").withSchema(TableName.PamResource).as("resourceName"),
         db.ref("resourceType").withSchema(TableName.PamResource),
         db.ref("encryptedRotationAccountCredentials").withSchema(TableName.PamResource)
       );
 
-    if (filter) {
-      /* eslint-disable @typescript-eslint/no-misused-promises */
-      void query.where(buildFindFilter(prependTableNameToFindFilter(TableName.PamAccount, filter)));
+      const direction = orderDirection === OrderByDirection.ASC ? "ASC" : "DESC";
+
+      void query.orderByRaw(`${TableName.PamAccount}.?? COLLATE "en-x-icu" ${direction}`, [orderBy]);
+
+      if (typeof limit === "number") {
+        void query.limit(limit).offset(offset);
+      }
+
+      const [results, countResult] = await Promise.all([query, countQuery]);
+      const totalCount = Number(countResult?.count || 0);
+
+      const accounts = results.map(
+        // @ts-expect-error resourceName, resourceType, encryptedRotationAccountCredentials are from joined table
+        ({ resourceId, resourceName, resourceType, encryptedRotationAccountCredentials, ...account }) => ({
+          ...account,
+          resourceId,
+          resource: {
+            id: resourceId,
+            name: resourceName as string,
+            resourceType,
+            encryptedRotationAccountCredentials
+          }
+        })
+      );
+      return { accounts, totalCount };
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find PAM accounts with resource details" });
     }
-
-    const accounts = await query;
-
-    return accounts.map(
-      ({ resourceId, resourceName, resourceType, encryptedRotationAccountCredentials, ...account }) => ({
-        ...account,
-        resourceId,
-        resource: {
-          id: resourceId,
-          name: resourceName,
-          resourceType,
-          encryptedRotationAccountCredentials
-        }
-      })
-    );
   };
 
   const findAccountsDueForRotation = async (tx?: Knex) => {
@@ -59,5 +113,9 @@ export const pamAccountDALFactory = (db: TDbClient) => {
     return accounts;
   };
 
-  return { ...orm, findWithResourceDetails, findAccountsDueForRotation };
+  return {
+    ...orm,
+    findByProjectIdWithResourceDetails,
+    findAccountsDueForRotation
+  };
 };

backend/src/ee/services/pam-account/pam-account-service.ts

@@ -1,6 +1,6 @@
 import { ForbiddenError, subject } from "@casl/ability";
 
-import { ActionProjectType, OrganizationActionScope, TPamAccounts, TPamResources } from "@app/db/schemas";
+import { ActionProjectType, OrganizationActionScope, TPamAccounts, TPamFolders, TPamResources } from "@app/db/schemas";
 import { PAM_RESOURCE_FACTORY_MAP } from "@app/ee/services/pam-resource/pam-resource-factory";
 import { decryptResource, decryptResourceConnectionDetails } from "@app/ee/services/pam-resource/pam-resource-fns";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
@@ -32,7 +32,13 @@ import { PamSessionStatus } from "../pam-session/pam-session-enums";
 import { OrgPermissionGatewayActions, OrgPermissionSubjects } from "../permission/org-permission";
 import { TPamAccountDALFactory } from "./pam-account-dal";
 import { decryptAccount, decryptAccountCredentials, encryptAccountCredentials } from "./pam-account-fns";
-import { TAccessAccountDTO, TCreateAccountDTO, TUpdateAccountDTO } from "./pam-account-types";
+import {
+  PamAccountView,
+  TAccessAccountDTO,
+  TCreateAccountDTO,
+  TListAccountsDTO,
+  TUpdateAccountDTO
+} from "./pam-account-types";
 
 type TPamAccountServiceFactoryDep = {
   pamResourceDAL: TPamResourceDALFactory;
@@ -334,21 +340,86 @@ export const pamAccountServiceFactory = ({
     };
   };
 
-  const list = async (projectId: string, actor: OrgServiceActor) => {
+  const list = async ({
+    projectId,
+    accountPath,
+    accountView,
+    actor,
+    actorId,
+    actorAuthMethod,
+    actorOrgId,
+    ...params
+  }: TListAccountsDTO) => {
     const { permission } = await permissionService.getProjectPermission({
-      actor: actor.type,
-      actorAuthMethod: actor.authMethod,
-      actorId: actor.id,
-      actorOrgId: actor.orgId,
+      actor,
+      actorId,
       projectId,
+      actorAuthMethod,
+      actorOrgId,
       actionProjectType: ActionProjectType.PAM
     });
 
-    const accountsWithResourceDetails = await pamAccountDAL.findWithResourceDetails({ projectId });
+    const limit = params.limit || 20;
+    const offset = params.offset || 0;
 
     const canReadFolders = permission.can(ProjectPermissionActions.Read, ProjectPermissionSub.PamFolders);
 
-    const folders = canReadFolders ? await pamFolderDAL.find({ projectId }) : [];
+    const folder = accountPath === "/" ? null : await pamFolderDAL.findByPath(projectId, accountPath);
+    if (accountPath !== "/" && !folder) {
+      return { accounts: [], folders: [], totalCount: 0, folderPaths: {} };
+    }
+    const folderId = folder?.id;
+
+    let totalFolderCount = 0;
+    if (canReadFolders && accountView === PamAccountView.Nested) {
+      const { totalCount } = await pamFolderDAL.findByProjectId({
+        projectId,
+        parentId: folderId
+      });
+      totalFolderCount = totalCount;
+    }
+    const { totalCount: totalAccountCount } = await pamAccountDAL.findByProjectIdWithResourceDetails({
+      projectId,
+      folderId,
+      accountView
+    });
+
+    const totalCount = totalFolderCount + totalAccountCount;
+
+    let folders: TPamFolders[] = [];
+    if (canReadFolders && accountView === PamAccountView.Nested && offset < totalFolderCount) {
+      const folderLimit = Math.min(limit, totalFolderCount - offset);
+      const { folders: foldersResp } = await pamFolderDAL.findByProjectId({
+        projectId,
+        parentId: folderId,
+        limit: folderLimit,
+        offset,
+        search: params.search,
+        orderBy: params.orderBy,
+        orderDirection: params.orderDirection
+      });
+
+      folders = foldersResp;
+    }
+
+    let accountsWithResourceDetails: Awaited<
+      ReturnType<typeof pamAccountDAL.findByProjectIdWithResourceDetails>
+    >["accounts"] = [];
+    const accountsToFetch = limit - folders.length;
+    if (accountsToFetch > 0) {
+      const accountOffset = Math.max(0, offset - totalFolderCount);
+      const { accounts: accountsResp } = await pamAccountDAL.findByProjectIdWithResourceDetails({
+        projectId,
+        folderId,
+        accountView,
+        offset: accountOffset,
+        limit: accountsToFetch,
+        search: params.search,
+        orderBy: params.orderBy,
+        orderDirection: params.orderDirection
+      });
+      accountsWithResourceDetails = accountsResp;
+    }
 
     const decryptedAndPermittedAccounts: Array<
       TPamAccounts & {
@@ -359,12 +430,6 @@ export const pamAccountServiceFactory = ({
     > = [];
 
     for await (const account of accountsWithResourceDetails) {
-      const accountPath = await getFullPamFolderPath({
-        pamFolderDAL,
-        folderId: account.folderId,
-        projectId: account.projectId
-      });
-
       // Check permission for each individual account
       if (
         permission.can(
@@ -391,9 +456,27 @@ export const pamAccountServiceFactory = ({
       }
     }
 
+    const folderPaths: Record<string, string> = {};
+    const accountFolderIds = [
+      ...new Set(decryptedAndPermittedAccounts.flatMap((a) => (a.folderId ? [a.folderId] : [])))
+    ];
+
+    await Promise.all(
+      accountFolderIds.map(async (fId) => {
+        folderPaths[fId] = await getFullPamFolderPath({
+          pamFolderDAL,
+          folderId: fId,
+          projectId
+        });
+      })
+    );
+
     return {
       accounts: decryptedAndPermittedAccounts,
-      folders
+      folders,
+      totalCount,
+      folderId,
+      folderPaths
     };
   };
 

backend/src/ee/services/pam-account/pam-account-types.ts

@@ -1,3 +1,5 @@
+import { OrderByDirection, TProjectPermission } from "@app/lib/types";
+
 import { TPamAccount } from "../pam-resource/pam-resource-types";
 
 // DTOs
@@ -18,3 +20,22 @@ export type TAccessAccountDTO = {
   actorUserAgent: string;
   duration: number;
 };
+
+export type TListAccountsDTO = {
+  accountPath: string;
+  accountView: PamAccountView;
+  search?: string;
+  orderBy?: PamAccountOrderBy;
+  orderDirection?: OrderByDirection;
+  limit?: number;
+  offset?: number;
+} & TProjectPermission;
+
+export enum PamAccountOrderBy {
+  Name = "name"
+}
+
+export enum PamAccountView {
+  Flat = "flat",
+  Nested = "nested"
+}

backend/src/ee/services/pam-folder/pam-folder-dal.ts

@@ -1,9 +1,104 @@
+import { Knex } from "knex";
+
 import { TDbClient } from "@app/db";
 import { TableName } from "@app/db/schemas";
-import { ormify } from "@app/lib/knex";
+import { DatabaseError } from "@app/lib/errors";
+import { ormify, selectAllTableCols } from "@app/lib/knex";
+import { OrderByDirection } from "@app/lib/types";
+
+import { PamAccountOrderBy } from "../pam-account/pam-account-types";
 
 export type TPamFolderDALFactory = ReturnType<typeof pamFolderDALFactory>;
 export const pamFolderDALFactory = (db: TDbClient) => {
   const orm = ormify(db, TableName.PamFolder);
-  return { ...orm };
+
+  const findByProjectId = async (
+    {
+      projectId,
+      parentId,
+      search,
+      limit,
+      offset = 0,
+      orderBy = PamAccountOrderBy.Name,
+      orderDirection = OrderByDirection.ASC
+    }: {
+      projectId: string;
+      parentId?: string | null;
+      search?: string;
+      limit?: number;
+      offset?: number;
+      orderBy?: PamAccountOrderBy;
+      orderDirection?: OrderByDirection;
+    },
+    tx?: Knex
+  ) => {
+    try {
+      const dbInstance = tx || db.replicaNode();
+      const query = dbInstance(TableName.PamFolder).where(`${TableName.PamFolder}.projectId`, projectId);
+
+      if (parentId) {
+        void query.where(`${TableName.PamFolder}.parentId`, parentId);
+      } else {
+        void query.whereNull(`${TableName.PamFolder}.parentId`);
+      }
+
+      if (search) {
+        void query.whereILike(`${TableName.PamFolder}.name`, `%${search}%`);
+      }
+
+      const countQuery = query.clone().count("*", { as: "count" }).first();
+
+      void query.select(selectAllTableCols(TableName.PamFolder));
+      const direction = orderDirection === OrderByDirection.ASC ? "ASC" : "DESC";
+
+      void query.orderByRaw(`${TableName.PamFolder}.?? COLLATE "en-x-icu" ${direction}`, [orderBy]);
+
+      if (typeof limit === "number") {
+        void query.limit(limit).offset(offset);
+      }
+
+      const [folders, countResult] = await Promise.all([query, countQuery]);
+      const totalCount = Number(countResult?.count || 0);
+
+      return { folders, totalCount };
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find PAM folders" });
+    }
+  };
+
+  const findByPath = async (projectId: string, path: string, tx?: Knex) => {
+    try {
+      const dbInstance = tx || db.replicaNode();
+      const pathSegments = path.split("/").filter(Boolean);
+
+      let parentId: string | null = null;
+      let currentFolder: Awaited<ReturnType<typeof orm.findOne>> | undefined;
+
+      for await (const segment of pathSegments) {
+        const query = dbInstance(TableName.PamFolder)
+          .where(`${TableName.PamFolder}.projectId`, projectId)
+          .where(`${TableName.PamFolder}.name`, segment);
+
+        if (parentId) {
+          void query.where(`${TableName.PamFolder}.parentId`, parentId);
+        } else {
+          void query.whereNull(`${TableName.PamFolder}.parentId`);
+        }
+
+        currentFolder = await query.first();
+
+        if (!currentFolder) {
+          return undefined;
+        }
+
+        parentId = currentFolder.id;
+      }
+
+      return currentFolder;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find PAM folder by path" });
+    }
+  };
+
+  return { ...orm, findByProjectId, findByPath };
 };