pam: resources table server-side filter, search, pagination

backend/src/ee/routes/v1/pam-resource-routers/pam-resource-router.ts

@@ -5,10 +5,12 @@ import {
   MySQLResourceListItemSchema,
   SanitizedMySQLResourceSchema
 } from "@app/ee/services/pam-resource/mysql/mysql-resource-schemas";
+import { PamResourceOrderBy } from "@app/ee/services/pam-resource/pam-resource-enums";
 import {
   PostgresResourceListItemSchema,
   SanitizedPostgresResourceSchema
 } from "@app/ee/services/pam-resource/postgres/postgres-resource-schemas";
+import { OrderByDirection } from "@app/lib/types";
 import { readLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
@@ -52,17 +54,36 @@ export const registerPamResourceRouter = async (server: FastifyZodProvider) => {
     schema: {
       description: "List PAM resources",
       querystring: z.object({
-        projectId: z.string().uuid()
+        projectId: z.string().uuid(),
+        offset: z.coerce.number().min(0).default(0),
+        limit: z.coerce.number().min(1).max(100).default(100),
+        orderBy: z.nativeEnum(PamResourceOrderBy).default(PamResourceOrderBy.Name),
+        orderDirection: z.nativeEnum(OrderByDirection).default(OrderByDirection.ASC),
+        search: z.string().trim().optional()
       }),
       response: {
         200: z.object({
-          resources: SanitizedResourceSchema.array()
+          resources: SanitizedResourceSchema.array(),
+          totalCount: z.number().default(0)
         })
       }
     },
     onRequest: verifyAuth([AuthMode.JWT]),
     handler: async (req) => {
-      const response = await server.services.pamResource.list(req.query.projectId, req.permission);
+      const { projectId, limit, offset, search, orderBy, orderDirection } = req.query;
+
+      const { resources, totalCount } = await server.services.pamResource.list({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        projectId,
+        limit,
+        offset,
+        search,
+        orderBy,
+        orderDirection
+      });
 
       await server.services.auditLog.createAuditLog({
         ...req.auditLogInfo,
@@ -71,12 +92,12 @@ export const registerPamResourceRouter = async (server: FastifyZodProvider) => {
         event: {
           type: EventType.PAM_RESOURCE_LIST,
           metadata: {
-            count: response.resources.length
+            count: totalCount
           }
         }
       });
 
-      return response;
+      return { resources, totalCount };
     }
   });
 };

backend/src/ee/services/pam-resource/pam-resource-dal.ts

@@ -2,7 +2,11 @@ import { Knex } from "knex";
 
 import { TDbClient } from "@app/db";
 import { TableName } from "@app/db/schemas";
+import { DatabaseError } from "@app/lib/errors";
 import { ormify, selectAllTableCols } from "@app/lib/knex";
+import { OrderByDirection } from "@app/lib/types";
+
+import { PamResourceOrderBy } from "./pam-resource-enums";
 
 export type TPamResourceDALFactory = ReturnType<typeof pamResourceDALFactory>;
 export const pamResourceDALFactory = (db: TDbClient) => {
@@ -20,5 +24,56 @@ export const pamResourceDALFactory = (db: TDbClient) => {
     return doc;
   };
 
-  return { ...orm, findById };
+  const findByProjectId = async (
+    {
+      projectId,
+      search,
+      limit,
+      offset = 0,
+      orderBy = PamResourceOrderBy.Name,
+      orderDirection = OrderByDirection.ASC
+    }: {
+      projectId: string;
+      search?: string;
+      limit?: number;
+      offset?: number;
+      orderBy?: PamResourceOrderBy;
+      orderDirection?: OrderByDirection;
+    },
+    tx?: Knex
+  ) => {
+    try {
+      const dbInstance = tx || db.replicaNode();
+      const query = dbInstance(TableName.PamResource).where(`${TableName.PamResource}.projectId`, projectId);
+
+      if (search) {
+        void query.where((q) => {
+          void q
+            .whereILike(`${TableName.PamResource}.name`, `%${search}%`)
+            .orWhereILike(`${TableName.PamResource}.resourceType`, `%${search}%`);
+        });
+      }
+
+      const countQuery = query.clone().count("*", { as: "count" }).first();
+
+      void query.select(selectAllTableCols(TableName.PamResource));
+
+      const direction = orderDirection === OrderByDirection.ASC ? "ASC" : "DESC";
+
+      void query.orderByRaw(`${TableName.PamResource}.?? COLLATE "en-x-icu" ${direction}`, [orderBy]);
+
+      if (typeof limit === "number") {
+        void query.limit(limit).offset(offset);
+      }
+
+      const [resources, countResult] = await Promise.all([query, countQuery]);
+      const totalCount = Number(countResult?.count || 0);
+
+      return { resources, totalCount };
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find PAM resources" });
+    }
+  };
+
+  return { ...orm, findById, findByProjectId };
 };

backend/src/ee/services/pam-resource/pam-resource-enums.ts

@@ -2,3 +2,7 @@ export enum PamResource {
   Postgres = "postgres",
   MySQL = "mysql"
 }
+
+export enum PamResourceOrderBy {
+  Name = "name"
+}

backend/src/ee/services/pam-resource/pam-resource-service.ts

@@ -20,7 +20,7 @@ import {
   encryptResourceConnectionDetails,
   listResourceOptions
 } from "./pam-resource-fns";
-import { TCreateResourceDTO, TUpdateResourceDTO } from "./pam-resource-types";
+import { TCreateResourceDTO, TListResourcesDTO, TUpdateResourceDTO } from "./pam-resource-types";
 
 type TPamResourceServiceFactoryDep = {
   pamResourceDAL: TPamResourceDALFactory;
@@ -268,22 +268,23 @@ export const pamResourceServiceFactory = ({
     }
   };
 
-  const list = async (projectId: string, actor: OrgServiceActor) => {
+  const list = async ({ projectId, actor, actorId, actorAuthMethod, actorOrgId, ...params }: TListResourcesDTO) => {
     const { permission } = await permissionService.getProjectPermission({
-      actor: actor.type,
+      actor,
-      actorAuthMethod: actor.authMethod,
+      actorId,
-      actorId: actor.id,
+      actorAuthMethod,
-      actorOrgId: actor.orgId,
+      actorOrgId,
       projectId,
       actionProjectType: ActionProjectType.PAM
     });
 
     ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.PamResources);
 
-    const resources = await pamResourceDAL.find({ projectId });
+    const { resources, totalCount } = await pamResourceDAL.findByProjectId({ projectId, ...params });
 
     return {
-      resources: await Promise.all(resources.map((resource) => decryptResource(resource, projectId, kmsService)))
+      resources: await Promise.all(resources.map((resource) => decryptResource(resource, projectId, kmsService))),
+      totalCount
     };
   };
 

backend/src/ee/services/pam-resource/pam-resource-types.ts

@@ -1,3 +1,5 @@
+import { OrderByDirection, TProjectPermission } from "@app/lib/types";
+
 import { TGatewayV2ServiceFactory } from "../gateway-v2/gateway-v2-service";
 import {
   TMySQLAccount,
@@ -5,7 +7,7 @@ import {
   TMySQLResource,
   TMySQLResourceConnectionDetails
 } from "./mysql/mysql-resource-types";
-import { PamResource } from "./pam-resource-enums";
+import { PamResource, PamResourceOrderBy } from "./pam-resource-enums";
 import {
   TPostgresAccount,
   TPostgresAccountCredentials,
@@ -32,6 +34,14 @@ export type TUpdateResourceDTO = Partial<Omit<TCreateResourceDTO, "resourceType"
   resourceId: string;
 };
 
+export type TListResourcesDTO = {
+  search?: string;
+  orderBy?: PamResourceOrderBy;
+  orderDirection?: OrderByDirection;
+  limit?: number;
+  offset?: number;
+} & TProjectPermission;
+
 // Resource factory
 export type TPamResourceFactoryValidateConnection<T extends TPamResourceConnectionDetails> = () => Promise<T>;
 export type TPamResourceFactoryValidateAccountCredentials<C extends TPamAccountCredentials> = (

frontend/src/hooks/api/pam/enums.ts

@@ -1,3 +1,4 @@
+// Resources
 export enum PamResourceType {
   Postgres = "postgres",
   MySQL = "mysql",
@@ -6,6 +7,11 @@ export enum PamResourceType {
   Kubernetes = "kubernetes"
 }
 
+export enum PamResourceOrderBy {
+  Name = "name"
+}
+
+// Sessions
 export enum PamSessionStatus {
   Starting = "starting",
   Active = "active",
@@ -13,6 +19,7 @@ export enum PamSessionStatus {
   Terminated = "terminated"
 }
 
+// Accounts
 export enum PamAccountOrderBy {
   Name = "name"
 }

frontend/src/hooks/api/pam/mutations.tsx

@@ -31,7 +31,7 @@ export const useCreatePamResource = () => {
       return data.resource;
     },
     onSuccess: ({ projectId }) => {
-      queryClient.invalidateQueries({ queryKey: pamKeys.listResources(projectId) });
+      queryClient.invalidateQueries({ queryKey: pamKeys.listResources({ projectId }) });
     }
   });
 };
@@ -48,7 +48,7 @@ export const useUpdatePamResource = () => {
       return data.resource;
     },
     onSuccess: ({ projectId }) => {
-      queryClient.invalidateQueries({ queryKey: pamKeys.listResources(projectId) });
+      queryClient.invalidateQueries({ queryKey: pamKeys.listResources({ projectId }) });
     }
   });
 };
@@ -64,7 +64,7 @@ export const useDeletePamResource = () => {
       return data.resource;
     },
     onSuccess: ({ projectId }) => {
-      queryClient.invalidateQueries({ queryKey: pamKeys.listResources(projectId) });
+      queryClient.invalidateQueries({ queryKey: pamKeys.listResources({ projectId }) });
     }
   });
 };

frontend/src/hooks/api/pam/queries.tsx

@@ -4,7 +4,14 @@ import { apiRequest } from "@app/config/request";
 
 import { TPamResourceOption } from "./types/resource-options";
 import { PamResourceType } from "./enums";
-import { TListPamAccountsDTO, TPamAccount, TPamFolder, TPamResource, TPamSession } from "./types";
+import {
+  TListPamAccountsDTO,
+  TListPamResourcesDTO,
+  TPamAccount,
+  TPamFolder,
+  TPamResource,
+  TPamSession
+} from "./types";
 
 export const pamKeys = {
   all: ["pam"] as const,
@@ -12,7 +19,12 @@ export const pamKeys = {
   account: () => [...pamKeys.all, "account"] as const,
   session: () => [...pamKeys.all, "session"] as const,
   listResourceOptions: () => [...pamKeys.resource(), "options"] as const,
-  listResources: (projectId: string) => [...pamKeys.resource(), "list", projectId],
+  listResources: ({ projectId, ...params }: TListPamResourcesDTO) => [
+    ...pamKeys.resource(),
+    "list",
+    projectId,
+    params
+  ],
   getResource: (resourceType: string, resourceId: string) => [
     ...pamKeys.resource(),
     "get",
@@ -54,27 +66,31 @@ export const useListPamResourceOptions = (
   });
 };
 
+type TListPamResourcesResponse = {
+  resources: TPamResource[];
+  totalCount: number;
+};
+
 export const useListPamResources = (
-  projectId: string,
+  params: TListPamResourcesDTO,
   options?: Omit<
     UseQueryOptions<
-      TPamResource[],
+      TListPamResourcesResponse,
       unknown,
-      TPamResource[],
+      TListPamResourcesResponse,
       ReturnType<typeof pamKeys.listResources>
     >,
     "queryKey" | "queryFn"
   >
 ) => {
   return useQuery({
-    queryKey: pamKeys.listResources(projectId),
+    queryKey: pamKeys.listResources(params),
     queryFn: async () => {
-      const { data } = await apiRequest.get<{ resources: TPamResource[] }>(
+      const { data } = await apiRequest.get<TListPamResourcesResponse>("/api/v1/pam/resources", {
-        "/api/v1/pam/resources",
+        params
-        { params: { projectId } }
+      });
-      );
 
-      return data.resources;
+      return data;
     },
     ...options
   });

frontend/src/hooks/api/pam/types/index.ts

@@ -1,5 +1,11 @@
 import { OrderByDirection } from "../../generic/types";
-import { PamAccountOrderBy, PamAccountView, PamResourceType, PamSessionStatus } from "../enums";
+import {
+  PamAccountOrderBy,
+  PamAccountView,
+  PamResourceOrderBy,
+  PamResourceType,
+  PamSessionStatus
+} from "../enums";
 import { TMySQLAccount, TMySQLResource } from "./mysql-resource";
 import { TPostgresAccount, TPostgresResource } from "./postgres-resource";
 
@@ -46,6 +52,15 @@ export type TPamSession = {
 };
 
 // Resource DTOs
+export type TListPamResourcesDTO = {
+  projectId: string;
+  offset?: number;
+  limit?: number;
+  orderBy?: PamResourceOrderBy;
+  orderDirection?: OrderByDirection;
+  search?: string;
+};
+
 export type TCreatePamResourceDTO = Pick<
   TPamResource,
   "name" | "connectionDetails" | "resourceType" | "gatewayId" | "projectId"

frontend/src/pages/pam/PamAccountsPage/components/ResourceSelect.tsx

@@ -1,11 +1,12 @@
+import { useState } from "react";
 import { Controller, FormProvider, useForm } from "react-hook-form";
 import { SingleValue } from "react-select";
 import { zodResolver } from "@hookform/resolvers/zod";
 import { z } from "zod";
 
-import { Button, FilterableSelect, FormControl, ModalClose, Spinner } from "@app/components/v2";
+import { Button, FilterableSelect, FormControl, ModalClose } from "@app/components/v2";
 import { ProjectPermissionActions, ProjectPermissionSub, useProjectPermission } from "@app/context";
-import { usePopUp } from "@app/hooks";
+import { useDebounce, usePopUp } from "@app/hooks";
 import { PamResourceType, useListPamResources } from "@app/hooks/api/pam";
 
 import { PamAddResourceModal } from "../../PamResourcesPage/components/PamAddResourceModal";
@@ -28,7 +29,17 @@ type FormData = z.infer<typeof formSchema>;
 
 export const ResourceSelect = ({ onSubmit, projectId }: Props) => {
   const { permission } = useProjectPermission();
-  const { isPending, data: resources } = useListPamResources(projectId);
+
+  const [search, setSearch] = useState("");
+  const [debouncedSearch] = useDebounce(search, 350);
+
+  const { isPending, data } = useListPamResources({
+    projectId,
+    limit: 100,
+    search: debouncedSearch
+  });
+
+  const resources = data?.resources || [];
 
   const { popUp, handlePopUpToggle, handlePopUpOpen } = usePopUp(["addResource"] as const);
 
@@ -43,15 +54,6 @@ export const ResourceSelect = ({ onSubmit, projectId }: Props) => {
     ProjectPermissionSub.PamResources
   );
 
-  if (isPending) {
-    return (
-      <div className="flex h-full flex-col items-center justify-center py-2.5">
-        <Spinner size="lg" className="text-mineshaft-500" />
-        <p className="mt-4 text-sm text-mineshaft-400">Loading options...</p>
-      </div>
-    );
-  }
-
   return (
     <>
       <FormProvider {...form}>
@@ -65,6 +67,12 @@ export const ResourceSelect = ({ onSubmit, projectId }: Props) => {
               <FormControl isError={Boolean(error)} errorText={error?.message} label="Resource">
                 <FilterableSelect
                   value={value}
+                  inputValue={search}
+                  onInputChange={(val, actionMeta) => {
+                    if (actionMeta.action === "input-change") {
+                      setSearch(val);
+                    }
+                  }}
                   onChange={(newValue) => {
                     if ((newValue as SingleValue<{ id: string }>)?.id === "_create") {
                       handlePopUpOpen("addResource");

frontend/src/pages/pam/PamResourcesPage/components/PamResourcesSection.tsx

@@ -1,17 +1,9 @@
-import { ContentLoader } from "@app/components/v2";
 import { useProject } from "@app/context";
-import { useListPamResources } from "@app/hooks/api/pam";
 
 import { PamResourcesTable } from "./PamResourcesTable";
 
 export const PamResourcesSection = () => {
   const { currentProject } = useProject();
 
-  const { data: resources = [], isPending } = useListPamResources(currentProject.id, {
+  return <PamResourcesTable projectId={currentProject.id} />;
-    refetchInterval: 30000
-  });
-
-  if (isPending) return <ContentLoader />;
-
-  return <PamResourcesTable resources={resources} projectId={currentProject.id} />;
 };

frontend/src/pages/pam/PamResourcesPage/components/PamResourcesTable.tsx

@@ -27,6 +27,7 @@ import {
   Pagination,
   Table,
   TableContainer,
+  TableSkeleton,
   TBody,
   Th,
   THead,
@@ -39,29 +40,34 @@ import {
   OrgGatewayPermissionActions,
   OrgPermissionSubjects
 } from "@app/context/OrgPermissionContext/types";
-import { usePagination, usePopUp, useResetPageHelper } from "@app/hooks";
+import {
+  getUserTablePreference,
+  PreferenceKey,
+  setUserTablePreference
+} from "@app/helpers/userTablePreferences";
+import { usePagination, usePopUp } from "@app/hooks";
 import { OrderByDirection } from "@app/hooks/api/generic/types";
-import { PAM_RESOURCE_TYPE_MAP, PamResourceType, TPamResource } from "@app/hooks/api/pam";
+import {
+  PAM_RESOURCE_TYPE_MAP,
+  PamResourceOrderBy,
+  PamResourceType,
+  useListPamResources
+} from "@app/hooks/api/pam";
 
 import { PamAddResourceModal } from "./PamAddResourceModal";
 import { PamDeleteResourceModal } from "./PamDeleteResourceModal";
 import { PamResourceRow } from "./PamResourceRow";
 import { PamUpdateResourceModal } from "./PamUpdateResourceModal";
 
-enum OrderBy {
-  Name = "name"
-}
-
 type Filters = {
   resourceType: PamResourceType[];
 };
 
 type Props = {
   projectId: string;
-  resources: TPamResource[];
 };
 
-export const PamResourcesTable = ({ projectId, resources }: Props) => {
+export const PamResourcesTable = ({ projectId }: Props) => {
   const navigate = useNavigate({ from: ROUTE_PATHS.Pam.ResourcesPage.path });
 
   const { popUp, handlePopUpOpen, handlePopUpToggle } = usePopUp([
@@ -80,6 +86,7 @@ export const PamResourcesTable = ({ projectId, resources }: Props) => {
 
   const {
     search,
+    debouncedSearch,
     setSearch,
     setPage,
     page,
@@ -91,51 +98,48 @@ export const PamResourcesTable = ({ projectId, resources }: Props) => {
     orderBy,
     setOrderDirection,
     setOrderBy
-  } = usePagination<OrderBy>(OrderBy.Name, { initPerPage: 20, initSearch });
+  } = usePagination<PamResourceOrderBy>(PamResourceOrderBy.Name, {
+    initPerPage: getUserTablePreference("pamResourcesTable", PreferenceKey.PerPage, 20),
+    initSearch
+  });
+
+  const handlePerPageChange = (newPerPage: number) => {
+    setPerPage(newPerPage);
+    setUserTablePreference("pamResourcesTable", PreferenceKey.PerPage, newPerPage);
+  };
+
+  const { data, isLoading } = useListPamResources({
+    projectId,
+    offset,
+    limit: perPage,
+    search: debouncedSearch,
+    orderBy,
+    orderDirection
+  });
+
+  const resources = data?.resources || [];
+  const totalCount = data?.totalCount || 0;
 
   const filteredResources = useMemo(
     () =>
-      resources
+      resources.filter((resource) => {
-        .filter((resource) => {
+        const { name, resourceType } = resource;
-          const { name, resourceType } = resource;
 
-          if (filters.resourceType.length && !filters.resourceType.includes(resourceType)) {
+        if (filters.resourceType.length && !filters.resourceType.includes(resourceType)) {
-            return false;
+          return false;
-          }
+        }
 
-          const searchValue = search.trim().toLowerCase();
+        const searchValue = search.trim().toLowerCase();
 
-          const { name: resourceTypeName } = PAM_RESOURCE_TYPE_MAP[resourceType];
+        return (
-
+          name.toLowerCase().includes(searchValue) ||
-          return (
+          resourceType.toLowerCase().includes(searchValue)
-            name.toLowerCase().includes(searchValue) ||
+        );
-            resourceTypeName.toLowerCase().includes(searchValue)
+      }),
-          );
+    [resources, search, filters]
-        })
-        .sort((a, b) => {
-          const [one, two] = orderDirection === OrderByDirection.ASC ? [a, b] : [b, a];
-
-          switch (orderBy) {
-            case OrderBy.Name:
-            default:
-              return one.name.toLowerCase().localeCompare(two.name.toLowerCase());
-          }
-        }),
-    [resources, orderDirection, search, orderBy, filters]
   );
 
-  useResetPageHelper({
+  const handleSort = (column: PamResourceOrderBy) => {
-    totalCount: filteredResources.length,
-    offset,
-    setPage
-  });
-
-  const currentPageData = useMemo(
-    () => filteredResources.slice(offset, perPage * page),
-    [filteredResources, offset, perPage, page]
-  );
-
-  const handleSort = (column: OrderBy) => {
     if (column === orderBy) {
       toggleOrderDirection();
       return;
@@ -145,9 +149,10 @@ export const PamResourcesTable = ({ projectId, resources }: Props) => {
     setOrderDirection(OrderByDirection.ASC);
   };
 
-  const getClassName = (col: OrderBy) => twMerge("ml-2", orderBy === col ? "" : "opacity-30");
+  const getClassName = (col: PamResourceOrderBy) =>
+    twMerge("ml-2", orderBy === col ? "" : "opacity-30");
 
-  const getColSortIcon = (col: OrderBy) =>
+  const getColSortIcon = (col: PamResourceOrderBy) =>
     orderDirection === OrderByDirection.DESC && orderBy === col ? faArrowUp : faArrowDown;
 
   const isTableFiltered = Boolean(filters.resourceType.length);
@@ -263,11 +268,11 @@ export const PamResourcesTable = ({ projectId, resources }: Props) => {
                   Resource
                   <IconButton
                     variant="plain"
-                    className={getClassName(OrderBy.Name)}
+                    className={getClassName(PamResourceOrderBy.Name)}
                     ariaLabel="sort"
-                    onClick={() => handleSort(OrderBy.Name)}
+                    onClick={() => handleSort(PamResourceOrderBy.Name)}
                   >
-                    <FontAwesomeIcon icon={getColSortIcon(OrderBy.Name)} />
+                    <FontAwesomeIcon icon={getColSortIcon(PamResourceOrderBy.Name)} />
                   </IconButton>
                 </div>
               </Th>
@@ -275,27 +280,29 @@ export const PamResourcesTable = ({ projectId, resources }: Props) => {
             </Tr>
           </THead>
           <TBody>
-            {currentPageData.map((resource) => (
+            {isLoading && <TableSkeleton columns={2} innerKey="pam-resources" />}
-              <PamResourceRow
+            {!isLoading &&
-                key={resource.id}
+              filteredResources.map((resource) => (
-                resource={resource}
+                <PamResourceRow
-                onUpdate={(e) => handlePopUpOpen("updateResource", e)}
+                  key={resource.id}
-                onDelete={(e) => handlePopUpOpen("deleteResource", e)}
+                  resource={resource}
-                search={search.trim().toLowerCase()}
+                  onUpdate={(e) => handlePopUpOpen("updateResource", e)}
-              />
+                  onDelete={(e) => handlePopUpOpen("deleteResource", e)}
-            ))}
+                  search={search.trim().toLowerCase()}
+                />
+              ))}
           </TBody>
         </Table>
-        {Boolean(filteredResources.length) && (
+        {Boolean(totalCount) && !isLoading && (
           <Pagination
-            count={filteredResources.length}
+            count={totalCount}
             page={page}
             perPage={perPage}
-            onChangePage={setPage}
+            onChangePage={(newPage) => setPage(newPage)}
-            onChangePerPage={setPerPage}
+            onChangePerPage={handlePerPageChange}
           />
         )}
-        {isContentEmpty && (
+        {!isLoading && isContentEmpty && (
           <EmptyState
             title={isSearchEmpty ? "No resources match search" : "No resources"}
             icon={isSearchEmpty ? faSearch : faCircleXmark}