improvement: standardize and update server side pagination for change requests

backend/src/ee/routes/v1/secret-approval-request-router.ts

@@ -30,6 +30,7 @@ export const registerSecretApprovalRequestRouter = async (server: FastifyZodProv
         workspaceId: z.string().trim(),
         environment: z.string().trim().optional(),
         committer: z.string().trim().optional(),
+        search: z.string().trim().optional(),
         status: z.nativeEnum(RequestState).optional(),
         limit: z.coerce.number().default(20),
         offset: z.coerce.number().default(0)
@@ -60,20 +61,20 @@ export const registerSecretApprovalRequestRouter = async (server: FastifyZodProv
             committerUser: approvalRequestUser,
             commits: z.object({ op: z.string(), secretId: z.string().nullable().optional() }).array(),
             environment: z.string(),
-            secretPath: z.string(),
             reviewers: z.object({ userId: z.string(), status: z.string() }).array(),
             approvers: z
               .object({
                 userId: z.string().nullable().optional()
               })
               .array()
-          }).array()
+          }).array(),
+          totalCount: z.number()
         })
       }
     },
     onRequest: verifyAuth([AuthMode.JWT]),
     handler: async (req) => {
-      const approvals = await server.services.secretApprovalRequest.getSecretApprovals({
+      const { approvals, totalCount } = await server.services.secretApprovalRequest.getSecretApprovals({
         actor: req.permission.type,
         actorId: req.permission.id,
         actorAuthMethod: req.permission.authMethod,
@@ -81,7 +82,7 @@ export const registerSecretApprovalRequestRouter = async (server: FastifyZodProv
         ...req.query,
         projectId: req.query.workspaceId
       });
-      return { approvals };
+      return { approvals, totalCount };
     }
   });
 

backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts

@@ -24,6 +24,7 @@ type TFindQueryFilter = {
   committer?: string;
   limit?: number;
   offset?: number;
+  search?: string;
 };
 
 export const secretApprovalRequestDALFactory = (db: TDbClient) => {
@@ -340,7 +341,7 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => {
   };
 
   const findByProjectId = async (
-    { status, limit = 20, offset = 0, projectId, committer, environment, userId }: TFindQueryFilter,
+    { status, limit = 20, offset = 0, projectId, committer, environment, userId, search }: TFindQueryFilter,
     tx?: Knex
   ) => {
     try {
@@ -433,16 +434,36 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => {
           db.ref("email").withSchema("committerUser").as("committerUserEmail"),
           db.ref("username").withSchema("committerUser").as("committerUserUsername"),
           db.ref("firstName").withSchema("committerUser").as("committerUserFirstName"),
-          db.ref("lastName").withSchema("committerUser").as("committerUserLastName")
+          db.ref("lastName").withSchema("committerUser").as("committerUserLastName"),
+
+          db.raw(`count(*) OVER() as total_count`)
         )
         .orderBy("createdAt", "desc");
 
+      if (search) {
+        void query
+          .whereRaw(`CONCAT_WS(' ', ??, ??) ilike ?`, [
+            db.ref("firstName").withSchema("committerUser"),
+            db.ref("lastName").withSchema("committerUser"),
+            `%${search}%`
+          ])
+          .orWhereRaw(`?? ilike ?`, [db.ref("username").withSchema("committerUser"), `%${search}%`])
+          .orWhereRaw(`?? ilike ?`, [db.ref("email").withSchema("committerUser"), `%${search}%`])
+          .orWhereILike(`${TableName.Environment}.name`, `%${search}%`)
+          .orWhereILike(`${TableName.Environment}.slug`, `%${search}%`)
+          .orWhereILike(`${TableName.SecretApprovalPolicy}.secretPath`, `%${search}%`);
+      }
+
       const docs = await (tx || db)
         .with("w", query)
         .select("*")
         .from<Awaited<typeof query>[number]>("w")
         .where("w.rank", ">=", offset)
         .andWhere("w.rank", "<", offset + limit);
+
+      // @ts-expect-error knex does not infer
+      const totalCount = Number(docs[0]?.total_count || 0);
+
       const formattedDoc = sqlNestRelationships({
         data: docs,
         key: "id",
@@ -504,17 +525,20 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => {
           }
         ]
       });
-      return formattedDoc.map((el) => ({
+      return {
-        ...el,
+        approvals: formattedDoc.map((el) => ({
-        policy: { ...el.policy, approvers: el.approvers, bypassers: el.bypassers }
+          ...el,
-      }));
+          policy: { ...el.policy, approvers: el.approvers, bypassers: el.bypassers }
+        })),
+        totalCount
+      };
     } catch (error) {
       throw new DatabaseError({ error, name: "FindSAR" });
     }
   };
 
   const findByProjectIdBridgeSecretV2 = async (
-    { status, limit = 20, offset = 0, projectId, committer, environment, userId }: TFindQueryFilter,
+    { status, limit = 20, offset = 0, projectId, committer, environment, userId, search }: TFindQueryFilter,
     tx?: Knex
   ) => {
     try {
@@ -607,16 +631,37 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => {
           db.ref("email").withSchema("committerUser").as("committerUserEmail"),
           db.ref("username").withSchema("committerUser").as("committerUserUsername"),
           db.ref("firstName").withSchema("committerUser").as("committerUserFirstName"),
-          db.ref("lastName").withSchema("committerUser").as("committerUserLastName")
+          db.ref("lastName").withSchema("committerUser").as("committerUserLastName"),
+
+          db.raw(`count(*) OVER() as total_count`)
         )
         .orderBy("createdAt", "desc");
 
+      if (search) {
+        void query
+          .whereRaw(`CONCAT_WS(' ', ??, ??) ilike ?`, [
+            db.ref("firstName").withSchema("committerUser"),
+            db.ref("lastName").withSchema("committerUser"),
+            `%${search}%`
+          ])
+          .orWhereRaw(`?? ilike ?`, [db.ref("username").withSchema("committerUser"), `%${search}%`])
+          .orWhereRaw(`?? ilike ?`, [db.ref("email").withSchema("committerUser"), `%${search}%`])
+          .orWhereILike(`${TableName.Environment}.name`, `%${search}%`)
+          .orWhereILike(`${TableName.Environment}.slug`, `%${search}%`)
+          .orWhereILike(`${TableName.SecretApprovalPolicy}.secretPath`, `%${search}%`);
+      }
+
+      const rankOffset = offset + 1;
       const docs = await (tx || db)
         .with("w", query)
         .select("*")
         .from<Awaited<typeof query>[number]>("w")
-        .where("w.rank", ">=", offset)
+        .where("w.rank", ">=", rankOffset)
-        .andWhere("w.rank", "<", offset + limit);
+        .andWhere("w.rank", "<", rankOffset + limit);
+
+      // @ts-expect-error knex does not infer
+      const totalCount = Number(docs[0]?.total_count || 0);
+
       const formattedDoc = sqlNestRelationships({
         data: docs,
         key: "id",
@@ -682,10 +727,13 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => {
           }
         ]
       });
-      return formattedDoc.map((el) => ({
+      return {
-        ...el,
+        approvals: formattedDoc.map((el) => ({
-        policy: { ...el.policy, approvers: el.approvers, bypassers: el.bypassers }
+          ...el,
-      }));
+          policy: { ...el.policy, approvers: el.approvers, bypassers: el.bypassers }
+        })),
+        totalCount
+      };
     } catch (error) {
       throw new DatabaseError({ error, name: "FindSAR" });
     }

backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts

@@ -194,7 +194,8 @@ export const secretApprovalRequestServiceFactory = ({
     environment,
     committer,
     limit,
-    offset
+    offset,
+    search
   }: TListApprovalsDTO) => {
     if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" });
 
@@ -209,47 +210,29 @@ export const secretApprovalRequestServiceFactory = ({
 
     const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
 
-    const getSecretMapPath = async (folderIds: string[]) => {
-      const secretPaths = await folderDAL.findSecretPathByFolderIds(projectId, folderIds);
-
-      const secretPathMap: Record<string, string> = {};
-
-      secretPaths.forEach((folder) => {
-        if (folder) secretPathMap[folder.id] = folder.path;
-      });
-
-      return secretPathMap;
-    };
-
     if (shouldUseSecretV2Bridge) {
-      const approvalsV2 = await secretApprovalRequestDAL.findByProjectIdBridgeSecretV2({
+      return secretApprovalRequestDAL.findByProjectIdBridgeSecretV2({
         projectId,
         committer,
         environment,
         status,
         userId: actorId,
         limit,
-        offset
+        offset,
+        search
       });
-
-      const secretPathMap = await getSecretMapPath([...new Set(approvalsV2.map((approval) => approval.folderId))]);
-
-      return approvalsV2.map((approval) => ({ ...approval, secretPath: secretPathMap[approval.folderId] }));
     }
 
-    const approvals = await secretApprovalRequestDAL.findByProjectId({
+    return secretApprovalRequestDAL.findByProjectId({
       projectId,
       committer,
       environment,
       status,
       userId: actorId,
       limit,
-      offset
+      offset,
+      search
     });
-
-    const secretPathMap = await getSecretMapPath([...new Set(approvals.map((approval) => approval.folderId))]);
-
-    return approvals.map((approval) => ({ ...approval, secretPath: secretPathMap[approval.folderId] }));
   };
 
   const getSecretApprovalDetails = async ({

backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts

@@ -93,6 +93,7 @@ export type TListApprovalsDTO = {
   committer?: string;
   limit?: number;
   offset?: number;
+  search?: string;
 } & TProjectPermission;
 
 export type TSecretApprovalDetailsDTO = {

frontend/src/hooks/api/secretApprovalRequest/queries.tsx

@@ -1,5 +1,5 @@
 /* eslint-disable no-param-reassign */
-import { useInfiniteQuery, useQuery, UseQueryOptions } from "@tanstack/react-query";
+import { useQuery, UseQueryOptions } from "@tanstack/react-query";
 
 import {
   decryptAssymmetric,
@@ -25,10 +25,11 @@ export const secretApprovalRequestKeys = {
     status,
     committer,
     offset,
-    limit
+    limit,
+    search
   }: TGetSecretApprovalRequestList) =>
     [
-      { workspaceId, environment, status, committer, offset, limit },
+      { workspaceId, environment, status, committer, offset, limit, search },
       "secret-approval-requests"
     ] as const,
   detail: ({ id }: Omit<TGetSecretApprovalRequestDetails, "decryptKey">) =>
@@ -118,23 +119,25 @@ const fetchSecretApprovalRequestList = async ({
   committer,
   status = "open",
   limit = 20,
-  offset
+  offset = 0,
+  search = ""
 }: TGetSecretApprovalRequestList) => {
-  const { data } = await apiRequest.get<{ approvals: TSecretApprovalRequest[] }>(
+  const { data } = await apiRequest.get<{
-    "/api/v1/secret-approval-requests",
+    approvals: TSecretApprovalRequest[];
-    {
+    totalCount: number;
-      params: {
+  }>("/api/v1/secret-approval-requests", {
-        workspaceId,
+    params: {
-        environment,
+      workspaceId,
-        committer,
+      environment,
-        status,
+      committer,
-        limit,
+      status,
-        offset
+      limit,
-      }
+      offset,
+      search
     }
-  );
+  });
 
-  return data.approvals;
+  return data;
 };
 
 export const useGetSecretApprovalRequests = ({
@@ -143,31 +146,32 @@ export const useGetSecretApprovalRequests = ({
   options = {},
   status,
   limit = 20,
+  offset = 0,
+  search,
   committer
 }: TGetSecretApprovalRequestList & TReactQueryOptions) =>
-  useInfiniteQuery({
+  useQuery({
-    initialPageParam: 0,
     queryKey: secretApprovalRequestKeys.list({
       workspaceId,
       environment,
       committer,
-      status
+      status,
+      limit,
+      search,
+      offset
     }),
-    queryFn: ({ pageParam }) =>
+    queryFn: () =>
       fetchSecretApprovalRequestList({
         workspaceId,
         environment,
         status,
         committer,
         limit,
-        offset: pageParam
+        offset,
+        search
       }),
     enabled: Boolean(workspaceId) && (options?.enabled ?? true),
-    getNextPageParam: (lastPage, pages) => {
+    placeholderData: (previousData) => previousData
-      if (lastPage.length && lastPage.length < limit) return undefined;
-
-      return lastPage?.length !== 0 ? pages.length * limit : undefined;
-    }
   });
 
 const fetchSecretApprovalRequestDetails = async ({

frontend/src/hooks/api/secretApprovalRequest/types.ts

@@ -113,6 +113,7 @@ export type TGetSecretApprovalRequestList = {
   committer?: string;
   limit?: number;
   offset?: number;
+  search?: string;
 };
 
 export type TGetSecretApprovalRequestCount = {

frontend/src/pages/secret-manager/SecretApprovalsPage/components/SecretApprovalRequest/SecretApprovalRequest.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useMemo, useState } from "react";
+import { useEffect, useState } from "react";
 import {
   faArrowUpRightFromSquare,
   faBookOpen,
@@ -24,6 +24,7 @@ import {
   DropdownMenuTrigger,
   EmptyState,
   Input,
+  Pagination,
   Skeleton
 } from "@app/components/v2";
 import { ROUTE_PATHS } from "@app/const/routes";
@@ -34,6 +35,12 @@ import {
   useUser,
   useWorkspace
 } from "@app/context";
+import {
+  getUserTablePreference,
+  PreferenceKey,
+  setUserTablePreference
+} from "@app/helpers/userTablePreferences";
+import { usePagination } from "@app/hooks";
 import {
   useGetSecretApprovalRequestCount,
   useGetSecretApprovalRequests,
@@ -58,18 +65,41 @@ export const SecretApprovalRequest = () => {
   const [usingUrlRequestId, setUsingUrlRequestId] = useState(false);
 
   const {
-    data: secretApprovalRequests,
+    debouncedSearch: debouncedSearchFilter,
-    isFetchingNextPage: isFetchingNextApprovalRequest,
+    search: searchFilter,
-    fetchNextPage: fetchNextApprovalRequest,
+    setSearch: setSearchFilter,
-    hasNextPage: hasNextApprovalPage,
+    setPage,
+    page,
+    perPage,
+    setPerPage,
+    offset,
+    limit
+  } = usePagination("", {
+    initPerPage: getUserTablePreference("changeRequestsTable", PreferenceKey.PerPage, 20)
+  });
+
+  const handlePerPageChange = (newPerPage: number) => {
+    setPerPage(newPerPage);
+    setUserTablePreference("changeRequestsTable", PreferenceKey.PerPage, newPerPage);
+  };
+
+  const {
+    data,
     isPending: isApprovalRequestLoading,
     refetch
   } = useGetSecretApprovalRequests({
     workspaceId,
     status: statusFilter,
     environment: envFilter,
-    committer: committerFilter
+    committer: committerFilter,
+    search: debouncedSearchFilter,
+    limit,
+    offset
   });
+
+  const totalApprovalCount = data?.totalCount ?? 0;
+  const secretApprovalRequests = data?.approvals ?? [];
+
   const { data: secretApprovalRequestCount, isSuccess: isSecretApprovalReqCountSuccess } =
     useGetSecretApprovalRequestCount({ workspaceId });
   const { user: userSession } = useUser();
@@ -94,30 +124,7 @@ export const SecretApprovalRequest = () => {
     refetch();
   };
 
-  const isRequestListEmpty =
+  const isRequestListEmpty = !isApprovalRequestLoading && secretApprovalRequests?.length === 0;
-    !isApprovalRequestLoading && secretApprovalRequests?.pages[0]?.length === 0;
-
-  const [searchFilter, setSearchFilter] = useState("");
-
-  const filteredRequests = useMemo(
-    () =>
-      secretApprovalRequests?.pages.flatMap((requests) =>
-        requests.filter((request) => {
-          const { environment, committerUser, secretPath } = request;
-
-          const searchValue = searchFilter.trim().toLowerCase();
-
-          return (
-            environment?.toLowerCase().includes(searchValue) ||
-            `${committerUser?.email ?? ""} ${committerUser?.firstName ?? ""} ${committerUser?.lastName ?? ""}`
-              .toLowerCase()
-              .includes(searchValue) ||
-            secretPath?.toLowerCase().includes(searchValue)
-          );
-        })
-      ) ?? [],
-    [secretApprovalRequests?.pages, searchFilter]
-  );
 
   return (
     <AnimatePresence mode="wait">
@@ -171,7 +178,7 @@ export const SecretApprovalRequest = () => {
               value={searchFilter}
               onChange={(e) => setSearchFilter(e.target.value)}
               leftIcon={<FontAwesomeIcon icon={faMagnifyingGlass} />}
-              placeholder="Search change requests by author, environment slug or secret path..."
+              placeholder="Search change requests by author, environment or policy path..."
               className="flex-1"
               containerClassName="mb-4"
             />
@@ -285,14 +292,14 @@ export const SecretApprovalRequest = () => {
               </div>
             </div>
             <div className="flex flex-col rounded-b-md border-x border-b border-t border-mineshaft-600 bg-mineshaft-800">
-              {isRequestListEmpty && (
+              {isRequestListEmpty && !searchFilter && (
                 <div className="py-12">
                   <EmptyState
                     title={`No ${statusFilter === "open" ? "Open" : "Closed"} Change Requests`}
                   />
                 </div>
               )}
-              {filteredRequests.map((secretApproval) => {
+              {secretApprovalRequests.map((secretApproval) => {
                 const {
                   id: reqId,
                   commits,
@@ -337,9 +344,23 @@ export const SecretApprovalRequest = () => {
                 );
               })}
               {Boolean(
-                !filteredRequests.length && !isRequestListEmpty && !isApprovalRequestLoading
+                !secretApprovalRequests.length && searchFilter && !isApprovalRequestLoading
-              ) && <EmptyState title="No Requests Match Search" icon={faSearch} />}
+              ) && (
-              {(isFetchingNextApprovalRequest || isApprovalRequestLoading) && (
+                <div className="py-12">
+                  <EmptyState title="No Requests Match Search" icon={faSearch} />
+                </div>
+              )}
+              {Boolean(totalApprovalCount) && (
+                <Pagination
+                  className="border-none"
+                  count={totalApprovalCount}
+                  page={page}
+                  perPage={perPage}
+                  onChangePage={setPage}
+                  onChangePerPage={handlePerPageChange}
+                />
+              )}
+              {isApprovalRequestLoading && (
                 <div>
                   {Array.apply(0, Array(3)).map((_x, index) => (
                     <div
@@ -356,18 +377,6 @@ export const SecretApprovalRequest = () => {
                 </div>
               )}
             </div>
-            {hasNextApprovalPage && (
-              <Button
-                className="mt-4 text-sm"
-                isFullWidth
-                colorSchema="secondary"
-                isLoading={isFetchingNextApprovalRequest}
-                isDisabled={isFetchingNextApprovalRequest || !hasNextApprovalPage}
-                onClick={() => fetchNextApprovalRequest()}
-              >
-                {hasNextApprovalPage ? "Load More" : "End of History"}
-              </Button>
-            )}
           </div>
         </motion.div>
       )}