Merge pull request #4335 from Infisical/ENG-3420

feat(app-connection): Make GitHub paginated requests concurrent

backend/src/services/app-connection/github/github-connection-fns.ts

@@ -142,16 +142,27 @@ export const getGitHubAppAuthToken = async (appConnection: TGitHubConnection) =>
   return token;
 };
 
+const parseGitHubLinkHeader = (linkHeader: string | undefined): Record<string, string> => {
+  if (!linkHeader) return {};
+
+  const links: Record<string, string> = {};
+  const segments = linkHeader.split(",");
+  const re = new RE2(/<([^>]+)>;\s*rel="([^"]+)"/);
+
+  for (const segment of segments) {
+    const match = re.exec(segment.trim());
+    if (match) {
+      const url = match[1];
+      const rel = match[2];
+      links[rel] = url;
+    }
+  }
+  return links;
+};
+
 function extractNextPageUrl(linkHeader: string | undefined): string | null {
-  if (!linkHeader) return null;
+  const links = parseGitHubLinkHeader(linkHeader);
-
+  return links.next || null;
-  const links = linkHeader.split(",");
-  const nextLink = links.find((link) => link.includes('rel="next"'));
-
-  if (!nextLink) return null;
-
-  const match = new RE2(/<([^>]+)>/).exec(nextLink);
-  return match ? match[1] : null;
 }
 
 export const makePaginatedGitHubRequest = async <T, R = T[]>(
@@ -164,27 +175,83 @@ export const makePaginatedGitHubRequest = async <T, R = T[]>(
 
   const token =
     method === GitHubConnectionMethod.OAuth ? credentials.accessToken : await getGitHubAppAuthToken(appConnection);
-  let url: string | null = `https://${await getGitHubInstanceApiUrl(appConnection)}${path}`;
+
+  const baseUrl = `https://${await getGitHubInstanceApiUrl(appConnection)}${path}`;
+  const initialUrlObj = new URL(baseUrl);
+  initialUrlObj.searchParams.set("per_page", "100");
+
   let results: T[] = [];
-  let i = 0;
+  const maxIterations = 1000;
 
-  while (url && i < 1000) {
+  // Make initial request to get link header
-    // eslint-disable-next-line no-await-in-loop
+  const firstResponse: AxiosResponse<R> = await requestWithGitHubGateway<R>(appConnection, gatewayService, {
-    const response: AxiosResponse<R> = await requestWithGitHubGateway<R>(appConnection, gatewayService, {
+    url: initialUrlObj.toString(),
-      url,
+    method: "GET",
-      method: "GET",
+    headers: {
-      headers: {
+      Accept: "application/vnd.github+json",
-        Accept: "application/vnd.github+json",
+      Authorization: `Bearer ${token}`,
-        Authorization: `Bearer ${token}`,
+      "X-GitHub-Api-Version": "2022-11-28"
-        "X-GitHub-Api-Version": "2022-11-28"
+    }
-      }
+  });
-    });
 
-    const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]);
+  const firstPageItems = dataMapper ? dataMapper(firstResponse.data) : (firstResponse.data as unknown as T[]);
-    results = results.concat(items);
+  results = results.concat(firstPageItems);
 
-    url = extractNextPageUrl(response.headers.link as string | undefined);
+  const linkHeader = parseGitHubLinkHeader(firstResponse.headers.link as string | undefined);
-    i += 1;
+  const lastPageUrl = linkHeader.last;
+
+  // If there's a last page URL, get its page number and concurrently fetch every page starting from 2 to last
+  if (lastPageUrl) {
+    const lastPageParam = new URL(lastPageUrl).searchParams.get("page");
+    const totalPages = lastPageParam ? parseInt(lastPageParam, 10) : 1;
+
+    const pageRequests: Promise<AxiosResponse<R>>[] = [];
+
+    for (let pageNum = 2; pageNum <= totalPages && pageNum - 1 < maxIterations; pageNum += 1) {
+      const pageUrlObj = new URL(initialUrlObj.toString());
+      pageUrlObj.searchParams.set("page", pageNum.toString());
+
+      pageRequests.push(
+        requestWithGitHubGateway<R>(appConnection, gatewayService, {
+          url: pageUrlObj.toString(),
+          method: "GET",
+          headers: {
+            Accept: "application/vnd.github+json",
+            Authorization: `Bearer ${token}`,
+            "X-GitHub-Api-Version": "2022-11-28"
+          }
+        })
+      );
+    }
+    const responses = await Promise.all(pageRequests);
+
+    for (const response of responses) {
+      const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]);
+      results = results.concat(items);
+    }
+  } else {
+    // Fallback in case last link isn't present
+    let url: string | null = extractNextPageUrl(firstResponse.headers.link as string | undefined);
+    let i = 1;
+
+    while (url && i < maxIterations) {
+      // eslint-disable-next-line no-await-in-loop
+      const response: AxiosResponse<R> = await requestWithGitHubGateway<R>(appConnection, gatewayService, {
+        url,
+        method: "GET",
+        headers: {
+          Accept: "application/vnd.github+json",
+          Authorization: `Bearer ${token}`,
+          "X-GitHub-Api-Version": "2022-11-28"
+        }
+      });
+
+      const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]);
+      results = results.concat(items);
+
+      url = extractNextPageUrl(response.headers.link as string | undefined);
+      i += 1;
+    }
   }
 
   return results;