From b6b9fb6ef5f9492736e30e02d909d340f1ad4a47 Mon Sep 17 00:00:00 2001 From: x032205 Date: Thu, 7 Aug 2025 20:20:48 -0400 Subject: [PATCH 1/2] feat(app-connection): Make GitHub paginated requests concurrent --- .../github/github-connection-fns.ts | 119 ++++++++++++++---- 1 file changed, 93 insertions(+), 26 deletions(-) diff --git a/backend/src/services/app-connection/github/github-connection-fns.ts b/backend/src/services/app-connection/github/github-connection-fns.ts index e4bd2e10d..b76987fc1 100644 --- a/backend/src/services/app-connection/github/github-connection-fns.ts +++ b/backend/src/services/app-connection/github/github-connection-fns.ts @@ -142,16 +142,27 @@ export const getGitHubAppAuthToken = async (appConnection: TGitHubConnection) => return token; }; +const parseGitHubLinkHeader = (linkHeader: string | undefined): Record => { + if (!linkHeader) return {}; + + const links: Record = {}; + const segments = linkHeader.split(","); + const re = new RE2(/<([^>]+)>;\s*rel="([^"]+)"/); + + for (const segment of segments) { + const match = re.exec(segment.trim()); + if (match) { + const url = match[1]; + const rel = match[2]; + links[rel] = url; + } + } + return links; +}; + function extractNextPageUrl(linkHeader: string | undefined): string | null { - if (!linkHeader) return null; - - const links = linkHeader.split(","); - const nextLink = links.find((link) => link.includes('rel="next"')); - - if (!nextLink) return null; - - const match = new RE2(/<([^>]+)>/).exec(nextLink); - return match ? match[1] : null; + const links = parseGitHubLinkHeader(linkHeader); + return links.next || null; } export const makePaginatedGitHubRequest = async ( @@ -164,27 +175,83 @@ export const makePaginatedGitHubRequest = async ( const token = method === GitHubConnectionMethod.OAuth ? credentials.accessToken : await getGitHubAppAuthToken(appConnection); - let url: string | null = `https://${await getGitHubInstanceApiUrl(appConnection)}${path}`; + + const baseUrl = `https://${await getGitHubInstanceApiUrl(appConnection)}${path}`; + const initialUrlObj = new URL(baseUrl); + initialUrlObj.searchParams.set("per_page", "100"); + let results: T[] = []; - let i = 0; + const maxIterations = 1000; - while (url && i < 1000) { - // eslint-disable-next-line no-await-in-loop - const response: AxiosResponse = await requestWithGitHubGateway(appConnection, gatewayService, { - url, - method: "GET", - headers: { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, - "X-GitHub-Api-Version": "2022-11-28" - } - }); + // Make initial request to get link header + const firstResponse: AxiosResponse = await requestWithGitHubGateway(appConnection, gatewayService, { + url: initialUrlObj.toString(), + method: "GET", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28" + } + }); - const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]); - results = results.concat(items); + const firstPageItems = dataMapper ? dataMapper(firstResponse.data) : (firstResponse.data as unknown as T[]); + results = results.concat(firstPageItems); - url = extractNextPageUrl(response.headers.link as string | undefined); - i += 1; + const linkHeader = parseGitHubLinkHeader(firstResponse.headers.link as string | undefined); + const lastPageUrl = linkHeader.last; + + // If there's a last page URL, get it's page number and concurrently fetch every page starting from 2 to last + if (lastPageUrl) { + const lastPageParam = new URL(lastPageUrl).searchParams.get("page"); + const totalPages = lastPageParam ? parseInt(lastPageParam, 10) : 1; + + const pageRequests: Promise>[] = []; + + for (let pageNum = 2; pageNum <= totalPages && pageNum - 1 < maxIterations; pageNum += 1) { + const pageUrlObj = new URL(initialUrlObj.toString()); + pageUrlObj.searchParams.set("page", pageNum.toString()); + + pageRequests.push( + requestWithGitHubGateway(appConnection, gatewayService, { + url: pageUrlObj.toString(), + method: "GET", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28" + } + }) + ); + } + const responses = await Promise.all(pageRequests); + + for (const response of responses) { + const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]); + results = results.concat(items); + } + } else { + // Fallback in case last link isn't present + let url: string | null = extractNextPageUrl(firstResponse.headers.link as string | undefined); + let i = 1; + + while (url && i < maxIterations) { + // eslint-disable-next-line no-await-in-loop + const response: AxiosResponse = await requestWithGitHubGateway(appConnection, gatewayService, { + url, + method: "GET", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28" + } + }); + + const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]); + results = results.concat(items); + + url = extractNextPageUrl(response.headers.link as string | undefined); + i += 1; + } } return results; From 352ef050c3fb014cf636e7947947d94607264417 Mon Sep 17 00:00:00 2001 From: x032205 <120525481+x032205@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:26:12 -0400 Subject: [PATCH 2/2] Update backend/src/services/app-connection/github/github-connection-fns.ts Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../src/services/app-connection/github/github-connection-fns.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/services/app-connection/github/github-connection-fns.ts b/backend/src/services/app-connection/github/github-connection-fns.ts index b76987fc1..a71036d82 100644 --- a/backend/src/services/app-connection/github/github-connection-fns.ts +++ b/backend/src/services/app-connection/github/github-connection-fns.ts @@ -200,7 +200,7 @@ export const makePaginatedGitHubRequest = async ( const linkHeader = parseGitHubLinkHeader(firstResponse.headers.link as string | undefined); const lastPageUrl = linkHeader.last; - // If there's a last page URL, get it's page number and concurrently fetch every page starting from 2 to last + // If there's a last page URL, get its page number and concurrently fetch every page starting from 2 to last if (lastPageUrl) { const lastPageParam = new URL(lastPageUrl).searchParams.get("page"); const totalPages = lastPageParam ? parseInt(lastPageParam, 10) : 1;