diff --git a/backend/src/services/app-connection/github/github-connection-fns.ts b/backend/src/services/app-connection/github/github-connection-fns.ts index e4bd2e10d..a71036d82 100644 --- a/backend/src/services/app-connection/github/github-connection-fns.ts +++ b/backend/src/services/app-connection/github/github-connection-fns.ts @@ -142,16 +142,27 @@ export const getGitHubAppAuthToken = async (appConnection: TGitHubConnection) => return token; }; +const parseGitHubLinkHeader = (linkHeader: string | undefined): Record => { + if (!linkHeader) return {}; + + const links: Record = {}; + const segments = linkHeader.split(","); + const re = new RE2(/<([^>]+)>;\s*rel="([^"]+)"/); + + for (const segment of segments) { + const match = re.exec(segment.trim()); + if (match) { + const url = match[1]; + const rel = match[2]; + links[rel] = url; + } + } + return links; +}; + function extractNextPageUrl(linkHeader: string | undefined): string | null { - if (!linkHeader) return null; - - const links = linkHeader.split(","); - const nextLink = links.find((link) => link.includes('rel="next"')); - - if (!nextLink) return null; - - const match = new RE2(/<([^>]+)>/).exec(nextLink); - return match ? match[1] : null; + const links = parseGitHubLinkHeader(linkHeader); + return links.next || null; } export const makePaginatedGitHubRequest = async ( @@ -164,27 +175,83 @@ export const makePaginatedGitHubRequest = async ( const token = method === GitHubConnectionMethod.OAuth ? credentials.accessToken : await getGitHubAppAuthToken(appConnection); - let url: string | null = `https://${await getGitHubInstanceApiUrl(appConnection)}${path}`; + + const baseUrl = `https://${await getGitHubInstanceApiUrl(appConnection)}${path}`; + const initialUrlObj = new URL(baseUrl); + initialUrlObj.searchParams.set("per_page", "100"); + let results: T[] = []; - let i = 0; + const maxIterations = 1000; - while (url && i < 1000) { - // eslint-disable-next-line no-await-in-loop - const response: AxiosResponse = await requestWithGitHubGateway(appConnection, gatewayService, { - url, - method: "GET", - headers: { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, - "X-GitHub-Api-Version": "2022-11-28" - } - }); + // Make initial request to get link header + const firstResponse: AxiosResponse = await requestWithGitHubGateway(appConnection, gatewayService, { + url: initialUrlObj.toString(), + method: "GET", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28" + } + }); - const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]); - results = results.concat(items); + const firstPageItems = dataMapper ? dataMapper(firstResponse.data) : (firstResponse.data as unknown as T[]); + results = results.concat(firstPageItems); - url = extractNextPageUrl(response.headers.link as string | undefined); - i += 1; + const linkHeader = parseGitHubLinkHeader(firstResponse.headers.link as string | undefined); + const lastPageUrl = linkHeader.last; + + // If there's a last page URL, get its page number and concurrently fetch every page starting from 2 to last + if (lastPageUrl) { + const lastPageParam = new URL(lastPageUrl).searchParams.get("page"); + const totalPages = lastPageParam ? parseInt(lastPageParam, 10) : 1; + + const pageRequests: Promise>[] = []; + + for (let pageNum = 2; pageNum <= totalPages && pageNum - 1 < maxIterations; pageNum += 1) { + const pageUrlObj = new URL(initialUrlObj.toString()); + pageUrlObj.searchParams.set("page", pageNum.toString()); + + pageRequests.push( + requestWithGitHubGateway(appConnection, gatewayService, { + url: pageUrlObj.toString(), + method: "GET", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28" + } + }) + ); + } + const responses = await Promise.all(pageRequests); + + for (const response of responses) { + const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]); + results = results.concat(items); + } + } else { + // Fallback in case last link isn't present + let url: string | null = extractNextPageUrl(firstResponse.headers.link as string | undefined); + let i = 1; + + while (url && i < maxIterations) { + // eslint-disable-next-line no-await-in-loop + const response: AxiosResponse = await requestWithGitHubGateway(appConnection, gatewayService, { + url, + method: "GET", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28" + } + }); + + const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]); + results = results.concat(items); + + url = extractNextPageUrl(response.headers.link as string | undefined); + i += 1; + } } return results;