Merge pull request #4240 from Infisical/ENG-3368

feat(app-connection, secret-sync): Gateway for GitHub App Connections & Secret Syncs
2026-10-15 16:52:38 +00:00 · 2025-07-29 16:53:36 -04:00
parent 9e76fa8230 0b7b32bdc3
commit 1f6a63fa71
19 changed files with 749 additions and 437 deletions
@@ -583,7 +583,7 @@ export const appConnectionServiceFactory = ({
    deleteAppConnection,
    connectAppConnectionById,
    listAvailableAppConnectionsForUser,
-    github: githubConnectionService(connectAppConnectionById),
+    github: githubConnectionService(connectAppConnectionById, gatewayService),
    githubRadar: githubRadarConnectionService(connectAppConnectionById),
    gcp: gcpConnectionService(connectAppConnectionById),
    databricks: databricksConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
@@ -1,10 +1,16 @@
 import { createAppAuth } from "@octokit/auth-app";
-import { Octokit } from "@octokit/rest";
-import { AxiosResponse } from "axios";
+import { AxiosError, AxiosRequestConfig, AxiosResponse } from "axios";
+import https from "https";
+import RE2 from "re2";

+import { verifyHostInputValidity } from "@app/ee/services/dynamic-secret/dynamic-secret-fns";
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { getConfig } from "@app/lib/config/env";
-import { request } from "@app/lib/config/request";
+import { request as httpRequest } from "@app/lib/config/request";
 import { BadRequestError, ForbiddenRequestError, InternalServerError } from "@app/lib/errors";
+import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
+import { logger } from "@app/lib/logger";
+import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
 import { getAppConnectionMethodName } from "@app/services/app-connection/app-connection-fns";
 import { IntegrationUrls } from "@app/services/integration-auth/integration-list";

@@ -24,123 +30,224 @@ export const getGitHubConnectionListItem = () => {
  };
 };

-export const getGitHubClient = (appConnection: TGitHubConnection) => {
+export const requestWithGitHubGateway = async <T>(
+  appConnection: { gatewayId?: string | null },
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
+  requestConfig: AxiosRequestConfig
+): Promise<AxiosResponse<T>> => {
+  const { gatewayId } = appConnection;
+
+  // If gateway isn't set up, don't proxy request
+  if (!gatewayId) {
+    return httpRequest.request(requestConfig);
+  }
+
+  const url = new URL(requestConfig.url as string);
+
+  await blockLocalAndPrivateIpAddresses(url.toString());
+
+  const [targetHost] = await verifyHostInputValidity(url.host, true);
+  const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(gatewayId);
+  const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
+
+  return withGatewayProxy(
+    async (proxyPort) => {
+      const httpsAgent = new https.Agent({
+        servername: targetHost
+      });
+
+      url.protocol = "https:";
+      url.host = `localhost:${proxyPort}`;
+
+      const finalRequestConfig: AxiosRequestConfig = {
+        ...requestConfig,
+        url: url.toString(),
+        httpsAgent,
+        headers: {
+          ...requestConfig.headers,
+          Host: targetHost
+        }
+      };
+
+      try {
+        return await httpRequest.request(finalRequestConfig);
+      } catch (error) {
+        const axiosError = error as AxiosError;
+        logger.error("Error during GitHub gateway request:", axiosError.message, axiosError.response?.data);
+        throw error;
+      }
+    },
+    {
+      protocol: GatewayProxyProtocol.Tcp,
+      targetHost,
+      targetPort: 443,
+      relayHost,
+      relayPort: Number(relayPort),
+      identityId: relayDetails.identityId,
+      orgId: relayDetails.orgId,
+      tlsOptions: {
+        ca: relayDetails.certChain,
+        cert: relayDetails.certificate,
+        key: relayDetails.privateKey.toString()
+      }
+    }
+  );
+};
+
+export const getGitHubAppAuthToken = async (appConnection: TGitHubConnection) => {
  const appCfg = getConfig();
-
-  const { method, credentials } = appConnection;
-
-  let client: Octokit;
-
  const appId = appCfg.INF_APP_CONNECTION_GITHUB_APP_ID;
  const appPrivateKey = appCfg.INF_APP_CONNECTION_GITHUB_APP_PRIVATE_KEY;

-  switch (method) {
-    case GitHubConnectionMethod.App:
-      if (!appId || !appPrivateKey) {
-        throw new InternalServerError({
-          message: `GitHub ${getAppConnectionMethodName(method).replace("GitHub", "")} has not been configured`
-        });
-      }
-
-      client = new Octokit({
-        authStrategy: createAppAuth,
-        auth: {
-          appId,
-          privateKey: appPrivateKey,
-          installationId: credentials.installationId
-        }
-      });
-      break;
-    case GitHubConnectionMethod.OAuth:
-      client = new Octokit({
-        auth: credentials.accessToken
-      });
-      break;
-    default:
-      throw new InternalServerError({
-        message: `Unhandled GitHub connection method: ${method as GitHubConnectionMethod}`
-      });
+  if (!appId || !appPrivateKey) {
+    throw new InternalServerError({
+      message: `GitHub App keys are not configured.`
+    });
  }

-  return client;
+  if (appConnection.method !== GitHubConnectionMethod.App) {
+    throw new InternalServerError({ message: "Cannot generate GitHub App token for non-app connection" });
+  }
+
+  const appAuth = createAppAuth({
+    appId,
+    privateKey: appPrivateKey,
+    installationId: appConnection.credentials.installationId
+  });
+
+  const { token } = await appAuth({ type: "installation" });
+  return token;
+};
+
+function extractNextPageUrl(linkHeader: string | undefined): string | null {
+  if (!linkHeader) return null;
+
+  const links = linkHeader.split(",");
+  const nextLink = links.find((link) => link.includes('rel="next"'));
+
+  if (!nextLink) return null;
+
+  const match = new RE2(/<([^>]+)>/).exec(nextLink);
+  return match ? match[1] : null;
+}
+
+export const makePaginatedGitHubRequest = async <T, R = T[]>(
+  appConnection: TGitHubConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
+  path: string,
+  dataMapper?: (data: R) => T[]
+): Promise<T[]> => {
+  const { credentials, method } = appConnection;
+
+  const token =
+    method === GitHubConnectionMethod.OAuth ? credentials.accessToken : await getGitHubAppAuthToken(appConnection);
+  let url: string | null = `https://api.${credentials.host || "github.com"}${path}`;
+  let results: T[] = [];
+  let i = 0;
+
+  while (url && i < 1000) {
+    // eslint-disable-next-line no-await-in-loop
+    const response: AxiosResponse<R> = await requestWithGitHubGateway<R>(appConnection, gatewayService, {
+      url,
+      method: "GET",
+      headers: {
+        Accept: "application/vnd.github+json",
+        Authorization: `Bearer ${token}`,
+        "X-GitHub-Api-Version": "2022-11-28"
+      }
+    });
+
+    const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]);
+    results = results.concat(items);
+
+    url = extractNextPageUrl(response.headers.link as string | undefined);
+    i += 1;
+  }
+
+  return results;
 };

 type GitHubOrganization = {
  login: string;
  id: number;
+  type: string;
 };

 type GitHubRepository = {
  id: number;
  name: string;
  owner: GitHubOrganization;
+  permissions?: {
+    admin: boolean;
+    maintain: boolean;
+    push: boolean;
+    triage: boolean;
+    pull: boolean;
+  };
 };

-export const getGitHubRepositories = async (appConnection: TGitHubConnection) => {
-  const client = getGitHubClient(appConnection);
+type GitHubEnvironment = {
+  id: number;
+  name: string;
+};

-  let repositories: GitHubRepository[];
-
-  switch (appConnection.method) {
-    case GitHubConnectionMethod.App:
-      repositories = await client.paginate("GET /installation/repositories");
-      break;
-    case GitHubConnectionMethod.OAuth:
-    default:
-      repositories = (await client.paginate("GET /user/repos")).filter((repo) => repo.permissions?.admin);
-      break;
+export const getGitHubRepositories = async (
+  appConnection: TGitHubConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
+  if (appConnection.method === GitHubConnectionMethod.App) {
+    return makePaginatedGitHubRequest<GitHubRepository, { repositories: GitHubRepository[] }>(
+      appConnection,
+      gatewayService,
+      "/installation/repositories",
+      (data) => data.repositories
+    );
  }

-  return repositories;
+  const repos = await makePaginatedGitHubRequest<GitHubRepository>(appConnection, gatewayService, "/user/repos");
+  return repos.filter((repo) => repo.permissions?.admin);
 };

-export const getGitHubOrganizations = async (appConnection: TGitHubConnection) => {
-  const client = getGitHubClient(appConnection);
+export const getGitHubOrganizations = async (
+  appConnection: TGitHubConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
+  if (appConnection.method === GitHubConnectionMethod.App) {
+    const installationRepositories = await makePaginatedGitHubRequest<
+      GitHubRepository,
+      { repositories: GitHubRepository[] }
+    >(appConnection, gatewayService, "/installation/repositories", (data) => data.repositories);

-  let organizations: GitHubOrganization[];
-
-  switch (appConnection.method) {
-    case GitHubConnectionMethod.App: {
-      const installationRepositories = await client.paginate("GET /installation/repositories");
-
-      const organizationMap: Record<string, GitHubOrganization> = {};
-
-      installationRepositories.forEach((repo) => {
-        if (repo.owner.type === "Organization") {
-          organizationMap[repo.owner.id] = repo.owner;
-        }
-      });
-
-      organizations = Object.values(organizationMap);
-
-      break;
-    }
-    case GitHubConnectionMethod.OAuth:
-    default:
-      organizations = await client.paginate("GET /user/orgs");
-      break;
-  }
-
-  return organizations;
-};
-
-export const getGitHubEnvironments = async (appConnection: TGitHubConnection, owner: string, repo: string) => {
-  const client = getGitHubClient(appConnection);
-
-  try {
-    const environments = await client.paginate("GET /repos/{owner}/{repo}/environments", {
-      owner,
-      repo
+    const organizationMap: Record<string, GitHubOrganization> = {};
+    installationRepositories.forEach((repo) => {
+      if (repo.owner.type === "Organization") {
+        organizationMap[repo.owner.id] = repo.owner;
+      }
    });

-    return environments;
-  } catch (e) {
-    // repo doesn't have envs
-    if ((e as { status: number }).status === 404) {
-      return [];
-    }
+    return Object.values(organizationMap);
+  }

-    throw e;
+  return makePaginatedGitHubRequest<GitHubOrganization>(appConnection, gatewayService, "/user/orgs");
+};
+
+export const getGitHubEnvironments = async (
+  appConnection: TGitHubConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
+  owner: string,
+  repo: string
+) => {
+  try {
+    return await makePaginatedGitHubRequest<GitHubEnvironment, { environments: GitHubEnvironment[] }>(
+      appConnection,
+      gatewayService,
+      `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/environments`,
+      (data) => data.environments
+    );
+  } catch (error) {
+    const axiosError = error as AxiosError;
+    if (axiosError.response?.status === 404) return [];
+    throw error;
  }
 };

@@ -159,9 +266,11 @@ export function isGithubErrorResponse(data: GithubTokenRespData): data is Github
  return "error" in data;
 }

-export const validateGitHubConnectionCredentials = async (config: TGitHubConnectionConfig) => {
+export const validateGitHubConnectionCredentials = async (
+  config: TGitHubConnectionConfig,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
  const { credentials, method } = config;
-
  const {
    INF_APP_CONNECTION_GITHUB_OAUTH_CLIENT_ID,
    INF_APP_CONNECTION_GITHUB_OAUTH_CLIENT_SECRET,
@@ -192,10 +301,13 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
  }

  let tokenResp: AxiosResponse<GithubTokenRespData>;
+  const host = credentials.host || "github.com";

  try {
-    tokenResp = await request.get<GithubTokenRespData>("https://github.com/login/oauth/access_token", {
-      params: {
+    tokenResp = await requestWithGitHubGateway<GithubTokenRespData>(config, gatewayService, {
+      url: `https://${host}/login/oauth/access_token`,
+      method: "POST",
+      data: {
        client_id: clientId,
        client_secret: clientSecret,
        code: credentials.code,
@@ -203,7 +315,7 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
      },
      headers: {
        Accept: "application/json",
-        "Accept-Encoding": "application/json"
+        "Content-Type": "application/json"
      }
    });

@@ -233,7 +345,7 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
      throw new InternalServerError({ message: `Missing access token: ${tokenResp.data.error}` });
    }

-    const installationsResp = await request.get<{
+    const installationsResp = await requestWithGitHubGateway<{
      installations: {
        id: number;
        account: {
@@ -242,7 +354,8 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
          id: number;
        };
      }[];
-    }>(IntegrationUrls.GITHUB_USER_INSTALLATIONS, {
+    }>(config, gatewayService, {
+      url: IntegrationUrls.GITHUB_USER_INSTALLATIONS.replace("api.github.com", `api.${host}`),
      headers: {
        Accept: "application/json",
        Authorization: `Bearer ${tokenResp.data.access_token}`,
@@ -11,20 +11,24 @@ import {
 import { GitHubConnectionMethod } from "./github-connection-enums";

 export const GitHubConnectionOAuthInputCredentialsSchema = z.object({
-  code: z.string().trim().min(1, "OAuth code required")
+  code: z.string().trim().min(1, "OAuth code required"),
+  host: z.string().trim().optional()
 });

 export const GitHubConnectionAppInputCredentialsSchema = z.object({
  code: z.string().trim().min(1, "GitHub App code required"),
-  installationId: z.string().min(1, "GitHub App Installation ID required")
+  installationId: z.string().min(1, "GitHub App Installation ID required"),
+  host: z.string().trim().optional()
 });

 export const GitHubConnectionOAuthOutputCredentialsSchema = z.object({
-  accessToken: z.string()
+  accessToken: z.string(),
+  host: z.string().trim().optional()
 });

 export const GitHubConnectionAppOutputCredentialsSchema = z.object({
-  installationId: z.string()
+  installationId: z.string(),
+  host: z.string().trim().optional()
 });

 export const ValidateGitHubConnectionCredentialsSchema = z.discriminatedUnion("method", [
@@ -43,7 +47,9 @@ export const ValidateGitHubConnectionCredentialsSchema = z.discriminatedUnion("m
 ]);

 export const CreateGitHubConnectionSchema = ValidateGitHubConnectionCredentialsSchema.and(
-  GenericCreateAppConnectionFieldsSchema(AppConnection.GitHub)
+  GenericCreateAppConnectionFieldsSchema(AppConnection.GitHub, {
+    supportsGateways: true
+  })
 );

 export const UpdateGitHubConnectionSchema = z
@@ -53,7 +59,11 @@ export const UpdateGitHubConnectionSchema = z
      .optional()
      .describe(AppConnections.UPDATE(AppConnection.GitHub).credentials)
  })
-  .and(GenericUpdateAppConnectionFieldsSchema(AppConnection.GitHub));
+  .and(
+    GenericUpdateAppConnectionFieldsSchema(AppConnection.GitHub, {
+      supportsGateways: true
+    })
+  );

 const BaseGitHubConnectionSchema = BaseAppConnectionSchema.extend({ app: z.literal(AppConnection.GitHub) });

@@ -1,3 +1,4 @@
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { OrgServiceActor } from "@app/lib/types";
 import { AppConnection } from "@app/services/app-connection/app-connection-enums";
 import {
@@ -19,11 +20,14 @@ type TListGitHubEnvironmentsDTO = {
  owner: string;
 };

-export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc) => {
+export const githubConnectionService = (
+  getAppConnection: TGetAppConnectionFunc,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
  const listRepositories = async (connectionId: string, actor: OrgServiceActor) => {
    const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);

-    const repositories = await getGitHubRepositories(appConnection);
+    const repositories = await getGitHubRepositories(appConnection, gatewayService);

    return repositories;
  };
@@ -31,7 +35,7 @@ export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc)
  const listOrganizations = async (connectionId: string, actor: OrgServiceActor) => {
    const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);

-    const organizations = await getGitHubOrganizations(appConnection);
+    const organizations = await getGitHubOrganizations(appConnection, gatewayService);

    return organizations;
  };
@@ -42,7 +46,7 @@ export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc)
  ) => {
    const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);

-    const environments = await getGitHubEnvironments(appConnection, owner, repo);
+    const environments = await getGitHubEnvironments(appConnection, gatewayService, owner, repo);

    return environments;
  };
@@ -17,4 +17,7 @@ export type TGitHubConnectionInput = z.infer<typeof CreateGitHubConnectionSchema

 export type TValidateGitHubConnectionCredentialsSchema = typeof ValidateGitHubConnectionCredentialsSchema;

-export type TGitHubConnectionConfig = DiscriminativePick<TGitHubConnectionInput, "method" | "app" | "credentials">;
+export type TGitHubConnectionConfig = DiscriminativePick<
+  TGitHubConnectionInput,
+  "method" | "app" | "credentials" | "gatewayId"
+>;