diff --git a/backend/src/lib/validator/validate-url.ts b/backend/src/lib/validator/validate-url.ts index 8f195e0b5..a4c07b37d 100644 --- a/backend/src/lib/validator/validate-url.ts +++ b/backend/src/lib/validator/validate-url.ts @@ -14,6 +14,11 @@ export const blockLocalAndPrivateIpAddresses = async (url: string) => { if (appCfg.isDevelopmentMode) return; const validUrl = new URL(url); + + if (validUrl.username || validUrl.password) { + throw new BadRequestError({ message: "URLs with user credentials (e.g., user:pass@) are not allowed" }); + } + const inputHostIps: string[] = []; if (isIPv4(validUrl.hostname)) { inputHostIps.push(validUrl.hostname); diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 0fb3191f8..f6bfd2b4a 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -1044,6 +1044,15 @@ export const registerRoutes = async ( kmsService }); + const gatewayService = gatewayServiceFactory({ + permissionService, + gatewayDAL, + kmsService, + licenseService, + orgGatewayConfigDAL, + keyStore + }); + const secretSyncQueue = secretSyncQueueFactory({ queueService, secretSyncDAL, @@ -1067,7 +1076,8 @@ export const registerRoutes = async ( secretVersionTagV2BridgeDAL, resourceMetadataDAL, appConnectionDAL, - licenseService + licenseService, + gatewayService }); const secretQueueService = secretQueueFactory({ @@ -1490,15 +1500,6 @@ export const registerRoutes = async ( licenseService }); - const gatewayService = gatewayServiceFactory({ - permissionService, - gatewayDAL, - kmsService, - licenseService, - orgGatewayConfigDAL, - keyStore - }); - const identityKubernetesAuthService = identityKubernetesAuthServiceFactory({ identityKubernetesAuthDAL, identityOrgMembershipDAL, diff --git a/backend/src/services/app-connection/app-connection-service.ts b/backend/src/services/app-connection/app-connection-service.ts index 98c9e5f4d..976d154b3 100644 --- a/backend/src/services/app-connection/app-connection-service.ts +++ b/backend/src/services/app-connection/app-connection-service.ts @@ -583,7 +583,7 @@ export const appConnectionServiceFactory = ({ deleteAppConnection, connectAppConnectionById, listAvailableAppConnectionsForUser, - github: githubConnectionService(connectAppConnectionById), + github: githubConnectionService(connectAppConnectionById, gatewayService), githubRadar: githubRadarConnectionService(connectAppConnectionById), gcp: gcpConnectionService(connectAppConnectionById), databricks: databricksConnectionService(connectAppConnectionById, appConnectionDAL, kmsService), diff --git a/backend/src/services/app-connection/github/github-connection-fns.ts b/backend/src/services/app-connection/github/github-connection-fns.ts index 360923e19..57d01be29 100644 --- a/backend/src/services/app-connection/github/github-connection-fns.ts +++ b/backend/src/services/app-connection/github/github-connection-fns.ts @@ -1,10 +1,16 @@ import { createAppAuth } from "@octokit/auth-app"; -import { Octokit } from "@octokit/rest"; -import { AxiosResponse } from "axios"; +import { AxiosError, AxiosRequestConfig, AxiosResponse } from "axios"; +import https from "https"; +import RE2 from "re2"; +import { verifyHostInputValidity } from "@app/ee/services/dynamic-secret/dynamic-secret-fns"; +import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service"; import { getConfig } from "@app/lib/config/env"; -import { request } from "@app/lib/config/request"; +import { request as httpRequest } from "@app/lib/config/request"; import { BadRequestError, ForbiddenRequestError, InternalServerError } from "@app/lib/errors"; +import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway"; +import { logger } from "@app/lib/logger"; +import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator"; import { getAppConnectionMethodName } from "@app/services/app-connection/app-connection-fns"; import { IntegrationUrls } from "@app/services/integration-auth/integration-list"; @@ -24,123 +30,224 @@ export const getGitHubConnectionListItem = () => { }; }; -export const getGitHubClient = (appConnection: TGitHubConnection) => { +export const requestWithGitHubGateway = async ( + appConnection: { gatewayId?: string | null }, + gatewayService: Pick, + requestConfig: AxiosRequestConfig +): Promise> => { + const { gatewayId } = appConnection; + + // If gateway isn't set up, don't proxy request + if (!gatewayId) { + return httpRequest.request(requestConfig); + } + + const url = new URL(requestConfig.url as string); + + await blockLocalAndPrivateIpAddresses(url.toString()); + + const [targetHost] = await verifyHostInputValidity(url.host, true); + const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(gatewayId); + const [relayHost, relayPort] = relayDetails.relayAddress.split(":"); + + return withGatewayProxy( + async (proxyPort) => { + const httpsAgent = new https.Agent({ + servername: targetHost + }); + + url.protocol = "https:"; + url.host = `localhost:${proxyPort}`; + + const finalRequestConfig: AxiosRequestConfig = { + ...requestConfig, + url: url.toString(), + httpsAgent, + headers: { + ...requestConfig.headers, + Host: targetHost + } + }; + + try { + return await httpRequest.request(finalRequestConfig); + } catch (error) { + const axiosError = error as AxiosError; + logger.error("Error during GitHub gateway request:", axiosError.message, axiosError.response?.data); + throw error; + } + }, + { + protocol: GatewayProxyProtocol.Tcp, + targetHost, + targetPort: 443, + relayHost, + relayPort: Number(relayPort), + identityId: relayDetails.identityId, + orgId: relayDetails.orgId, + tlsOptions: { + ca: relayDetails.certChain, + cert: relayDetails.certificate, + key: relayDetails.privateKey.toString() + } + } + ); +}; + +export const getGitHubAppAuthToken = async (appConnection: TGitHubConnection) => { const appCfg = getConfig(); - - const { method, credentials } = appConnection; - - let client: Octokit; - const appId = appCfg.INF_APP_CONNECTION_GITHUB_APP_ID; const appPrivateKey = appCfg.INF_APP_CONNECTION_GITHUB_APP_PRIVATE_KEY; - switch (method) { - case GitHubConnectionMethod.App: - if (!appId || !appPrivateKey) { - throw new InternalServerError({ - message: `GitHub ${getAppConnectionMethodName(method).replace("GitHub", "")} has not been configured` - }); - } - - client = new Octokit({ - authStrategy: createAppAuth, - auth: { - appId, - privateKey: appPrivateKey, - installationId: credentials.installationId - } - }); - break; - case GitHubConnectionMethod.OAuth: - client = new Octokit({ - auth: credentials.accessToken - }); - break; - default: - throw new InternalServerError({ - message: `Unhandled GitHub connection method: ${method as GitHubConnectionMethod}` - }); + if (!appId || !appPrivateKey) { + throw new InternalServerError({ + message: `GitHub App keys are not configured.` + }); } - return client; + if (appConnection.method !== GitHubConnectionMethod.App) { + throw new InternalServerError({ message: "Cannot generate GitHub App token for non-app connection" }); + } + + const appAuth = createAppAuth({ + appId, + privateKey: appPrivateKey, + installationId: appConnection.credentials.installationId + }); + + const { token } = await appAuth({ type: "installation" }); + return token; +}; + +function extractNextPageUrl(linkHeader: string | undefined): string | null { + if (!linkHeader) return null; + + const links = linkHeader.split(","); + const nextLink = links.find((link) => link.includes('rel="next"')); + + if (!nextLink) return null; + + const match = new RE2(/<([^>]+)>/).exec(nextLink); + return match ? match[1] : null; +} + +export const makePaginatedGitHubRequest = async ( + appConnection: TGitHubConnection, + gatewayService: Pick, + path: string, + dataMapper?: (data: R) => T[] +): Promise => { + const { credentials, method } = appConnection; + + const token = + method === GitHubConnectionMethod.OAuth ? credentials.accessToken : await getGitHubAppAuthToken(appConnection); + let url: string | null = `https://api.${credentials.host || "github.com"}${path}`; + let results: T[] = []; + let i = 0; + + while (url && i < 1000) { + // eslint-disable-next-line no-await-in-loop + const response: AxiosResponse = await requestWithGitHubGateway(appConnection, gatewayService, { + url, + method: "GET", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28" + } + }); + + const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]); + results = results.concat(items); + + url = extractNextPageUrl(response.headers.link as string | undefined); + i += 1; + } + + return results; }; type GitHubOrganization = { login: string; id: number; + type: string; }; type GitHubRepository = { id: number; name: string; owner: GitHubOrganization; + permissions?: { + admin: boolean; + maintain: boolean; + push: boolean; + triage: boolean; + pull: boolean; + }; }; -export const getGitHubRepositories = async (appConnection: TGitHubConnection) => { - const client = getGitHubClient(appConnection); +type GitHubEnvironment = { + id: number; + name: string; +}; - let repositories: GitHubRepository[]; - - switch (appConnection.method) { - case GitHubConnectionMethod.App: - repositories = await client.paginate("GET /installation/repositories"); - break; - case GitHubConnectionMethod.OAuth: - default: - repositories = (await client.paginate("GET /user/repos")).filter((repo) => repo.permissions?.admin); - break; +export const getGitHubRepositories = async ( + appConnection: TGitHubConnection, + gatewayService: Pick +) => { + if (appConnection.method === GitHubConnectionMethod.App) { + return makePaginatedGitHubRequest( + appConnection, + gatewayService, + "/installation/repositories", + (data) => data.repositories + ); } - return repositories; + const repos = await makePaginatedGitHubRequest(appConnection, gatewayService, "/user/repos"); + return repos.filter((repo) => repo.permissions?.admin); }; -export const getGitHubOrganizations = async (appConnection: TGitHubConnection) => { - const client = getGitHubClient(appConnection); +export const getGitHubOrganizations = async ( + appConnection: TGitHubConnection, + gatewayService: Pick +) => { + if (appConnection.method === GitHubConnectionMethod.App) { + const installationRepositories = await makePaginatedGitHubRequest< + GitHubRepository, + { repositories: GitHubRepository[] } + >(appConnection, gatewayService, "/installation/repositories", (data) => data.repositories); - let organizations: GitHubOrganization[]; - - switch (appConnection.method) { - case GitHubConnectionMethod.App: { - const installationRepositories = await client.paginate("GET /installation/repositories"); - - const organizationMap: Record = {}; - - installationRepositories.forEach((repo) => { - if (repo.owner.type === "Organization") { - organizationMap[repo.owner.id] = repo.owner; - } - }); - - organizations = Object.values(organizationMap); - - break; - } - case GitHubConnectionMethod.OAuth: - default: - organizations = await client.paginate("GET /user/orgs"); - break; - } - - return organizations; -}; - -export const getGitHubEnvironments = async (appConnection: TGitHubConnection, owner: string, repo: string) => { - const client = getGitHubClient(appConnection); - - try { - const environments = await client.paginate("GET /repos/{owner}/{repo}/environments", { - owner, - repo + const organizationMap: Record = {}; + installationRepositories.forEach((repo) => { + if (repo.owner.type === "Organization") { + organizationMap[repo.owner.id] = repo.owner; + } }); - return environments; - } catch (e) { - // repo doesn't have envs - if ((e as { status: number }).status === 404) { - return []; - } + return Object.values(organizationMap); + } - throw e; + return makePaginatedGitHubRequest(appConnection, gatewayService, "/user/orgs"); +}; + +export const getGitHubEnvironments = async ( + appConnection: TGitHubConnection, + gatewayService: Pick, + owner: string, + repo: string +) => { + try { + return await makePaginatedGitHubRequest( + appConnection, + gatewayService, + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/environments`, + (data) => data.environments + ); + } catch (error) { + const axiosError = error as AxiosError; + if (axiosError.response?.status === 404) return []; + throw error; } }; @@ -159,9 +266,11 @@ export function isGithubErrorResponse(data: GithubTokenRespData): data is Github return "error" in data; } -export const validateGitHubConnectionCredentials = async (config: TGitHubConnectionConfig) => { +export const validateGitHubConnectionCredentials = async ( + config: TGitHubConnectionConfig, + gatewayService: Pick +) => { const { credentials, method } = config; - const { INF_APP_CONNECTION_GITHUB_OAUTH_CLIENT_ID, INF_APP_CONNECTION_GITHUB_OAUTH_CLIENT_SECRET, @@ -192,10 +301,13 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect } let tokenResp: AxiosResponse; + const host = credentials.host || "github.com"; try { - tokenResp = await request.get("https://github.com/login/oauth/access_token", { - params: { + tokenResp = await requestWithGitHubGateway(config, gatewayService, { + url: `https://${host}/login/oauth/access_token`, + method: "POST", + data: { client_id: clientId, client_secret: clientSecret, code: credentials.code, @@ -203,7 +315,7 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect }, headers: { Accept: "application/json", - "Accept-Encoding": "application/json" + "Content-Type": "application/json" } }); @@ -233,7 +345,7 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect throw new InternalServerError({ message: `Missing access token: ${tokenResp.data.error}` }); } - const installationsResp = await request.get<{ + const installationsResp = await requestWithGitHubGateway<{ installations: { id: number; account: { @@ -242,7 +354,8 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect id: number; }; }[]; - }>(IntegrationUrls.GITHUB_USER_INSTALLATIONS, { + }>(config, gatewayService, { + url: IntegrationUrls.GITHUB_USER_INSTALLATIONS.replace("api.github.com", `api.${host}`), headers: { Accept: "application/json", Authorization: `Bearer ${tokenResp.data.access_token}`, diff --git a/backend/src/services/app-connection/github/github-connection-schemas.ts b/backend/src/services/app-connection/github/github-connection-schemas.ts index e98b9169d..bf92ec155 100644 --- a/backend/src/services/app-connection/github/github-connection-schemas.ts +++ b/backend/src/services/app-connection/github/github-connection-schemas.ts @@ -11,20 +11,24 @@ import { import { GitHubConnectionMethod } from "./github-connection-enums"; export const GitHubConnectionOAuthInputCredentialsSchema = z.object({ - code: z.string().trim().min(1, "OAuth code required") + code: z.string().trim().min(1, "OAuth code required"), + host: z.string().trim().optional() }); export const GitHubConnectionAppInputCredentialsSchema = z.object({ code: z.string().trim().min(1, "GitHub App code required"), - installationId: z.string().min(1, "GitHub App Installation ID required") + installationId: z.string().min(1, "GitHub App Installation ID required"), + host: z.string().trim().optional() }); export const GitHubConnectionOAuthOutputCredentialsSchema = z.object({ - accessToken: z.string() + accessToken: z.string(), + host: z.string().trim().optional() }); export const GitHubConnectionAppOutputCredentialsSchema = z.object({ - installationId: z.string() + installationId: z.string(), + host: z.string().trim().optional() }); export const ValidateGitHubConnectionCredentialsSchema = z.discriminatedUnion("method", [ @@ -43,7 +47,9 @@ export const ValidateGitHubConnectionCredentialsSchema = z.discriminatedUnion("m ]); export const CreateGitHubConnectionSchema = ValidateGitHubConnectionCredentialsSchema.and( - GenericCreateAppConnectionFieldsSchema(AppConnection.GitHub) + GenericCreateAppConnectionFieldsSchema(AppConnection.GitHub, { + supportsGateways: true + }) ); export const UpdateGitHubConnectionSchema = z @@ -53,7 +59,11 @@ export const UpdateGitHubConnectionSchema = z .optional() .describe(AppConnections.UPDATE(AppConnection.GitHub).credentials) }) - .and(GenericUpdateAppConnectionFieldsSchema(AppConnection.GitHub)); + .and( + GenericUpdateAppConnectionFieldsSchema(AppConnection.GitHub, { + supportsGateways: true + }) + ); const BaseGitHubConnectionSchema = BaseAppConnectionSchema.extend({ app: z.literal(AppConnection.GitHub) }); diff --git a/backend/src/services/app-connection/github/github-connection-service.ts b/backend/src/services/app-connection/github/github-connection-service.ts index b4e95c5a7..f1198ddfa 100644 --- a/backend/src/services/app-connection/github/github-connection-service.ts +++ b/backend/src/services/app-connection/github/github-connection-service.ts @@ -1,3 +1,4 @@ +import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service"; import { OrgServiceActor } from "@app/lib/types"; import { AppConnection } from "@app/services/app-connection/app-connection-enums"; import { @@ -19,11 +20,14 @@ type TListGitHubEnvironmentsDTO = { owner: string; }; -export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc) => { +export const githubConnectionService = ( + getAppConnection: TGetAppConnectionFunc, + gatewayService: Pick +) => { const listRepositories = async (connectionId: string, actor: OrgServiceActor) => { const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor); - const repositories = await getGitHubRepositories(appConnection); + const repositories = await getGitHubRepositories(appConnection, gatewayService); return repositories; }; @@ -31,7 +35,7 @@ export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc) const listOrganizations = async (connectionId: string, actor: OrgServiceActor) => { const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor); - const organizations = await getGitHubOrganizations(appConnection); + const organizations = await getGitHubOrganizations(appConnection, gatewayService); return organizations; }; @@ -42,7 +46,7 @@ export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc) ) => { const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor); - const environments = await getGitHubEnvironments(appConnection, owner, repo); + const environments = await getGitHubEnvironments(appConnection, gatewayService, owner, repo); return environments; }; diff --git a/backend/src/services/app-connection/github/github-connection-types.ts b/backend/src/services/app-connection/github/github-connection-types.ts index 600506277..c4aed54b5 100644 --- a/backend/src/services/app-connection/github/github-connection-types.ts +++ b/backend/src/services/app-connection/github/github-connection-types.ts @@ -17,4 +17,7 @@ export type TGitHubConnectionInput = z.infer; +export type TGitHubConnectionConfig = DiscriminativePick< + TGitHubConnectionInput, + "method" | "app" | "credentials" | "gatewayId" +>; diff --git a/backend/src/services/secret-sync/github/github-sync-fns.ts b/backend/src/services/secret-sync/github/github-sync-fns.ts index f06f0cfc2..b37d5e90e 100644 --- a/backend/src/services/secret-sync/github/github-sync-fns.ts +++ b/backend/src/services/secret-sync/github/github-sync-fns.ts @@ -1,7 +1,12 @@ -import { Octokit } from "@octokit/rest"; import sodium from "libsodium-wrappers"; -import { getGitHubClient } from "@app/services/app-connection/github"; +import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service"; +import { + getGitHubAppAuthToken, + GitHubConnectionMethod, + makePaginatedGitHubRequest, + requestWithGitHubGateway +} from "@app/services/app-connection/github"; import { GitHubSyncScope, GitHubSyncVisibility } from "@app/services/secret-sync/github/github-sync-enums"; import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors"; import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns"; @@ -12,155 +17,165 @@ import { TGitHubPublicKey, TGitHubSecret, TGitHubSecretPayload, TGitHubSyncWithC // TODO: rate limit handling -const getEncryptedSecrets = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => { - let encryptedSecrets: TGitHubSecret[]; - - const { destinationConfig } = secretSync; +const getEncryptedSecrets = async ( + secretSync: TGitHubSyncWithCredentials, + gatewayService: Pick +) => { + const { destinationConfig, connection } = secretSync; + let path: string; switch (destinationConfig.scope) { case GitHubSyncScope.Organization: { - encryptedSecrets = await client.paginate("GET /orgs/{org}/actions/secrets", { - org: destinationConfig.org - }); + path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets`; break; } case GitHubSyncScope.Repository: { - encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/actions/secrets", { - owner: destinationConfig.owner, - repo: destinationConfig.repo - }); - + path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets`; break; } case GitHubSyncScope.RepositoryEnvironment: default: { - encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets", { - owner: destinationConfig.owner, - repo: destinationConfig.repo, - environment_name: destinationConfig.env - }); + path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets`; break; } } - return encryptedSecrets; + return makePaginatedGitHubRequest( + connection, + gatewayService, + path, + (data) => data.secrets + ); }; -const getPublicKey = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => { - let publicKey: TGitHubPublicKey; - - const { destinationConfig } = secretSync; +const getPublicKey = async ( + secretSync: TGitHubSyncWithCredentials, + gatewayService: Pick, + token: string +) => { + const { destinationConfig, connection } = secretSync; + let path: string; switch (destinationConfig.scope) { case GitHubSyncScope.Organization: { - publicKey = ( - await client.request("GET /orgs/{org}/actions/secrets/public-key", { - org: destinationConfig.org - }) - ).data; + path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/public-key`; break; } case GitHubSyncScope.Repository: { - publicKey = ( - await client.request("GET /repos/{owner}/{repo}/actions/secrets/public-key", { - owner: destinationConfig.owner, - repo: destinationConfig.repo - }) - ).data; + path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/public-key`; break; } case GitHubSyncScope.RepositoryEnvironment: default: { - publicKey = ( - await client.request("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets/public-key", { - owner: destinationConfig.owner, - repo: destinationConfig.repo, - environment_name: destinationConfig.env - }) - ).data; + path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/public-key`; break; } } - return publicKey; + const response = await requestWithGitHubGateway(connection, gatewayService, { + url: `https://api.${connection.credentials.host || "github.com"}${path}`, + method: "GET", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28" + } + }); + + return response.data; }; const deleteSecret = async ( - client: Octokit, secretSync: TGitHubSyncWithCredentials, + gatewayService: Pick, + token: string, encryptedSecret: TGitHubSecret ) => { - const { destinationConfig } = secretSync; + const { destinationConfig, connection } = secretSync; + let path: string; switch (destinationConfig.scope) { case GitHubSyncScope.Organization: { - await client.request(`DELETE /orgs/{org}/actions/secrets/{secret_name}`, { - org: destinationConfig.org, - secret_name: encryptedSecret.name - }); + path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/${encodeURIComponent(encryptedSecret.name)}`; break; } case GitHubSyncScope.Repository: { - await client.request("DELETE /repos/{owner}/{repo}/actions/secrets/{secret_name}", { - owner: destinationConfig.owner, - repo: destinationConfig.repo, - secret_name: encryptedSecret.name - }); + path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/${encodeURIComponent(encryptedSecret.name)}`; break; } case GitHubSyncScope.RepositoryEnvironment: default: { - await client.request("DELETE /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", { - owner: destinationConfig.owner, - repo: destinationConfig.repo, - environment_name: destinationConfig.env, - secret_name: encryptedSecret.name - }); + path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/${encodeURIComponent(encryptedSecret.name)}`; break; } } + + await requestWithGitHubGateway(connection, gatewayService, { + url: `https://api.${connection.credentials.host || "github.com"}${path}`, + method: "DELETE", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28" + } + }); }; -const putSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, payload: TGitHubSecretPayload) => { - const { destinationConfig } = secretSync; +const putSecret = async ( + secretSync: TGitHubSyncWithCredentials, + gatewayService: Pick, + token: string, + payload: TGitHubSecretPayload +) => { + const { destinationConfig, connection } = secretSync; + + let path: string; + let body: Record = payload; switch (destinationConfig.scope) { case GitHubSyncScope.Organization: { const { visibility, selectedRepositoryIds } = destinationConfig; - - await client.request(`PUT /orgs/{org}/actions/secrets/{secret_name}`, { - org: destinationConfig.org, + path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/${encodeURIComponent(payload.secret_name)}`; + body = { ...payload, visibility, ...(visibility === GitHubSyncVisibility.Selected && { selected_repository_ids: selectedRepositoryIds }) - }); + }; break; } case GitHubSyncScope.Repository: { - await client.request("PUT /repos/{owner}/{repo}/actions/secrets/{secret_name}", { - owner: destinationConfig.owner, - repo: destinationConfig.repo, - ...payload - }); + path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/${encodeURIComponent(payload.secret_name)}`; break; } case GitHubSyncScope.RepositoryEnvironment: default: { - await client.request("PUT /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", { - owner: destinationConfig.owner, - repo: destinationConfig.repo, - environment_name: destinationConfig.env, - ...payload - }); + path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/${encodeURIComponent(payload.secret_name)}`; break; } } + + await requestWithGitHubGateway(connection, gatewayService, { + url: `https://api.${connection.credentials.host || "github.com"}${path}`, + method: "PUT", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28" + }, + data: body + }); }; export const GithubSyncFns = { - syncSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => { + syncSecrets: async ( + secretSync: TGitHubSyncWithCredentials, + ogSecretMap: TSecretMap, + gatewayService: Pick + ) => { + const secretMap = Object.fromEntries(Object.entries(ogSecretMap).map(([i, v]) => [i.toUpperCase(), v])); + switch (secretSync.destinationConfig.scope) { case GitHubSyncScope.Organization: if (Object.values(secretMap).length > 1000) { @@ -187,38 +202,40 @@ export const GithubSyncFns = { ); } - const client = getGitHubClient(secretSync.connection); + const { connection } = secretSync; + const token = + connection.method === GitHubConnectionMethod.OAuth + ? connection.credentials.accessToken + : await getGitHubAppAuthToken(connection); - const encryptedSecrets = await getEncryptedSecrets(client, secretSync); + const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService); + const publicKey = await getPublicKey(secretSync, gatewayService, token); - const publicKey = await getPublicKey(client, secretSync); + await sodium.ready; + for await (const key of Object.keys(secretMap)) { + // convert secret & base64 key to Uint8Array. + const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL); + const binarySecretValue = sodium.from_string(secretMap[key].value); - await sodium.ready.then(async () => { - for await (const key of Object.keys(secretMap)) { - // convert secret & base64 key to Uint8Array. - const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL); - const binarySecretValue = sodium.from_string(secretMap[key].value); + // encrypt secret using libsodium + const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey); - // encrypt secret using libsodium - const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey); + // convert encrypted Uint8Array to base64 + const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL); - // convert encrypted Uint8Array to base64 - const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL); - - try { - await putSecret(client, secretSync, { - secret_name: key, - encrypted_value: encryptedSecretValue, - key_id: publicKey.key_id - }); - } catch (error) { - throw new SecretSyncError({ - error, - secretKey: key - }); - } + try { + await putSecret(secretSync, gatewayService, token, { + secret_name: key, + encrypted_value: encryptedSecretValue, + key_id: publicKey.key_id + }); + } catch (error) { + throw new SecretSyncError({ + error, + secretKey: key + }); } - }); + } if (secretSync.syncOptions.disableSecretDeletion) return; @@ -228,21 +245,31 @@ export const GithubSyncFns = { continue; if (!(encryptedSecret.name in secretMap)) { - await deleteSecret(client, secretSync, encryptedSecret); + await deleteSecret(secretSync, gatewayService, token, encryptedSecret); } } }, getSecrets: async (secretSync: TGitHubSyncWithCredentials) => { throw new Error(`${SECRET_SYNC_NAME_MAP[secretSync.destination]} does not support importing secrets.`); }, - removeSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => { - const client = getGitHubClient(secretSync.connection); + removeSecrets: async ( + secretSync: TGitHubSyncWithCredentials, + ogSecretMap: TSecretMap, + gatewayService: Pick + ) => { + const secretMap = Object.fromEntries(Object.entries(ogSecretMap).map(([i, v]) => [i.toUpperCase(), v])); - const encryptedSecrets = await getEncryptedSecrets(client, secretSync); + const { connection } = secretSync; + const token = + connection.method === GitHubConnectionMethod.OAuth + ? connection.credentials.accessToken + : await getGitHubAppAuthToken(connection); + + const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService); for await (const encryptedSecret of encryptedSecrets) { if (encryptedSecret.name in secretMap) { - await deleteSecret(client, secretSync, encryptedSecret); + await deleteSecret(secretSync, gatewayService, token, encryptedSecret); } } } diff --git a/backend/src/services/secret-sync/secret-sync-fns.ts b/backend/src/services/secret-sync/secret-sync-fns.ts index caa6ddcee..6fa46f1e6 100644 --- a/backend/src/services/secret-sync/secret-sync-fns.ts +++ b/backend/src/services/secret-sync/secret-sync-fns.ts @@ -1,6 +1,7 @@ import { AxiosError } from "axios"; import handlebars from "handlebars"; +import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { OCI_VAULT_SYNC_LIST_OPTION, OCIVaultSyncFns } from "@app/ee/services/secret-sync/oci-vault"; import { BadRequestError } from "@app/lib/errors"; @@ -97,6 +98,7 @@ export const listSecretSyncOptions = () => { type TSyncSecretDeps = { appConnectionDAL: Pick; kmsService: Pick; + gatewayService: Pick; }; // Add schema to secret keys @@ -191,7 +193,7 @@ export const SecretSyncFns = { syncSecrets: ( secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap, - { kmsService, appConnectionDAL }: TSyncSecretDeps + { kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps ): Promise => { const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema); @@ -201,7 +203,7 @@ export const SecretSyncFns = { case SecretSync.AWSSecretsManager: return AwsSecretsManagerSyncFns.syncSecrets(secretSync, schemaSecretMap); case SecretSync.GitHub: - return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap); + return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap, gatewayService); case SecretSync.GCPSecretManager: return GcpSyncFns.syncSecrets(secretSync, schemaSecretMap); case SecretSync.AzureKeyVault: @@ -395,7 +397,7 @@ export const SecretSyncFns = { removeSecrets: ( secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap, - { kmsService, appConnectionDAL }: TSyncSecretDeps + { kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps ): Promise => { const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema); @@ -405,7 +407,7 @@ export const SecretSyncFns = { case SecretSync.AWSSecretsManager: return AwsSecretsManagerSyncFns.removeSecrets(secretSync, schemaSecretMap); case SecretSync.GitHub: - return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap); + return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap, gatewayService); case SecretSync.GCPSecretManager: return GcpSyncFns.removeSecrets(secretSync, schemaSecretMap); case SecretSync.AzureKeyVault: diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts index 5d788ea88..7bef7d8c7 100644 --- a/backend/src/services/secret-sync/secret-sync-queue.ts +++ b/backend/src/services/secret-sync/secret-sync-queue.ts @@ -4,6 +4,7 @@ import { Job } from "bullmq"; import { ProjectMembershipRole, SecretType } from "@app/db/schemas"; import { EventType, TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-types"; +import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; @@ -96,6 +97,7 @@ type TSecretSyncQueueFactoryDep = { resourceMetadataDAL: Pick; folderCommitService: Pick; licenseService: Pick; + gatewayService: Pick; }; type SecretSyncActionJob = Job< @@ -138,7 +140,8 @@ export const secretSyncQueueFactory = ({ secretVersionTagV2BridgeDAL, resourceMetadataDAL, folderCommitService, - licenseService + licenseService, + gatewayService }: TSecretSyncQueueFactoryDep) => { const appCfg = getConfig(); @@ -353,7 +356,8 @@ export const secretSyncQueueFactory = ({ const importedSecrets = await SecretSyncFns.getSecrets(secretSync, { appConnectionDAL, - kmsService + kmsService, + gatewayService }); if (!Object.keys(importedSecrets).length) return {}; @@ -481,7 +485,8 @@ export const secretSyncQueueFactory = ({ await SecretSyncFns.syncSecrets(secretSyncWithCredentials, secretMap, { appConnectionDAL, - kmsService + kmsService, + gatewayService }); isSynced = true; @@ -730,7 +735,8 @@ export const secretSyncQueueFactory = ({ secretMap, { appConnectionDAL, - kmsService + kmsService, + gatewayService } ); diff --git a/docs/images/app-connections/github/create-github-app-method.png b/docs/images/app-connections/github/create-github-app-method.png index 640fb0213..e69de29bb 100644 Binary files a/docs/images/app-connections/github/create-github-app-method.png and b/docs/images/app-connections/github/create-github-app-method.png differ diff --git a/docs/integrations/app-connections/github.mdx b/docs/integrations/app-connections/github.mdx index 8f4283ae9..9a952f815 100644 --- a/docs/integrations/app-connections/github.mdx +++ b/docs/integrations/app-connections/github.mdx @@ -94,6 +94,11 @@ Infisical supports two methods for connecting to GitHub. Select the **GitHub App** method and click **Connect to GitHub**. + + You may optionally configure GitHub Enterprise options: + - **Gateway:** The gateway connected to your private network + - **Hostname:** The hostname at which to access your GitHub Enterprise instance + ![Connect via GitHub App](/images/app-connections/github/create-github-app-method.png) diff --git a/frontend/src/hooks/api/appConnections/types/github-connection.ts b/frontend/src/hooks/api/appConnections/types/github-connection.ts index d00936cda..a8b734aa9 100644 --- a/frontend/src/hooks/api/appConnections/types/github-connection.ts +++ b/frontend/src/hooks/api/appConnections/types/github-connection.ts @@ -11,6 +11,7 @@ export type TGitHubConnection = TRootAppConnection & { app: AppConnection.GitHub method: GitHubConnectionMethod.OAuth; credentials: { code: string; + host?: string; }; } | { @@ -18,6 +19,7 @@ export type TGitHubConnection = TRootAppConnection & { app: AppConnection.GitHub credentials: { code: string; installationId: string; + host?: string; }; } ); diff --git a/frontend/src/pages/organization/AppConnections/AppConnectionsPage/components/AppConnectionForm/GitHubConnectionForm.tsx b/frontend/src/pages/organization/AppConnections/AppConnectionsPage/components/AppConnectionForm/GitHubConnectionForm.tsx index ea64d1751..eb93993af 100644 --- a/frontend/src/pages/organization/AppConnections/AppConnectionsPage/components/AppConnectionForm/GitHubConnectionForm.tsx +++ b/frontend/src/pages/organization/AppConnections/AppConnectionsPage/components/AppConnectionForm/GitHubConnectionForm.tsx @@ -3,11 +3,31 @@ import crypto from "crypto"; import { useState } from "react"; import { Controller, FormProvider, useForm } from "react-hook-form"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useQuery } from "@tanstack/react-query"; import { z } from "zod"; -import { Button, FormControl, ModalClose, Select, SelectItem } from "@app/components/v2"; +import { OrgPermissionCan } from "@app/components/permissions"; +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, + Button, + FormControl, + Input, + ModalClose, + Select, + SelectItem, + Tooltip +} from "@app/components/v2"; +import { useSubscription } from "@app/context"; +import { + OrgGatewayPermissionActions, + OrgPermissionSubjects +} from "@app/context/OrgPermissionContext/types"; import { APP_CONNECTION_MAP, getAppConnectionMethodDetails } from "@app/helpers/appConnections"; import { isInfisicalCloud } from "@app/helpers/platform"; +import { gatewaysQueryKeys } from "@app/hooks/api"; import { GitHubConnectionMethod, TGitHubConnection, @@ -26,7 +46,12 @@ type Props = { const formSchema = genericAppConnectionFieldsSchema.extend({ app: z.literal(AppConnection.GitHub), - method: z.nativeEnum(GitHubConnectionMethod) + method: z.nativeEnum(GitHubConnectionMethod), + credentials: z + .object({ + host: z.string().optional() + }) + .optional() }); type FormData = z.infer; @@ -44,7 +69,8 @@ export const GitHubConnectionForm = ({ appConnection }: Props) => { resolver: zodResolver(formSchema), defaultValues: appConnection ?? { app: AppConnection.GitHub, - method: GitHubConnectionMethod.App + method: GitHubConnectionMethod.App, + gatewayId: null } }); @@ -55,6 +81,9 @@ export const GitHubConnectionForm = ({ appConnection }: Props) => { formState: { isSubmitting, isDirty } } = form; + const { subscription } = useSubscription(); + const { data: gateways, isPending: isGatewaysLoading } = useQuery(gatewaysQueryKeys.list()); + const selectedMethod = watch("method"); const onSubmit = (formData: FormData) => { @@ -66,15 +95,20 @@ export const GitHubConnectionForm = ({ appConnection }: Props) => { JSON.stringify({ ...formData, connectionId: appConnection?.id }) ); + const githubHost = + formData.credentials?.host && formData.credentials.host.length > 0 + ? `https://${formData.credentials.host}` + : "https://github.com"; + switch (formData.method) { case GitHubConnectionMethod.App: window.location.assign( - `https://github.com/apps/${appClientSlug}/installations/new?state=${state}` + `${githubHost}/apps/${appClientSlug}/installations/new?state=${state}` ); break; case GitHubConnectionMethod.OAuth: window.location.assign( - `https://github.com/login/oauth/authorize?client_id=${oauthClientId}&response_type=code&scope=repo,admin:org&redirect_uri=${window.location.origin}/organization/app-connections/github/oauth/callback&state=${state}` + `${githubHost}/login/oauth/authorize?client_id=${oauthClientId}&response_type=code&scope=repo,admin:org&redirect_uri=${window.location.origin}/organization/app-connections/github/oauth/callback&state=${state}` ); break; default: @@ -141,6 +175,81 @@ export const GitHubConnectionForm = ({ appConnection }: Props) => { )} /> + {subscription.gateway && ( + + + +
GitHub Enterprise Options
+ + + + {(isAllowed) => ( + ( + + +
+ +
+ + + )} + /> + )} + + ( + + + + )} + /> + + + + )}
+ + + )} + /> + )} + + )} { formState: { isSubmitting, isDirty } } = form; + const { subscription } = useSubscription(); const isPlatformManagedCredentials = appConnection?.isPlatformManagedCredentials ?? false; const { data: gateways, isPending: isGatewaysLoading } = useQuery(gatewaysQueryKeys.list()); @@ -96,55 +98,57 @@ export const MySqlConnectionForm = ({ appConnection, onSubmit }: Props) => { }} > {!isUpdate && } - - {(isAllowed) => ( - ( - - + {(isAllowed) => ( + ( + -
- - Internet Gateway - - {gateways?.map((el) => ( - - {el.name} + onChange(undefined)} + > + Internet Gateway - ))} - -
- - - )} - /> - )} - + {gateways?.map((el) => ( + + {el.name} + + ))} + + + + + )} + /> + )} + + )} { formState: { isSubmitting, isDirty } } = form; + const { subscription } = useSubscription(); const isPlatformManagedCredentials = appConnection?.isPlatformManagedCredentials ?? false; const { data: gateways, isPending: isGatewaysLoading } = useQuery(gatewaysQueryKeys.list()); @@ -96,55 +98,57 @@ export const OracleDBConnectionForm = ({ appConnection, onSubmit }: Props) => { }} > {!isUpdate && } - - {(isAllowed) => ( - ( - - + {(isAllowed) => ( + ( + -
- - Internet Gateway - - {gateways?.map((el) => ( - - {el.name} + onChange(undefined)} + > + Internet Gateway - ))} - -
- - - )} - /> - )} - + {gateways?.map((el) => ( + + {el.name} + + ))} + + + + + )} + /> + )} + + )} { formState: { isSubmitting, isDirty } } = form; + const { subscription } = useSubscription(); const isPlatformManagedCredentials = appConnection?.isPlatformManagedCredentials ?? false; const { data: gateways, isPending: isGatewaysLoading } = useQuery(gatewaysQueryKeys.list()); @@ -96,55 +98,57 @@ export const PostgresConnectionForm = ({ appConnection, onSubmit }: Props) => { }} > {!isUpdate && } - - {(isAllowed) => ( - ( - - + {(isAllowed) => ( + ( + -
- - Internet Gateway - - {gateways?.map((el) => ( - - {el.name} + onChange(undefined)} + > + Internet Gateway - ))} - -
- - - )} - /> - )} - + {gateways?.map((el) => ( + + {el.name} + + ))} + + + + + )} + /> + )} + + )} ; +type GithubFormData = BaseFormData & + Pick; type GithubRadarFormData = BaseFormData & Pick; @@ -395,7 +396,7 @@ export const OAuthCallbackPage = () => { clearState(AppConnection.GitHub); - const { connectionId, name, description, returnUrl } = formData; + const { connectionId, name, description, returnUrl, gatewayId, credentials } = formData; try { if (connectionId) { @@ -406,14 +407,18 @@ export const OAuthCallbackPage = () => { connectionId, credentials: { code: code as string, - installationId: installationId as string - } + installationId: installationId as string, + host: credentials.host + }, + gatewayId } : { connectionId, credentials: { - code: code as string - } + code: code as string, + host: credentials.host + }, + gatewayId }) }); } else { @@ -426,14 +431,18 @@ export const OAuthCallbackPage = () => { method: GitHubConnectionMethod.App, credentials: { code: code as string, - installationId: installationId as string - } + installationId: installationId as string, + host: credentials.host + }, + gatewayId } : { method: GitHubConnectionMethod.OAuth, credentials: { - code: code as string - } + code: code as string, + host: credentials.host + }, + gatewayId }) }); }