Merge pull request #4240 from Infisical/ENG-3368

feat(app-connection, secret-sync): Gateway for GitHub App Connections & Secret Syncs
This commit is contained in:
x032205
2025-07-29 16:53:36 -04:00
committed by GitHub
19 changed files with 749 additions and 437 deletions
@@ -14,6 +14,11 @@ export const blockLocalAndPrivateIpAddresses = async (url: string) => {
if (appCfg.isDevelopmentMode) return;
const validUrl = new URL(url);
if (validUrl.username || validUrl.password) {
throw new BadRequestError({ message: "URLs with user credentials (e.g., user:pass@) are not allowed" });
}
const inputHostIps: string[] = [];
if (isIPv4(validUrl.hostname)) {
inputHostIps.push(validUrl.hostname);
+11 -10
View File
@@ -1044,6 +1044,15 @@ export const registerRoutes = async (
kmsService
});
const gatewayService = gatewayServiceFactory({
permissionService,
gatewayDAL,
kmsService,
licenseService,
orgGatewayConfigDAL,
keyStore
});
const secretSyncQueue = secretSyncQueueFactory({
queueService,
secretSyncDAL,
@@ -1067,7 +1076,8 @@ export const registerRoutes = async (
secretVersionTagV2BridgeDAL,
resourceMetadataDAL,
appConnectionDAL,
licenseService
licenseService,
gatewayService
});
const secretQueueService = secretQueueFactory({
@@ -1490,15 +1500,6 @@ export const registerRoutes = async (
licenseService
});
const gatewayService = gatewayServiceFactory({
permissionService,
gatewayDAL,
kmsService,
licenseService,
orgGatewayConfigDAL,
keyStore
});
const identityKubernetesAuthService = identityKubernetesAuthServiceFactory({
identityKubernetesAuthDAL,
identityOrgMembershipDAL,
@@ -583,7 +583,7 @@ export const appConnectionServiceFactory = ({
deleteAppConnection,
connectAppConnectionById,
listAvailableAppConnectionsForUser,
github: githubConnectionService(connectAppConnectionById),
github: githubConnectionService(connectAppConnectionById, gatewayService),
githubRadar: githubRadarConnectionService(connectAppConnectionById),
gcp: gcpConnectionService(connectAppConnectionById),
databricks: databricksConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
@@ -1,10 +1,16 @@
import { createAppAuth } from "@octokit/auth-app";
import { Octokit } from "@octokit/rest";
import { AxiosResponse } from "axios";
import { AxiosError, AxiosRequestConfig, AxiosResponse } from "axios";
import https from "https";
import RE2 from "re2";
import { verifyHostInputValidity } from "@app/ee/services/dynamic-secret/dynamic-secret-fns";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { getConfig } from "@app/lib/config/env";
import { request } from "@app/lib/config/request";
import { request as httpRequest } from "@app/lib/config/request";
import { BadRequestError, ForbiddenRequestError, InternalServerError } from "@app/lib/errors";
import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
import { logger } from "@app/lib/logger";
import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
import { getAppConnectionMethodName } from "@app/services/app-connection/app-connection-fns";
import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
@@ -24,123 +30,224 @@ export const getGitHubConnectionListItem = () => {
};
};
export const getGitHubClient = (appConnection: TGitHubConnection) => {
export const requestWithGitHubGateway = async <T>(
appConnection: { gatewayId?: string | null },
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
requestConfig: AxiosRequestConfig
): Promise<AxiosResponse<T>> => {
const { gatewayId } = appConnection;
// If gateway isn't set up, don't proxy request
if (!gatewayId) {
return httpRequest.request(requestConfig);
}
const url = new URL(requestConfig.url as string);
await blockLocalAndPrivateIpAddresses(url.toString());
const [targetHost] = await verifyHostInputValidity(url.host, true);
const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(gatewayId);
const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
return withGatewayProxy(
async (proxyPort) => {
const httpsAgent = new https.Agent({
servername: targetHost
});
url.protocol = "https:";
url.host = `localhost:${proxyPort}`;
const finalRequestConfig: AxiosRequestConfig = {
...requestConfig,
url: url.toString(),
httpsAgent,
headers: {
...requestConfig.headers,
Host: targetHost
}
};
try {
return await httpRequest.request(finalRequestConfig);
} catch (error) {
const axiosError = error as AxiosError;
logger.error("Error during GitHub gateway request:", axiosError.message, axiosError.response?.data);
throw error;
}
},
{
protocol: GatewayProxyProtocol.Tcp,
targetHost,
targetPort: 443,
relayHost,
relayPort: Number(relayPort),
identityId: relayDetails.identityId,
orgId: relayDetails.orgId,
tlsOptions: {
ca: relayDetails.certChain,
cert: relayDetails.certificate,
key: relayDetails.privateKey.toString()
}
}
);
};
export const getGitHubAppAuthToken = async (appConnection: TGitHubConnection) => {
const appCfg = getConfig();
const { method, credentials } = appConnection;
let client: Octokit;
const appId = appCfg.INF_APP_CONNECTION_GITHUB_APP_ID;
const appPrivateKey = appCfg.INF_APP_CONNECTION_GITHUB_APP_PRIVATE_KEY;
switch (method) {
case GitHubConnectionMethod.App:
if (!appId || !appPrivateKey) {
throw new InternalServerError({
message: `GitHub ${getAppConnectionMethodName(method).replace("GitHub", "")} has not been configured`
});
}
client = new Octokit({
authStrategy: createAppAuth,
auth: {
appId,
privateKey: appPrivateKey,
installationId: credentials.installationId
}
});
break;
case GitHubConnectionMethod.OAuth:
client = new Octokit({
auth: credentials.accessToken
});
break;
default:
throw new InternalServerError({
message: `Unhandled GitHub connection method: ${method as GitHubConnectionMethod}`
});
if (!appId || !appPrivateKey) {
throw new InternalServerError({
message: `GitHub App keys are not configured.`
});
}
return client;
if (appConnection.method !== GitHubConnectionMethod.App) {
throw new InternalServerError({ message: "Cannot generate GitHub App token for non-app connection" });
}
const appAuth = createAppAuth({
appId,
privateKey: appPrivateKey,
installationId: appConnection.credentials.installationId
});
const { token } = await appAuth({ type: "installation" });
return token;
};
function extractNextPageUrl(linkHeader: string | undefined): string | null {
if (!linkHeader) return null;
const links = linkHeader.split(",");
const nextLink = links.find((link) => link.includes('rel="next"'));
if (!nextLink) return null;
const match = new RE2(/<([^>]+)>/).exec(nextLink);
return match ? match[1] : null;
}
export const makePaginatedGitHubRequest = async <T, R = T[]>(
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
path: string,
dataMapper?: (data: R) => T[]
): Promise<T[]> => {
const { credentials, method } = appConnection;
const token =
method === GitHubConnectionMethod.OAuth ? credentials.accessToken : await getGitHubAppAuthToken(appConnection);
let url: string | null = `https://api.${credentials.host || "github.com"}${path}`;
let results: T[] = [];
let i = 0;
while (url && i < 1000) {
// eslint-disable-next-line no-await-in-loop
const response: AxiosResponse<R> = await requestWithGitHubGateway<R>(appConnection, gatewayService, {
url,
method: "GET",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
}
});
const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]);
results = results.concat(items);
url = extractNextPageUrl(response.headers.link as string | undefined);
i += 1;
}
return results;
};
type GitHubOrganization = {
login: string;
id: number;
type: string;
};
type GitHubRepository = {
id: number;
name: string;
owner: GitHubOrganization;
permissions?: {
admin: boolean;
maintain: boolean;
push: boolean;
triage: boolean;
pull: boolean;
};
};
export const getGitHubRepositories = async (appConnection: TGitHubConnection) => {
const client = getGitHubClient(appConnection);
type GitHubEnvironment = {
id: number;
name: string;
};
let repositories: GitHubRepository[];
switch (appConnection.method) {
case GitHubConnectionMethod.App:
repositories = await client.paginate("GET /installation/repositories");
break;
case GitHubConnectionMethod.OAuth:
default:
repositories = (await client.paginate("GET /user/repos")).filter((repo) => repo.permissions?.admin);
break;
export const getGitHubRepositories = async (
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
if (appConnection.method === GitHubConnectionMethod.App) {
return makePaginatedGitHubRequest<GitHubRepository, { repositories: GitHubRepository[] }>(
appConnection,
gatewayService,
"/installation/repositories",
(data) => data.repositories
);
}
return repositories;
const repos = await makePaginatedGitHubRequest<GitHubRepository>(appConnection, gatewayService, "/user/repos");
return repos.filter((repo) => repo.permissions?.admin);
};
export const getGitHubOrganizations = async (appConnection: TGitHubConnection) => {
const client = getGitHubClient(appConnection);
export const getGitHubOrganizations = async (
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
if (appConnection.method === GitHubConnectionMethod.App) {
const installationRepositories = await makePaginatedGitHubRequest<
GitHubRepository,
{ repositories: GitHubRepository[] }
>(appConnection, gatewayService, "/installation/repositories", (data) => data.repositories);
let organizations: GitHubOrganization[];
switch (appConnection.method) {
case GitHubConnectionMethod.App: {
const installationRepositories = await client.paginate("GET /installation/repositories");
const organizationMap: Record<string, GitHubOrganization> = {};
installationRepositories.forEach((repo) => {
if (repo.owner.type === "Organization") {
organizationMap[repo.owner.id] = repo.owner;
}
});
organizations = Object.values(organizationMap);
break;
}
case GitHubConnectionMethod.OAuth:
default:
organizations = await client.paginate("GET /user/orgs");
break;
}
return organizations;
};
export const getGitHubEnvironments = async (appConnection: TGitHubConnection, owner: string, repo: string) => {
const client = getGitHubClient(appConnection);
try {
const environments = await client.paginate("GET /repos/{owner}/{repo}/environments", {
owner,
repo
const organizationMap: Record<string, GitHubOrganization> = {};
installationRepositories.forEach((repo) => {
if (repo.owner.type === "Organization") {
organizationMap[repo.owner.id] = repo.owner;
}
});
return environments;
} catch (e) {
// repo doesn't have envs
if ((e as { status: number }).status === 404) {
return [];
}
return Object.values(organizationMap);
}
throw e;
return makePaginatedGitHubRequest<GitHubOrganization>(appConnection, gatewayService, "/user/orgs");
};
export const getGitHubEnvironments = async (
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
owner: string,
repo: string
) => {
try {
return await makePaginatedGitHubRequest<GitHubEnvironment, { environments: GitHubEnvironment[] }>(
appConnection,
gatewayService,
`/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/environments`,
(data) => data.environments
);
} catch (error) {
const axiosError = error as AxiosError;
if (axiosError.response?.status === 404) return [];
throw error;
}
};
@@ -159,9 +266,11 @@ export function isGithubErrorResponse(data: GithubTokenRespData): data is Github
return "error" in data;
}
export const validateGitHubConnectionCredentials = async (config: TGitHubConnectionConfig) => {
export const validateGitHubConnectionCredentials = async (
config: TGitHubConnectionConfig,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const { credentials, method } = config;
const {
INF_APP_CONNECTION_GITHUB_OAUTH_CLIENT_ID,
INF_APP_CONNECTION_GITHUB_OAUTH_CLIENT_SECRET,
@@ -192,10 +301,13 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
}
let tokenResp: AxiosResponse<GithubTokenRespData>;
const host = credentials.host || "github.com";
try {
tokenResp = await request.get<GithubTokenRespData>("https://github.com/login/oauth/access_token", {
params: {
tokenResp = await requestWithGitHubGateway<GithubTokenRespData>(config, gatewayService, {
url: `https://${host}/login/oauth/access_token`,
method: "POST",
data: {
client_id: clientId,
client_secret: clientSecret,
code: credentials.code,
@@ -203,7 +315,7 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
},
headers: {
Accept: "application/json",
"Accept-Encoding": "application/json"
"Content-Type": "application/json"
}
});
@@ -233,7 +345,7 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
throw new InternalServerError({ message: `Missing access token: ${tokenResp.data.error}` });
}
const installationsResp = await request.get<{
const installationsResp = await requestWithGitHubGateway<{
installations: {
id: number;
account: {
@@ -242,7 +354,8 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
id: number;
};
}[];
}>(IntegrationUrls.GITHUB_USER_INSTALLATIONS, {
}>(config, gatewayService, {
url: IntegrationUrls.GITHUB_USER_INSTALLATIONS.replace("api.github.com", `api.${host}`),
headers: {
Accept: "application/json",
Authorization: `Bearer ${tokenResp.data.access_token}`,
@@ -11,20 +11,24 @@ import {
import { GitHubConnectionMethod } from "./github-connection-enums";
export const GitHubConnectionOAuthInputCredentialsSchema = z.object({
code: z.string().trim().min(1, "OAuth code required")
code: z.string().trim().min(1, "OAuth code required"),
host: z.string().trim().optional()
});
export const GitHubConnectionAppInputCredentialsSchema = z.object({
code: z.string().trim().min(1, "GitHub App code required"),
installationId: z.string().min(1, "GitHub App Installation ID required")
installationId: z.string().min(1, "GitHub App Installation ID required"),
host: z.string().trim().optional()
});
export const GitHubConnectionOAuthOutputCredentialsSchema = z.object({
accessToken: z.string()
accessToken: z.string(),
host: z.string().trim().optional()
});
export const GitHubConnectionAppOutputCredentialsSchema = z.object({
installationId: z.string()
installationId: z.string(),
host: z.string().trim().optional()
});
export const ValidateGitHubConnectionCredentialsSchema = z.discriminatedUnion("method", [
@@ -43,7 +47,9 @@ export const ValidateGitHubConnectionCredentialsSchema = z.discriminatedUnion("m
]);
export const CreateGitHubConnectionSchema = ValidateGitHubConnectionCredentialsSchema.and(
GenericCreateAppConnectionFieldsSchema(AppConnection.GitHub)
GenericCreateAppConnectionFieldsSchema(AppConnection.GitHub, {
supportsGateways: true
})
);
export const UpdateGitHubConnectionSchema = z
@@ -53,7 +59,11 @@ export const UpdateGitHubConnectionSchema = z
.optional()
.describe(AppConnections.UPDATE(AppConnection.GitHub).credentials)
})
.and(GenericUpdateAppConnectionFieldsSchema(AppConnection.GitHub));
.and(
GenericUpdateAppConnectionFieldsSchema(AppConnection.GitHub, {
supportsGateways: true
})
);
const BaseGitHubConnectionSchema = BaseAppConnectionSchema.extend({ app: z.literal(AppConnection.GitHub) });
@@ -1,3 +1,4 @@
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { OrgServiceActor } from "@app/lib/types";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import {
@@ -19,11 +20,14 @@ type TListGitHubEnvironmentsDTO = {
owner: string;
};
export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc) => {
export const githubConnectionService = (
getAppConnection: TGetAppConnectionFunc,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const listRepositories = async (connectionId: string, actor: OrgServiceActor) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const repositories = await getGitHubRepositories(appConnection);
const repositories = await getGitHubRepositories(appConnection, gatewayService);
return repositories;
};
@@ -31,7 +35,7 @@ export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc)
const listOrganizations = async (connectionId: string, actor: OrgServiceActor) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const organizations = await getGitHubOrganizations(appConnection);
const organizations = await getGitHubOrganizations(appConnection, gatewayService);
return organizations;
};
@@ -42,7 +46,7 @@ export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc)
) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const environments = await getGitHubEnvironments(appConnection, owner, repo);
const environments = await getGitHubEnvironments(appConnection, gatewayService, owner, repo);
return environments;
};
@@ -17,4 +17,7 @@ export type TGitHubConnectionInput = z.infer<typeof CreateGitHubConnectionSchema
export type TValidateGitHubConnectionCredentialsSchema = typeof ValidateGitHubConnectionCredentialsSchema;
export type TGitHubConnectionConfig = DiscriminativePick<TGitHubConnectionInput, "method" | "app" | "credentials">;
export type TGitHubConnectionConfig = DiscriminativePick<
TGitHubConnectionInput,
"method" | "app" | "credentials" | "gatewayId"
>;
@@ -1,7 +1,12 @@
import { Octokit } from "@octokit/rest";
import sodium from "libsodium-wrappers";
import { getGitHubClient } from "@app/services/app-connection/github";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import {
getGitHubAppAuthToken,
GitHubConnectionMethod,
makePaginatedGitHubRequest,
requestWithGitHubGateway
} from "@app/services/app-connection/github";
import { GitHubSyncScope, GitHubSyncVisibility } from "@app/services/secret-sync/github/github-sync-enums";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns";
@@ -12,155 +17,165 @@ import { TGitHubPublicKey, TGitHubSecret, TGitHubSecretPayload, TGitHubSyncWithC
// TODO: rate limit handling
const getEncryptedSecrets = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
let encryptedSecrets: TGitHubSecret[];
const { destinationConfig } = secretSync;
const getEncryptedSecrets = async (
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const { destinationConfig, connection } = secretSync;
let path: string;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
encryptedSecrets = await client.paginate("GET /orgs/{org}/actions/secrets", {
org: destinationConfig.org
});
path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets`;
break;
}
case GitHubSyncScope.Repository: {
encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/actions/secrets", {
owner: destinationConfig.owner,
repo: destinationConfig.repo
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets`;
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets`;
break;
}
}
return encryptedSecrets;
return makePaginatedGitHubRequest<TGitHubSecret, { secrets: TGitHubSecret[] }>(
connection,
gatewayService,
path,
(data) => data.secrets
);
};
const getPublicKey = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
let publicKey: TGitHubPublicKey;
const { destinationConfig } = secretSync;
const getPublicKey = async (
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
token: string
) => {
const { destinationConfig, connection } = secretSync;
let path: string;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
publicKey = (
await client.request("GET /orgs/{org}/actions/secrets/public-key", {
org: destinationConfig.org
})
).data;
path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/public-key`;
break;
}
case GitHubSyncScope.Repository: {
publicKey = (
await client.request("GET /repos/{owner}/{repo}/actions/secrets/public-key", {
owner: destinationConfig.owner,
repo: destinationConfig.repo
})
).data;
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/public-key`;
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
publicKey = (
await client.request("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets/public-key", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env
})
).data;
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/public-key`;
break;
}
}
return publicKey;
const response = await requestWithGitHubGateway<TGitHubPublicKey>(connection, gatewayService, {
url: `https://api.${connection.credentials.host || "github.com"}${path}`,
method: "GET",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
}
});
return response.data;
};
const deleteSecret = async (
client: Octokit,
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
token: string,
encryptedSecret: TGitHubSecret
) => {
const { destinationConfig } = secretSync;
const { destinationConfig, connection } = secretSync;
let path: string;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
await client.request(`DELETE /orgs/{org}/actions/secrets/{secret_name}`, {
org: destinationConfig.org,
secret_name: encryptedSecret.name
});
path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/${encodeURIComponent(encryptedSecret.name)}`;
break;
}
case GitHubSyncScope.Repository: {
await client.request("DELETE /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
secret_name: encryptedSecret.name
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/${encodeURIComponent(encryptedSecret.name)}`;
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
await client.request("DELETE /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env,
secret_name: encryptedSecret.name
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/${encodeURIComponent(encryptedSecret.name)}`;
break;
}
}
await requestWithGitHubGateway(connection, gatewayService, {
url: `https://api.${connection.credentials.host || "github.com"}${path}`,
method: "DELETE",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
}
});
};
const putSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, payload: TGitHubSecretPayload) => {
const { destinationConfig } = secretSync;
const putSecret = async (
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
token: string,
payload: TGitHubSecretPayload
) => {
const { destinationConfig, connection } = secretSync;
let path: string;
let body: Record<string, string | number[]> = payload;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
const { visibility, selectedRepositoryIds } = destinationConfig;
await client.request(`PUT /orgs/{org}/actions/secrets/{secret_name}`, {
org: destinationConfig.org,
path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/${encodeURIComponent(payload.secret_name)}`;
body = {
...payload,
visibility,
...(visibility === GitHubSyncVisibility.Selected && {
selected_repository_ids: selectedRepositoryIds
})
});
};
break;
}
case GitHubSyncScope.Repository: {
await client.request("PUT /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
...payload
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/${encodeURIComponent(payload.secret_name)}`;
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
await client.request("PUT /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env,
...payload
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/${encodeURIComponent(payload.secret_name)}`;
break;
}
}
await requestWithGitHubGateway(connection, gatewayService, {
url: `https://api.${connection.credentials.host || "github.com"}${path}`,
method: "PUT",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
},
data: body
});
};
export const GithubSyncFns = {
syncSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => {
syncSecrets: async (
secretSync: TGitHubSyncWithCredentials,
ogSecretMap: TSecretMap,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const secretMap = Object.fromEntries(Object.entries(ogSecretMap).map(([i, v]) => [i.toUpperCase(), v]));
switch (secretSync.destinationConfig.scope) {
case GitHubSyncScope.Organization:
if (Object.values(secretMap).length > 1000) {
@@ -187,38 +202,40 @@ export const GithubSyncFns = {
);
}
const client = getGitHubClient(secretSync.connection);
const { connection } = secretSync;
const token =
connection.method === GitHubConnectionMethod.OAuth
? connection.credentials.accessToken
: await getGitHubAppAuthToken(connection);
const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService);
const publicKey = await getPublicKey(secretSync, gatewayService, token);
const publicKey = await getPublicKey(client, secretSync);
await sodium.ready;
for await (const key of Object.keys(secretMap)) {
// convert secret & base64 key to Uint8Array.
const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL);
const binarySecretValue = sodium.from_string(secretMap[key].value);
await sodium.ready.then(async () => {
for await (const key of Object.keys(secretMap)) {
// convert secret & base64 key to Uint8Array.
const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL);
const binarySecretValue = sodium.from_string(secretMap[key].value);
// encrypt secret using libsodium
const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey);
// encrypt secret using libsodium
const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey);
// convert encrypted Uint8Array to base64
const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL);
// convert encrypted Uint8Array to base64
const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL);
try {
await putSecret(client, secretSync, {
secret_name: key,
encrypted_value: encryptedSecretValue,
key_id: publicKey.key_id
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
try {
await putSecret(secretSync, gatewayService, token, {
secret_name: key,
encrypted_value: encryptedSecretValue,
key_id: publicKey.key_id
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
});
}
if (secretSync.syncOptions.disableSecretDeletion) return;
@@ -228,21 +245,31 @@ export const GithubSyncFns = {
continue;
if (!(encryptedSecret.name in secretMap)) {
await deleteSecret(client, secretSync, encryptedSecret);
await deleteSecret(secretSync, gatewayService, token, encryptedSecret);
}
}
},
getSecrets: async (secretSync: TGitHubSyncWithCredentials) => {
throw new Error(`${SECRET_SYNC_NAME_MAP[secretSync.destination]} does not support importing secrets.`);
},
removeSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => {
const client = getGitHubClient(secretSync.connection);
removeSecrets: async (
secretSync: TGitHubSyncWithCredentials,
ogSecretMap: TSecretMap,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const secretMap = Object.fromEntries(Object.entries(ogSecretMap).map(([i, v]) => [i.toUpperCase(), v]));
const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
const { connection } = secretSync;
const token =
connection.method === GitHubConnectionMethod.OAuth
? connection.credentials.accessToken
: await getGitHubAppAuthToken(connection);
const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService);
for await (const encryptedSecret of encryptedSecrets) {
if (encryptedSecret.name in secretMap) {
await deleteSecret(client, secretSync, encryptedSecret);
await deleteSecret(secretSync, gatewayService, token, encryptedSecret);
}
}
}
@@ -1,6 +1,7 @@
import { AxiosError } from "axios";
import handlebars from "handlebars";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { OCI_VAULT_SYNC_LIST_OPTION, OCIVaultSyncFns } from "@app/ee/services/secret-sync/oci-vault";
import { BadRequestError } from "@app/lib/errors";
@@ -97,6 +98,7 @@ export const listSecretSyncOptions = () => {
type TSyncSecretDeps = {
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update" | "updateById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
};
// Add schema to secret keys
@@ -191,7 +193,7 @@ export const SecretSyncFns = {
syncSecrets: (
secretSync: TSecretSyncWithCredentials,
secretMap: TSecretMap,
{ kmsService, appConnectionDAL }: TSyncSecretDeps
{ kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps
): Promise<void> => {
const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema);
@@ -201,7 +203,7 @@ export const SecretSyncFns = {
case SecretSync.AWSSecretsManager:
return AwsSecretsManagerSyncFns.syncSecrets(secretSync, schemaSecretMap);
case SecretSync.GitHub:
return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap);
return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap, gatewayService);
case SecretSync.GCPSecretManager:
return GcpSyncFns.syncSecrets(secretSync, schemaSecretMap);
case SecretSync.AzureKeyVault:
@@ -395,7 +397,7 @@ export const SecretSyncFns = {
removeSecrets: (
secretSync: TSecretSyncWithCredentials,
secretMap: TSecretMap,
{ kmsService, appConnectionDAL }: TSyncSecretDeps
{ kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps
): Promise<void> => {
const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema);
@@ -405,7 +407,7 @@ export const SecretSyncFns = {
case SecretSync.AWSSecretsManager:
return AwsSecretsManagerSyncFns.removeSecrets(secretSync, schemaSecretMap);
case SecretSync.GitHub:
return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap);
return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap, gatewayService);
case SecretSync.GCPSecretManager:
return GcpSyncFns.removeSecrets(secretSync, schemaSecretMap);
case SecretSync.AzureKeyVault:
@@ -4,6 +4,7 @@ import { Job } from "bullmq";
import { ProjectMembershipRole, SecretType } from "@app/db/schemas";
import { EventType, TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-types";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
@@ -96,6 +97,7 @@ type TSecretSyncQueueFactoryDep = {
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
folderCommitService: Pick<TFolderCommitServiceFactory, "createCommit">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
};
type SecretSyncActionJob = Job<
@@ -138,7 +140,8 @@ export const secretSyncQueueFactory = ({
secretVersionTagV2BridgeDAL,
resourceMetadataDAL,
folderCommitService,
licenseService
licenseService,
gatewayService
}: TSecretSyncQueueFactoryDep) => {
const appCfg = getConfig();
@@ -353,7 +356,8 @@ export const secretSyncQueueFactory = ({
const importedSecrets = await SecretSyncFns.getSecrets(secretSync, {
appConnectionDAL,
kmsService
kmsService,
gatewayService
});
if (!Object.keys(importedSecrets).length) return {};
@@ -481,7 +485,8 @@ export const secretSyncQueueFactory = ({
await SecretSyncFns.syncSecrets(secretSyncWithCredentials, secretMap, {
appConnectionDAL,
kmsService
kmsService,
gatewayService
});
isSynced = true;
@@ -730,7 +735,8 @@ export const secretSyncQueueFactory = ({
secretMap,
{
appConnectionDAL,
kmsService
kmsService,
gatewayService
}
);