Merge pull request #4240 from Infisical/ENG-3368

feat(app-connection, secret-sync): Gateway for GitHub App Connections & Secret Syncs

backend/src/services/secret-sync/github/github-sync-fns.ts

@@ -1,7 +1,12 @@
-import { Octokit } from "@octokit/rest";
 import sodium from "libsodium-wrappers";
 
-import { getGitHubClient } from "@app/services/app-connection/github";
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
+import {
+  getGitHubAppAuthToken,
+  GitHubConnectionMethod,
+  makePaginatedGitHubRequest,
+  requestWithGitHubGateway
+} from "@app/services/app-connection/github";
 import { GitHubSyncScope, GitHubSyncVisibility } from "@app/services/secret-sync/github/github-sync-enums";
 import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
 import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns";
@@ -12,155 +17,165 @@ import { TGitHubPublicKey, TGitHubSecret, TGitHubSecretPayload, TGitHubSyncWithC
 
 // TODO: rate limit handling
 
-const getEncryptedSecrets = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
-  let encryptedSecrets: TGitHubSecret[];
-
-  const { destinationConfig } = secretSync;
+const getEncryptedSecrets = async (
+  secretSync: TGitHubSyncWithCredentials,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
+  const { destinationConfig, connection } = secretSync;
 
+  let path: string;
   switch (destinationConfig.scope) {
     case GitHubSyncScope.Organization: {
-      encryptedSecrets = await client.paginate("GET /orgs/{org}/actions/secrets", {
-        org: destinationConfig.org
-      });
+      path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets`;
       break;
     }
     case GitHubSyncScope.Repository: {
-      encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/actions/secrets", {
-        owner: destinationConfig.owner,
-        repo: destinationConfig.repo
-      });
-
+      path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets`;
       break;
     }
     case GitHubSyncScope.RepositoryEnvironment:
     default: {
-      encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets", {
-        owner: destinationConfig.owner,
-        repo: destinationConfig.repo,
-        environment_name: destinationConfig.env
-      });
+      path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets`;
       break;
     }
   }
 
-  return encryptedSecrets;
+  return makePaginatedGitHubRequest<TGitHubSecret, { secrets: TGitHubSecret[] }>(
+    connection,
+    gatewayService,
+    path,
+    (data) => data.secrets
+  );
 };
 
-const getPublicKey = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
-  let publicKey: TGitHubPublicKey;
-
-  const { destinationConfig } = secretSync;
+const getPublicKey = async (
+  secretSync: TGitHubSyncWithCredentials,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
+  token: string
+) => {
+  const { destinationConfig, connection } = secretSync;
 
+  let path: string;
   switch (destinationConfig.scope) {
     case GitHubSyncScope.Organization: {
-      publicKey = (
-        await client.request("GET /orgs/{org}/actions/secrets/public-key", {
-          org: destinationConfig.org
-        })
-      ).data;
+      path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/public-key`;
       break;
     }
     case GitHubSyncScope.Repository: {
-      publicKey = (
-        await client.request("GET /repos/{owner}/{repo}/actions/secrets/public-key", {
-          owner: destinationConfig.owner,
-          repo: destinationConfig.repo
-        })
-      ).data;
+      path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/public-key`;
       break;
     }
     case GitHubSyncScope.RepositoryEnvironment:
     default: {
-      publicKey = (
-        await client.request("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets/public-key", {
-          owner: destinationConfig.owner,
-          repo: destinationConfig.repo,
-          environment_name: destinationConfig.env
-        })
-      ).data;
+      path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/public-key`;
       break;
     }
   }
 
-  return publicKey;
+  const response = await requestWithGitHubGateway<TGitHubPublicKey>(connection, gatewayService, {
+    url: `https://api.${connection.credentials.host || "github.com"}${path}`,
+    method: "GET",
+    headers: {
+      Accept: "application/vnd.github+json",
+      Authorization: `Bearer ${token}`,
+      "X-GitHub-Api-Version": "2022-11-28"
+    }
+  });
+
+  return response.data;
 };
 
 const deleteSecret = async (
-  client: Octokit,
   secretSync: TGitHubSyncWithCredentials,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
+  token: string,
   encryptedSecret: TGitHubSecret
 ) => {
-  const { destinationConfig } = secretSync;
+  const { destinationConfig, connection } = secretSync;
 
+  let path: string;
   switch (destinationConfig.scope) {
     case GitHubSyncScope.Organization: {
-      await client.request(`DELETE /orgs/{org}/actions/secrets/{secret_name}`, {
-        org: destinationConfig.org,
-        secret_name: encryptedSecret.name
-      });
+      path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/${encodeURIComponent(encryptedSecret.name)}`;
       break;
     }
     case GitHubSyncScope.Repository: {
-      await client.request("DELETE /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
-        owner: destinationConfig.owner,
-        repo: destinationConfig.repo,
-        secret_name: encryptedSecret.name
-      });
+      path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/${encodeURIComponent(encryptedSecret.name)}`;
       break;
     }
     case GitHubSyncScope.RepositoryEnvironment:
     default: {
-      await client.request("DELETE /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", {
-        owner: destinationConfig.owner,
-        repo: destinationConfig.repo,
-        environment_name: destinationConfig.env,
-        secret_name: encryptedSecret.name
-      });
+      path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/${encodeURIComponent(encryptedSecret.name)}`;
       break;
     }
   }
+
+  await requestWithGitHubGateway(connection, gatewayService, {
+    url: `https://api.${connection.credentials.host || "github.com"}${path}`,
+    method: "DELETE",
+    headers: {
+      Accept: "application/vnd.github+json",
+      Authorization: `Bearer ${token}`,
+      "X-GitHub-Api-Version": "2022-11-28"
+    }
+  });
 };
 
-const putSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, payload: TGitHubSecretPayload) => {
-  const { destinationConfig } = secretSync;
+const putSecret = async (
+  secretSync: TGitHubSyncWithCredentials,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
+  token: string,
+  payload: TGitHubSecretPayload
+) => {
+  const { destinationConfig, connection } = secretSync;
+
+  let path: string;
+  let body: Record<string, string | number[]> = payload;
 
   switch (destinationConfig.scope) {
     case GitHubSyncScope.Organization: {
       const { visibility, selectedRepositoryIds } = destinationConfig;
-
-      await client.request(`PUT /orgs/{org}/actions/secrets/{secret_name}`, {
-        org: destinationConfig.org,
+      path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/${encodeURIComponent(payload.secret_name)}`;
+      body = {
         ...payload,
         visibility,
         ...(visibility === GitHubSyncVisibility.Selected && {
           selected_repository_ids: selectedRepositoryIds
         })
-      });
+      };
       break;
     }
     case GitHubSyncScope.Repository: {
-      await client.request("PUT /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
-        owner: destinationConfig.owner,
-        repo: destinationConfig.repo,
-        ...payload
-      });
+      path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/${encodeURIComponent(payload.secret_name)}`;
       break;
     }
     case GitHubSyncScope.RepositoryEnvironment:
     default: {
-      await client.request("PUT /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", {
-        owner: destinationConfig.owner,
-        repo: destinationConfig.repo,
-        environment_name: destinationConfig.env,
-        ...payload
-      });
+      path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/${encodeURIComponent(payload.secret_name)}`;
       break;
     }
   }
+
+  await requestWithGitHubGateway(connection, gatewayService, {
+    url: `https://api.${connection.credentials.host || "github.com"}${path}`,
+    method: "PUT",
+    headers: {
+      Accept: "application/vnd.github+json",
+      Authorization: `Bearer ${token}`,
+      "X-GitHub-Api-Version": "2022-11-28"
+    },
+    data: body
+  });
 };
 
 export const GithubSyncFns = {
-  syncSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => {
+  syncSecrets: async (
+    secretSync: TGitHubSyncWithCredentials,
+    ogSecretMap: TSecretMap,
+    gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+  ) => {
+    const secretMap = Object.fromEntries(Object.entries(ogSecretMap).map(([i, v]) => [i.toUpperCase(), v]));
+
     switch (secretSync.destinationConfig.scope) {
       case GitHubSyncScope.Organization:
         if (Object.values(secretMap).length > 1000) {
@@ -187,38 +202,40 @@ export const GithubSyncFns = {
         );
     }
 
-    const client = getGitHubClient(secretSync.connection);
+    const { connection } = secretSync;
+    const token =
+      connection.method === GitHubConnectionMethod.OAuth
+        ? connection.credentials.accessToken
+        : await getGitHubAppAuthToken(connection);
 
-    const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
+    const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService);
+    const publicKey = await getPublicKey(secretSync, gatewayService, token);
 
-    const publicKey = await getPublicKey(client, secretSync);
+    await sodium.ready;
+    for await (const key of Object.keys(secretMap)) {
+      // convert secret & base64 key to Uint8Array.
+      const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL);
+      const binarySecretValue = sodium.from_string(secretMap[key].value);
 
-    await sodium.ready.then(async () => {
-      for await (const key of Object.keys(secretMap)) {
-        // convert secret & base64 key to Uint8Array.
-        const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL);
-        const binarySecretValue = sodium.from_string(secretMap[key].value);
+      // encrypt secret using libsodium
+      const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey);
 
-        // encrypt secret using libsodium
-        const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey);
+      // convert encrypted Uint8Array to base64
+      const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL);
 
-        // convert encrypted Uint8Array to base64
-        const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL);
-
-        try {
-          await putSecret(client, secretSync, {
-            secret_name: key,
-            encrypted_value: encryptedSecretValue,
-            key_id: publicKey.key_id
-          });
-        } catch (error) {
-          throw new SecretSyncError({
-            error,
-            secretKey: key
-          });
-        }
+      try {
+        await putSecret(secretSync, gatewayService, token, {
+          secret_name: key,
+          encrypted_value: encryptedSecretValue,
+          key_id: publicKey.key_id
+        });
+      } catch (error) {
+        throw new SecretSyncError({
+          error,
+          secretKey: key
+        });
       }
-    });
+    }
 
     if (secretSync.syncOptions.disableSecretDeletion) return;
 
@@ -228,21 +245,31 @@ export const GithubSyncFns = {
         continue;
 
       if (!(encryptedSecret.name in secretMap)) {
-        await deleteSecret(client, secretSync, encryptedSecret);
+        await deleteSecret(secretSync, gatewayService, token, encryptedSecret);
       }
     }
   },
   getSecrets: async (secretSync: TGitHubSyncWithCredentials) => {
     throw new Error(`${SECRET_SYNC_NAME_MAP[secretSync.destination]} does not support importing secrets.`);
   },
-  removeSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => {
-    const client = getGitHubClient(secretSync.connection);
+  removeSecrets: async (
+    secretSync: TGitHubSyncWithCredentials,
+    ogSecretMap: TSecretMap,
+    gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+  ) => {
+    const secretMap = Object.fromEntries(Object.entries(ogSecretMap).map(([i, v]) => [i.toUpperCase(), v]));
 
-    const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
+    const { connection } = secretSync;
+    const token =
+      connection.method === GitHubConnectionMethod.OAuth
+        ? connection.credentials.accessToken
+        : await getGitHubAppAuthToken(connection);
+
+    const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService);
 
     for await (const encryptedSecret of encryptedSecrets) {
       if (encryptedSecret.name in secretMap) {
-        await deleteSecret(client, secretSync, encryptedSecret);
+        await deleteSecret(secretSync, gatewayService, token, encryptedSecret);
       }
     }
   }

backend/src/services/secret-sync/secret-sync-fns.ts

@@ -1,6 +1,7 @@
 import { AxiosError } from "axios";
 import handlebars from "handlebars";
 
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { OCI_VAULT_SYNC_LIST_OPTION, OCIVaultSyncFns } from "@app/ee/services/secret-sync/oci-vault";
 import { BadRequestError } from "@app/lib/errors";
@@ -97,6 +98,7 @@ export const listSecretSyncOptions = () => {
 type TSyncSecretDeps = {
   appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update" | "updateById">;
   kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
 };
 
 // Add schema to secret keys
@@ -191,7 +193,7 @@ export const SecretSyncFns = {
   syncSecrets: (
     secretSync: TSecretSyncWithCredentials,
     secretMap: TSecretMap,
-    { kmsService, appConnectionDAL }: TSyncSecretDeps
+    { kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps
   ): Promise<void> => {
     const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema);
 
@@ -201,7 +203,7 @@ export const SecretSyncFns = {
       case SecretSync.AWSSecretsManager:
         return AwsSecretsManagerSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.GitHub:
-        return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap);
+        return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap, gatewayService);
       case SecretSync.GCPSecretManager:
         return GcpSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.AzureKeyVault:
@@ -395,7 +397,7 @@ export const SecretSyncFns = {
   removeSecrets: (
     secretSync: TSecretSyncWithCredentials,
     secretMap: TSecretMap,
-    { kmsService, appConnectionDAL }: TSyncSecretDeps
+    { kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps
   ): Promise<void> => {
     const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema);
 
@@ -405,7 +407,7 @@ export const SecretSyncFns = {
       case SecretSync.AWSSecretsManager:
         return AwsSecretsManagerSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.GitHub:
-        return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap);
+        return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap, gatewayService);
       case SecretSync.GCPSecretManager:
         return GcpSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.AzureKeyVault:

backend/src/services/secret-sync/secret-sync-queue.ts

@@ -4,6 +4,7 @@ import { Job } from "bullmq";
 
 import { ProjectMembershipRole, SecretType } from "@app/db/schemas";
 import { EventType, TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-types";
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig } from "@app/lib/config/env";
@@ -96,6 +97,7 @@ type TSecretSyncQueueFactoryDep = {
   resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
   folderCommitService: Pick<TFolderCommitServiceFactory, "createCommit">;
   licenseService: Pick<TLicenseServiceFactory, "getPlan">;
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
 };
 
 type SecretSyncActionJob = Job<
@@ -138,7 +140,8 @@ export const secretSyncQueueFactory = ({
   secretVersionTagV2BridgeDAL,
   resourceMetadataDAL,
   folderCommitService,
-  licenseService
+  licenseService,
+  gatewayService
 }: TSecretSyncQueueFactoryDep) => {
   const appCfg = getConfig();
 
@@ -353,7 +356,8 @@ export const secretSyncQueueFactory = ({
 
     const importedSecrets = await SecretSyncFns.getSecrets(secretSync, {
       appConnectionDAL,
-      kmsService
+      kmsService,
+      gatewayService
     });
 
     if (!Object.keys(importedSecrets).length) return {};
@@ -481,7 +485,8 @@ export const secretSyncQueueFactory = ({
 
       await SecretSyncFns.syncSecrets(secretSyncWithCredentials, secretMap, {
         appConnectionDAL,
-        kmsService
+        kmsService,
+        gatewayService
       });
 
       isSynced = true;
@@ -730,7 +735,8 @@ export const secretSyncQueueFactory = ({
         secretMap,
         {
           appConnectionDAL,
-          kmsService
+          kmsService,
+          gatewayService
         }
       );