feature: secret scanning architecture and github data source (wip)

backend/src/services/app-connection/app-connection-enums.ts

@@ -1,5 +1,6 @@
 export enum AppConnection {
   GitHub = "github",
+  GitHubRadar = "github-radar",
   AWS = "aws",
   Databricks = "databricks",
   GCP = "gcp",

backend/src/services/app-connection/app-connection-fns.ts

@@ -41,6 +41,11 @@ import {
 } from "./databricks";
 import { GcpConnectionMethod, getGcpConnectionListItem, validateGcpConnectionCredentials } from "./gcp";
 import { getGitHubConnectionListItem, GitHubConnectionMethod, validateGitHubConnectionCredentials } from "./github";
+import {
+  getGitHubRadarConnectionListItem,
+  GitHubRadarConnectionMethod,
+  validateGitHubRadarConnectionCredentials
+} from "./github-radar";
 import {
   getHCVaultConnectionListItem,
   HCVaultConnectionMethod,
@@ -77,6 +82,7 @@ export const listAppConnectionOptions = () => {
   return [
     getAwsConnectionListItem(),
     getGitHubConnectionListItem(),
+    getGitHubRadarConnectionListItem(),
     getGcpConnectionListItem(),
     getAzureKeyVaultConnectionListItem(),
     getAzureAppConfigurationConnectionListItem(),
@@ -146,6 +152,7 @@ export const validateAppConnectionCredentials = async (
     [AppConnection.AWS]: validateAwsConnectionCredentials as TAppConnectionCredentialsValidator,
     [AppConnection.Databricks]: validateDatabricksConnectionCredentials as TAppConnectionCredentialsValidator,
     [AppConnection.GitHub]: validateGitHubConnectionCredentials as TAppConnectionCredentialsValidator,
+    [AppConnection.GitHubRadar]: validateGitHubRadarConnectionCredentials as TAppConnectionCredentialsValidator,
     [AppConnection.GCP]: validateGcpConnectionCredentials as TAppConnectionCredentialsValidator,
     [AppConnection.AzureKeyVault]: validateAzureKeyVaultConnectionCredentials as TAppConnectionCredentialsValidator,
     [AppConnection.AzureAppConfiguration]:
@@ -173,6 +180,8 @@ export const getAppConnectionMethodName = (method: TAppConnection["method"]) =>
   switch (method) {
     case GitHubConnectionMethod.App:
       return "GitHub App";
+    case GitHubRadarConnectionMethod.App:
+      return "GitHub App";
     case AzureKeyVaultConnectionMethod.OAuth:
     case AzureAppConfigurationConnectionMethod.OAuth:
     case AzureClientSecretsConnectionMethod.OAuth:
@@ -240,6 +249,7 @@ export const TRANSITION_CONNECTION_CREDENTIALS_TO_PLATFORM: Record<
   [AppConnection.AWS]: platformManagedCredentialsNotSupported,
   [AppConnection.Databricks]: platformManagedCredentialsNotSupported,
   [AppConnection.GitHub]: platformManagedCredentialsNotSupported,
+  [AppConnection.GitHubRadar]: platformManagedCredentialsNotSupported,
   [AppConnection.GCP]: platformManagedCredentialsNotSupported,
   [AppConnection.AzureKeyVault]: platformManagedCredentialsNotSupported,
   [AppConnection.AzureAppConfiguration]: platformManagedCredentialsNotSupported,

backend/src/services/app-connection/app-connection-maps.ts

@@ -3,6 +3,7 @@ import { AppConnection } from "./app-connection-enums";
 export const APP_CONNECTION_NAME_MAP: Record<AppConnection, string> = {
   [AppConnection.AWS]: "AWS",
   [AppConnection.GitHub]: "GitHub",
+  [AppConnection.GitHubRadar]: "GitHub Radar",
   [AppConnection.GCP]: "GCP",
   [AppConnection.AzureKeyVault]: "Azure Key Vault",
   [AppConnection.AzureAppConfiguration]: "Azure App Configuration",

backend/src/services/app-connection/app-connection-service.ts

@@ -15,6 +15,7 @@ import {
   validateAppConnectionCredentials
 } from "@app/services/app-connection/app-connection-fns";
 import { auth0ConnectionService } from "@app/services/app-connection/auth0/auth0-connection-service";
+import { githubRadarConnectionService } from "@app/services/app-connection/github-radar/github-radar-connection-service";
 import { TKmsServiceFactory } from "@app/services/kms/kms-service";
 
 import { TAppConnectionDALFactory } from "./app-connection-dal";
@@ -43,6 +44,7 @@ import { ValidateGcpConnectionCredentialsSchema } from "./gcp";
 import { gcpConnectionService } from "./gcp/gcp-connection-service";
 import { ValidateGitHubConnectionCredentialsSchema } from "./github";
 import { githubConnectionService } from "./github/github-connection-service";
+import { ValidateGitHubRadarConnectionCredentialsSchema } from "./github-radar";
 import { ValidateHCVaultConnectionCredentialsSchema } from "./hc-vault";
 import { hcVaultConnectionService } from "./hc-vault/hc-vault-connection-service";
 import { ValidateHumanitecConnectionCredentialsSchema } from "./humanitec";
@@ -72,6 +74,7 @@ export type TAppConnectionServiceFactory = ReturnType<typeof appConnectionServic
 const VALIDATE_APP_CONNECTION_CREDENTIALS_MAP: Record<AppConnection, TValidateAppConnectionCredentialsSchema> = {
   [AppConnection.AWS]: ValidateAwsConnectionCredentialsSchema,
   [AppConnection.GitHub]: ValidateGitHubConnectionCredentialsSchema,
+  [AppConnection.GitHubRadar]: ValidateGitHubRadarConnectionCredentialsSchema,
   [AppConnection.GCP]: ValidateGcpConnectionCredentialsSchema,
   [AppConnection.AzureKeyVault]: ValidateAzureKeyVaultConnectionCredentialsSchema,
   [AppConnection.AzureAppConfiguration]: ValidateAzureAppConfigurationConnectionCredentialsSchema,
@@ -456,6 +459,7 @@ export const appConnectionServiceFactory = ({
     connectAppConnectionById,
     listAvailableAppConnectionsForUser,
     github: githubConnectionService(connectAppConnectionById),
+    githubRadar: githubRadarConnectionService(connectAppConnectionById),
     gcp: gcpConnectionService(connectAppConnectionById),
     databricks: databricksConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
     aws: awsConnectionService(connectAppConnectionById),

backend/src/services/app-connection/app-connection-types.ts

@@ -57,6 +57,12 @@ import {
   TGitHubConnectionInput,
   TValidateGitHubConnectionCredentialsSchema
 } from "./github";
+import {
+  TGitHubRadarConnection,
+  TGitHubRadarConnectionConfig,
+  TGitHubRadarConnectionInput,
+  TValidateGitHubRadarConnectionCredentialsSchema
+} from "./github-radar";
 import {
   THCVaultConnection,
   THCVaultConnectionConfig,
@@ -115,6 +121,7 @@ import {
 export type TAppConnection = { id: string } & (
   | TAwsConnection
   | TGitHubConnection
+  | TGitHubRadarConnection
   | TGcpConnection
   | TAzureKeyVaultConnection
   | TAzureAppConfigurationConnection
@@ -141,6 +148,7 @@ export type TSqlConnection = TPostgresConnection | TMsSqlConnection;
 export type TAppConnectionInput = { id: string } & (
   | TAwsConnectionInput
   | TGitHubConnectionInput
+  | TGitHubRadarConnectionInput
   | TGcpConnectionInput
   | TAzureKeyVaultConnectionInput
   | TAzureAppConfigurationConnectionInput
@@ -174,6 +182,7 @@ export type TUpdateAppConnectionDTO = Partial<Omit<TCreateAppConnectionDTO, "met
 export type TAppConnectionConfig =
   | TAwsConnectionConfig
   | TGitHubConnectionConfig
+  | TGitHubRadarConnectionConfig
   | TGcpConnectionConfig
   | TAzureKeyVaultConnectionConfig
   | TAzureAppConfigurationConnectionConfig
@@ -194,6 +203,7 @@ export type TAppConnectionConfig =
 export type TValidateAppConnectionCredentialsSchema =
   | TValidateAwsConnectionCredentialsSchema
   | TValidateGitHubConnectionCredentialsSchema
+  | TValidateGitHubRadarConnectionCredentialsSchema
   | TValidateGcpConnectionCredentialsSchema
   | TValidateAzureKeyVaultConnectionCredentialsSchema
   | TValidateAzureAppConfigurationConnectionCredentialsSchema

backend/src/services/app-connection/github-radar/github-radar-connection-enums.ts

@@ -0,0 +1,3 @@
+export enum GitHubRadarConnectionMethod {
+  App = "github-app"
+}

backend/src/services/app-connection/github-radar/github-radar-connection-fns.ts

@@ -0,0 +1,166 @@
+import { createAppAuth } from "@octokit/auth-app";
+import { Octokit } from "@octokit/rest";
+import { AxiosResponse } from "axios";
+
+import { getConfig } from "@app/lib/config/env";
+import { request } from "@app/lib/config/request";
+import { BadRequestError, ForbiddenRequestError, InternalServerError } from "@app/lib/errors";
+import { getAppConnectionMethodName } from "@app/services/app-connection/app-connection-fns";
+import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
+
+import { AppConnection } from "../app-connection-enums";
+import { GitHubRadarConnectionMethod } from "./github-radar-connection-enums";
+import {
+  TGitHubRadarConnection,
+  TGitHubRadarConnectionConfig,
+  TGitHubRadarRepository
+} from "./github-radar-connection-types";
+
+export const getGitHubRadarConnectionListItem = () => {
+  const { INF_APP_CONNECTION_GITHUB_RADAR_APP_SLUG } = getConfig();
+
+  return {
+    name: "GitHub Radar" as const,
+    app: AppConnection.GitHubRadar as const,
+    methods: Object.values(GitHubRadarConnectionMethod) as [GitHubRadarConnectionMethod.App],
+    appClientSlug: INF_APP_CONNECTION_GITHUB_RADAR_APP_SLUG
+  };
+};
+
+export const getGitHubRadarClient = (appConnection: TGitHubRadarConnection) => {
+  const appCfg = getConfig();
+
+  const { method, credentials } = appConnection;
+
+  let client: Octokit;
+
+  switch (method) {
+    case GitHubRadarConnectionMethod.App:
+      if (!appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID || !appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY) {
+        throw new InternalServerError({
+          message: `GitHub ${getAppConnectionMethodName(method).replace(
+            "GitHub",
+            ""
+          )} environment variables have not been configured`
+        });
+      }
+
+      client = new Octokit({
+        authStrategy: createAppAuth,
+        auth: {
+          appId: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID,
+          privateKey: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY,
+          installationId: credentials.installationId
+        }
+      });
+      break;
+    default:
+      throw new InternalServerError({
+        message: `Unhandled GitHub Radar connection method: ${method as GitHubRadarConnectionMethod}`
+      });
+  }
+
+  return client;
+};
+
+export const listGitHubRadarRepositories = async (appConnection: TGitHubRadarConnection) => {
+  const client = getGitHubRadarClient(appConnection);
+
+  const repositories: TGitHubRadarRepository[] = await client.paginate("GET /installation/repositories");
+
+  return repositories;
+};
+
+type TokenRespData = {
+  access_token: string;
+  scope: string;
+  token_type: string;
+  error?: string;
+};
+
+export const validateGitHubRadarConnectionCredentials = async (config: TGitHubRadarConnectionConfig) => {
+  const { credentials, method } = config;
+
+  const { INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_ID, INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_SECRET, SITE_URL } =
+    getConfig();
+
+  if (!INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_ID || !INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_SECRET) {
+    throw new InternalServerError({
+      message: `GitHub ${getAppConnectionMethodName(method).replace(
+        "GitHub",
+        ""
+      )} environment variables have not been configured`
+    });
+  }
+
+  let tokenResp: AxiosResponse<TokenRespData>;
+
+  try {
+    tokenResp = await request.get<TokenRespData>("https://github.com/login/oauth/access_token", {
+      params: {
+        client_id: INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_ID,
+        client_secret: INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_SECRET,
+        code: credentials.code,
+        redirect_uri: `${SITE_URL}/organization/app-connections/github-radar/oauth/callback`
+      },
+      headers: {
+        Accept: "application/json",
+        "Accept-Encoding": "application/json"
+      }
+    });
+  } catch (e: unknown) {
+    throw new BadRequestError({
+      message: `Unable to validate connection: verify credentials`
+    });
+  }
+
+  if (tokenResp.status !== 200) {
+    throw new BadRequestError({
+      message: `Unable to validate credentials: GitHub responded with a status code of ${tokenResp.status} (${tokenResp.statusText}). Verify credentials and try again.`
+    });
+  }
+
+  if (method === GitHubRadarConnectionMethod.App) {
+    const installationsResp = await request.get<{
+      installations: {
+        id: number;
+        account: {
+          login: string;
+          type: string;
+          id: number;
+        };
+      }[];
+    }>(IntegrationUrls.GITHUB_USER_INSTALLATIONS, {
+      headers: {
+        Accept: "application/json",
+        Authorization: `Bearer ${tokenResp.data.access_token}`,
+        "Accept-Encoding": "application/json"
+      }
+    });
+
+    const matchingInstallation = installationsResp.data.installations.find(
+      (installation) => installation.id === +credentials.installationId
+    );
+
+    if (!matchingInstallation) {
+      throw new ForbiddenRequestError({
+        message: "User does not have access to the provided installation"
+      });
+    }
+  }
+
+  if (!tokenResp.data.access_token) {
+    throw new InternalServerError({ message: `Missing access token: ${tokenResp.data.error}` });
+  }
+
+  switch (method) {
+    case GitHubRadarConnectionMethod.App:
+      return {
+        installationId: credentials.installationId
+      };
+    default:
+      throw new InternalServerError({
+        message: `Unhandled GitHub connection method: ${method as GitHubRadarConnectionMethod}`
+      });
+  }
+};

backend/src/services/app-connection/github-radar/github-radar-connection-schemas.ts

@@ -0,0 +1,66 @@
+import { z } from "zod";
+
+import { AppConnections } from "@app/lib/api-docs";
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+import {
+  BaseAppConnectionSchema,
+  GenericCreateAppConnectionFieldsSchema,
+  GenericUpdateAppConnectionFieldsSchema
+} from "@app/services/app-connection/app-connection-schemas";
+
+import { GitHubRadarConnectionMethod } from "./github-radar-connection-enums";
+
+export const GitHubRadarConnectionInputCredentialsSchema = z.object({
+  code: z.string().trim().min(1, "GitHub Radar App code required"),
+  installationId: z.string().min(1, "GitHub Radar App Installation ID required")
+});
+
+export const GitHubRadarConnectionOutputCredentialsSchema = z.object({
+  installationId: z.string()
+});
+
+export const ValidateGitHubRadarConnectionCredentialsSchema = z.discriminatedUnion("method", [
+  z.object({
+    method: z
+      .literal(GitHubRadarConnectionMethod.App)
+      .describe(AppConnections.CREATE(AppConnection.GitHubRadar).method),
+    credentials: GitHubRadarConnectionInputCredentialsSchema.describe(
+      AppConnections.CREATE(AppConnection.GitHubRadar).credentials
+    )
+  })
+]);
+
+export const CreateGitHubRadarConnectionSchema = ValidateGitHubRadarConnectionCredentialsSchema.and(
+  GenericCreateAppConnectionFieldsSchema(AppConnection.GitHubRadar)
+);
+
+export const UpdateGitHubRadarConnectionSchema = z
+  .object({
+    credentials: GitHubRadarConnectionInputCredentialsSchema.optional().describe(
+      AppConnections.UPDATE(AppConnection.GitHubRadar).credentials
+    )
+  })
+  .and(GenericUpdateAppConnectionFieldsSchema(AppConnection.GitHubRadar));
+
+const BaseGitHubRadarConnectionSchema = BaseAppConnectionSchema.extend({ app: z.literal(AppConnection.GitHubRadar) });
+
+export const GitHubRadarConnectionSchema = BaseGitHubRadarConnectionSchema.extend({
+  method: z.literal(GitHubRadarConnectionMethod.App),
+  credentials: GitHubRadarConnectionOutputCredentialsSchema
+});
+
+export const SanitizedGitHubRadarConnectionSchema = z.discriminatedUnion("method", [
+  BaseGitHubRadarConnectionSchema.extend({
+    method: z.literal(GitHubRadarConnectionMethod.App),
+    credentials: GitHubRadarConnectionOutputCredentialsSchema.pick({})
+  })
+]);
+
+export const GitHubRadarConnectionListItemSchema = z.object({
+  name: z.literal("GitHub Radar"),
+  app: z.literal(AppConnection.GitHubRadar),
+  // the below is preferable but currently breaks with our zod to json schema parser
+  // methods: z.tuple([z.literal(GitHubConnectionMethod.App), z.literal(GitHubConnectionMethod.OAuth)]),
+  methods: z.nativeEnum(GitHubRadarConnectionMethod).array(),
+  appClientSlug: z.string().optional()
+});

backend/src/services/app-connection/github-radar/github-radar-connection-service.ts

@@ -0,0 +1,24 @@
+import { OrgServiceActor } from "@app/lib/types";
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+import { listGitHubRadarRepositories } from "@app/services/app-connection/github-radar/github-radar-connection-fns";
+import { TGitHubRadarConnection } from "@app/services/app-connection/github-radar/github-radar-connection-types";
+
+type TGetAppConnectionFunc = (
+  app: AppConnection,
+  connectionId: string,
+  actor: OrgServiceActor
+) => Promise<TGitHubRadarConnection>;
+
+export const githubRadarConnectionService = (getAppConnection: TGetAppConnectionFunc) => {
+  const listRepositories = async (connectionId: string, actor: OrgServiceActor) => {
+    const appConnection = await getAppConnection(AppConnection.GitHubRadar, connectionId, actor);
+
+    const repositories = await listGitHubRadarRepositories(appConnection);
+
+    return repositories.map((repo) => ({ id: repo.id, name: repo.full_name }));
+  };
+
+  return {
+    listRepositories
+  };
+};

backend/src/services/app-connection/github-radar/github-radar-connection-types.ts

@@ -0,0 +1,28 @@
+import { z } from "zod";
+
+import { DiscriminativePick } from "@app/lib/types";
+
+import { AppConnection } from "../app-connection-enums";
+import {
+  CreateGitHubRadarConnectionSchema,
+  GitHubRadarConnectionSchema,
+  ValidateGitHubRadarConnectionCredentialsSchema
+} from "./github-radar-connection-schemas";
+
+export type TGitHubRadarConnection = z.infer<typeof GitHubRadarConnectionSchema>;
+
+export type TGitHubRadarConnectionInput = z.infer<typeof CreateGitHubRadarConnectionSchema> & {
+  app: AppConnection.GitHubRadar;
+};
+
+export type TValidateGitHubRadarConnectionCredentialsSchema = typeof ValidateGitHubRadarConnectionCredentialsSchema;
+
+export type TGitHubRadarConnectionConfig = DiscriminativePick<
+  TGitHubRadarConnectionInput,
+  "method" | "app" | "credentials"
+>;
+
+export type TGitHubRadarRepository = {
+  id: number;
+  full_name: string;
+};

backend/src/services/app-connection/github-radar/index.ts

@@ -0,0 +1,4 @@
+export * from "./github-radar-connection-enums";
+export * from "./github-radar-connection-fns";
+export * from "./github-radar-connection-schemas";
+export * from "./github-radar-connection-types";

backend/src/services/certificate/certificate-service.ts

@@ -8,6 +8,7 @@ import {
   ProjectPermissionCertificateActions,
   ProjectPermissionSub
 } from "@app/ee/services/permission/project-permission";
+import { NotFoundError } from "@app/lib/errors";
 import { TCertificateBodyDALFactory } from "@app/services/certificate/certificate-body-dal";
 import { TCertificateDALFactory } from "@app/services/certificate/certificate-dal";
 import { TCertificateAuthorityCertDALFactory } from "@app/services/certificate-authority/certificate-authority-cert-dal";
@@ -29,7 +30,6 @@ import {
   TGetCertPrivateKeyDTO,
   TRevokeCertDTO
 } from "./certificate-types";
-import { NotFoundError } from "@app/lib/errors";
 
 type TCertificateServiceFactoryDep = {
   certificateDAL: Pick<TCertificateDALFactory, "findOne" | "deleteById" | "update" | "find">;