feature: secret scanning architecture and github data source (wip)

backend/src/server/plugins/secret-scanner-v2.ts

@@ -0,0 +1,60 @@
+import type { EmitterWebhookEventName } from "@octokit/webhooks/dist-types/types";
+import { PushEvent } from "@octokit/webhooks-types";
+import { Probot } from "probot";
+
+import { getConfig } from "@app/lib/config/env";
+import { logger } from "@app/lib/logger";
+import { writeLimit } from "@app/server/config/rateLimiter";
+
+export const registerSecretScanningV2Webhooks = async (server: FastifyZodProvider) => {
+  const probotApp = (app: Probot) => {
+    app.on("installation.deleted", async (context) => {
+      const { payload } = context;
+      const { installation } = payload;
+
+      await server.services.secretScanningV2.github.handleInstallationDeleted(installation.id);
+    });
+
+    app.on("installation", async (context) => {
+      const { payload } = context;
+      logger.info({ repositories: payload.repositories }, "Installed secret scanner to");
+    });
+
+    app.on("push", async (context) => {
+      const { payload } = context;
+      await server.services.secretScanningV2.github.handlePushEvent(payload as PushEvent);
+    });
+  };
+
+  const appCfg = getConfig();
+
+  const probot = new Probot({
+    appId: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID as string,
+    privateKey: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY as string,
+    secret: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_WEBHOOK_SECRET as string
+  });
+
+  await probot.load(probotApp);
+
+  server.route({
+    method: "POST",
+    url: "/github",
+    config: {
+      rateLimit: writeLimit
+    },
+    handler: async (req, res) => {
+      const eventName = req.headers["x-github-event"] as EmitterWebhookEventName;
+      const signatureSHA256 = req.headers["x-hub-signature-256"] as string;
+      const id = req.headers["x-github-delivery"] as string;
+
+      await probot.webhooks.verifyAndReceive({
+        id,
+        name: eventName,
+        payload: JSON.stringify(req.body),
+        signature: signatureSHA256
+      });
+
+      return res.send("ok");
+    }
+  });
+};

backend/src/server/routes/index.ts

@@ -86,6 +86,9 @@ import { gitAppInstallSessionDALFactory } from "@app/ee/services/secret-scanning
 import { secretScanningDALFactory } from "@app/ee/services/secret-scanning/secret-scanning-dal";
 import { secretScanningQueueFactory } from "@app/ee/services/secret-scanning/secret-scanning-queue";
 import { secretScanningServiceFactory } from "@app/ee/services/secret-scanning/secret-scanning-service";
+import { secretScanningV2DALFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-dal";
+import { secretScanningV2QueueServiceFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-queue";
+import { secretScanningV2ServiceFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-service";
 import { secretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
 import { snapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal";
 import { snapshotFolderDALFactory } from "@app/ee/services/secret-snapshot/snapshot-folder-dal";
@@ -112,6 +115,7 @@ import { getConfig, TEnvConfig } from "@app/lib/config/env";
 import { logger } from "@app/lib/logger";
 import { TQueueServiceFactory } from "@app/queue";
 import { readLimit } from "@app/server/config/rateLimiter";
+import { registerSecretScanningV2Webhooks } from "@app/server/plugins/secret-scanner-v2";
 import { accessTokenQueueServiceFactory } from "@app/services/access-token-queue/access-token-queue";
 import { apiKeyDALFactory } from "@app/services/api-key/api-key-dal";
 import { apiKeyServiceFactory } from "@app/services/api-key/api-key-service";
@@ -299,6 +303,9 @@ export const registerRoutes = async (
 ) => {
   const appCfg = getConfig();
   await server.register(registerSecretScannerGhApp, { prefix: "/ss-webhook" });
+  await server.register(registerSecretScanningV2Webhooks, {
+    prefix: "/secret-scanning/webhooks"
+  });
 
   // db layers
   const userDAL = userDALFactory(db);
@@ -444,6 +451,7 @@ export const registerRoutes = async (
   const secretRotationV2DAL = secretRotationV2DALFactory(db, folderDAL);
   const microsoftTeamsIntegrationDAL = microsoftTeamsIntegrationDALFactory(db);
   const projectMicrosoftTeamsConfigDAL = projectMicrosoftTeamsConfigDALFactory(db);
+  const secretScanningV2DAL = secretScanningV2DALFactory(db);
 
   const permissionService = permissionServiceFactory({
     permissionDAL,
@@ -1694,6 +1702,28 @@ export const registerRoutes = async (
     smtpService
   });
 
+  const secretScanningV2Queue = await secretScanningV2QueueServiceFactory({
+    secretScanningV2DAL,
+    queueService,
+    // projectDAL,
+    // projectMembershipDAL,
+    // smtpService,
+    kmsService
+  });
+
+  const secretScanningV2Service = secretScanningV2ServiceFactory({
+    // appConnectionDAL,
+    permissionService,
+    appConnectionService,
+    licenseService,
+    // auditLogService,
+    // keyStore,
+    // queueService,
+    secretScanningV2DAL,
+    secretScanningV2Queue,
+    kmsService
+  });
+
   await superAdminService.initServerCfg();
 
   // setup the communication with license key server
@@ -1805,7 +1835,8 @@ export const registerRoutes = async (
     secretRotationV2: secretRotationV2Service,
     microsoftTeams: microsoftTeamsService,
     assumePrivileges: assumePrivilegeService,
-    githubOrgSync: githubOrgSyncConfigService
+    githubOrgSync: githubOrgSyncConfigService,
+    secretScanningV2: secretScanningV2Service
   });
 
   const cronJobs: CronJob[] = [];

backend/src/server/routes/v1/app-connection-routers/app-connection-router.ts

@@ -28,6 +28,10 @@ import {
 } from "@app/services/app-connection/databricks";
 import { GcpConnectionListItemSchema, SanitizedGcpConnectionSchema } from "@app/services/app-connection/gcp";
 import { GitHubConnectionListItemSchema, SanitizedGitHubConnectionSchema } from "@app/services/app-connection/github";
+import {
+  GitHubRadarConnectionListItemSchema,
+  SanitizedGitHubRadarConnectionSchema
+} from "@app/services/app-connection/github-radar";
 import {
   HCVaultConnectionListItemSchema,
   SanitizedHCVaultConnectionSchema
@@ -62,6 +66,7 @@ import { AuthMode } from "@app/services/auth/auth-type";
 const SanitizedAppConnectionSchema = z.union([
   ...SanitizedAwsConnectionSchema.options,
   ...SanitizedGitHubConnectionSchema.options,
+  ...SanitizedGitHubRadarConnectionSchema.options,
   ...SanitizedGcpConnectionSchema.options,
   ...SanitizedAzureKeyVaultConnectionSchema.options,
   ...SanitizedAzureAppConfigurationConnectionSchema.options,
@@ -84,6 +89,7 @@ const SanitizedAppConnectionSchema = z.union([
 const AppConnectionOptionsSchema = z.discriminatedUnion("app", [
   AwsConnectionListItemSchema,
   GitHubConnectionListItemSchema,
+  GitHubRadarConnectionListItemSchema,
   GcpConnectionListItemSchema,
   AzureKeyVaultConnectionListItemSchema,
   AzureAppConfigurationConnectionListItemSchema,

backend/src/server/routes/v1/app-connection-routers/github-radar-connection-router.ts

@@ -0,0 +1,54 @@
+import { z } from "zod";
+
+import { readLimit } from "@app/server/config/rateLimiter";
+import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+import {
+  CreateGitHubRadarConnectionSchema,
+  SanitizedGitHubRadarConnectionSchema,
+  UpdateGitHubRadarConnectionSchema
+} from "@app/services/app-connection/github-radar";
+import { AuthMode } from "@app/services/auth/auth-type";
+
+import { registerAppConnectionEndpoints } from "./app-connection-endpoints";
+
+export const registerGitHubRadarConnectionRouter = async (server: FastifyZodProvider) => {
+  registerAppConnectionEndpoints({
+    app: AppConnection.GitHubRadar,
+    server,
+    sanitizedResponseSchema: SanitizedGitHubRadarConnectionSchema,
+    createSchema: CreateGitHubRadarConnectionSchema,
+    updateSchema: UpdateGitHubRadarConnectionSchema
+  });
+
+  // The below endpoints are not exposed and for Infisical App use
+
+  server.route({
+    method: "GET",
+    url: `/:connectionId/repositories`,
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      params: z.object({
+        connectionId: z.string().uuid()
+      }),
+      response: {
+        200: z.object({
+          repositories: z.object({ id: z.number(), name: z.string() }).array()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { connectionId } = req.params;
+
+      const repositories = await server.services.appConnection.githubRadar.listRepositories(
+        connectionId,
+        req.permission
+      );
+
+      return { repositories };
+    }
+  });
+};

backend/src/server/routes/v1/app-connection-routers/index.ts

@@ -9,6 +9,7 @@ import { registerCamundaConnectionRouter } from "./camunda-connection-router";
 import { registerDatabricksConnectionRouter } from "./databricks-connection-router";
 import { registerGcpConnectionRouter } from "./gcp-connection-router";
 import { registerGitHubConnectionRouter } from "./github-connection-router";
+import { registerGitHubRadarConnectionRouter } from "./github-radar-connection-router";
 import { registerHCVaultConnectionRouter } from "./hc-vault-connection-router";
 import { registerHumanitecConnectionRouter } from "./humanitec-connection-router";
 import { registerLdapConnectionRouter } from "./ldap-connection-router";
@@ -26,6 +27,7 @@ export const APP_CONNECTION_REGISTER_ROUTER_MAP: Record<AppConnection, (server:
   {
     [AppConnection.AWS]: registerAwsConnectionRouter,
     [AppConnection.GitHub]: registerGitHubConnectionRouter,
+    [AppConnection.GitHubRadar]: registerGitHubRadarConnectionRouter,
     [AppConnection.GCP]: registerGcpConnectionRouter,
     [AppConnection.AzureKeyVault]: registerAzureKeyVaultConnectionRouter,
     [AppConnection.AzureAppConfiguration]: registerAzureAppConfigurationConnectionRouter,

backend/src/server/routes/v1/project-router.ts

@@ -160,7 +160,14 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
           .default("false")
           .transform((value) => value === "true"),
         type: z
-          .enum([ProjectType.SecretManager, ProjectType.KMS, ProjectType.CertificateManager, ProjectType.SSH, "all"])
+          .enum([
+            ProjectType.SecretManager,
+            ProjectType.KMS,
+            ProjectType.CertificateManager,
+            ProjectType.SSH,
+            ProjectType.SecretScanning,
+            "all"
+          ])
           .optional()
       }),
       response: {