Merge pull request #3863 from Infisical/remove-secret-scanning-v1-backend

chore(secret-scanning-v1): remove secret scanning v1 queue and webhook endpoint

backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts

@@ -1,17 +1,11 @@
-import { ProbotOctokit } from "probot";
-
-import { OrgMembershipRole, TableName } from "@app/db/schemas";
-import { getConfig } from "@app/lib/config/env";
-import { logger } from "@app/lib/logger";
-import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
+import { InternalServerError } from "@app/lib/errors";
+import { TQueueServiceFactory } from "@app/queue";
 import { TOrgDALFactory } from "@app/services/org/org-dal";
-import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
+import { TSmtpService } from "@app/services/smtp/smtp-service";
 import { TTelemetryServiceFactory } from "@app/services/telemetry/telemetry-service";
-import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
 
 import { TSecretScanningDALFactory } from "../secret-scanning-dal";
-import { scanContentAndGetFindings, scanFullRepoContentAndGetFindings } from "./secret-scanning-fns";
-import { SecretMatch, TScanFullRepoEventPayload, TScanPushEventPayload } from "./secret-scanning-queue-types";
+import { TScanFullRepoEventPayload, TScanPushEventPayload } from "./secret-scanning-queue-types";
 
 type TSecretScanningQueueFactoryDep = {
   queueService: TQueueServiceFactory;
@@ -23,227 +17,21 @@ type TSecretScanningQueueFactoryDep = {
 
 export type TSecretScanningQueueFactory = ReturnType<typeof secretScanningQueueFactory>;
 
-export const secretScanningQueueFactory = ({
-  queueService,
-  secretScanningDAL,
-  smtpService,
-  telemetryService,
-  orgMembershipDAL: orgMemberDAL
-}: TSecretScanningQueueFactoryDep) => {
-  const startFullRepoScan = async (payload: TScanFullRepoEventPayload) => {
-    await queueService.queue(QueueName.SecretFullRepoScan, QueueJobs.SecretScan, payload, {
-      attempts: 3,
-      backoff: {
-        type: "exponential",
-        delay: 5000
-      },
-      removeOnComplete: true,
-      removeOnFail: {
-        count: 20 // keep the most recent 20 jobs
-      }
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+export const secretScanningQueueFactory = (_props: TSecretScanningQueueFactoryDep) => {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const startFullRepoScan = async (_payload: TScanFullRepoEventPayload) => {
+    throw new InternalServerError({
+      message: "Secret Scanning V1 has been deprecated. Please migrate to Secret Scanning V2"
     });
   };
 
-  const startPushEventScan = async (payload: TScanPushEventPayload) => {
-    await queueService.queue(QueueName.SecretPushEventScan, QueueJobs.SecretScan, payload, {
-      attempts: 3,
-      backoff: {
-        type: "exponential",
-        delay: 5000
-      },
-      removeOnComplete: true,
-      removeOnFail: {
-        count: 20 // keep the most recent 20 jobs
-      }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const startPushEventScan = async (_payload: TScanPushEventPayload) => {
+    throw new InternalServerError({
+      message: "Secret Scanning V1 has been deprecated. Please migrate to Secret Scanning V2"
     });
   };
 
-  const getOrgAdminEmails = async (organizationId: string) => {
-    // get emails of admins
-    const adminsOfWork = await orgMemberDAL.findMembership({
-      [`${TableName.Organization}.id` as string]: organizationId,
-      role: OrgMembershipRole.Admin
-    });
-    return adminsOfWork.filter((userObject) => userObject.email).map((userObject) => userObject.email as string);
-  };
-
-  queueService.start(QueueName.SecretPushEventScan, async (job) => {
-    const appCfg = getConfig();
-    const { organizationId, commits, pusher, repository, installationId } = job.data;
-    const [owner, repo] = repository.fullName.split("/");
-    const octokit = new ProbotOctokit({
-      auth: {
-        appId: appCfg.SECRET_SCANNING_GIT_APP_ID,
-        privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY,
-        installationId
-      }
-    });
-    const allFindingsByFingerprint: { [key: string]: SecretMatch } = {};
-
-    for (const commit of commits) {
-      for (const filepath of [...commit.added, ...commit.modified]) {
-        // eslint-disable-next-line
-        const fileContentsResponse = await octokit.repos.getContent({
-          owner,
-          repo,
-          path: filepath
-        });
-
-        const { data } = fileContentsResponse;
-        const fileContent = Buffer.from((data as { content: string }).content, "base64").toString();
-
-        // eslint-disable-next-line
-        const findings = await scanContentAndGetFindings(`\n${fileContent}`); // extra line to count lines correctly
-
-        for (const finding of findings) {
-          const fingerPrintWithCommitId = `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`;
-          const fingerPrintWithoutCommitId = `${filepath}:${finding.RuleID}:${finding.StartLine}`;
-          finding.Fingerprint = fingerPrintWithCommitId;
-          finding.FingerPrintWithoutCommitId = fingerPrintWithoutCommitId;
-          finding.Commit = commit.id;
-          finding.File = filepath;
-          finding.Author = commit.author.name;
-          finding.Email = commit?.author?.email ? commit?.author?.email : "";
-
-          allFindingsByFingerprint[fingerPrintWithCommitId] = finding;
-        }
-      }
-    }
-    await secretScanningDAL.transaction(async (tx) => {
-      if (!Object.keys(allFindingsByFingerprint).length) return;
-      await secretScanningDAL.upsert(
-        Object.keys(allFindingsByFingerprint).map((key) => ({
-          installationId,
-          email: allFindingsByFingerprint[key].Email,
-          author: allFindingsByFingerprint[key].Author,
-          date: allFindingsByFingerprint[key].Date,
-          file: allFindingsByFingerprint[key].File,
-          tags: allFindingsByFingerprint[key].Tags,
-          commit: allFindingsByFingerprint[key].Commit,
-          ruleID: allFindingsByFingerprint[key].RuleID,
-          endLine: String(allFindingsByFingerprint[key].EndLine),
-          entropy: String(allFindingsByFingerprint[key].Entropy),
-          message: allFindingsByFingerprint[key].Message,
-          endColumn: String(allFindingsByFingerprint[key].EndColumn),
-          startLine: String(allFindingsByFingerprint[key].StartLine),
-          startColumn: String(allFindingsByFingerprint[key].StartColumn),
-          fingerPrintWithoutCommitId: allFindingsByFingerprint[key].FingerPrintWithoutCommitId,
-          description: allFindingsByFingerprint[key].Description,
-          symlinkFile: allFindingsByFingerprint[key].SymlinkFile,
-          orgId: organizationId,
-          pusherEmail: pusher.email,
-          pusherName: pusher.name,
-          repositoryFullName: repository.fullName,
-          repositoryId: String(repository.id),
-          fingerprint: allFindingsByFingerprint[key].Fingerprint
-        })),
-        tx
-      );
-    });
-
-    const adminEmails = await getOrgAdminEmails(organizationId);
-    if (pusher?.email) {
-      adminEmails.push(pusher.email);
-    }
-    if (Object.keys(allFindingsByFingerprint).length) {
-      await smtpService.sendMail({
-        template: SmtpTemplates.SecretLeakIncident,
-        subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`,
-        recipients: adminEmails.filter((email) => email).map((email) => email),
-        substitutions: {
-          numberOfSecrets: Object.keys(allFindingsByFingerprint).length,
-          pusher_email: pusher.email,
-          pusher_name: pusher.name
-        }
-      });
-    }
-
-    await telemetryService.sendPostHogEvents({
-      event: PostHogEventTypes.SecretScannerPush,
-      distinctId: repository.fullName,
-      properties: {
-        numberOfRisks: Object.keys(allFindingsByFingerprint).length
-      }
-    });
-  });
-
-  queueService.start(QueueName.SecretFullRepoScan, async (job) => {
-    const appCfg = getConfig();
-    const { organizationId, repository, installationId } = job.data;
-    const octokit = new ProbotOctokit({
-      auth: {
-        appId: appCfg.SECRET_SCANNING_GIT_APP_ID,
-        privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY,
-        installationId
-      }
-    });
-
-    const findings = await scanFullRepoContentAndGetFindings(
-      // this is because of collision of octokit in probot and github
-      // eslint-disable-next-line
-      octokit as any,
-      installationId,
-      repository.fullName
-    );
-    await secretScanningDAL.transaction(async (tx) => {
-      if (!findings.length) return;
-      // eslint-disable-next-line
-      await secretScanningDAL.upsert(
-        findings.map((finding) => ({
-          installationId,
-          email: finding.Email,
-          author: finding.Author,
-          date: finding.Date,
-          file: finding.File,
-          tags: finding.Tags,
-          commit: finding.Commit,
-          ruleID: finding.RuleID,
-          endLine: String(finding.EndLine),
-          entropy: String(finding.Entropy),
-          message: finding.Message,
-          endColumn: String(finding.EndColumn),
-          startLine: String(finding.StartLine),
-          startColumn: String(finding.StartColumn),
-          fingerPrintWithoutCommitId: finding.FingerPrintWithoutCommitId,
-          description: finding.Description,
-          symlinkFile: finding.SymlinkFile,
-          orgId: organizationId,
-          repositoryFullName: repository.fullName,
-          repositoryId: String(repository.id),
-          fingerprint: finding.Fingerprint
-        })),
-        tx
-      );
-    });
-
-    const adminEmails = await getOrgAdminEmails(organizationId);
-    if (findings.length) {
-      await smtpService.sendMail({
-        template: SmtpTemplates.SecretLeakIncident,
-        subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`,
-        recipients: adminEmails.filter((email) => email).map((email) => email),
-        substitutions: {
-          numberOfSecrets: findings.length
-        }
-      });
-    }
-
-    await telemetryService.sendPostHogEvents({
-      event: PostHogEventTypes.SecretScannerFull,
-      distinctId: repository.fullName,
-      properties: {
-        numberOfRisks: findings.length
-      }
-    });
-  });
-
-  queueService.listen(QueueName.SecretPushEventScan, "failed", (job, err) => {
-    logger.error(err, "Failed to secret scan on push", job?.data);
-  });
-
-  queueService.listen(QueueName.SecretFullRepoScan, "failed", (job, err) => {
-    logger.error(err, "Failed to do full repo secret scan", job?.data);
-  });
-
   return { startFullRepoScan, startPushEventScan };
 };

backend/src/ee/services/secret-scanning/secret-scanning-service.ts

@@ -98,6 +98,7 @@ export const secretScanningServiceFactory = ({
     if (canUseSecretScanning(actorOrgId)) {
       await Promise.all(
         repositories.map(({ id, full_name }) =>
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-call,@typescript-eslint/no-unsafe-return,@typescript-eslint/no-unsafe-member-access
           secretScanningQueue.startFullRepoScan({
             organizationId: session.orgId,
             installationId,
@@ -180,6 +181,7 @@ export const secretScanningServiceFactory = ({
     if (!installationLink) return;
 
     if (canUseSecretScanning(installationLink.orgId)) {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-call,@typescript-eslint/no-unsafe-return,@typescript-eslint/no-unsafe-member-access
       await secretScanningQueue.startPushEventScan({
         commits,
         pusher: { name: pusher.name, email: pusher.email },

backend/src/server/routes/index.ts

@@ -297,7 +297,6 @@ import { injectAssumePrivilege } from "../plugins/auth/inject-assume-privilege";
 import { injectIdentity } from "../plugins/auth/inject-identity";
 import { injectPermission } from "../plugins/auth/inject-permission";
 import { injectRateLimits } from "../plugins/inject-rate-limits";
-import { registerSecretScannerGhApp } from "../plugins/secret-scanner";
 import { registerV1Routes } from "./v1";
 import { registerV2Routes } from "./v2";
 import { registerV3Routes } from "./v3";
@@ -326,7 +325,6 @@ export const registerRoutes = async (
   }
 ) => {
   const appCfg = getConfig();
-  await server.register(registerSecretScannerGhApp, { prefix: "/ss-webhook" });
   await server.register(registerSecretScanningV2Webhooks, {
     prefix: "/secret-scanning/webhooks"
   });

backend/src/server/routes/v1/secret-sync-routers/cloudflare-pages-sync-router.ts

@@ -1,10 +1,11 @@
-import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
-import { registerSyncSecretsEndpoints } from "./secret-sync-endpoints";
 import {
   CloudflarePagesSyncSchema,
   CreateCloudflarePagesSyncSchema,
   UpdateCloudflarePagesSyncSchema
 } from "@app/services/secret-sync/cloudflare-pages/cloudflare-pages-schema";
+import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
+
+import { registerSyncSecretsEndpoints } from "./secret-sync-endpoints";
 
 export const registerCloudflarePagesSyncRouter = async (server: FastifyZodProvider) =>
   registerSyncSecretsEndpoints({

backend/src/services/app-connection/cloudflare/cloudflare-connection-schema.ts

@@ -1,6 +1,7 @@
 import z from "zod";
 
 import { AppConnections } from "@app/lib/api-docs";
+import { CharacterType, characterValidator } from "@app/lib/validator/validate-string";
 import { AppConnection } from "@app/services/app-connection/app-connection-enums";
 import {
   BaseAppConnectionSchema,
@@ -9,7 +10,6 @@ import {
 } from "@app/services/app-connection/app-connection-schemas";
 
 import { CloudflareConnectionMethod } from "./cloudflare-connection-enum";
-import { CharacterType, characterValidator } from "@app/lib/validator/validate-string";
 
 const accountIdCharacterValidator = characterValidator([
   CharacterType.AlphaNumeric,