From d7913a75c2ef5777de003ab51decfeb849c44d49 Mon Sep 17 00:00:00 2001 From: Scott Wilson Date: Thu, 26 Jun 2025 11:32:45 -0700 Subject: [PATCH] chore: remove secret scanning v1 queue and webhook endpoint --- .../secret-scanning-queue.ts | 240 +----------------- .../secret-scanning-service.ts | 2 + backend/src/server/routes/index.ts | 2 - .../cloudflare-pages-sync-router.ts | 5 +- .../cloudflare-connection-schema.ts | 2 +- 5 files changed, 20 insertions(+), 231 deletions(-) diff --git a/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts b/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts index 42ff90055..87e39e5b4 100644 --- a/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts +++ b/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts @@ -1,17 +1,11 @@ -import { ProbotOctokit } from "probot"; - -import { OrgMembershipRole, TableName } from "@app/db/schemas"; -import { getConfig } from "@app/lib/config/env"; -import { logger } from "@app/lib/logger"; -import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { InternalServerError } from "@app/lib/errors"; +import { TQueueServiceFactory } from "@app/queue"; import { TOrgDALFactory } from "@app/services/org/org-dal"; -import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; +import { TSmtpService } from "@app/services/smtp/smtp-service"; import { TTelemetryServiceFactory } from "@app/services/telemetry/telemetry-service"; -import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types"; import { TSecretScanningDALFactory } from "../secret-scanning-dal"; -import { scanContentAndGetFindings, scanFullRepoContentAndGetFindings } from "./secret-scanning-fns"; -import { SecretMatch, TScanFullRepoEventPayload, TScanPushEventPayload } from "./secret-scanning-queue-types"; +import { TScanFullRepoEventPayload, TScanPushEventPayload } from "./secret-scanning-queue-types"; type TSecretScanningQueueFactoryDep = { queueService: TQueueServiceFactory; @@ -23,227 +17,21 @@ type TSecretScanningQueueFactoryDep = { export type TSecretScanningQueueFactory = ReturnType; -export const secretScanningQueueFactory = ({ - queueService, - secretScanningDAL, - smtpService, - telemetryService, - orgMembershipDAL: orgMemberDAL -}: TSecretScanningQueueFactoryDep) => { - const startFullRepoScan = async (payload: TScanFullRepoEventPayload) => { - await queueService.queue(QueueName.SecretFullRepoScan, QueueJobs.SecretScan, payload, { - attempts: 3, - backoff: { - type: "exponential", - delay: 5000 - }, - removeOnComplete: true, - removeOnFail: { - count: 20 // keep the most recent 20 jobs - } +// eslint-disable-next-line @typescript-eslint/no-unused-vars +export const secretScanningQueueFactory = (_props: TSecretScanningQueueFactoryDep) => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const startFullRepoScan = async (_payload: TScanFullRepoEventPayload) => { + throw new InternalServerError({ + message: "Secret Scanning V1 has been deprecated. Please migrate to Secret Scanning V2" }); }; - const startPushEventScan = async (payload: TScanPushEventPayload) => { - await queueService.queue(QueueName.SecretPushEventScan, QueueJobs.SecretScan, payload, { - attempts: 3, - backoff: { - type: "exponential", - delay: 5000 - }, - removeOnComplete: true, - removeOnFail: { - count: 20 // keep the most recent 20 jobs - } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const startPushEventScan = async (_payload: TScanPushEventPayload) => { + throw new InternalServerError({ + message: "Secret Scanning V1 has been deprecated. Please migrate to Secret Scanning V2" }); }; - const getOrgAdminEmails = async (organizationId: string) => { - // get emails of admins - const adminsOfWork = await orgMemberDAL.findMembership({ - [`${TableName.Organization}.id` as string]: organizationId, - role: OrgMembershipRole.Admin - }); - return adminsOfWork.filter((userObject) => userObject.email).map((userObject) => userObject.email as string); - }; - - queueService.start(QueueName.SecretPushEventScan, async (job) => { - const appCfg = getConfig(); - const { organizationId, commits, pusher, repository, installationId } = job.data; - const [owner, repo] = repository.fullName.split("/"); - const octokit = new ProbotOctokit({ - auth: { - appId: appCfg.SECRET_SCANNING_GIT_APP_ID, - privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY, - installationId - } - }); - const allFindingsByFingerprint: { [key: string]: SecretMatch } = {}; - - for (const commit of commits) { - for (const filepath of [...commit.added, ...commit.modified]) { - // eslint-disable-next-line - const fileContentsResponse = await octokit.repos.getContent({ - owner, - repo, - path: filepath - }); - - const { data } = fileContentsResponse; - const fileContent = Buffer.from((data as { content: string }).content, "base64").toString(); - - // eslint-disable-next-line - const findings = await scanContentAndGetFindings(`\n${fileContent}`); // extra line to count lines correctly - - for (const finding of findings) { - const fingerPrintWithCommitId = `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`; - const fingerPrintWithoutCommitId = `${filepath}:${finding.RuleID}:${finding.StartLine}`; - finding.Fingerprint = fingerPrintWithCommitId; - finding.FingerPrintWithoutCommitId = fingerPrintWithoutCommitId; - finding.Commit = commit.id; - finding.File = filepath; - finding.Author = commit.author.name; - finding.Email = commit?.author?.email ? commit?.author?.email : ""; - - allFindingsByFingerprint[fingerPrintWithCommitId] = finding; - } - } - } - await secretScanningDAL.transaction(async (tx) => { - if (!Object.keys(allFindingsByFingerprint).length) return; - await secretScanningDAL.upsert( - Object.keys(allFindingsByFingerprint).map((key) => ({ - installationId, - email: allFindingsByFingerprint[key].Email, - author: allFindingsByFingerprint[key].Author, - date: allFindingsByFingerprint[key].Date, - file: allFindingsByFingerprint[key].File, - tags: allFindingsByFingerprint[key].Tags, - commit: allFindingsByFingerprint[key].Commit, - ruleID: allFindingsByFingerprint[key].RuleID, - endLine: String(allFindingsByFingerprint[key].EndLine), - entropy: String(allFindingsByFingerprint[key].Entropy), - message: allFindingsByFingerprint[key].Message, - endColumn: String(allFindingsByFingerprint[key].EndColumn), - startLine: String(allFindingsByFingerprint[key].StartLine), - startColumn: String(allFindingsByFingerprint[key].StartColumn), - fingerPrintWithoutCommitId: allFindingsByFingerprint[key].FingerPrintWithoutCommitId, - description: allFindingsByFingerprint[key].Description, - symlinkFile: allFindingsByFingerprint[key].SymlinkFile, - orgId: organizationId, - pusherEmail: pusher.email, - pusherName: pusher.name, - repositoryFullName: repository.fullName, - repositoryId: String(repository.id), - fingerprint: allFindingsByFingerprint[key].Fingerprint - })), - tx - ); - }); - - const adminEmails = await getOrgAdminEmails(organizationId); - if (pusher?.email) { - adminEmails.push(pusher.email); - } - if (Object.keys(allFindingsByFingerprint).length) { - await smtpService.sendMail({ - template: SmtpTemplates.SecretLeakIncident, - subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`, - recipients: adminEmails.filter((email) => email).map((email) => email), - substitutions: { - numberOfSecrets: Object.keys(allFindingsByFingerprint).length, - pusher_email: pusher.email, - pusher_name: pusher.name - } - }); - } - - await telemetryService.sendPostHogEvents({ - event: PostHogEventTypes.SecretScannerPush, - distinctId: repository.fullName, - properties: { - numberOfRisks: Object.keys(allFindingsByFingerprint).length - } - }); - }); - - queueService.start(QueueName.SecretFullRepoScan, async (job) => { - const appCfg = getConfig(); - const { organizationId, repository, installationId } = job.data; - const octokit = new ProbotOctokit({ - auth: { - appId: appCfg.SECRET_SCANNING_GIT_APP_ID, - privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY, - installationId - } - }); - - const findings = await scanFullRepoContentAndGetFindings( - // this is because of collision of octokit in probot and github - // eslint-disable-next-line - octokit as any, - installationId, - repository.fullName - ); - await secretScanningDAL.transaction(async (tx) => { - if (!findings.length) return; - // eslint-disable-next-line - await secretScanningDAL.upsert( - findings.map((finding) => ({ - installationId, - email: finding.Email, - author: finding.Author, - date: finding.Date, - file: finding.File, - tags: finding.Tags, - commit: finding.Commit, - ruleID: finding.RuleID, - endLine: String(finding.EndLine), - entropy: String(finding.Entropy), - message: finding.Message, - endColumn: String(finding.EndColumn), - startLine: String(finding.StartLine), - startColumn: String(finding.StartColumn), - fingerPrintWithoutCommitId: finding.FingerPrintWithoutCommitId, - description: finding.Description, - symlinkFile: finding.SymlinkFile, - orgId: organizationId, - repositoryFullName: repository.fullName, - repositoryId: String(repository.id), - fingerprint: finding.Fingerprint - })), - tx - ); - }); - - const adminEmails = await getOrgAdminEmails(organizationId); - if (findings.length) { - await smtpService.sendMail({ - template: SmtpTemplates.SecretLeakIncident, - subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`, - recipients: adminEmails.filter((email) => email).map((email) => email), - substitutions: { - numberOfSecrets: findings.length - } - }); - } - - await telemetryService.sendPostHogEvents({ - event: PostHogEventTypes.SecretScannerFull, - distinctId: repository.fullName, - properties: { - numberOfRisks: findings.length - } - }); - }); - - queueService.listen(QueueName.SecretPushEventScan, "failed", (job, err) => { - logger.error(err, "Failed to secret scan on push", job?.data); - }); - - queueService.listen(QueueName.SecretFullRepoScan, "failed", (job, err) => { - logger.error(err, "Failed to do full repo secret scan", job?.data); - }); - return { startFullRepoScan, startPushEventScan }; }; diff --git a/backend/src/ee/services/secret-scanning/secret-scanning-service.ts b/backend/src/ee/services/secret-scanning/secret-scanning-service.ts index 1c12f8b4e..a65bb9e68 100644 --- a/backend/src/ee/services/secret-scanning/secret-scanning-service.ts +++ b/backend/src/ee/services/secret-scanning/secret-scanning-service.ts @@ -98,6 +98,7 @@ export const secretScanningServiceFactory = ({ if (canUseSecretScanning(actorOrgId)) { await Promise.all( repositories.map(({ id, full_name }) => + // eslint-disable-next-line @typescript-eslint/no-unsafe-call,@typescript-eslint/no-unsafe-return,@typescript-eslint/no-unsafe-member-access secretScanningQueue.startFullRepoScan({ organizationId: session.orgId, installationId, @@ -180,6 +181,7 @@ export const secretScanningServiceFactory = ({ if (!installationLink) return; if (canUseSecretScanning(installationLink.orgId)) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-call,@typescript-eslint/no-unsafe-return,@typescript-eslint/no-unsafe-member-access await secretScanningQueue.startPushEventScan({ commits, pusher: { name: pusher.name, email: pusher.email }, diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index bde1b805e..36e9aa0a5 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -297,7 +297,6 @@ import { injectAssumePrivilege } from "../plugins/auth/inject-assume-privilege"; import { injectIdentity } from "../plugins/auth/inject-identity"; import { injectPermission } from "../plugins/auth/inject-permission"; import { injectRateLimits } from "../plugins/inject-rate-limits"; -import { registerSecretScannerGhApp } from "../plugins/secret-scanner"; import { registerV1Routes } from "./v1"; import { registerV2Routes } from "./v2"; import { registerV3Routes } from "./v3"; @@ -326,7 +325,6 @@ export const registerRoutes = async ( } ) => { const appCfg = getConfig(); - await server.register(registerSecretScannerGhApp, { prefix: "/ss-webhook" }); await server.register(registerSecretScanningV2Webhooks, { prefix: "/secret-scanning/webhooks" }); diff --git a/backend/src/server/routes/v1/secret-sync-routers/cloudflare-pages-sync-router.ts b/backend/src/server/routes/v1/secret-sync-routers/cloudflare-pages-sync-router.ts index 6a70b5837..59b95323c 100644 --- a/backend/src/server/routes/v1/secret-sync-routers/cloudflare-pages-sync-router.ts +++ b/backend/src/server/routes/v1/secret-sync-routers/cloudflare-pages-sync-router.ts @@ -1,10 +1,11 @@ -import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; -import { registerSyncSecretsEndpoints } from "./secret-sync-endpoints"; import { CloudflarePagesSyncSchema, CreateCloudflarePagesSyncSchema, UpdateCloudflarePagesSyncSchema } from "@app/services/secret-sync/cloudflare-pages/cloudflare-pages-schema"; +import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; + +import { registerSyncSecretsEndpoints } from "./secret-sync-endpoints"; export const registerCloudflarePagesSyncRouter = async (server: FastifyZodProvider) => registerSyncSecretsEndpoints({ diff --git a/backend/src/services/app-connection/cloudflare/cloudflare-connection-schema.ts b/backend/src/services/app-connection/cloudflare/cloudflare-connection-schema.ts index 64dee9dd5..f3b26a2d6 100644 --- a/backend/src/services/app-connection/cloudflare/cloudflare-connection-schema.ts +++ b/backend/src/services/app-connection/cloudflare/cloudflare-connection-schema.ts @@ -1,6 +1,7 @@ import z from "zod"; import { AppConnections } from "@app/lib/api-docs"; +import { CharacterType, characterValidator } from "@app/lib/validator/validate-string"; import { AppConnection } from "@app/services/app-connection/app-connection-enums"; import { BaseAppConnectionSchema, @@ -9,7 +10,6 @@ import { } from "@app/services/app-connection/app-connection-schemas"; import { CloudflareConnectionMethod } from "./cloudflare-connection-enum"; -import { CharacterType, characterValidator } from "@app/lib/validator/validate-string"; const accountIdCharacterValidator = characterValidator([ CharacterType.AlphaNumeric,