import { ProbotOctokit } from "probot"; import { OrgMembershipRole } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { TOrgDALFactory } from "@app/services/org/org-dal"; import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; import { TTelemetryServiceFactory } from "@app/services/telemetry/telemetry-service"; import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types"; import { TSecretScanningDALFactory } from "../secret-scanning-dal"; import { scanContentAndGetFindings, scanFullRepoContentAndGetFindings } from "./secret-scanning-fns"; import { SecretMatch, TScanFullRepoEventPayload, TScanPushEventPayload } from "./secret-scanning-queue-types"; type TSecretScanningQueueFactoryDep = { queueService: TQueueServiceFactory; secretScanningDAL: TSecretScanningDALFactory; smtpService: Pick; orgMembershipDAL: Pick; telemetryService: Pick; }; export type TSecretScanningQueueFactory = ReturnType; export const secretScanningQueueFactory = ({ queueService, secretScanningDAL, smtpService, telemetryService, orgMembershipDAL: orgMemberDAL }: TSecretScanningQueueFactoryDep) => { const startFullRepoScan = async (payload: TScanFullRepoEventPayload) => { await queueService.queue(QueueName.SecretFullRepoScan, QueueJobs.SecretScan, payload, { attempts: 3, backoff: { type: "exponential", delay: 5000 }, removeOnComplete: true, removeOnFail: { count: 20 // keep the most recent 20 jobs } }); }; const startPushEventScan = async (payload: TScanPushEventPayload) => { await queueService.queue(QueueName.SecretPushEventScan, QueueJobs.SecretScan, payload, { attempts: 3, backoff: { type: "exponential", delay: 5000 }, removeOnComplete: true, removeOnFail: { count: 20 // keep the most recent 20 jobs } }); }; const getOrgAdminEmails = async (organizationId: string) => { // get emails of admins const adminsOfWork = await orgMemberDAL.findMembership({ orgId: organizationId, role: OrgMembershipRole.Admin }); return adminsOfWork.map((userObject) => userObject.email); }; queueService.start(QueueName.SecretPushEventScan, async (job) => { const appCfg = getConfig(); const { organizationId, commits, pusher, repository, installationId } = job.data; const [owner, repo] = repository.fullName.split("/"); const octokit = new ProbotOctokit({ auth: { appId: appCfg.SECRET_SCANNING_GIT_APP_ID, privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY, installationId } }); const allFindingsByFingerprint: { [key: string]: SecretMatch } = {}; for (const commit of commits) { for (const filepath of [...commit.added, ...commit.modified]) { // eslint-disable-next-line const fileContentsResponse = await octokit.repos.getContent({ owner, repo, path: filepath }); const { data } = fileContentsResponse; const fileContent = Buffer.from((data as { content: string }).content, "base64").toString(); // eslint-disable-next-line const findings = await scanContentAndGetFindings(`\n${fileContent}`); // extra line to count lines correctly for (const finding of findings) { const fingerPrintWithCommitId = `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`; const fingerPrintWithoutCommitId = `${filepath}:${finding.RuleID}:${finding.StartLine}`; finding.Fingerprint = fingerPrintWithCommitId; finding.FingerPrintWithoutCommitId = fingerPrintWithoutCommitId; finding.Commit = commit.id; finding.File = filepath; finding.Author = commit.author.name; finding.Email = commit?.author?.email ? commit?.author?.email : ""; allFindingsByFingerprint[fingerPrintWithCommitId] = finding; } } } await secretScanningDAL.transaction(async (tx) => { if (!Object.keys(allFindingsByFingerprint).length) return; await secretScanningDAL.upsert( Object.keys(allFindingsByFingerprint).map((key) => ({ installationId, email: allFindingsByFingerprint[key].Email, author: allFindingsByFingerprint[key].Author, date: allFindingsByFingerprint[key].Date, file: allFindingsByFingerprint[key].File, tags: allFindingsByFingerprint[key].Tags, commit: allFindingsByFingerprint[key].Commit, ruleID: allFindingsByFingerprint[key].RuleID, endLine: String(allFindingsByFingerprint[key].EndLine), entropy: String(allFindingsByFingerprint[key].Entropy), message: allFindingsByFingerprint[key].Message, endColumn: String(allFindingsByFingerprint[key].EndColumn), startLine: String(allFindingsByFingerprint[key].StartLine), startColumn: String(allFindingsByFingerprint[key].StartColumn), fingerPrintWithoutCommitId: allFindingsByFingerprint[key].FingerPrintWithoutCommitId, description: allFindingsByFingerprint[key].Description, symlinkFile: allFindingsByFingerprint[key].SymlinkFile, orgId: organizationId, pusherEmail: pusher.email, pusherName: pusher.name, repositoryFullName: repository.fullName, repositoryId: String(repository.id), fingerprint: allFindingsByFingerprint[key].Fingerprint })), tx ); }); const adminEmails = await getOrgAdminEmails(organizationId); if (pusher?.email) { adminEmails.push(pusher.email); } if (Object.keys(allFindingsByFingerprint).length) { await smtpService.sendMail({ template: SmtpTemplates.SecretLeakIncident, subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`, recipients: adminEmails, substitutions: { numberOfSecrets: Object.keys(allFindingsByFingerprint).length, pusher_email: pusher.email, pusher_name: pusher.name } }); } telemetryService.sendPostHogEvents({ event: PostHogEventTypes.SecretScannerPush, distinctId: repository.fullName, properties: { numberOfRisks: Object.keys(allFindingsByFingerprint).length } }); }); queueService.start(QueueName.SecretFullRepoScan, async (job) => { const appCfg = getConfig(); const { organizationId, repository, installationId } = job.data; const octokit = new ProbotOctokit({ auth: { appId: appCfg.SECRET_SCANNING_GIT_APP_ID, privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY, installationId } }); const findings = await scanFullRepoContentAndGetFindings( // this is because of collision of octokit in probot and github // eslint-disable-next-line octokit as any, installationId, repository.fullName ); await secretScanningDAL.transaction(async (tx) => { if (!findings.length) return; // eslint-disable-next-line await secretScanningDAL.upsert( findings.map((finding) => ({ installationId, email: finding.Email, author: finding.Author, date: finding.Date, file: finding.File, tags: finding.Tags, commit: finding.Commit, ruleID: finding.RuleID, endLine: String(finding.EndLine), entropy: String(finding.Entropy), message: finding.Message, endColumn: String(finding.EndColumn), startLine: String(finding.StartLine), startColumn: String(finding.StartColumn), fingerPrintWithoutCommitId: finding.FingerPrintWithoutCommitId, description: finding.Description, symlinkFile: finding.SymlinkFile, orgId: organizationId, repositoryFullName: repository.fullName, repositoryId: String(repository.id), fingerprint: finding.Fingerprint })), tx ); }); const adminEmails = await getOrgAdminEmails(organizationId); if (findings.length) { await smtpService.sendMail({ template: SmtpTemplates.SecretLeakIncident, subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`, recipients: adminEmails, substitutions: { numberOfSecrets: findings.length } }); } telemetryService.sendPostHogEvents({ event: PostHogEventTypes.SecretScannerFull, distinctId: repository.fullName, properties: { numberOfRisks: findings.length } }); }); queueService.listen(QueueName.SecretPushEventScan, "failed", (job, err) => { logger.error("Failed to secret scan on push", job?.data, err); }); queueService.listen(QueueName.SecretFullRepoScan, "failed", (job, err) => { logger.error("Failed to do full repo secret scan", job?.data, err); }); return { startFullRepoScan, startPushEventScan }; };