diff --git a/backend/Dockerfile b/backend/Dockerfile index 249ece4f6..4c3abfcc1 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -23,7 +23,7 @@ COPY --from=build /app . RUN apk add --no-cache bash curl && curl -1sLf \ 'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.alpine.sh' | bash \ - && apk add infisical=0.8.1 + && apk add infisical=0.8.1 && apk add --no-cache git HEALTHCHECK --interval=10s --timeout=3s --start-period=10s \ CMD node healthcheck.js diff --git a/backend/src/controllers/v1/secretScanningController.ts b/backend/src/controllers/v1/secretScanningController.ts index 3acd7e293..ee55a4fc5 100644 --- a/backend/src/controllers/v1/secretScanningController.ts +++ b/backend/src/controllers/v1/secretScanningController.ts @@ -5,7 +5,10 @@ import { Types } from "mongoose"; import { UnauthorizedRequestError } from "../../utils/errors"; import GitAppOrganizationInstallation from "../../ee/models/gitAppOrganizationInstallation"; import { MembershipOrg } from "../../models"; +import { scanGithubFullRepoForSecretLeaks } from "../../queues/secret-scanning/githubScanFullRepository" +import { getSecretScanningGitAppId, getSecretScanningPrivateKey } from "../../config"; import GitRisks, { STATUS_RESOLVED_FALSE_POSITIVE, STATUS_RESOLVED_NOT_REVOKED, STATUS_RESOLVED_REVOKED } from "../../ee/models/gitRisks"; +import { ProbotOctokit } from "probot"; export const createInstallationSession = async (req: Request, res: Response) => { const sessionId = crypto.randomBytes(16).toString("hex"); @@ -47,6 +50,18 @@ export const linkInstallationToOrganization = async (req: Request, res: Response upsert: true }).lean() + const octokit = new ProbotOctokit({ + auth: { + appId: await getSecretScanningGitAppId(), + privateKey: await getSecretScanningPrivateKey(), + installationId: installationId.toString() + }, + }); + + const { data: { repositories }}= await octokit.apps.listReposAccessibleToInstallation() + for (const repository of repositories) { + scanGithubFullRepoForSecretLeaks({organizationId: installationSession.organization.toString(), installationId, repository: {id: repository.id, fullName: repository.full_name}}) + } res.json(installationLink) } diff --git a/backend/src/ee/services/GithubSecretScanning/helper.ts b/backend/src/ee/services/GithubSecretScanning/helper.ts index aea025410..550580a7a 100644 --- a/backend/src/ee/services/GithubSecretScanning/helper.ts +++ b/backend/src/ee/services/GithubSecretScanning/helper.ts @@ -5,6 +5,21 @@ import { join } from "path" import { SecretMatch } from "./types"; import { Octokit } from "@octokit/rest"; +export async function scanFullContentAndGetFindings(octokit: Octokit, installationId: number, repositoryFullName: string): Promise { + const tempFolder = await createTempFolder(); + const findingsPath = join(tempFolder, "findings.json"); + const repoPath = join(tempFolder, "repo.git") + try { + const { data: { token }} = await octokit.apps.createInstallationAccessToken({installation_id: installationId}) + await cloneRepo(token, repositoryFullName, repoPath) + await runInfisicalScanOnRepo(repoPath, findingsPath); + const findingsData = await readFindingsFile(findingsPath); + return JSON.parse(findingsData); + } finally { + await deleteTempFolder(tempFolder); + } +} + export async function scanContentAndGetFindings(textContent: string): Promise { const tempFolder = await createTempFolder(); const filePath = join(tempFolder, "content.txt"); @@ -36,6 +51,8 @@ export function createTempFolder(): Promise { }); } + + export function writeTextToFile(filePath: string, content: string): Promise { return new Promise((resolve, reject) => { writeFile(filePath, content, (err) => { @@ -48,6 +65,33 @@ export function writeTextToFile(filePath: string, content: string): Promise { + const cloneUrl = `https://x-access-token:${installationAcccessToken}@github.com/${repositoryFullName}.git`; + const command = `git clone ${cloneUrl} ${repoPath} --bare` + return new Promise((resolve, reject) => { + exec(command, (error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }) +} + +export function runInfisicalScanOnRepo(repoPath: string, outputPath: string): Promise { + return new Promise((resolve, reject) => { + const command = `cd ${repoPath} && infisical scan --exit-code=77 -r "${outputPath}"`; + exec(command, (error) => { + if (error && error.code != 77) { + reject(error); + } else { + resolve(); + } + }); + }); +} + export function runInfisicalScan(inputPath: string, outputPath: string): Promise { return new Promise((resolve, reject) => { const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}"`; @@ -96,30 +140,4 @@ export function convertKeysToLowercase(obj: T): T { } return convertedObj; -} - -export async function getCommits(octokit: Octokit, owner: string, repo: string) { - let commits: { sha: string }[] = []; - let page = 1; - while (true) { - const response = await octokit.repos.listCommits({ - owner, - repo, - per_page: 100, - page, - }); - - commits = commits.concat(response.data); - if (response.data.length == 0) break; - page++; - } - return commits; -} - -export async function getFilesFromCommit(octokit: any, owner: string, repo: string, sha: string) { - const response = await octokit.repos.getCommit({ - owner, - repo, - ref: sha, - }); } \ No newline at end of file diff --git a/backend/src/queues/secret-scanning/githubScanFullRepository.ts b/backend/src/queues/secret-scanning/githubScanFullRepository.ts index bd2054d90..bd464369a 100644 --- a/backend/src/queues/secret-scanning/githubScanFullRepository.ts +++ b/backend/src/queues/secret-scanning/githubScanFullRepository.ts @@ -1,201 +1,105 @@ -// import Queue, { Job } from "bull"; -// import { ProbotOctokit } from "probot" -// import { Commit, Committer, Repository } from "@octokit/webhooks-types"; -// import TelemetryService from "../../services/TelemetryService"; -// import { sendMail } from "../../helpers"; -// import GitRisks from "../../ee/models/gitRisks"; -// import { MembershipOrg, User } from "../../models"; -// import { OWNER, ADMIN } from "../../variables"; -// import { convertKeysToLowercase, getFilesFromCommit, scanContentAndGetFindings } from "../../ee/services/GithubSecretScanning/helper"; -// import { getSecretScanningGitAppId, getSecretScanningPrivateKey } from "../../config"; +import Queue, { Job } from "bull"; +import { ProbotOctokit } from "probot" +import TelemetryService from "../../services/TelemetryService"; +import { sendMail } from "../../helpers"; +import GitRisks from "../../ee/models/gitRisks"; +import { MembershipOrg, User } from "../../models"; +import { ADMIN, OWNER } from "../../variables"; +import { convertKeysToLowercase, scanFullContentAndGetFindings } from "../../ee/services/GithubSecretScanning/helper"; +import { getSecretScanningGitAppId, getSecretScanningPrivateKey } from "../../config"; +import { SecretMatch } from "../../ee/services/GithubSecretScanning/types"; -// const githubFullRepositoryScan = new Queue('github-historical-secret-scanning', 'redis://redis:6379'); +export const githubFullRepositorySecretScan = new Queue("github-full-repository-secret-scanning", "redis://redis:6379"); -// type TScanFullRepositoryDetails = { -// organizationId: string, -// repositories: { -// id: number; -// node_id: string; -// name: string; -// full_name: string; -// private: boolean; -// }[] | undefined -// installationId: number -// } +type TScanPushEventQueueDetails = { + organizationId: string, + installationId: number, + repository: { + id: number, + fullName: string, + }, +} -// type SecretMatch = { -// Description: string; -// StartLine: number; -// EndLine: number; -// StartColumn: number; -// EndColumn: number; -// Match: string; -// Secret: string; -// File: string; -// SymlinkFile: string; -// Commit: string; -// Entropy: number; -// Author: string; -// Email: string; -// Date: string; -// Message: string; -// Tags: string[]; -// RuleID: string; -// Fingerprint: string; -// FingerPrintWithoutCommitId: string -// }; +githubFullRepositorySecretScan.process(async (job: Job, done: Queue.DoneCallback) => { + const { organizationId, repository, installationId }: TScanPushEventQueueDetails = job.data + const octokit = new ProbotOctokit({ + auth: { + appId: await getSecretScanningGitAppId(), + privateKey: await getSecretScanningPrivateKey(), + installationId: installationId + }, + }); +try { + const findings : SecretMatch[] = await scanFullContentAndGetFindings(octokit, installationId, repository.fullName) -// type Helllo = { -// url: string; -// sha: string; -// node_id: string; -// html_url: string; -// comments_url: string; -// commit: { -// url: string; -// author: { -// name?: string | undefined; -// email?: string | undefined; -// date?: string | undefined; -// } | null; -// verification?: { -// } | undefined; -// }; -// files?: {}[] | undefined; -// }[] + for (const finding of findings) { + await GitRisks.findOneAndUpdate({ fingerprint: finding.Fingerprint}, + { + ...convertKeysToLowercase(finding), + installationId: installationId, + organization: organizationId, + repositoryFullName: repository.fullName, + repositoryId: repository.id + }, { + upsert: true + }).lean() + } + // get emails of admins + const adminsOfWork = await MembershipOrg.find({ + organization: organizationId, + $or: [ + { role: OWNER }, + { role: ADMIN } + ] + }).lean() -// githubFullRepositoryScan.process(async (job: Job, done: Queue.DoneCallback) => { -// const { organizationId, repositories, installationId }: TScanFullRepositoryDetails = job.data -// const repositoryFullNamesList = repositories ? repositories.map(repoDetails => repoDetails.full_name) : [] -// const octokit = new ProbotOctokit({ -// auth: { -// appId: await getSecretScanningGitAppId(), -// privateKey: await getSecretScanningPrivateKey(), -// installationId: installationId -// }, -// }); + const userEmails = await User.find({ + _id: { + $in: [adminsOfWork.map(orgMembership => orgMembership.user)] + } + }).select("email").lean() -// for (const repositoryFullName of repositoryFullNamesList) { -// const [owner, repo] = repositoryFullName.split("/"); + const usersToNotify = userEmails.map(userObject => userObject.email) -// let page = 1; -// while (true) { -// // octokit.repos.getco -// const { data } = await octokit.repos.listCommits({ -// owner, -// repo, -// per_page: 100, -// page -// }); + if (findings.length) { + await sendMail({ + template: "historicalSecretLeakIncident.handlebars", + subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`, + recipients: usersToNotify, + substitutions: { + numberOfSecrets: findings.length, + } + }); + } + const postHogClient = await TelemetryService.getPostHogClient(); + if (postHogClient) { + postHogClient.capture({ + event: "historical cloud secret scan", + distinctId: repository.fullName, + properties: { + numberOfRisksFound: findings.length, + } + }); + } + done(null, findings) +} catch (error) { + done(new Error(`gitHubHistoricalScanning.process: an error occurred ${error}`), null) + } -// await getFilesFromCommit(octokit, owner, repo, "646b386605177ed0a2cc0a596eeee0cf57666342") - - -// page++; -// } - -// } - -// done() - -// // const allFindingsByFingerprint: { [key: string]: SecretMatch; } = {} -// // for (const commit of commits) { -// // for (const filepath of [...commit.added, ...commit.modified]) { -// // try { -// // const fileContentsResponse = await octokit.repos.getContent({ -// // owner, -// // repo, -// // path: filepath, -// // }); - -// // const data: any = fileContentsResponse.data; -// // const fileContent = Buffer.from(data.content, "base64").toString(); - -// // const findings = await scanContentAndGetFindings(`\n${fileContent}`) // extra line to count lines correctly - -// // for (const finding of findings) { -// // const fingerPrintWithCommitId = `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}` -// // const fingerPrintWithoutCommitId = `${filepath}:${finding.RuleID}:${finding.StartLine}` -// // finding.Fingerprint = fingerPrintWithCommitId -// // finding.FingerPrintWithoutCommitId = fingerPrintWithoutCommitId -// // finding.Commit = commit.id -// // finding.File = filepath -// // finding.Author = commit.author.name -// // finding.Email = commit?.author?.email ? commit?.author?.email : "" - -// // allFindingsByFingerprint[fingerPrintWithCommitId] = finding -// // } - -// // } catch (error) { -// // done(new Error(`gitHubHistoricalScanning.process: unable to fetch content for [filepath=${filepath}] because [error=${error}]`), null) -// // } -// // } -// // } - -// // // change to update -// // for (const key in allFindingsByFingerprint) { -// // await GitRisks.findOneAndUpdate({ fingerprint: allFindingsByFingerprint[key].Fingerprint }, -// // { -// // ...convertKeysToLowercase(allFindingsByFingerprint[key]), -// // installationId: installationId, -// // organization: organizationId, -// // repositoryFullName: repository.fullName, -// // repositoryId: repository.id -// // }, { -// // upsert: true -// // }).lean() -// // } -// // // get emails of admins -// // const adminsOfWork = await MembershipOrg.find({ -// // organization: organizationId, -// // $or: [ -// // { role: OWNER }, -// // { role: ADMIN } -// // ] -// // }).lean() - -// // const userEmails = await User.find({ -// // _id: { -// // $in: [adminsOfWork.map(orgMembership => orgMembership.user)] -// // } -// // }).select("email").lean() - -// // const adminOrOwnerEmails = userEmails.map(userObject => userObject.email) - -// // const usersToNotify = pusher?.email ? [pusher.email, ...adminOrOwnerEmails] : [...adminOrOwnerEmails] -// // if (Object.keys(allFindingsByFingerprint).length) { -// // await sendMail({ -// // template: "secretLeakIncident.handlebars", -// // subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`, -// // recipients: usersToNotify, -// // substitutions: { -// // numberOfSecrets: Object.keys(allFindingsByFingerprint).length, -// // pusher_email: pusher.email, -// // pusher_name: pusher.name -// // } -// // }); -// // } - -// // const postHogClient = await TelemetryService.getPostHogClient(); -// // if (postHogClient) { -// // postHogClient.capture({ -// // event: "cloud secret scan", -// // distinctId: pusher.email, -// // properties: { -// // numberOfCommitsScanned: commits.length, -// // numberOfRisksFound: Object.keys(allFindingsByFingerprint).length, -// // } -// // }); -// // } - -// // done(null, allFindingsByFingerprint) - -// }) - -// export const scanGithubFullRepositoryForSecretLeaks = (scanFullRepositoryDetails: TScanFullRepositoryDetails) => { -// console.log("full repo scan started") -// githubFullRepositoryScan.add(scanFullRepositoryDetails) -// } +}) +export const scanGithubFullRepoForSecretLeaks = (pushEventPayload: TScanPushEventQueueDetails) => { + githubFullRepositorySecretScan.add(pushEventPayload, { + attempts: 3, + backoff: { + type: "exponential", + delay: 5000 + }, + removeOnComplete: true, + removeOnFail: { + count: 20 // keep the most recent 20 jobs + } + }) +} \ No newline at end of file diff --git a/backend/src/templates/historicalSecretLeakIncident.handlebars b/backend/src/templates/historicalSecretLeakIncident.handlebars new file mode 100644 index 000000000..3cb517a57 --- /dev/null +++ b/backend/src/templates/historicalSecretLeakIncident.handlebars @@ -0,0 +1,21 @@ + + + + + + + Incident alert: secrets potentially leaked + + + +

Infisical has uncovered {{numberOfSecrets}} secret(s) from historical commits to your repo

+

View leaked secrets

+ +

If these are production secrets, please rotate them immediately.

+ +

Once you have taken action, be sure to update the status of the risk in your Infisical + dashboard.

+ + + \ No newline at end of file