Initial implementation

backend/Dockerfile

@@ -23,7 +23,7 @@ COPY --from=build /app .
 
 RUN apk add --no-cache bash curl && curl -1sLf \
   'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.alpine.sh' | bash \
-  && apk add infisical=0.8.1
+  && apk add infisical=0.8.1 && apk add --no-cache git
 
 HEALTHCHECK --interval=10s --timeout=3s --start-period=10s \  
   CMD node healthcheck.js

backend/src/controllers/v1/secretScanningController.ts

@@ -5,7 +5,10 @@ import { Types } from "mongoose";
 import { UnauthorizedRequestError } from "../../utils/errors";
 import GitAppOrganizationInstallation from "../../ee/models/gitAppOrganizationInstallation";
 import { MembershipOrg } from "../../models";
+import { scanGithubFullRepoForSecretLeaks  } from "../../queues/secret-scanning/githubScanFullRepository"
+import { getSecretScanningGitAppId, getSecretScanningPrivateKey } from "../../config";
 import GitRisks, { STATUS_RESOLVED_FALSE_POSITIVE, STATUS_RESOLVED_NOT_REVOKED, STATUS_RESOLVED_REVOKED } from "../../ee/models/gitRisks";
+import { ProbotOctokit } from "probot";
 
 export const createInstallationSession = async (req: Request, res: Response) => {
   const sessionId = crypto.randomBytes(16).toString("hex");
@@ -47,6 +50,18 @@ export const linkInstallationToOrganization = async (req: Request, res: Response
     upsert: true
   }).lean()
 
+  const octokit = new ProbotOctokit({
+    auth: {
+      appId: await getSecretScanningGitAppId(),
+      privateKey: await getSecretScanningPrivateKey(),
+      installationId: installationId.toString()
+    },
+  });
+
+  const { data: { repositories }}= await octokit.apps.listReposAccessibleToInstallation()
+  for (const repository of repositories) {
+    scanGithubFullRepoForSecretLeaks({organizationId: installationSession.organization.toString(), installationId, repository: {id: repository.id, fullName: repository.full_name}})
+  }
   res.json(installationLink)
 }
 

backend/src/ee/services/GithubSecretScanning/helper.ts

@@ -5,6 +5,21 @@ import { join } from "path"
 import { SecretMatch } from "./types";
 import { Octokit } from "@octokit/rest";
 
+export async function scanFullContentAndGetFindings(octokit: Octokit, installationId: number, repositoryFullName: string): Promise<SecretMatch[]> {
+  const tempFolder = await createTempFolder();
+  const findingsPath = join(tempFolder, "findings.json");
+  const repoPath = join(tempFolder, "repo.git")
+  try {
+    const { data: { token }} = await octokit.apps.createInstallationAccessToken({installation_id: installationId})
+    await cloneRepo(token, repositoryFullName, repoPath)
+    await runInfisicalScanOnRepo(repoPath, findingsPath);
+    const findingsData = await readFindingsFile(findingsPath);
+    return JSON.parse(findingsData);
+  } finally {
+    await deleteTempFolder(tempFolder);
+  }
+}
+
 export async function scanContentAndGetFindings(textContent: string): Promise<SecretMatch[]> {
   const tempFolder = await createTempFolder();
   const filePath = join(tempFolder, "content.txt");
@@ -36,6 +51,8 @@ export function createTempFolder(): Promise<string> {
   });
 }
 
+
+
 export function writeTextToFile(filePath: string, content: string): Promise<void> {
   return new Promise((resolve, reject) => {
     writeFile(filePath, content, (err) => {
@@ -48,6 +65,33 @@ export function writeTextToFile(filePath: string, content: string): Promise<void
   });
 }
 
+export async function cloneRepo(installationAcccessToken: string, repositoryFullName: string, repoPath: string): Promise<void> {
+  const cloneUrl = `https://x-access-token:${installationAcccessToken}@github.com/${repositoryFullName}.git`;
+  const command = `git clone ${cloneUrl} ${repoPath} --bare`
+  return new Promise((resolve, reject) => {
+    exec(command, (error) => {
+      if (error) {
+        reject(error);
+      } else {
+        resolve();
+      }
+    });
+  }) 
+}
+
+export function runInfisicalScanOnRepo(repoPath: string, outputPath: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const command = `cd ${repoPath} && infisical scan --exit-code=77 -r "${outputPath}"`;
+    exec(command, (error) => {
+      if (error && error.code != 77) {
+        reject(error);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
 export function runInfisicalScan(inputPath: string, outputPath: string): Promise<void> {
   return new Promise((resolve, reject) => {
     const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}"`;
@@ -96,30 +140,4 @@ export function convertKeysToLowercase<T>(obj: T): T {
   }
 
   return convertedObj;
-}
-
-export async function getCommits(octokit: Octokit, owner: string, repo: string) {
-  let commits: { sha: string }[] = [];
-  let page = 1;
-  while (true) {
-    const response = await octokit.repos.listCommits({
-      owner,
-      repo,
-      per_page: 100,
-      page,
-    });
-
-    commits = commits.concat(response.data);
-    if (response.data.length == 0) break;
-    page++;
-  }
-  return commits;
-}
-
-export async function getFilesFromCommit(octokit: any, owner: string, repo: string, sha: string) {
-  const response = await octokit.repos.getCommit({
-    owner,
-    repo,
-    ref: sha,
-  });
 }

backend/src/queues/secret-scanning/githubScanFullRepository.ts

@@ -1,201 +1,105 @@
-// import Queue, { Job } from "bull";
+import Queue, { Job } from "bull";
-// import { ProbotOctokit } from "probot"
+import { ProbotOctokit } from "probot"
-// import { Commit, Committer, Repository } from "@octokit/webhooks-types";
+import TelemetryService from "../../services/TelemetryService";
-// import TelemetryService from "../../services/TelemetryService";
+import { sendMail } from "../../helpers";
-// import { sendMail } from "../../helpers";
+import GitRisks from "../../ee/models/gitRisks";
-// import GitRisks from "../../ee/models/gitRisks";
+import { MembershipOrg, User } from "../../models";
-// import { MembershipOrg, User } from "../../models";
+import { ADMIN, OWNER } from "../../variables";
-// import { OWNER, ADMIN } from "../../variables";
+import { convertKeysToLowercase, scanFullContentAndGetFindings } from "../../ee/services/GithubSecretScanning/helper";
-// import { convertKeysToLowercase, getFilesFromCommit, scanContentAndGetFindings } from "../../ee/services/GithubSecretScanning/helper";
+import { getSecretScanningGitAppId, getSecretScanningPrivateKey } from "../../config";
-// import { getSecretScanningGitAppId, getSecretScanningPrivateKey } from "../../config";
+import { SecretMatch } from "../../ee/services/GithubSecretScanning/types";
 
-// const githubFullRepositoryScan = new Queue('github-historical-secret-scanning', 'redis://redis:6379');
+export const githubFullRepositorySecretScan = new Queue("github-full-repository-secret-scanning", "redis://redis:6379");
 
-// type TScanFullRepositoryDetails = {
+type TScanPushEventQueueDetails = {
-//   organizationId: string,
+  organizationId: string,
-//   repositories: {
+  installationId: number, 
-//     id: number;
+  repository: {
-//     node_id: string;
+    id: number,
-//     name: string;
+    fullName: string,
-//     full_name: string;
+  },
-//     private: boolean;
+}
-//   }[] | undefined
-//   installationId: number
-// }
 
-// type SecretMatch = {
+githubFullRepositorySecretScan.process(async (job: Job, done: Queue.DoneCallback) => {
-//   Description: string;
+  const { organizationId, repository, installationId }: TScanPushEventQueueDetails = job.data
-//   StartLine: number;
+  const octokit = new ProbotOctokit({
-//   EndLine: number;
+    auth: {
-//   StartColumn: number;
+      appId: await getSecretScanningGitAppId(),
-//   EndColumn: number;
+      privateKey: await getSecretScanningPrivateKey(),
-//   Match: string;
+      installationId: installationId
-//   Secret: string;
+    },
-//   File: string;
+  });
-//   SymlinkFile: string;
+try {
-//   Commit: string;
+  const findings : SecretMatch[] = await scanFullContentAndGetFindings(octokit, installationId, repository.fullName)
-//   Entropy: number;
-//   Author: string;
-//   Email: string;
-//   Date: string;
-//   Message: string;
-//   Tags: string[];
-//   RuleID: string;
-//   Fingerprint: string;
-//   FingerPrintWithoutCommitId: string
-// };
 
-// type Helllo = {
+  for (const finding of findings) {
-//   url: string;
+    await GitRisks.findOneAndUpdate({ fingerprint: finding.Fingerprint},
-//   sha: string;
+      {
-//   node_id: string;
+        ...convertKeysToLowercase(finding),
-//   html_url: string;
+        installationId: installationId,
-//   comments_url: string;
+        organization: organizationId,
-//   commit: {
+        repositoryFullName: repository.fullName,
-//     url: string;
+        repositoryId: repository.id
-//     author: {
+      }, {
-//       name?: string | undefined;
+      upsert: true
-//       email?: string | undefined;
+    }).lean()
-//       date?: string | undefined;
+  }
-//     } | null;
-//     verification?: {
-//     } | undefined;
-//   };
-//   files?: {}[] | undefined;
-// }[]
 
+  // get emails of admins
+  const adminsOfWork = await MembershipOrg.find({
+    organization: organizationId,
+    $or: [
+      { role: OWNER },
+      { role: ADMIN }
+    ]
+  }).lean()
 
-// githubFullRepositoryScan.process(async (job: Job, done: Queue.DoneCallback) => {
+  const userEmails = await User.find({
-//   const { organizationId, repositories, installationId }: TScanFullRepositoryDetails = job.data
+    _id: {
-//   const repositoryFullNamesList = repositories ? repositories.map(repoDetails => repoDetails.full_name) : []
+      $in: [adminsOfWork.map(orgMembership => orgMembership.user)]
-//   const octokit = new ProbotOctokit({
+    }
-//     auth: {
+  }).select("email").lean()
-//       appId: await getSecretScanningGitAppId(),
-//       privateKey: await getSecretScanningPrivateKey(),
-//       installationId: installationId
-//     },
-//   });
 
-//   for (const repositoryFullName of repositoryFullNamesList) {
+  const usersToNotify = userEmails.map(userObject => userObject.email)
-//     const [owner, repo] = repositoryFullName.split("/");
 
-//     let page = 1;
+  if (findings.length) {
-//     while (true) {
+    await sendMail({
-//       // octokit.repos.getco
+      template: "historicalSecretLeakIncident.handlebars",
-//       const { data } = await octokit.repos.listCommits({
+      subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`,
-//         owner,
+      recipients: usersToNotify,
-//         repo,
+      substitutions: {
-//         per_page: 100,
+        numberOfSecrets: findings.length,
-//         page
+      }
-//       });
+    });
+  }
 
+  const postHogClient = await TelemetryService.getPostHogClient();
+  if (postHogClient) {
+    postHogClient.capture({
+      event: "historical cloud secret scan",
+      distinctId: repository.fullName,
+      properties: {
+        numberOfRisksFound: findings.length,
+      }
+    });
+  }
+  done(null, findings)
+} catch (error) {
+    done(new Error(`gitHubHistoricalScanning.process: an error occurred ${error}`), null)
+  }
 
-//       await getFilesFromCommit(octokit, owner, repo, "646b386605177ed0a2cc0a596eeee0cf57666342")
+})
-
-
-//       page++;
-//     }
-
-//   }
-
-//   done()
-
-//   // const allFindingsByFingerprint: { [key: string]: SecretMatch; } = {}
-//   // for (const commit of commits) {
-//   //   for (const filepath of [...commit.added, ...commit.modified]) {
-//   //     try {
-//   //       const fileContentsResponse = await octokit.repos.getContent({
-//   //         owner,
-//   //         repo,
-//   //         path: filepath,
-//   //       });
-
-//   //       const data: any = fileContentsResponse.data;
-//   //       const fileContent = Buffer.from(data.content, "base64").toString();
-
-//   //       const findings = await scanContentAndGetFindings(`\n${fileContent}`) // extra line to count lines correctly
-
-//   //       for (const finding of findings) {
-//   //         const fingerPrintWithCommitId = `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`
-//   //         const fingerPrintWithoutCommitId = `${filepath}:${finding.RuleID}:${finding.StartLine}`
-//   //         finding.Fingerprint = fingerPrintWithCommitId
-//   //         finding.FingerPrintWithoutCommitId = fingerPrintWithoutCommitId
-//   //         finding.Commit = commit.id
-//   //         finding.File = filepath
-//   //         finding.Author = commit.author.name
-//   //         finding.Email = commit?.author?.email ? commit?.author?.email : ""
-
-//   //         allFindingsByFingerprint[fingerPrintWithCommitId] = finding
-//   //       }
-
-//   //     } catch (error) {
-//   //       done(new Error(`gitHubHistoricalScanning.process: unable to fetch content for [filepath=${filepath}] because [error=${error}]`), null)
-//   //     }
-//   //   }
-//   // }
-
-//   // // change to update
-//   // for (const key in allFindingsByFingerprint) {
-//   //   await GitRisks.findOneAndUpdate({ fingerprint: allFindingsByFingerprint[key].Fingerprint },
-//   //     {
-//   //       ...convertKeysToLowercase(allFindingsByFingerprint[key]),
-//   //       installationId: installationId,
-//   //       organization: organizationId,
-//   //       repositoryFullName: repository.fullName,
-//   //       repositoryId: repository.id
-//   //     }, {
-//   //     upsert: true
-//   //   }).lean()
-//   // }
-//   // // get emails of admins
-//   // const adminsOfWork = await MembershipOrg.find({
-//   //   organization: organizationId,
-//   //   $or: [
-//   //     { role: OWNER },
-//   //     { role: ADMIN }
-//   //   ]
-//   // }).lean()
-
-//   // const userEmails = await User.find({
-//   //   _id: {
-//   //     $in: [adminsOfWork.map(orgMembership => orgMembership.user)]
-//   //   }
-//   // }).select("email").lean()
-
-//   // const adminOrOwnerEmails = userEmails.map(userObject => userObject.email)
-
-//   // const usersToNotify = pusher?.email ? [pusher.email, ...adminOrOwnerEmails] : [...adminOrOwnerEmails]
-//   // if (Object.keys(allFindingsByFingerprint).length) {
-//   //   await sendMail({
-//   //     template: "secretLeakIncident.handlebars",
-//   //     subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`,
-//   //     recipients: usersToNotify,
-//   //     substitutions: {
-//   //       numberOfSecrets: Object.keys(allFindingsByFingerprint).length,
-//   //       pusher_email: pusher.email,
-//   //       pusher_name: pusher.name
-//   //     }
-//   //   });
-//   // }
-
-//   // const postHogClient = await TelemetryService.getPostHogClient();
-//   // if (postHogClient) {
-//   //   postHogClient.capture({
-//   //     event: "cloud secret scan",
-//   //     distinctId: pusher.email,
-//   //     properties: {
-//   //       numberOfCommitsScanned: commits.length,
-//   //       numberOfRisksFound: Object.keys(allFindingsByFingerprint).length,
-//   //     }
-//   //   });
-//   // }
-
-//   // done(null, allFindingsByFingerprint)
-
-// })
-
-// export const scanGithubFullRepositoryForSecretLeaks = (scanFullRepositoryDetails: TScanFullRepositoryDetails) => {
-//   console.log("full repo scan started")
-//   githubFullRepositoryScan.add(scanFullRepositoryDetails)
-// }
 
+export const scanGithubFullRepoForSecretLeaks = (pushEventPayload: TScanPushEventQueueDetails) => {
+  githubFullRepositorySecretScan.add(pushEventPayload, {
+    attempts: 3,
+    backoff: {
+      type: "exponential",
+      delay: 5000
+    },
+    removeOnComplete: true,
+    removeOnFail: {
+      count: 20 // keep the most recent 20 jobs
+    }
+  })
+}

backend/src/templates/historicalSecretLeakIncident.handlebars

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="x-ua-compatible" content="ie=edge">
+  <title>Incident alert: secrets potentially leaked</title>
+</head>
+
+<body>
+  <h3>Infisical has uncovered {{numberOfSecrets}} secret(s) from historical commits to your repo</h3>
+  <p><a href="https://app.infisical.com/secret-scanning"><strong>View leaked secrets</strong></a></p>
+
+  <p>If these are production secrets, please rotate them immediately.</p>
+
+  <p>Once you have taken action, be sure to update the status of the risk in your <a
+      href="https://app.infisical.com/">Infisical
+      dashboard</a>.</p>
+</body>
+
+</html>