Initial implementation

2026-09-22 13:39:35 +00:00 · 2023-09-01 22:36:52 +01:00
parent 0616f24923
commit 35f3d6c776
5 changed files with 175 additions and 217 deletions
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -23,7 +23,7 @@ COPY --from=build /app .

 RUN apk add --no-cache bash curl && curl -1sLf \
  'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.alpine.sh' | bash \
-  && apk add infisical=0.8.1
+  && apk add infisical=0.8.1 && apk add --no-cache git

 HEALTHCHECK --interval=10s --timeout=3s --start-period=10s \  
  CMD node healthcheck.js
--- a/backend/src/controllers/v1/secretScanningController.ts
+++ b/backend/src/controllers/v1/secretScanningController.ts
@@ -5,7 +5,10 @@ import { Types } from "mongoose";
 import { UnauthorizedRequestError } from "../../utils/errors";
 import GitAppOrganizationInstallation from "../../ee/models/gitAppOrganizationInstallation";
 import { MembershipOrg } from "../../models";
+import { scanGithubFullRepoForSecretLeaks  } from "../../queues/secret-scanning/githubScanFullRepository"
+import { getSecretScanningGitAppId, getSecretScanningPrivateKey } from "../../config";
 import GitRisks, { STATUS_RESOLVED_FALSE_POSITIVE, STATUS_RESOLVED_NOT_REVOKED, STATUS_RESOLVED_REVOKED } from "../../ee/models/gitRisks";
+import { ProbotOctokit } from "probot";

 export const createInstallationSession = async (req: Request, res: Response) => {
  const sessionId = crypto.randomBytes(16).toString("hex");
@@ -47,6 +50,18 @@ export const linkInstallationToOrganization = async (req: Request, res: Response
    upsert: true
  }).lean()

+  const octokit = new ProbotOctokit({
+    auth: {
+      appId: await getSecretScanningGitAppId(),
+      privateKey: await getSecretScanningPrivateKey(),
+      installationId: installationId.toString()
+    },
+  });
+
+  const { data: { repositories }}= await octokit.apps.listReposAccessibleToInstallation()
+  for (const repository of repositories) {
+    scanGithubFullRepoForSecretLeaks({organizationId: installationSession.organization.toString(), installationId, repository: {id: repository.id, fullName: repository.full_name}})
+  }
  res.json(installationLink)
 }

--- a/backend/src/ee/services/GithubSecretScanning/helper.ts
+++ b/backend/src/ee/services/GithubSecretScanning/helper.ts
@@ -5,6 +5,21 @@ import { join } from "path"
 import { SecretMatch } from "./types";
 import { Octokit } from "@octokit/rest";

+export async function scanFullContentAndGetFindings(octokit: Octokit, installationId: number, repositoryFullName: string): Promise<SecretMatch[]> {
+  const tempFolder = await createTempFolder();
+  const findingsPath = join(tempFolder, "findings.json");
+  const repoPath = join(tempFolder, "repo.git")
+  try {
+    const { data: { token }} = await octokit.apps.createInstallationAccessToken({installation_id: installationId})
+    await cloneRepo(token, repositoryFullName, repoPath)
+    await runInfisicalScanOnRepo(repoPath, findingsPath);
+    const findingsData = await readFindingsFile(findingsPath);
+    return JSON.parse(findingsData);
+  } finally {
+    await deleteTempFolder(tempFolder);
+  }
+}
+
 export async function scanContentAndGetFindings(textContent: string): Promise<SecretMatch[]> {
  const tempFolder = await createTempFolder();
  const filePath = join(tempFolder, "content.txt");
@@ -36,6 +51,8 @@ export function createTempFolder(): Promise<string> {
  });
 }

+
+
 export function writeTextToFile(filePath: string, content: string): Promise<void> {
  return new Promise((resolve, reject) => {
    writeFile(filePath, content, (err) => {
@@ -48,6 +65,33 @@ export function writeTextToFile(filePath: string, content: string): Promise<void
  });
 }

+export async function cloneRepo(installationAcccessToken: string, repositoryFullName: string, repoPath: string): Promise<void> {
+  const cloneUrl = `https://x-access-token:${installationAcccessToken}@github.com/${repositoryFullName}.git`;
+  const command = `git clone ${cloneUrl} ${repoPath} --bare`
+  return new Promise((resolve, reject) => {
+    exec(command, (error) => {
+      if (error) {
+        reject(error);
+      } else {
+        resolve();
+      }
+    });
+  }) 
+}
+
+export function runInfisicalScanOnRepo(repoPath: string, outputPath: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const command = `cd ${repoPath} && infisical scan --exit-code=77 -r "${outputPath}"`;
+    exec(command, (error) => {
+      if (error && error.code != 77) {
+        reject(error);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
 export function runInfisicalScan(inputPath: string, outputPath: string): Promise<void> {
  return new Promise((resolve, reject) => {
    const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}"`;
@@ -96,30 +140,4 @@ export function convertKeysToLowercase<T>(obj: T): T {
  }

  return convertedObj;
-}
-
-export async function getCommits(octokit: Octokit, owner: string, repo: string) {
-  let commits: { sha: string }[] = [];
-  let page = 1;
-  while (true) {
-    const response = await octokit.repos.listCommits({
-      owner,
-      repo,
-      per_page: 100,
-      page,
-    });
-
-    commits = commits.concat(response.data);
-    if (response.data.length == 0) break;
-    page++;
-  }
-  return commits;
-}
-
-export async function getFilesFromCommit(octokit: any, owner: string, repo: string, sha: string) {
-  const response = await octokit.repos.getCommit({
-    owner,
-    repo,
-    ref: sha,
-  });
 }
--- a/backend/src/queues/secret-scanning/githubScanFullRepository.ts
+++ b/backend/src/queues/secret-scanning/githubScanFullRepository.ts
@@ -1,201 +1,105 @@
-// import Queue, { Job } from "bull";
-// import { ProbotOctokit } from "probot"
-// import { Commit, Committer, Repository } from "@octokit/webhooks-types";
-// import TelemetryService from "../../services/TelemetryService";
-// import { sendMail } from "../../helpers";
-// import GitRisks from "../../ee/models/gitRisks";
-// import { MembershipOrg, User } from "../../models";
-// import { OWNER, ADMIN } from "../../variables";
-// import { convertKeysToLowercase, getFilesFromCommit, scanContentAndGetFindings } from "../../ee/services/GithubSecretScanning/helper";
-// import { getSecretScanningGitAppId, getSecretScanningPrivateKey } from "../../config";
+import Queue, { Job } from "bull";
+import { ProbotOctokit } from "probot"
+import TelemetryService from "../../services/TelemetryService";
+import { sendMail } from "../../helpers";
+import GitRisks from "../../ee/models/gitRisks";
+import { MembershipOrg, User } from "../../models";
+import { ADMIN, OWNER } from "../../variables";
+import { convertKeysToLowercase, scanFullContentAndGetFindings } from "../../ee/services/GithubSecretScanning/helper";
+import { getSecretScanningGitAppId, getSecretScanningPrivateKey } from "../../config";
+import { SecretMatch } from "../../ee/services/GithubSecretScanning/types";

-// const githubFullRepositoryScan = new Queue('github-historical-secret-scanning', 'redis://redis:6379');
+export const githubFullRepositorySecretScan = new Queue("github-full-repository-secret-scanning", "redis://redis:6379");

-// type TScanFullRepositoryDetails = {
-//   organizationId: string,
-//   repositories: {
-//     id: number;
-//     node_id: string;
-//     name: string;
-//     full_name: string;
-//     private: boolean;
-//   }[] | undefined
-//   installationId: number
-// }
+type TScanPushEventQueueDetails = {
+  organizationId: string,
+  installationId: number, 
+  repository: {
+    id: number,
+    fullName: string,
+  },
+}

-// type SecretMatch = {
-//   Description: string;
-//   StartLine: number;
-//   EndLine: number;
-//   StartColumn: number;
-//   EndColumn: number;
-//   Match: string;
-//   Secret: string;
-//   File: string;
-//   SymlinkFile: string;
-//   Commit: string;
-//   Entropy: number;
-//   Author: string;
-//   Email: string;
-//   Date: string;
-//   Message: string;
-//   Tags: string[];
-//   RuleID: string;
-//   Fingerprint: string;
-//   FingerPrintWithoutCommitId: string
-// };
+githubFullRepositorySecretScan.process(async (job: Job, done: Queue.DoneCallback) => {
+  const { organizationId, repository, installationId }: TScanPushEventQueueDetails = job.data
+  const octokit = new ProbotOctokit({
+    auth: {
+      appId: await getSecretScanningGitAppId(),
+      privateKey: await getSecretScanningPrivateKey(),
+      installationId: installationId
+    },
+  });
+try {
+  const findings : SecretMatch[] = await scanFullContentAndGetFindings(octokit, installationId, repository.fullName)

-// type Helllo = {
-//   url: string;
-//   sha: string;
-//   node_id: string;
-//   html_url: string;
-//   comments_url: string;
-//   commit: {
-//     url: string;
-//     author: {
-//       name?: string | undefined;
-//       email?: string | undefined;
-//       date?: string | undefined;
-//     } | null;
-//     verification?: {
-//     } | undefined;
-//   };
-//   files?: {}[] | undefined;
-// }[]
+  for (const finding of findings) {
+    await GitRisks.findOneAndUpdate({ fingerprint: finding.Fingerprint},
+      {
+        ...convertKeysToLowercase(finding),
+        installationId: installationId,
+        organization: organizationId,
+        repositoryFullName: repository.fullName,
+        repositoryId: repository.id
+      }, {
+      upsert: true
+    }).lean()
+  }

+  // get emails of admins
+  const adminsOfWork = await MembershipOrg.find({
+    organization: organizationId,
+    $or: [
+      { role: OWNER },
+      { role: ADMIN }
+    ]
+  }).lean()

-// githubFullRepositoryScan.process(async (job: Job, done: Queue.DoneCallback) => {
-//   const { organizationId, repositories, installationId }: TScanFullRepositoryDetails = job.data
-//   const repositoryFullNamesList = repositories ? repositories.map(repoDetails => repoDetails.full_name) : []
-//   const octokit = new ProbotOctokit({
-//     auth: {
-//       appId: await getSecretScanningGitAppId(),
-//       privateKey: await getSecretScanningPrivateKey(),
-//       installationId: installationId
-//     },
-//   });
+  const userEmails = await User.find({
+    _id: {
+      $in: [adminsOfWork.map(orgMembership => orgMembership.user)]
+    }
+  }).select("email").lean()

-//   for (const repositoryFullName of repositoryFullNamesList) {
-//     const [owner, repo] = repositoryFullName.split("/");
+  const usersToNotify = userEmails.map(userObject => userObject.email)

-//     let page = 1;
-//     while (true) {
-//       // octokit.repos.getco
-//       const { data } = await octokit.repos.listCommits({
-//         owner,
-//         repo,
-//         per_page: 100,
-//         page
-//       });
+  if (findings.length) {
+    await sendMail({
+      template: "historicalSecretLeakIncident.handlebars",
+      subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`,
+      recipients: usersToNotify,
+      substitutions: {
+        numberOfSecrets: findings.length,
+      }
+    });
+  }

+  const postHogClient = await TelemetryService.getPostHogClient();
+  if (postHogClient) {
+    postHogClient.capture({
+      event: "historical cloud secret scan",
+      distinctId: repository.fullName,
+      properties: {
+        numberOfRisksFound: findings.length,
+      }
+    });
+  }
+  done(null, findings)
+} catch (error) {
+    done(new Error(`gitHubHistoricalScanning.process: an error occurred ${error}`), null)
+  }

-//       await getFilesFromCommit(octokit, owner, repo, "646b386605177ed0a2cc0a596eeee0cf57666342")
-
-
-//       page++;
-//     }
-
-//   }
-
-//   done()
-
-//   // const allFindingsByFingerprint: { [key: string]: SecretMatch; } = {}
-//   // for (const commit of commits) {
-//   //   for (const filepath of [...commit.added, ...commit.modified]) {
-//   //     try {
-//   //       const fileContentsResponse = await octokit.repos.getContent({
-//   //         owner,
-//   //         repo,
-//   //         path: filepath,
-//   //       });
-
-//   //       const data: any = fileContentsResponse.data;
-//   //       const fileContent = Buffer.from(data.content, "base64").toString();
-
-//   //       const findings = await scanContentAndGetFindings(`\n${fileContent}`) // extra line to count lines correctly
-
-//   //       for (const finding of findings) {
-//   //         const fingerPrintWithCommitId = `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`
-//   //         const fingerPrintWithoutCommitId = `${filepath}:${finding.RuleID}:${finding.StartLine}`
-//   //         finding.Fingerprint = fingerPrintWithCommitId
-//   //         finding.FingerPrintWithoutCommitId = fingerPrintWithoutCommitId
-//   //         finding.Commit = commit.id
-//   //         finding.File = filepath
-//   //         finding.Author = commit.author.name
-//   //         finding.Email = commit?.author?.email ? commit?.author?.email : ""
-
-//   //         allFindingsByFingerprint[fingerPrintWithCommitId] = finding
-//   //       }
-
-//   //     } catch (error) {
-//   //       done(new Error(`gitHubHistoricalScanning.process: unable to fetch content for [filepath=${filepath}] because [error=${error}]`), null)
-//   //     }
-//   //   }
-//   // }
-
-//   // // change to update
-//   // for (const key in allFindingsByFingerprint) {
-//   //   await GitRisks.findOneAndUpdate({ fingerprint: allFindingsByFingerprint[key].Fingerprint },
-//   //     {
-//   //       ...convertKeysToLowercase(allFindingsByFingerprint[key]),
-//   //       installationId: installationId,
-//   //       organization: organizationId,
-//   //       repositoryFullName: repository.fullName,
-//   //       repositoryId: repository.id
-//   //     }, {
-//   //     upsert: true
-//   //   }).lean()
-//   // }
-//   // // get emails of admins
-//   // const adminsOfWork = await MembershipOrg.find({
-//   //   organization: organizationId,
-//   //   $or: [
-//   //     { role: OWNER },
-//   //     { role: ADMIN }
-//   //   ]
-//   // }).lean()
-
-//   // const userEmails = await User.find({
-//   //   _id: {
-//   //     $in: [adminsOfWork.map(orgMembership => orgMembership.user)]
-//   //   }
-//   // }).select("email").lean()
-
-//   // const adminOrOwnerEmails = userEmails.map(userObject => userObject.email)
-
-//   // const usersToNotify = pusher?.email ? [pusher.email, ...adminOrOwnerEmails] : [...adminOrOwnerEmails]
-//   // if (Object.keys(allFindingsByFingerprint).length) {
-//   //   await sendMail({
-//   //     template: "secretLeakIncident.handlebars",
-//   //     subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`,
-//   //     recipients: usersToNotify,
-//   //     substitutions: {
-//   //       numberOfSecrets: Object.keys(allFindingsByFingerprint).length,
-//   //       pusher_email: pusher.email,
-//   //       pusher_name: pusher.name
-//   //     }
-//   //   });
-//   // }
-
-//   // const postHogClient = await TelemetryService.getPostHogClient();
-//   // if (postHogClient) {
-//   //   postHogClient.capture({
-//   //     event: "cloud secret scan",
-//   //     distinctId: pusher.email,
-//   //     properties: {
-//   //       numberOfCommitsScanned: commits.length,
-//   //       numberOfRisksFound: Object.keys(allFindingsByFingerprint).length,
-//   //     }
-//   //   });
-//   // }
-
-//   // done(null, allFindingsByFingerprint)
-
-// })
-
-// export const scanGithubFullRepositoryForSecretLeaks = (scanFullRepositoryDetails: TScanFullRepositoryDetails) => {
-//   console.log("full repo scan started")
-//   githubFullRepositoryScan.add(scanFullRepositoryDetails)
-// }
+})

+export const scanGithubFullRepoForSecretLeaks = (pushEventPayload: TScanPushEventQueueDetails) => {
+  githubFullRepositorySecretScan.add(pushEventPayload, {
+    attempts: 3,
+    backoff: {
+      type: "exponential",
+      delay: 5000
+    },
+    removeOnComplete: true,
+    removeOnFail: {
+      count: 20 // keep the most recent 20 jobs
+    }
+  })
+}
--- a/backend/src/templates/historicalSecretLeakIncident.handlebars
+++ b/backend/src/templates/historicalSecretLeakIncident.handlebars
@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="x-ua-compatible" content="ie=edge">
+  <title>Incident alert: secrets potentially leaked</title>
+</head>
+
+<body>
+  <h3>Infisical has uncovered {{numberOfSecrets}} secret(s) from historical commits to your repo</h3>
+  <p><a href="https://app.infisical.com/secret-scanning"><strong>View leaked secrets</strong></a></p>
+
+  <p>If these are production secrets, please rotate them immediately.</p>
+
+  <p>Once you have taken action, be sure to update the status of the risk in your <a
+      href="https://app.infisical.com/">Infisical
+      dashboard</a>.</p>
+</body>
+
+</html>