Merge branch 'main' into feat/secret-import

backend/.eslintrc

@@ -10,7 +10,6 @@
     "plugin:@typescript-eslint/recommended"
   ],
   "rules": {
-    "@typescript-eslint/no-empty-function": "off",
     "no-console": 2,
     "quotes": [
       "error",
@@ -35,11 +34,6 @@
         "argsIgnorePattern": "^_"
       }
     ],
-    "sort-imports": [
-      "error",
-      {
-        "ignoreDeclarationSort": true
-      }
-    ]
+    "sort-imports": 1
   }
 }

backend/Dockerfile

@@ -19,6 +19,10 @@ RUN npm ci --only-production
 
 COPY --from=build /app .
 
+RUN apk add --no-cache bash curl && curl -1sLf \
+  'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.alpine.sh' | bash \
+  && apk add infisical=0.8.1
+
 HEALTHCHECK --interval=10s --timeout=3s --start-period=10s \  
   CMD node healthcheck.js
 

backend/package.json

@@ -36,6 +36,7 @@
     "passport": "^0.6.0",
     "passport-google-oauth20": "^2.0.0",
     "posthog-node": "^2.6.0",
+    "probot": "^12.3.1",
     "query-string": "^7.1.3",
     "rate-limit-mongo": "^2.3.2",
     "rimraf": "^3.0.2",
@@ -103,6 +104,7 @@
     "jest-junit": "^15.0.0",
     "nodemon": "^2.0.19",
     "npm": "^8.19.3",
+    "smee-client": "^1.2.3",
     "supertest": "^6.3.3",
     "ts-jest": "^29.0.3",
     "ts-node": "^10.9.1"

backend/src/config/index.ts

@@ -10,7 +10,7 @@ export const getEncryptionKey = async () => {
   return secretValue === "" ? undefined : secretValue;
 }
 export const getRootEncryptionKey = async () => {
-  const secretValue = (await client.getSecret("ROOT_ENCRYPTION_KEY")).secretValue; 
+  const secretValue = (await client.getSecret("ROOT_ENCRYPTION_KEY")).secretValue;
   return secretValue === "" ? undefined : secretValue;
 }
 export const getInviteOnlySignup = async () => (await client.getSecret("INVITE_ONLY_SIGNUP")).secretValue === "true"
@@ -57,6 +57,11 @@ export const getSmtpPassword = async () => (await client.getSecret("SMTP_PASSWOR
 export const getSmtpFromAddress = async () => (await client.getSecret("SMTP_FROM_ADDRESS")).secretValue;
 export const getSmtpFromName = async () => (await client.getSecret("SMTP_FROM_NAME")).secretValue || "Infisical";
 
+export const getSecretScanningWebhookProxy = async () => (await client.getSecret("SECRET_SCANNING_WEBHOOK_PROXY")).secretValue;
+export const getSecretScanningWebhookSecret = async () => (await client.getSecret("SECRET_SCANNING_WEBHOOK_SECRET")).secretValue;
+export const getSecretScanningGitAppId = async () => (await client.getSecret("SECRET_SCANNING_GIT_APP_ID")).secretValue;
+export const getSecretScanningPrivateKey = async () => (await client.getSecret("SECRET_SCANNING_PRIVATE_KEY")).secretValue;
+
 export const getLicenseKey = async () => {
   const secretValue = (await client.getSecret("LICENSE_KEY")).secretValue;
   return secretValue === "" ? undefined : secretValue;

backend/src/controllers/v1/secretScanningController.ts

@@ -5,7 +5,7 @@ import { Types } from "mongoose";
 import { UnauthorizedRequestError } from "../../utils/errors";
 import GitAppOrganizationInstallation from "../../models/gitAppOrganizationInstallation";
 import { MembershipOrg } from "../../models";
-import GitRisks, { STATUS_UNRESOLVED } from "../../models/gitRisks";
+import GitRisks, { STATUS_RESOLVED_FALSE_POSITIVE, STATUS_RESOLVED_NOT_REVOKED, STATUS_RESOLVED_REVOKED } from "../../models/gitRisks";
 
 export const createInstallationSession = async (req: Request, res: Response) => {
   const sessionId = crypto.randomBytes(16).toString("hex");
@@ -72,7 +72,7 @@ export const getCurrentOrganizationInstallationStatus = async (req: Request, res
 
 export const getRisksForOrganization = async (req: Request, res: Response) => {
   const { organizationId } = req.params
-  const risks = await GitRisks.find({ organization: organizationId, status: STATUS_UNRESOLVED }).lean()
+  const risks = await GitRisks.find({ organization: organizationId }).sort({ createdAt: -1 }).lean()
   res.json({
     risks: risks
   })
@@ -81,9 +81,11 @@ export const getRisksForOrganization = async (req: Request, res: Response) => {
 export const updateRisksStatus = async (req: Request, res: Response) => {
   const { riskId } = req.params
   const { status } = req.body
-  const risks = await GitRisks.findByIdAndUpdate(riskId, {
-    sttaus: status
+  const isRiskResolved = status == STATUS_RESOLVED_FALSE_POSITIVE || status == STATUS_RESOLVED_REVOKED || status == STATUS_RESOLVED_NOT_REVOKED ? true : false
+  const risk = await GitRisks.findByIdAndUpdate(riskId, {
+    status: status,
+    isResolved: isRiskResolved
   }).lean()
 
-  res.json(risks)
+  res.json(risk)
 }

backend/src/index.ts

@@ -5,11 +5,12 @@ import express from "express";
 require("express-async-errors");
 import helmet from "helmet";
 import cors from "cors";
-import { DatabaseService } from "./services";
+import { DatabaseService, GithubSecretScanningService } from "./services";
 import { EELicenseService } from "./ee/services";
 import { setUpHealthEndpoint } from "./services/health";
 import cookieParser from "cookie-parser";
 import swaggerUi = require("swagger-ui-express");
+import { Probot, createNodeMiddleware } from "probot";
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const swaggerFile = require("../spec.json");
 // eslint-disable-next-line @typescript-eslint/no-var-requires
@@ -46,9 +47,9 @@ import {
 } from "./routes/v1";
 import {
   auth as v2AuthRouter,
+  organizations as v2OrganizationsRouter,
   signup as v2SignupRouter,
   users as v2UsersRouter,
-  organizations as v2OrganizationsRouter,
   workspace as v2WorkspaceRouter,
   secret as v2SecretRouter, // begin to phase out
   secrets as v2SecretsRouter,
@@ -67,10 +68,12 @@ import { healthCheck } from "./routes/status";
 import { getLogger } from "./utils/logger";
 import { RouteNotFoundError } from "./utils/errors";
 import { requestErrorHandler } from "./middleware/requestErrorHandler";
-import { getNodeEnv, getPort, getSiteURL } from "./config";
+import { getNodeEnv, getPort, getSecretScanningGitAppId, getSecretScanningPrivateKey, getSecretScanningWebhookProxy, getSecretScanningWebhookSecret, getSiteURL } from "./config";
 import { setup } from "./utils/setup";
+const SmeeClient = require('smee-client') // eslint-disable-line
 
 const main = async () => {
+
   await setup();
 
   await EELicenseService.initGlobalFeatureSet();
@@ -86,6 +89,26 @@ const main = async () => {
     })
   );
 
+  if (await getSecretScanningGitAppId() && await getSecretScanningWebhookSecret() && await getSecretScanningPrivateKey()) {
+    const probot = new Probot({
+      appId: await getSecretScanningGitAppId(),
+      privateKey: await getSecretScanningPrivateKey(),
+      secret: await getSecretScanningWebhookSecret(),
+    });
+
+    if ((await getNodeEnv()) != "production") {
+      const smee = new SmeeClient({
+        source: await getSecretScanningWebhookProxy(),
+        target: "http://backend:4000/ss-webhook",
+        logger: console
+      })
+
+      smee.start()
+    }
+
+    app.use(createNodeMiddleware(GithubSecretScanningService, { probot, webhooksPath: "/ss-webhook" })); // secret scanning webhook
+  }
+
   if ((await getNodeEnv()) === "production") {
     // enable app-wide rate-limiting + helmet security
     // in production

backend/src/models/gitRisks.ts

@@ -25,6 +25,7 @@ export type GitRisks = {
   tags: string[];
   ruleID: string;
   fingerprint: string;
+  fingerPrintWithoutCommitId: string
 
   isFalsePositive: boolean; // New field for marking risks as false positives
   isResolved: boolean; // New field for marking risks as resolved
@@ -94,6 +95,9 @@ const gitRisks = new Schema<GitRisks>({
     type: String,
     unique: true
   },
+  fingerPrintWithoutCommitId: {
+    type: String,
+  },
   isFalsePositive: {
     type: Boolean,
     default: false

backend/src/models/index.ts

@@ -16,13 +16,14 @@ import ServiceAccountKey, { IServiceAccountKey } from "./serviceAccountKey"; //
 import ServiceAccountOrganizationPermission, { IServiceAccountOrganizationPermission } from "./serviceAccountOrganizationPermission"; // new
 import ServiceAccountWorkspacePermission, { IServiceAccountWorkspacePermission } from "./serviceAccountWorkspacePermission"; // new
 import TokenData, { ITokenData } from "./tokenData";
-import User,{ AuthProvider, IUser } from "./user";
+import User, { AuthProvider, IUser } from "./user";
 import UserAction, { IUserAction } from "./userAction";
 import Workspace, { IWorkspace } from "./workspace";
 import ServiceTokenData, { IServiceTokenData } from "./serviceTokenData";
 import APIKeyData, { IAPIKeyData } from "./apiKeyData";
 import LoginSRPDetail, { ILoginSRPDetail } from "./loginSRPDetail";
 import TokenVersion, { ITokenVersion } from "./tokenVersion";
+import GitRisks, { STATUS_RESOLVED_FALSE_POSITIVE } from "./gitRisks";
 
 export {
 	AuthProvider,
@@ -76,4 +77,6 @@ export {
 	ILoginSRPDetail,
 	TokenVersion,
 	ITokenVersion,
+	GitRisks,
+	STATUS_RESOLVED_FALSE_POSITIVE
 };

backend/src/routes/status/status.ts

@@ -1,5 +1,5 @@
 import express, { Request, Response } from "express";
-import { getSmtpConfigured } from "../../config";
+import { getSecretScanningGitAppId, getSecretScanningPrivateKey, getSecretScanningWebhookSecret, getSmtpConfigured } from "../../config";
 
 const router = express.Router();
 
@@ -10,6 +10,7 @@ router.get(
       date: new Date(),
       message: "Ok",
       emailConfigured: await getSmtpConfigured(),
+      secretScanningConfigured: await getSecretScanningGitAppId() && await getSecretScanningWebhookSecret() && await getSecretScanningPrivateKey(),
     })
   }
 );

backend/src/routes/v1/secretScanning.ts

@@ -69,6 +69,7 @@ router.post(
   }),
   param("organizationId").exists().trim(),
   param("riskId").exists().trim(),
+  body("status").exists(),
   requireOrganizationAuth({
     acceptedRoles: [OWNER, ADMIN, MEMBER],
     acceptedStatuses: [ACCEPTED],

backend/src/services/GithubSecretScanningService.ts

@@ -0,0 +1,248 @@
+import { Probot } from "probot";
+import { exec } from "child_process";
+import { mkdir, readFile, rm, writeFile } from "fs";
+import { tmpdir } from "os";
+import { join } from "path"
+import GitRisks from "../models/gitRisks";
+import GitAppOrganizationInstallation from "../models/gitAppOrganizationInstallation";
+import MembershipOrg from "../models/membershipOrg";
+import { ADMIN, OWNER } from "../variables";
+import User from "../models/user";
+import { sendMail } from "../helpers";
+import TelemetryService from "./TelemetryService";
+
+type SecretMatch = {
+  Description: string;
+  StartLine: number;
+  EndLine: number;
+  StartColumn: number;
+  EndColumn: number;
+  Match: string;
+  Secret: string;
+  File: string;
+  SymlinkFile: string;
+  Commit: string;
+  Entropy: number;
+  Author: string;
+  Email: string;
+  Date: string;
+  Message: string;
+  Tags: string[];
+  RuleID: string;
+  Fingerprint: string;
+  FingerPrintWithoutCommitId: string
+};
+
+export default async (app: Probot) => {
+  app.on("installation.deleted", async (context) => {
+    const { payload } = context;
+    const { installation, repositories } = payload;
+    if (installation.repository_selection == "all") {
+      await GitRisks.deleteMany({ installationId: installation.id })
+      await GitAppOrganizationInstallation.deleteOne({ installationId: installation.id })
+    } else {
+      if (repositories) {
+        for (const repository of repositories) {
+          await GitRisks.deleteMany({ repositoryId: repository.id })
+        }
+      }
+    }
+  })
+
+  app.on("push", async (context) => {
+    const { payload } = context;
+    const { commits, repository, installation, pusher } = payload;
+    const [owner, repo] = repository.full_name.split("/");
+
+    if (!commits || !repository || !installation || !pusher) {
+      return
+    }
+
+    const installationLinkToOrgExists = await GitAppOrganizationInstallation.findOne({ installationId: installation?.id }).lean()
+    if (!installationLinkToOrgExists) {
+      return
+    }
+
+    const allFindingsByFingerprint: { [key: string]: SecretMatch; } = {}
+
+    for (const commit of commits) {
+      for (const filepath of [...commit.added, ...commit.modified]) {
+        try {
+          const fileContentsResponse = await context.octokit.repos.getContent({
+            owner,
+            repo,
+            path: filepath,
+          });
+
+          const data: any = fileContentsResponse.data;
+          const fileContent = Buffer.from(data.content, "base64").toString();
+
+          const findings = await scanContentAndGetFindings(`\n${fileContent}`) // extra line to count lines correctly
+
+          for (const finding of findings) {
+            const fingerPrintWithCommitId = `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`
+            const fingerPrintWithoutCommitId = `${filepath}:${finding.RuleID}:${finding.StartLine}`
+            finding.Fingerprint = fingerPrintWithCommitId
+            finding.FingerPrintWithoutCommitId = fingerPrintWithoutCommitId
+            finding.Commit = commit.id
+            finding.File = filepath
+            finding.Author = commit.author.name
+            finding.Email = commit?.author?.email ? commit?.author?.email : ""
+
+            allFindingsByFingerprint[fingerPrintWithCommitId] = finding
+          }
+
+        } catch (error) {
+          console.error(`Error fetching content for ${filepath}`, error); // eslint-disable-line
+        }
+      }
+    }
+
+    // change to update
+    for (const key in allFindingsByFingerprint) {
+      const risk = await GitRisks.findOneAndUpdate({ fingerprint: allFindingsByFingerprint[key].Fingerprint },
+        {
+          ...convertKeysToLowercase(allFindingsByFingerprint[key]),
+          installationId: installation.id,
+          organization: installationLinkToOrgExists.organizationId,
+          repositoryFullName: repository.full_name,
+          repositoryId: repository.id
+        }, {
+        upsert: true
+      }).lean()
+    }
+    // get emails of admins
+    const adminsOfWork = await MembershipOrg.find({
+      organization: installationLinkToOrgExists.organizationId,
+      $or: [
+        { role: OWNER },
+        { role: ADMIN }
+      ]
+    }).lean()
+
+    const userEmails = await User.find({
+      _id: {
+        $in: [adminsOfWork.map(orgMembership => orgMembership.user)]
+      }
+    }).select("email").lean()
+
+    const adminOrOwnerEmails = userEmails.map(userObject => userObject.email)
+
+    const usersToNotify = pusher?.email ? [pusher.email, ...adminOrOwnerEmails] : [...adminOrOwnerEmails]
+    await sendMail({
+      template: "secretLeakIncident.handlebars",
+      subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.full_name}`,
+      recipients: usersToNotify,
+      substitutions: {
+        numberOfSecrets: Object.keys(allFindingsByFingerprint).length,
+        pusher_email: pusher.email,
+        pusher_name: pusher.name
+      }
+    });
+
+    const postHogClient = await TelemetryService.getPostHogClient();
+    if (postHogClient) {
+      postHogClient.capture({
+        event: "cloud secret scan",
+        distinctId: pusher.email,
+        properties: {
+          numberOfCommitsScanned: commits.length,
+          numberOfRisksFound: Object.keys(allFindingsByFingerprint).length,
+        }
+      });
+    }
+  });
+};
+
+async function scanContentAndGetFindings(textContent: string): Promise<SecretMatch[]> {
+  const tempFolder = await createTempFolder();
+  const filePath = join(tempFolder, "content.txt");
+  const findingsPath = join(tempFolder, "findings.json");
+
+  try {
+    await writeTextToFile(filePath, textContent);
+    await runInfisicalScan(filePath, findingsPath);
+    const findingsData = await readFindingsFile(findingsPath);
+    return JSON.parse(findingsData);
+  } finally {
+    await deleteTempFolder(tempFolder);
+  }
+}
+
+function createTempFolder(): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const tempDir = tmpdir()
+    const tempFolderName = Math.random().toString(36).substring(2);
+    const tempFolderPath = join(tempDir, tempFolderName);
+
+    mkdir(tempFolderPath, (err: any) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(tempFolderPath);
+      }
+    });
+  });
+}
+
+function writeTextToFile(filePath: string, content: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    writeFile(filePath, content, (err) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+function runInfisicalScan(inputPath: string, outputPath: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}"`;
+    exec(command, (error) => {
+      if (error && error.code != 77) {
+        reject(error);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+function readFindingsFile(filePath: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    readFile(filePath, "utf8", (err, data) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(data);
+      }
+    });
+  });
+}
+
+function deleteTempFolder(folderPath: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    rm(folderPath, { recursive: true }, (err) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+function convertKeysToLowercase<T>(obj: T): T {
+  const convertedObj = {} as T;
+
+  for (const key in obj) {
+    if (Object.prototype.hasOwnProperty.call(obj, key)) {
+      const lowercaseKey = key.charAt(0).toLowerCase() + key.slice(1);
+      convertedObj[lowercaseKey as keyof T] = obj[key];
+    }
+  }
+
+  return convertedObj;
+}

backend/src/services/index.ts

@@ -6,6 +6,7 @@ import EventService from "./EventService";
 import IntegrationService from "./IntegrationService";
 import TokenService from "./TokenService";
 import SecretService from "./SecretService";
+import GithubSecretScanningService from "./GithubSecretScanningService"
 
 export {
   TelemetryService,
@@ -14,5 +15,6 @@ export {
   EventService,
   IntegrationService,
   TokenService,
-  SecretService
-};
+  SecretService,
+  GithubSecretScanningService
+};

backend/src/templates/secretLeakIncident.handlebars

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="x-ua-compatible" content="ie=edge">
+  <title>Incident alert: secret leaked</title>
+</head>
+
+<body>
+  <h3>Infisical has uncovered {{numberOfSecrets}} secret(s) from your recent push</h3>
+  <p><a href="https://app.infisical.com/secret-scanning"><strong>View leaked secrets</strong></a></p>
+  <p>You are receiving this notification because one or more secret leaks have been detected in a recent commit pushed
+    by {{pusher_name}} ({{pusher_email}}). If
+    these are test secrets, please add `infisical-scan:ignore` at the end of the line containing the secret as comment
+    in the given programming. This will prevent future notifications from being sent out for those secret(s).</p>
+
+  <p>If these are production secrets, please rotate them immediately.</p>
+
+  <p>Once you have taken action, be sure to update the status of the risk in your <a
+      href="https://app.infisical.com/">Infisical
+      dashboard</a>.</p>
+</body>
+
+</html>