Fix merge conflicts

backend/src/ee/models/gitAppInstallationSession.ts

@@ -0,0 +1,34 @@
+import { Schema, Types, model } from "mongoose";
+
+type GitAppInstallationSession = {
+  id: string;
+  sessionId: string;
+  organization: Types.ObjectId;
+  user: Types.ObjectId;
+}
+
+const gitAppInstallationSession = new Schema<GitAppInstallationSession>({
+  id: {
+    required: true,
+    type: String,
+  },
+  sessionId: {
+    type: String,
+    required: true,
+    unique: true
+  },
+  organization: {
+    type: Schema.Types.ObjectId,
+    required: true,
+    unique: true
+  },
+  user: {
+    type: Schema.Types.ObjectId,
+    ref: "User"
+  }
+});
+
+
+const GitAppInstallationSession = model<GitAppInstallationSession>("git_app_installation_session", gitAppInstallationSession);
+
+export default GitAppInstallationSession;

backend/src/ee/models/gitAppOrganizationInstallation.ts

@@ -0,0 +1,31 @@
+import { Schema, model } from "mongoose";
+
+type Installation = {
+  installationId: string
+  organizationId: string
+  user: Schema.Types.ObjectId
+};
+
+
+const gitAppOrganizationInstallation = new Schema<Installation>({
+  installationId: {
+    type: String,
+    required: true,
+    unique: true
+  },
+  organizationId: {
+    type: String,
+    required: true,
+    unique: true
+  },
+  user: {
+    type: Schema.Types.ObjectId,
+    ref: "User",
+    required: true,
+  }
+});
+
+
+const GitAppOrganizationInstallation = model<Installation>("git_app_organization_installation", gitAppOrganizationInstallation);
+
+export default GitAppOrganizationInstallation;

backend/src/ee/models/gitRisks.ts

@@ -0,0 +1,152 @@
+import { Schema, model } from "mongoose";
+
+export const STATUS_RESOLVED_FALSE_POSITIVE = "RESOLVED_FALSE_POSITIVE";
+export const STATUS_RESOLVED_REVOKED = "RESOLVED_REVOKED";
+export const STATUS_RESOLVED_NOT_REVOKED = "RESOLVED_NOT_REVOKED";
+export const STATUS_UNRESOLVED = "UNRESOLVED";
+
+export type GitRisks = {
+  id: string;
+  description: string;
+  startLine: string;
+  endLine: string;
+  startColumn: string;
+  endColumn: string;
+  match: string;
+  secret: string;
+  file: string;
+  symlinkFile: string;
+  commit: string;
+  entropy: string;
+  author: string;
+  email: string;
+  date: string;
+  message: string;
+  tags: string[];
+  ruleID: string;
+  fingerprint: string;
+  fingerPrintWithoutCommitId: string
+
+  isFalsePositive: boolean; // New field for marking risks as false positives
+  isResolved: boolean; // New field for marking risks as resolved
+  riskOwner: string | null; // New field for setting a risk owner (nullable string)
+  installationId: string,
+  repositoryId: string,
+  repositoryLink: string
+  repositoryFullName: string
+  status: string
+  pusher: {
+    name: string,
+    email: string
+  },
+  organization: Schema.Types.ObjectId,
+}
+
+const gitRisks = new Schema<GitRisks>({
+  id: {
+    type: String,
+  },
+  description: {
+    type: String,
+  },
+  startLine: {
+    type: String,
+  },
+  endLine: {
+    type: String,
+  },
+  startColumn: {
+    type: String,
+  },
+  endColumn: {
+    type: String,
+  },
+  file: {
+    type: String,
+  },
+  symlinkFile: {
+    type: String,
+  },
+  commit: {
+    type: String,
+  },
+  entropy: {
+    type: String,
+  },
+  author: {
+    type: String,
+  },
+  email: {
+    type: String,
+  },
+  date: {
+    type: String,
+  },
+  message: {
+    type: String,
+  },
+  tags: {
+    type: [String],
+  },
+  ruleID: {
+    type: String,
+  },
+  fingerprint: {
+    type: String,
+    unique: true
+  },
+  fingerPrintWithoutCommitId: {
+    type: String,
+  },
+  isFalsePositive: {
+    type: Boolean,
+    default: false
+  },
+  isResolved: {
+    type: Boolean,
+    default: false
+  },
+  riskOwner: {
+    type: String,
+    default: null
+  },
+  installationId: {
+    type: String,
+    require: true
+  },
+  repositoryId: {
+    type: String
+  },
+  repositoryLink: {
+    type: String
+  },
+  repositoryFullName: {
+    type: String
+  },
+  pusher: {
+    name: {
+      type: String
+    },
+    email: {
+      type: String
+    },
+  },
+  organization: {
+    type: Schema.Types.ObjectId,
+    ref: "Organization",
+  },
+  status: {
+    type: String,
+    enum: [
+      STATUS_RESOLVED_FALSE_POSITIVE,
+      STATUS_RESOLVED_REVOKED,
+      STATUS_RESOLVED_NOT_REVOKED,
+      STATUS_UNRESOLVED
+    ],
+    default: STATUS_UNRESOLVED
+  }
+}, { timestamps: true });
+
+const GitRisks = model<GitRisks>("GitRisks", gitRisks);
+
+export default GitRisks;

backend/src/ee/models/index.ts

@@ -6,3 +6,6 @@ export * from "./action";
 export * from "./ssoConfig";
 export * from "./trustedIp";
 export * from "./auditLog";
+export * from  "./gitRisks";
+export * from  "./gitAppOrganizationInstallation";
+export * from  "./gitAppInstallationSession";

backend/src/ee/routes/v1/index.ts

@@ -6,6 +6,7 @@ import users from "./users";
 import workspace from "./workspace";
 import action from "./action";
 import cloudProducts from "./cloudProducts";
+import secretScanning from "./secretScanning";
 
 export {
     secret,
@@ -16,4 +17,5 @@ export {
     workspace,
     action,
     cloudProducts,
+    secretScanning
 }

backend/src/ee/routes/v1/secretScanning.ts

@@ -0,0 +1,81 @@
+import express from "express";
+const router = express.Router();
+import {
+  requireAuth,
+  requireOrganizationAuth,
+  validateRequest,
+} from "../../../middleware";
+import { body, param } from "express-validator";
+import { createInstallationSession, getCurrentOrganizationInstallationStatus, getRisksForOrganization, linkInstallationToOrganization, updateRisksStatus } from "../../../controllers/v1/secretScanningController";
+import { ACCEPTED, ADMIN, AuthMode, MEMBER, OWNER } from "../../../variables";
+
+router.post(
+  "/create-installation-session/organization/:organizationId",
+  requireAuth({
+    acceptedAuthModes: [AuthMode.JWT],
+  }),
+  param("organizationId").exists().trim(),
+  requireOrganizationAuth({
+    acceptedRoles: [OWNER, ADMIN, MEMBER],
+    acceptedStatuses: [ACCEPTED],
+  }),
+  validateRequest,
+  createInstallationSession
+);
+
+router.post(
+  "/link-installation",
+  requireAuth({
+    acceptedAuthModes: [AuthMode.JWT],
+  }),
+  body("installationId").exists().trim(),
+  body("sessionId").exists().trim(),
+  validateRequest,
+  linkInstallationToOrganization
+);
+
+router.get(
+  "/installation-status/organization/:organizationId",
+  requireAuth({
+    acceptedAuthModes: [AuthMode.JWT],
+  }),
+  param("organizationId").exists().trim(),
+  requireOrganizationAuth({
+    acceptedRoles: [OWNER, ADMIN, MEMBER],
+    acceptedStatuses: [ACCEPTED],
+  }),
+  validateRequest,
+  getCurrentOrganizationInstallationStatus
+);
+
+router.get(
+  "/organization/:organizationId/risks",
+  requireAuth({
+    acceptedAuthModes: [AuthMode.JWT],
+  }),
+  param("organizationId").exists().trim(),
+  requireOrganizationAuth({
+    acceptedRoles: [OWNER, ADMIN, MEMBER],
+    acceptedStatuses: [ACCEPTED],
+  }),
+  validateRequest,
+  getRisksForOrganization
+);
+
+router.post(
+  "/organization/:organizationId/risks/:riskId/status",
+  requireAuth({
+    acceptedAuthModes: [AuthMode.JWT],
+  }),
+  param("organizationId").exists().trim(),
+  param("riskId").exists().trim(),
+  body("status").exists(),
+  requireOrganizationAuth({
+    acceptedRoles: [OWNER, ADMIN, MEMBER],
+    acceptedStatuses: [ACCEPTED],
+  }),
+  validateRequest,
+  updateRisksStatus
+);
+
+export default router;

backend/src/ee/services/GithubSecretScanning/GithubSecretScanningService.ts

@@ -0,0 +1,250 @@
+import { Probot } from "probot";
+import { exec } from "child_process";
+import { mkdir, readFile, rm, writeFile } from "fs";
+import { tmpdir } from "os";
+import { join } from "path"
+import GitRisks from "../../models/gitRisks";
+import GitAppOrganizationInstallation from "../../models/gitAppOrganizationInstallation";
+import MembershipOrg from "../../../models/membershipOrg";
+import { ADMIN, OWNER } from "../../../variables";
+import User from "../../../models/user";
+import { sendMail } from "../../../helpers";
+import TelemetryService from "../../../services/TelemetryService";
+
+type SecretMatch = {
+  Description: string;
+  StartLine: number;
+  EndLine: number;
+  StartColumn: number;
+  EndColumn: number;
+  Match: string;
+  Secret: string;
+  File: string;
+  SymlinkFile: string;
+  Commit: string;
+  Entropy: number;
+  Author: string;
+  Email: string;
+  Date: string;
+  Message: string;
+  Tags: string[];
+  RuleID: string;
+  Fingerprint: string;
+  FingerPrintWithoutCommitId: string
+};
+
+export default async (app: Probot) => {
+  app.on("installation.deleted", async (context) => {
+    const { payload } = context;
+    const { installation, repositories } = payload;
+    if (installation.repository_selection == "all") {
+      await GitRisks.deleteMany({ installationId: installation.id })
+      await GitAppOrganizationInstallation.deleteOne({ installationId: installation.id })
+    } else {
+      if (repositories) {
+        for (const repository of repositories) {
+          await GitRisks.deleteMany({ repositoryId: repository.id })
+        }
+      }
+    }
+  })
+
+  app.on("push", async (context) => {
+    const { payload } = context;
+    const { commits, repository, installation, pusher } = payload;
+    const [owner, repo] = repository.full_name.split("/");
+
+    if (!commits || !repository || !installation || !pusher) {
+      return
+    }
+
+    const installationLinkToOrgExists = await GitAppOrganizationInstallation.findOne({ installationId: installation?.id }).lean()
+    if (!installationLinkToOrgExists) {
+      return
+    }
+
+    const allFindingsByFingerprint: { [key: string]: SecretMatch; } = {}
+
+    for (const commit of commits) {
+      for (const filepath of [...commit.added, ...commit.modified]) {
+        try {
+          const fileContentsResponse = await context.octokit.repos.getContent({
+            owner,
+            repo,
+            path: filepath,
+          });
+
+          const data: any = fileContentsResponse.data;
+          const fileContent = Buffer.from(data.content, "base64").toString();
+
+          const findings = await scanContentAndGetFindings(`\n${fileContent}`) // extra line to count lines correctly
+
+          for (const finding of findings) {
+            const fingerPrintWithCommitId = `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`
+            const fingerPrintWithoutCommitId = `${filepath}:${finding.RuleID}:${finding.StartLine}`
+            finding.Fingerprint = fingerPrintWithCommitId
+            finding.FingerPrintWithoutCommitId = fingerPrintWithoutCommitId
+            finding.Commit = commit.id
+            finding.File = filepath
+            finding.Author = commit.author.name
+            finding.Email = commit?.author?.email ? commit?.author?.email : ""
+
+            allFindingsByFingerprint[fingerPrintWithCommitId] = finding
+          }
+
+        } catch (error) {
+          console.error(`Error fetching content for ${filepath}`, error); // eslint-disable-line
+        }
+      }
+    }
+
+    // change to update
+    for (const key in allFindingsByFingerprint) {
+      const risk = await GitRisks.findOneAndUpdate({ fingerprint: allFindingsByFingerprint[key].Fingerprint },
+        {
+          ...convertKeysToLowercase(allFindingsByFingerprint[key]),
+          installationId: installation.id,
+          organization: installationLinkToOrgExists.organizationId,
+          repositoryFullName: repository.full_name,
+          repositoryId: repository.id
+        }, {
+        upsert: true
+      }).lean()
+    }
+    // get emails of admins
+    const adminsOfWork = await MembershipOrg.find({
+      organization: installationLinkToOrgExists.organizationId,
+      $or: [
+        { role: OWNER },
+        { role: ADMIN }
+      ]
+    }).lean()
+
+    const userEmails = await User.find({
+      _id: {
+        $in: [adminsOfWork.map(orgMembership => orgMembership.user)]
+      }
+    }).select("email").lean()
+
+    const adminOrOwnerEmails = userEmails.map(userObject => userObject.email)
+
+    const usersToNotify = pusher?.email ? [pusher.email, ...adminOrOwnerEmails] : [...adminOrOwnerEmails]
+    if (Object.keys(allFindingsByFingerprint).length) {
+      await sendMail({
+        template: "secretLeakIncident.handlebars",
+        subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.full_name}`,
+        recipients: usersToNotify,
+        substitutions: {
+          numberOfSecrets: Object.keys(allFindingsByFingerprint).length,
+          pusher_email: pusher.email,
+          pusher_name: pusher.name
+        }
+      });
+    }
+
+    const postHogClient = await TelemetryService.getPostHogClient();
+    if (postHogClient) {
+      postHogClient.capture({
+        event: "cloud secret scan",
+        distinctId: pusher.email,
+        properties: {
+          numberOfCommitsScanned: commits.length,
+          numberOfRisksFound: Object.keys(allFindingsByFingerprint).length,
+        }
+      });
+    }
+  });
+};
+
+async function scanContentAndGetFindings(textContent: string): Promise<SecretMatch[]> {
+  const tempFolder = await createTempFolder();
+  const filePath = join(tempFolder, "content.txt");
+  const findingsPath = join(tempFolder, "findings.json");
+
+  try {
+    await writeTextToFile(filePath, textContent);
+    await runInfisicalScan(filePath, findingsPath);
+    const findingsData = await readFindingsFile(findingsPath);
+    return JSON.parse(findingsData);
+  } finally {
+    await deleteTempFolder(tempFolder);
+  }
+}
+
+function createTempFolder(): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const tempDir = tmpdir()
+    const tempFolderName = Math.random().toString(36).substring(2);
+    const tempFolderPath = join(tempDir, tempFolderName);
+
+    mkdir(tempFolderPath, (err: any) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(tempFolderPath);
+      }
+    });
+  });
+}
+
+function writeTextToFile(filePath: string, content: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    writeFile(filePath, content, (err) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+function runInfisicalScan(inputPath: string, outputPath: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}"`;
+    exec(command, (error) => {
+      if (error && error.code != 77) {
+        reject(error);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+function readFindingsFile(filePath: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    readFile(filePath, "utf8", (err, data) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(data);
+      }
+    });
+  });
+}
+
+function deleteTempFolder(folderPath: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    rm(folderPath, { recursive: true }, (err) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+function convertKeysToLowercase<T>(obj: T): T {
+  const convertedObj = {} as T;
+
+  for (const key in obj) {
+    if (Object.prototype.hasOwnProperty.call(obj, key)) {
+      const lowercaseKey = key.charAt(0).toLowerCase() + key.slice(1);
+      convertedObj[lowercaseKey as keyof T] = obj[key];
+    }
+  }
+
+  return convertedObj;
+}

backend/src/ee/services/index.ts

@@ -2,10 +2,12 @@ import EELicenseService from "./EELicenseService";
 import EESecretService from "./EESecretService";
 import EELogService from "./EELogService";
 import EEAuditLogService from "./EEAuditLogService";
+import GithubSecretScanningService from "./GithubSecretScanning/GithubSecretScanningService"
 
 export {
     EELicenseService,
     EESecretService,
     EELogService,
-    EEAuditLogService
+    EEAuditLogService,
+    GithubSecretScanningService
 }