feat(infisical-pg): completed secret scanner

backend-pg/Dockerfile.dev

@@ -1,5 +1,9 @@
 FROM node:20-alpine
 
+RUN apk add --no-cache bash curl && curl -1sLf \
+  'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.alpine.sh' | bash \
+  && apk add infisical=0.8.1 && apk add --no-cache git
+
 WORKDIR /app
 
 COPY package.json package.json

backend-pg/package.json

@@ -31,6 +31,7 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
+    "@octokit/webhooks-types": "^7.3.1",
     "@types/bcrypt": "^5.0.2",
     "@types/jmespath": "^0.15.2",
     "@types/jsonwebtoken": "^9.0.5",
@@ -108,6 +109,8 @@
     "pg": "^8.11.3",
     "picomatch": "^3.0.1",
     "pino": "^8.16.2",
+    "probot": "^12.3.3",
+    "smee-client": "^2.0.0",
     "tweetnacl": "^1.0.3",
     "tweetnacl-util": "^0.15.1",
     "zod": "^3.22.4",

backend-pg/scripts/generate-schema-types.ts

@@ -85,9 +85,7 @@ const main = async () => {
       .whereRaw("table_schema =  current_schema()")
       .select<{ tableName: string }[]>("table_name as tableName")
       .orderBy("table_name")
-  ).filter(
+  ).filter((el) => el.tableName.includes("migration"));
-    (el) => el.tableName !== "infisical_migrations_lock" && el.tableName !== "infisical_migrations"
-  );
 
   console.log("Select a table to generate schema");
   console.table(tables);

backend-pg/src/@types/fastify.d.ts

@@ -8,6 +8,7 @@ import { TSamlConfigServiceFactory } from "@app/ee/services/saml-config/saml-con
 import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
 import { TSecretApprovalRequestServiceFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-service";
 import { TSecretRotationServiceFactory } from "@app/ee/services/secret-rotation/secret-rotation-service";
+import { TSecretScanningServiceFactory } from "@app/ee/services/secret-scanning/secret-scanning-service";
 import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
 import { TApiKeyServiceFactory } from "@app/services/api-key/api-key-service";
 import { TAuthLoginFactory } from "@app/services/auth/auth-login-service";
@@ -105,6 +106,7 @@ declare module "fastify" {
       snapshot: TSecretSnapshotServiceFactory;
       saml: TSamlConfigServiceFactory;
       auditLog: TAuditLogServiceFactory;
+      secretScanning: TSecretScanningServiceFactory;
     };
 
     // this is exclusive use for middlewares in which we need to inject data

backend-pg/src/@types/knex.d.ts

@@ -16,6 +16,12 @@ import {
   TBackupPrivateKey,
   TBackupPrivateKeyInsert,
   TBackupPrivateKeyUpdate,
+  TGitAppInstallSessions,
+  TGitAppInstallSessionsInsert,
+  TGitAppInstallSessionsUpdate,
+  TGitAppOrg,
+  TGitAppOrgInsert,
+  TGitAppOrgUpdate,
   TIdentities,
   TIdentitiesInsert,
   TIdentitiesUpdate,
@@ -113,6 +119,9 @@ import {
   TSecretRotationsInsert,
   TSecretRotationsUpdate,
   TSecrets,
+  TSecretScanningGitRisks,
+  TSecretScanningGitRisksInsert,
+  TSecretScanningGitRisksUpdate,
   TSecretsInsert,
   TSecretSnapshotFolders,
   TSecretSnapshotFoldersInsert,
@@ -373,6 +382,17 @@ declare module "knex/types/tables" {
     >;
     [TableName.OrgBot]: Knex.CompositeTableType<TOrgBots, TOrgBotsInsert, TOrgBotsUpdate>;
     [TableName.AuditLog]: Knex.CompositeTableType<TAuditLogs, TAuditLogsInsert, TAuditLogsUpdate>;
+    [TableName.GitAppInstallSession]: Knex.CompositeTableType<
+      TGitAppInstallSessions,
+      TGitAppInstallSessionsInsert,
+      TGitAppInstallSessionsUpdate
+    >;
+    [TableName.GitAppOrg]: Knex.CompositeTableType<TGitAppOrg, TGitAppOrgInsert, TGitAppOrgUpdate>;
+    [TableName.SecretScanningGitRisk]: Knex.CompositeTableType<
+      TSecretScanningGitRisks,
+      TSecretScanningGitRisksInsert,
+      TSecretScanningGitRisksUpdate
+    >;
     // Junction tables
     [TableName.JnSecretTag]: Knex.CompositeTableType<
       TSecretTagJunction,

backend-pg/src/db/migrations/20240111051011_secret-scanning.ts

@@ -0,0 +1,79 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.GitAppInstallSession))) {
+    await knex.schema.createTable(TableName.GitAppInstallSession, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("sessionId").notNullable().unique();
+      t.uuid("userId");
+      // one to one relationship
+      t.uuid("orgId").notNullable().unique();
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+  }
+  createOnUpdateTrigger(knex, TableName.GitAppInstallSession);
+
+  if (!(await knex.schema.hasTable(TableName.GitAppOrg))) {
+    await knex.schema.createTable(TableName.GitAppOrg, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("installationId").notNullable().unique();
+      t.uuid("userId").notNullable();
+      // one to one relationship
+      t.uuid("orgId").notNullable().unique();
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+  }
+  createOnUpdateTrigger(knex, TableName.GitAppOrg);
+
+  if (!(await knex.schema.hasTable(TableName.SecretScanningGitRisk))) {
+    await knex.schema.createTable(TableName.SecretScanningGitRisk, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("description");
+      t.string("startLine");
+      t.string("endLine");
+      t.string("startColumn");
+      t.string("endColumn");
+      t.string("file");
+      t.string("symlinkFile");
+      t.string("commit");
+      t.string("entropy");
+      t.string("author");
+      t.string("email");
+      t.string("date");
+      t.text("message");
+      t.specificType("tags", "text[]");
+      t.string("ruleID");
+      t.string("fingerprint").unique();
+      t.string("fingerPrintWithoutCommitId");
+      t.boolean("isFalsePositive").defaultTo(false);
+      t.boolean("isResolved").defaultTo(false);
+      t.string("riskOwner");
+      t.string("installationId").notNullable();
+      t.string("repositoryId");
+      t.string("repositoryLink");
+      t.string("repositoryFullName");
+      t.string("pusherName");
+      t.string("pusherEmail");
+      t.string("status");
+      // one to one relationship
+      t.uuid("orgId").notNullable();
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+  }
+  createOnUpdateTrigger(knex, TableName.SecretScanningGitRisk);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.SecretScanningGitRisk);
+  await knex.schema.dropTableIfExists(TableName.GitAppOrg);
+  await knex.schema.dropTableIfExists(TableName.GitAppInstallSession);
+  await dropOnUpdateTrigger(knex, TableName.SecretScanningGitRisk);
+  await dropOnUpdateTrigger(knex, TableName.GitAppOrg);
+  await dropOnUpdateTrigger(knex, TableName.GitAppInstallSession);
+}

backend-pg/src/db/schemas/git-app-install-sessions.ts

@@ -0,0 +1,21 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const GitAppInstallSessionsSchema = z.object({
+  id: z.string().uuid(),
+  sessionId: z.string(),
+  userId: z.string().uuid().nullable().optional(),
+  orgId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+});
+
+export type TGitAppInstallSessions = z.infer<typeof GitAppInstallSessionsSchema>;
+export type TGitAppInstallSessionsInsert = Omit<TGitAppInstallSessions, TImmutableDBKeys>;
+export type TGitAppInstallSessionsUpdate = Partial<Omit<TGitAppInstallSessions, TImmutableDBKeys>>;

backend-pg/src/db/schemas/git-app-org.ts

@@ -0,0 +1,21 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const GitAppOrgSchema = z.object({
+  id: z.string().uuid(),
+  installationId: z.string(),
+  userId: z.string().uuid(),
+  orgId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+});
+
+export type TGitAppOrg = z.infer<typeof GitAppOrgSchema>;
+export type TGitAppOrgInsert = Omit<TGitAppOrg, TImmutableDBKeys>;
+export type TGitAppOrgUpdate = Partial<Omit<TGitAppOrg, TImmutableDBKeys>>;

backend-pg/src/db/schemas/index.ts

@@ -3,6 +3,8 @@ export * from "./audit-logs";
 export * from "./auth-token-sessions";
 export * from "./auth-tokens";
 export * from "./backup-private-key";
+export * from "./git-app-install-sessions";
+export * from "./git-app-org";
 export * from "./identities";
 export * from "./identity-access-tokens";
 export * from "./identity-org-memberships";
@@ -36,6 +38,7 @@ export * from "./secret-folders";
 export * from "./secret-imports";
 export * from "./secret-rotation-outputs";
 export * from "./secret-rotations";
+export * from "./secret-scanning-git-risks";
 export * from "./secret-snapshot-folders";
 export * from "./secret-snapshot-secrets";
 export * from "./secret-snapshots";

backend-pg/src/db/schemas/knex-migrations-lock.ts

@@ -1,17 +0,0 @@
-// Code generated by automation script, DO NOT EDIT.
-// Automated by pulling database and generating zod schema
-// To update. Just run npm run generate:schema
-// Written by akhilmhdh.
-
-import { z } from "zod";
-
-import { TImmutableDBKeys } from "./models";
-
-export const KnexMigrationsLockSchema = z.object({
-  index: z.number(),
-  is_locked: z.number().nullable().optional(),
-});
-
-export type TKnexMigrationsLock = z.infer<typeof KnexMigrationsLockSchema>;
-export type TKnexMigrationsLockInsert = Omit<TKnexMigrationsLock, TImmutableDBKeys>;
-export type TKnexMigrationsLockUpdate = Partial<Omit<TKnexMigrationsLock, TImmutableDBKeys>>;

backend-pg/src/db/schemas/knex-migrations.ts

@@ -1,19 +0,0 @@
-// Code generated by automation script, DO NOT EDIT.
-// Automated by pulling database and generating zod schema
-// To update. Just run npm run generate:schema
-// Written by akhilmhdh.
-
-import { z } from "zod";
-
-import { TImmutableDBKeys } from "./models";
-
-export const KnexMigrationsSchema = z.object({
-  id: z.number(),
-  name: z.string().nullable().optional(),
-  batch: z.number().nullable().optional(),
-  migration_time: z.date().nullable().optional(),
-});
-
-export type TKnexMigrations = z.infer<typeof KnexMigrationsSchema>;
-export type TKnexMigrationsInsert = Omit<TKnexMigrations, TImmutableDBKeys>;
-export type TKnexMigrationsUpdate = Partial<Omit<TKnexMigrations, TImmutableDBKeys>>;

backend-pg/src/db/schemas/models.ts

@@ -50,6 +50,9 @@ export enum TableName {
   SecretRotationOutput = "secret_rotation_outputs",
   SamlConfig = "saml_configs",
   AuditLog = "audit_logs",
+  GitAppInstallSession = "git_app_install_sessions",
+  GitAppOrg = "git_app_org",
+  SecretScanningGitRisk = "secret_scanning_git_risks",
   // junction tables
   JnSecretTag = "secret_tag_junction",
   JnSecretVersionTag = "secret_version_tag_junction"

backend-pg/src/db/schemas/secret-scanning-git-risks.ts

@@ -0,0 +1,46 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const SecretScanningGitRisksSchema = z.object({
+  id: z.string().uuid(),
+  description: z.string().nullable().optional(),
+  startLine: z.string().nullable().optional(),
+  endLine: z.string().nullable().optional(),
+  startColumn: z.string().nullable().optional(),
+  endColumn: z.string().nullable().optional(),
+  file: z.string().nullable().optional(),
+  symlinkFile: z.string().nullable().optional(),
+  commit: z.string().nullable().optional(),
+  entropy: z.string().nullable().optional(),
+  author: z.string().nullable().optional(),
+  email: z.string().nullable().optional(),
+  date: z.string().nullable().optional(),
+  message: z.string().nullable().optional(),
+  tags: z.string().array().nullable().optional(),
+  ruleID: z.string().nullable().optional(),
+  fingerprint: z.string().nullable().optional(),
+  fingerPrintWithoutCommitId: z.string().nullable().optional(),
+  isFalsePositive: z.boolean().default(false).nullable().optional(),
+  isResolved: z.boolean().default(false).nullable().optional(),
+  riskOwner: z.string().nullable().optional(),
+  installationId: z.string(),
+  repositoryId: z.string().nullable().optional(),
+  repositoryLink: z.string().nullable().optional(),
+  repositoryFullName: z.string().nullable().optional(),
+  pusherName: z.string().nullable().optional(),
+  pusherEmail: z.string().nullable().optional(),
+  status: z.string().nullable().optional(),
+  orgId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+});
+
+export type TSecretScanningGitRisks = z.infer<typeof SecretScanningGitRisksSchema>;
+export type TSecretScanningGitRisksInsert = Omit<TSecretScanningGitRisks, TImmutableDBKeys>;
+export type TSecretScanningGitRisksUpdate = Partial<Omit<TSecretScanningGitRisks, TImmutableDBKeys>>;

backend-pg/src/ee/routes/v1/index.ts

@@ -6,6 +6,7 @@ import { registerSecretApprovalPolicyRouter } from "./secret-approval-policy-rou
 import { registerSecretApprovalRequestRouter } from "./secret-approval-request-router";
 import { registerSecretRotationProviderRouter } from "./secret-rotation-provider-router";
 import { registerSecretRotationRouter } from "./secret-rotation-router";
+import { registerSecretScanningRouter } from "./secret-scanning-router";
 import { registerSnapshotRouter } from "./snapshot-router";
 
 export const registerV1EERoutes = async (server: FastifyZodProvider) => {
@@ -27,5 +28,6 @@ export const registerV1EERoutes = async (server: FastifyZodProvider) => {
     prefix: "/secret-rotation-providers"
   });
   await server.register(registerSamlRouter, { prefix: "/sso" });
+  await server.register(registerSecretScanningRouter, { prefix: "/secret-scanning" });
   await server.register(registerSecretRotationRouter, { prefix: "/secret-rotations" });
 };

backend-pg/src/ee/routes/v1/secret-scanning-router.ts

@@ -0,0 +1,117 @@
+import { z } from "zod";
+
+import { GitAppOrgSchema, SecretScanningGitRisksSchema } from "@app/db/schemas";
+import { SecretScanningRiskStatus } from "@app/ee/services/secret-scanning/secret-scanning-types";
+import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
+import { AuthMode } from "@app/services/auth/auth-type";
+
+export const registerSecretScanningRouter = async (server: FastifyZodProvider) => {
+  server.route({
+    url: "/create-installation-session/organization",
+    method: "POST",
+    schema: {
+      body: z.object({ organizationId: z.string().trim() }),
+      response: {
+        200: z.object({
+          sessionId: z.string()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const session = await server.services.secretScanning.createInstallationSession({
+        actor: req.permission.type,
+        actorId: req.permission.id,
+        orgId: req.body.organizationId
+      });
+      return session;
+    }
+  });
+
+  server.route({
+    url: "/link-installation",
+    method: "POST",
+    schema: {
+      body: z.object({
+        installationId: z.string(),
+        sessionId: z.string().trim()
+      }),
+      response: {
+        200: GitAppOrgSchema
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { installatedApp } = await server.services.secretScanning.linkInstallationToOrg({
+        actor: req.permission.type,
+        actorId: req.permission.id,
+        ...req.body
+      });
+      return installatedApp;
+    }
+  });
+
+  server.route({
+    url: "/installation-status/organization/:organizationId",
+    method: "GET",
+    schema: {
+      params: z.object({ organizationId: z.string().trim() }),
+      response: {
+        200: z.object({ appInstallationCompleted: z.boolean() })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const appInstallationCompleted =
+        await server.services.secretScanning.getOrgInstallationStatus({
+          actor: req.permission.type,
+          actorId: req.permission.id,
+          orgId: req.params.organizationId
+        });
+      return { appInstallationCompleted };
+    }
+  });
+
+  server.route({
+    url: "/organization/:organizationId/risks",
+    method: "GET",
+    schema: {
+      params: z.object({ organizationId: z.string().trim() }),
+      response: {
+        200: z.object({ risks: SecretScanningGitRisksSchema.array() })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { risks } = await server.services.secretScanning.getRisksByOrg({
+        actor: req.permission.type,
+        actorId: req.permission.id,
+        orgId: req.params.organizationId
+      });
+      return { risks };
+    }
+  });
+
+  server.route({
+    url: "/organization/:organizationId/risks/:riskId/status",
+    method: "POST",
+    schema: {
+      params: z.object({ organizationId: z.string().trim(), riskId: z.string().trim() }),
+      body: z.object({ status: z.nativeEnum(SecretScanningRiskStatus) }),
+      response: {
+        200: SecretScanningGitRisksSchema
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { risk } = await server.services.secretScanning.updateRiskStatus({
+        actor: req.permission.type,
+        actorId: req.permission.id,
+        orgId: req.params.organizationId,
+        riskId: req.params.riskId,
+        ...req.body
+      });
+      return risk;
+    }
+  });
+};

backend-pg/src/ee/services/secret-scanning/git-app-dal.ts

@@ -0,0 +1,27 @@
+import { Knex } from "knex";
+
+import { TDbClient } from "@app/db";
+import { TableName,TGitAppOrgInsert } from "@app/db/schemas";
+import { DatabaseError } from "@app/lib/errors";
+import { ormify } from "@app/lib/knex";
+
+export type TGitAppDalFactory = ReturnType<typeof gitAppDalFactory>;
+
+export const gitAppDalFactory = (db: TDbClient) => {
+  const gitAppOrm = ormify(db, TableName.GitAppOrg);
+
+  const upsert = async (data: TGitAppOrgInsert, tx?: Knex) => {
+    try {
+      const [doc] = await (tx || db)(TableName.GitAppOrg)
+        .insert(data)
+        .onConflict("orgId")
+        .merge()
+        .returning("*");
+      return doc;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "UpsertGitAppOrm" });
+    }
+  };
+
+  return { ...gitAppOrm, upsert };
+};

backend-pg/src/ee/services/secret-scanning/git-app-install-session-dal.ts

@@ -0,0 +1,27 @@
+import { Knex } from "knex";
+
+import { TDbClient } from "@app/db";
+import { TableName,TGitAppInstallSessionsInsert } from "@app/db/schemas";
+import { DatabaseError } from "@app/lib/errors";
+import { ormify } from "@app/lib/knex";
+
+export type TGitAppInstallSessionDalFactory = ReturnType<typeof gitAppInstallSessionDalFactory>;
+
+export const gitAppInstallSessionDalFactory = (db: TDbClient) => {
+  const gitAppInstallSessionOrm = ormify(db, TableName.GitAppInstallSession);
+
+  const upsert = async (data: TGitAppInstallSessionsInsert, tx?: Knex) => {
+    try {
+      const [doc] = await (tx || db)(TableName.GitAppInstallSession)
+        .insert(data)
+        .onConflict("orgId")
+        .merge()
+        .returning("*");
+      return doc;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "UpsertGitAppOrm" });
+    }
+  };
+
+  return { ...gitAppInstallSessionOrm, upsert };
+};

backend-pg/src/ee/services/secret-scanning/secret-scanning-dal.ts

@@ -0,0 +1,26 @@
+import { Knex } from "knex";
+
+import { TDbClient } from "@app/db";
+import { TableName,TSecretScanningGitRisksInsert } from "@app/db/schemas";
+import { DatabaseError } from "@app/lib/errors";
+import { ormify } from "@app/lib/knex";
+
+export type TSecretScanningDalFactory = ReturnType<typeof secretScanningDalFactory>;
+
+export const secretScanningDalFactory = (db: TDbClient) => {
+  const gitRiskOrm = ormify(db, TableName.SecretScanningGitRisk);
+
+  const upsert = async (data: TSecretScanningGitRisksInsert[], tx?: Knex) => {
+    try {
+      const docs = await (tx || db)(TableName.SecretScanningGitRisk)
+        .insert(data)
+        .onConflict("fingerprint")
+        .merge();
+      return docs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "GitRiskUpsert" });
+    }
+  };
+
+  return { ...gitRiskOrm, upsert };
+};

backend-pg/src/ee/services/secret-scanning/secret-scanning-queue/index.ts

@@ -0,0 +1 @@
+export * from "./secret-scanning-queue";

backend-pg/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns.ts

@@ -0,0 +1,151 @@
+import { exec } from "child_process";
+import { mkdir, readFile, rm, writeFile } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+
+import { SecretMatch } from "./secret-scanning-queue-types";
+
+export function createTempFolder(): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const tempDir = tmpdir();
+    const tempFolderName = Math.random().toString(36).substring(2);
+    const tempFolderPath = join(tempDir, tempFolderName);
+
+    mkdir(tempFolderPath, (err: any) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(tempFolderPath);
+      }
+    });
+  });
+}
+
+export function writeTextToFile(filePath: string, content: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    writeFile(filePath, content, (err) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+export async function cloneRepo(
+  installationAcccessToken: string,
+  repositoryFullName: string,
+  repoPath: string
+): Promise<void> {
+  const cloneUrl = `https://x-access-token:${installationAcccessToken}@github.com/${repositoryFullName}.git`;
+  const command = `git clone ${cloneUrl} ${repoPath} --bare`;
+  return new Promise((resolve, reject) => {
+    exec(command, (error) => {
+      if (error) {
+        reject(error);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+export function runInfisicalScanOnRepo(repoPath: string, outputPath: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const command = `cd ${repoPath} && infisical scan --exit-code=77 -r "${outputPath}"`;
+    exec(command, (error) => {
+      if (error && error.code !== 77) {
+        reject(error);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+export function runInfisicalScan(inputPath: string, outputPath: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}"`;
+    exec(command, (error) => {
+      if (error && error.code !== 77) {
+        reject(error);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+export function readFindingsFile(filePath: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    readFile(filePath, "utf8", (err, data) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(data);
+      }
+    });
+  });
+}
+
+export function deleteTempFolder(folderPath: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    rm(folderPath, { recursive: true }, (err) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+export function convertKeysToLowercase<T>(obj: T): T {
+  const convertedObj = {} as T;
+
+  for (const key in obj) {
+    if (Object.prototype.hasOwnProperty.call(obj, key)) {
+      const lowercaseKey = key.charAt(0).toLowerCase() + key.slice(1);
+      convertedObj[lowercaseKey as keyof T] = obj[key];
+    }
+  }
+
+  return convertedObj;
+}
+
+export async function scanFullRepoContentAndGetFindings(
+  octokit: any,
+  installationId: string,
+  repositoryFullName: string
+): Promise<SecretMatch[]> {
+  const tempFolder = await createTempFolder();
+  const findingsPath = join(tempFolder, "findings.json");
+  const repoPath = join(tempFolder, "repo.git");
+  try {
+    const {
+      data: { token }
+    } = await octokit.apps.createInstallationAccessToken({ installation_id: installationId });
+    await cloneRepo(token, repositoryFullName, repoPath);
+    await runInfisicalScanOnRepo(repoPath, findingsPath);
+    const findingsData = await readFindingsFile(findingsPath);
+    return JSON.parse(findingsData);
+  } finally {
+    await deleteTempFolder(tempFolder);
+  }
+}
+
+export async function scanContentAndGetFindings(textContent: string): Promise<SecretMatch[]> {
+  const tempFolder = await createTempFolder();
+  const filePath = join(tempFolder, "content.txt");
+  const findingsPath = join(tempFolder, "findings.json");
+
+  try {
+    await writeTextToFile(filePath, textContent);
+    await runInfisicalScan(filePath, findingsPath);
+    const findingsData = await readFindingsFile(findingsPath);
+    return JSON.parse(findingsData);
+  } finally {
+    await deleteTempFolder(tempFolder);
+  }
+}

backend-pg/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types.ts

@@ -0,0 +1,46 @@
+import { Commit } from "@octokit/webhooks-types";
+
+export type SecretMatch = {
+  Description: string;
+  StartLine: number;
+  EndLine: number;
+  StartColumn: number;
+  EndColumn: number;
+  Match: string;
+  Secret: string;
+  File: string;
+  SymlinkFile: string;
+  Commit: string;
+  Entropy: number;
+  Author: string;
+  Email: string;
+  Date: string;
+  Message: string;
+  Tags: string[];
+  RuleID: string;
+  Fingerprint: string;
+  FingerPrintWithoutCommitId: string;
+};
+
+export type TScanPushEventPayload = {
+  organizationId: string;
+  commits: Commit[];
+  pusher: {
+    name: string;
+    email: string | null;
+  };
+  repository: {
+    id: number;
+    fullName: string;
+  };
+  installationId: string;
+};
+
+export type TScanFullRepoEventPayload = {
+  organizationId: string;
+  installationId: string;
+  repository: {
+    id: number;
+    fullName: string;
+  };
+};

backend-pg/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts

@@ -0,0 +1,241 @@
+import { ProbotOctokit } from "probot";
+
+import { OrgMembershipRole } from "@app/db/schemas";
+import { getConfig } from "@app/lib/config/env";
+import { logger } from "@app/lib/logger";
+import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
+import { TOrgDalFactory } from "@app/services/org/org-dal";
+import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
+import { TUserDalFactory } from "@app/services/user/user-dal";
+
+import { TSecretScanningDalFactory } from "../secret-scanning-dal";
+import {
+  scanContentAndGetFindings,
+  scanFullRepoContentAndGetFindings} from "./secret-scanning-fns";
+import {
+  SecretMatch,
+  TScanFullRepoEventPayload,
+  TScanPushEventPayload
+} from "./secret-scanning-queue-types";
+
+type TSecretScanningQueueFactoryDep = {
+  queueService: TQueueServiceFactory;
+  secretScanningDal: TSecretScanningDalFactory;
+  smtpService: Pick<TSmtpService, "sendMail">;
+  orgMembershipDal: Pick<TOrgDalFactory, "findMembership">;
+  userDal: Pick<TUserDalFactory, "find">;
+};
+
+export type TSecretScanningQueueFactory = ReturnType<typeof secretScanningQueueFactory>;
+
+export const secretScanningQueueFactory = ({
+  queueService,
+  secretScanningDal,
+  smtpService,
+  orgMembershipDal: orgMemberDal,
+  userDal
+}: TSecretScanningQueueFactoryDep) => {
+  const startFullRepoScan = async (payload: TScanFullRepoEventPayload) => {
+    await queueService.queue(QueueName.SecretFullRepoScan, QueueJobs.SecretScan, payload, {
+      attempts: 3,
+      backoff: {
+        type: "exponential",
+        delay: 5000
+      },
+      removeOnComplete: true,
+      removeOnFail: {
+        count: 20 // keep the most recent 20 jobs
+      }
+    });
+  };
+
+  const startPushEventScan = async (payload: TScanPushEventPayload) => {
+    await queueService.queue(QueueName.SecretPushEventScan, QueueJobs.SecretScan, payload, {
+      attempts: 3,
+      backoff: {
+        type: "exponential",
+        delay: 5000
+      },
+      removeOnComplete: true,
+      removeOnFail: {
+        count: 20 // keep the most recent 20 jobs
+      }
+    });
+  };
+
+  const getOrgAdminEmails = async (organizationId: string) => {
+    // get emails of admins
+    const adminsOfWork = await orgMemberDal.findMembership({
+      orgId: organizationId,
+      role: OrgMembershipRole.Admin
+    });
+    const userEmails = await userDal.find({
+      $in: {
+        id: adminsOfWork.map(({ userId }) => userId).filter(Boolean) as string[]
+      }
+    });
+    return userEmails.map((userObject) => userObject.email);
+  };
+
+  queueService.start(QueueName.SecretPushEventScan, async (job) => {
+    const appCfg = getConfig();
+    const { organizationId, commits, pusher, repository, installationId } = job.data;
+    const [owner, repo] = repository.fullName.split("/");
+    const octokit = new ProbotOctokit({
+      auth: {
+        appId: appCfg.SECRET_SCANNING_GIT_APP_ID,
+        privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY,
+        installationId
+      }
+    });
+    const allFindingsByFingerprint: { [key: string]: SecretMatch } = {};
+
+    for (const commit of commits) {
+      for (const filepath of [...commit.added, ...commit.modified]) {
+        // eslint-disable-next-line
+        const fileContentsResponse = await octokit.repos.getContent({
+          owner,
+          repo,
+          path: filepath
+        });
+
+        const { data }: any = fileContentsResponse;
+        const fileContent = Buffer.from(data.content, "base64").toString();
+
+        // eslint-disable-next-line
+        const findings = await scanContentAndGetFindings(`\n${fileContent}`); // extra line to count lines correctly
+
+        for (const finding of findings) {
+          const fingerPrintWithCommitId = `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`;
+          const fingerPrintWithoutCommitId = `${filepath}:${finding.RuleID}:${finding.StartLine}`;
+          finding.Fingerprint = fingerPrintWithCommitId;
+          finding.FingerPrintWithoutCommitId = fingerPrintWithoutCommitId;
+          finding.Commit = commit.id;
+          finding.File = filepath;
+          finding.Author = commit.author.name;
+          finding.Email = commit?.author?.email ? commit?.author?.email : "";
+
+          allFindingsByFingerprint[fingerPrintWithCommitId] = finding;
+        }
+      }
+    }
+    await secretScanningDal.transaction(async (tx) => {
+      if (!Object.keys(allFindingsByFingerprint).length) return;
+      secretScanningDal.upsert(
+        Object.keys(allFindingsByFingerprint).map((key) => ({
+          installationId,
+          email: allFindingsByFingerprint[key].Email,
+          author: allFindingsByFingerprint[key].Author,
+          date: allFindingsByFingerprint[key].Date,
+          file: allFindingsByFingerprint[key].File,
+          tags: allFindingsByFingerprint[key].Tags,
+          commit: allFindingsByFingerprint[key].Commit,
+          ruleID: allFindingsByFingerprint[key].RuleID,
+          endLine: String(allFindingsByFingerprint[key].EndLine),
+          entropy: String(allFindingsByFingerprint[key].Entropy),
+          message: allFindingsByFingerprint[key].Message,
+          endColumn: String(allFindingsByFingerprint[key].EndColumn),
+          startLine: String(allFindingsByFingerprint[key].StartLine),
+          startColumn: String(allFindingsByFingerprint[key].StartColumn),
+          fingerPrintWithoutCommitId: allFindingsByFingerprint[key].FingerPrintWithoutCommitId,
+          description: allFindingsByFingerprint[key].Description,
+          symlinkFile: allFindingsByFingerprint[key].SymlinkFile,
+          orgId: organizationId,
+          pusherEmail: pusher.email,
+          pusherName: pusher.name,
+          repositoryFullName: repository.fullName,
+          repositoryId: String(repository.id),
+          fingerprint: allFindingsByFingerprint[key].Fingerprint
+        })),
+        tx
+      );
+    });
+
+    const adminEmails = await getOrgAdminEmails(organizationId);
+    if (pusher?.email) {
+      adminEmails.push(pusher.email);
+    }
+    if (Object.keys(allFindingsByFingerprint).length) {
+      await smtpService.sendMail({
+        template: SmtpTemplates.SecretLeakIncident,
+        subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`,
+        recipients: adminEmails,
+        substitutions: {
+          numberOfSecrets: Object.keys(allFindingsByFingerprint).length,
+          pusher_email: pusher.email,
+          pusher_name: pusher.name
+        }
+      });
+    }
+  });
+
+  queueService.start(QueueName.SecretFullRepoScan, async (job) => {
+    const appCfg = getConfig();
+    const { organizationId, repository, installationId } = job.data;
+    const octokit = new ProbotOctokit({
+      auth: {
+        appId: appCfg.SECRET_SCANNING_GIT_APP_ID,
+        privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY,
+        installationId
+      }
+    });
+
+    const findings = await scanFullRepoContentAndGetFindings(
+      octokit,
+      installationId,
+      repository.fullName
+    );
+    await secretScanningDal.transaction(async (tx) => {
+      if (!findings.length) return;
+      // eslint-disable-next-line
+      await secretScanningDal.upsert(
+        findings.map((finding) => ({
+          installationId,
+          email: finding.Email,
+          author: finding.Author,
+          date: finding.Date,
+          file: finding.File,
+          tags: finding.Tags,
+          commit: finding.Commit,
+          ruleID: finding.RuleID,
+          endLine: String(finding.EndLine),
+          entropy: String(finding.Entropy),
+          message: finding.Message,
+          endColumn: String(finding.EndColumn),
+          startLine: String(finding.StartLine),
+          startColumn: String(finding.StartColumn),
+          fingerPrintWithoutCommitId: finding.FingerPrintWithoutCommitId,
+          description: finding.Description,
+          symlinkFile: finding.SymlinkFile,
+          orgId: organizationId,
+          repositoryFullName: repository.fullName,
+          repositoryId: String(repository.id),
+          fingerprint: finding.Fingerprint
+        })),
+        tx
+      );
+    });
+
+    const adminEmails = await getOrgAdminEmails(organizationId);
+    if (findings.length) {
+      await smtpService.sendMail({
+        template: SmtpTemplates.SecretLeakIncident,
+        subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`,
+        recipients: adminEmails,
+        substitutions: {
+          numberOfSecrets: findings.length
+        }
+      });
+    }
+  });
+
+  queueService.listen(QueueName.SecretPushEventScan, "failed", (job, err) => {
+    logger.error("Failed to secret scan on push", job?.data, err);
+  });
+
+  queueService.listen(QueueName.SecretFullRepoScan, "failed", (job, err) => {
+    logger.error("Failed to do full repo secret scan", job?.data, err);
+  });
+
+  return { startFullRepoScan, startPushEventScan };
+};

backend-pg/src/ee/services/secret-scanning/secret-scanning-service.ts

@@ -0,0 +1,191 @@
+import crypto from "node:crypto";
+
+import { ForbiddenError } from "@casl/ability";
+import { PushEvent } from "@octokit/webhooks-types";
+import { ProbotOctokit } from "probot";
+
+import {
+  OrgPermissionActions,
+  OrgPermissionSubjects
+} from "@app/ee/services/permission/org-permission";
+import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
+import { getConfig } from "@app/lib/config/env";
+import { UnauthorizedError } from "@app/lib/errors";
+
+import { TGitAppDalFactory } from "./git-app-dal";
+import { TGitAppInstallSessionDalFactory } from "./git-app-install-session-dal";
+import { TSecretScanningDalFactory } from "./secret-scanning-dal";
+import { TSecretScanningQueueFactory } from "./secret-scanning-queue";
+import {
+  SecretScanningRiskStatus,
+  TGetOrgInstallStatusDTO,
+  TGetOrgRisksDTO,
+  TInstallAppSessionDTO,
+  TLinkInstallSessionDTO,
+  TUpdateRiskStatusDTO
+} from "./secret-scanning-types";
+
+type TSecretScanningServiceFactoryDep = {
+  permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
+  secretScanningDal: TSecretScanningDalFactory;
+  gitAppInstallSessionDal: TGitAppInstallSessionDalFactory;
+  gitAppOrgDal: TGitAppDalFactory;
+  secretScanningQueue: TSecretScanningQueueFactory;
+};
+
+export type TSecretScanningServiceFactory = ReturnType<typeof secretScanningServiceFactory>;
+
+export const secretScanningServiceFactory = ({
+  secretScanningDal,
+  gitAppOrgDal,
+  gitAppInstallSessionDal,
+  permissionService,
+  secretScanningQueue
+}: TSecretScanningServiceFactoryDep) => {
+  const createInstallationSession = async ({ actor, orgId, actorId }: TInstallAppSessionDTO) => {
+    const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId);
+    ForbiddenError.from(permission).throwUnlessCan(
+      OrgPermissionActions.Create,
+      OrgPermissionSubjects.SecretScanning
+    );
+
+    const sessionId = crypto.randomBytes(16).toString("hex");
+    await gitAppInstallSessionDal.upsert({ orgId, sessionId, userId: actorId });
+    return { sessionId };
+  };
+
+  const linkInstallationToOrg = async ({
+    sessionId,
+    actorId,
+    installationId,
+    actor
+  }: TLinkInstallSessionDTO) => {
+    const session = await gitAppInstallSessionDal.findOne({ sessionId });
+    if (!session) throw new UnauthorizedError({ message: "Session not found" });
+
+    const { permission } = await permissionService.getOrgPermission(actor, actorId, session.orgId);
+    ForbiddenError.from(permission).throwUnlessCan(
+      OrgPermissionActions.Create,
+      OrgPermissionSubjects.SecretScanning
+    );
+    const installatedApp = await gitAppOrgDal.transaction(async (tx) => {
+      await gitAppInstallSessionDal.deleteById(session.id, tx);
+      return gitAppOrgDal.upsert({ orgId: session.orgId, installationId, userId: actorId }, tx);
+    });
+
+    const appCfg = getConfig();
+    const octokit = new ProbotOctokit({
+      auth: {
+        appId: appCfg.SECRET_SCANNING_GIT_APP_ID,
+        privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY,
+        installationId: installationId.toString()
+      }
+    });
+
+    const {
+      data: { repositories }
+    } = await octokit.apps.listReposAccessibleToInstallation();
+    await Promise.all(
+      repositories.map(({ id, full_name }) =>
+        secretScanningQueue.startFullRepoScan({
+          organizationId: session.orgId,
+          installationId,
+          repository: { id, fullName: full_name }
+        })
+      )
+    );
+    return { installatedApp };
+  };
+
+  const getOrgInstallationStatus = async ({ actorId, orgId, actor }: TGetOrgInstallStatusDTO) => {
+    const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId);
+    ForbiddenError.from(permission).throwUnlessCan(
+      OrgPermissionActions.Read,
+      OrgPermissionSubjects.SecretScanning
+    );
+
+    const appInstallation = await gitAppOrgDal.findOne({ orgId });
+    return Boolean(appInstallation);
+  };
+
+  const getRisksByOrg = async ({ actor, orgId, actorId }: TGetOrgRisksDTO) => {
+    const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId);
+    ForbiddenError.from(permission).throwUnlessCan(
+      OrgPermissionActions.Read,
+      OrgPermissionSubjects.SecretScanning
+    );
+    const risks = await secretScanningDal.find({ orgId }, { sort: [["createdAt", "desc"]] });
+    return { risks };
+  };
+
+  const updateRiskStatus = async ({
+    actorId,
+    orgId,
+    actor,
+    riskId,
+    status
+  }: TUpdateRiskStatusDTO) => {
+    const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId);
+    ForbiddenError.from(permission).throwUnlessCan(
+      OrgPermissionActions.Edit,
+      OrgPermissionSubjects.SecretScanning
+    );
+
+    const isRiskResolved = Boolean(
+      [
+        SecretScanningRiskStatus.FalsePositive,
+        SecretScanningRiskStatus.Revoked,
+        SecretScanningRiskStatus.NotRevoked
+      ].includes(status)
+    );
+
+    const risk = await secretScanningDal.updateById(riskId, {
+      status,
+      isResolved: isRiskResolved
+    });
+    return { risk };
+  };
+
+  const handleRepoPushEvent = async (payload: PushEvent) => {
+    const { commits, repository, installation, pusher } = payload;
+    if (!commits || !repository || !installation || !pusher) {
+      return;
+    }
+
+    const installationLink = await gitAppOrgDal.findOne({
+      installationId: String(installation.id)
+    });
+    if (!installationLink) return;
+
+    await secretScanningQueue.startPushEventScan({
+      commits,
+      pusher: { name: pusher.name, email: pusher.email },
+      repository: { fullName: repository.full_name, id: repository.id },
+      organizationId: installationLink.orgId,
+      installationId: String(installation?.id)
+    });
+  };
+
+  const handleRepoDeleteEvent = async (installationId: string, repositoryIds: string[]) => {
+    await secretScanningDal.transaction(async (tx) => {
+      if (repositoryIds.length) {
+        await Promise.all(
+          Object.keys(repositoryIds).map((key) =>
+            secretScanningDal.delete({ repositoryId: key }, tx)
+          )
+        );
+      }
+      await gitAppOrgDal.delete({ installationId }, tx);
+    });
+  };
+
+  return {
+    createInstallationSession,
+    linkInstallationToOrg,
+    getOrgInstallationStatus,
+    getRisksByOrg,
+    updateRiskStatus,
+    handleRepoPushEvent,
+    handleRepoDeleteEvent
+  };
+};

backend-pg/src/ee/services/secret-scanning/secret-scanning-types.ts

@@ -0,0 +1,24 @@
+import { TOrgPermission } from "@app/lib/types";
+
+export enum SecretScanningRiskStatus {
+  FalsePositive = "RESOLVED_FALSE_POSITIVE",
+  Revoked = "RESOLVED_REVOKED",
+  NotRevoked = "RESOLVED_NOT_REVOKED",
+  Unresolved = "UNRESOLVED"
+}
+
+export type TInstallAppSessionDTO = TOrgPermission;
+
+export type TLinkInstallSessionDTO = {
+  installationId: string;
+  sessionId: string;
+} & Omit<TOrgPermission, "orgId">;
+
+export type TGetOrgInstallStatusDTO = TOrgPermission;
+
+export type TGetOrgRisksDTO = TOrgPermission;
+
+export type TUpdateRiskStatusDTO = {
+  riskId: string;
+  status: SecretScanningRiskStatus;
+} & TOrgPermission;

backend-pg/src/lib/config/env.ts

@@ -76,9 +76,21 @@ const envSchema = z
     // google
     CLIENT_ID_GITLAB: zpStr(z.string().optional()),
     CLIENT_SECRET_GITLAB: zpStr(z.string().optional()),
-    URL_GITLAB_URL: zpStr(z.string().optional().default(GITLAB_URL))
+    URL_GITLAB_URL: zpStr(z.string().optional().default(GITLAB_URL)),
+    // SECRET-SCANNING
+    SECRET_SCANNING_WEBHOOK_PROXY: zpStr(z.string().optional()),
+    SECRET_SCANNING_WEBHOOK_SECRET: zpStr(z.string().optional()),
+    SECRET_SCANNING_GIT_APP_ID: zpStr(z.string().optional()),
+    SECRET_SCANNING_PRIVATE_KEY: zpStr(z.string().optional())
   })
-  .transform((data) => ({ ...data, isSmtpConfigured: Boolean(data.SMTP_HOST) }));
+  .transform((data) => ({
+    ...data,
+    isSmtpConfigured: Boolean(data.SMTP_HOST),
+    isSecretScanningConfigured:
+      Boolean(data.SECRET_SCANNING_GIT_APP_ID) &&
+      Boolean(data.SECRET_SCANNING_PRIVATE_KEY) &&
+      Boolean(data.SECRET_SCANNING_WEBHOOK_SECRET)
+  }));
 
 let envCfg: Readonly<z.infer<typeof envSchema>>;
 

backend-pg/src/queue/queue-service.ts

@@ -2,19 +2,26 @@ import { Job, JobsOptions, Queue, Worker, WorkerListener } from "bullmq";
 import Redis from "ioredis";
 
 import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
+import {
+  TScanFullRepoEventPayload,
+  TScanPushEventPayload
+} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
 
 export enum QueueName {
   SecretRotation = "secret-rotation",
   AuditLog = "audit-log",
   IntegrationSync = "sync-integrations",
-  SecretWebhook = "secret-webhook"
+  SecretWebhook = "secret-webhook",
+  SecretFullRepoScan = "secret-full-repo-scan",
+  SecretPushEventScan = "secret-push-event-scan"
 }
 
 export enum QueueJobs {
   SecretRotation = "secret-rotation-job",
   AuditLog = "audit-log-job",
   SecWebhook = "secret-webhook-trigger",
-  IntegrationSync = "secret-integration-pull"
+  IntegrationSync = "secret-integration-pull",
+  SecretScan = "secret-scan"
 }
 
 export type TQueueJobTypes = {
@@ -34,6 +41,11 @@ export type TQueueJobTypes = {
     name: QueueJobs.IntegrationSync;
     payload: { projectId: string; environment: string; secretPath: string };
   };
+  [QueueName.SecretFullRepoScan]: {
+    name: QueueJobs.SecretScan;
+    payload: TScanFullRepoEventPayload;
+  };
+  [QueueName.SecretPushEventScan]: { name: QueueJobs.SecretScan; payload: TScanPushEventPayload };
 };
 
 export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;

backend-pg/src/server/app.ts

@@ -53,12 +53,16 @@ export const main = async ({ db, smtp, logger, queue }: TMain) => {
 
     await server.register(fastifySwagger);
     await server.register(fastifyFormBody);
+    // allow empty body on post request
+    // server.addContentTypeParser("application/json", { bodyLimit: 0 }, (_request, _payload, done) =>
+    //   done(null, null)
+    // );
 
     // Rate limiters and security headers
     await server.register<FastifyRateLimitOptions>(ratelimiter, globalRateLimiterCfg);
     await server.register(helmet, { contentSecurityPolicy: false });
 
-    await server.register(registerRoutes, { prefix: "/api", smtp, queue, db });
+    await server.register(registerRoutes, { smtp, queue, db });
     await server.ready();
     server.swagger();
     return server;

backend-pg/src/server/plugins/secret-scanner.ts

@@ -0,0 +1,65 @@
+import { Probot } from "probot";
+import SmeeClient from "smee-client";
+
+import { getConfig } from "@app/lib/config/env";
+import { logger } from "@app/lib/logger";
+
+export const registerSecretScannerGhApp = async (server: FastifyZodProvider) => {
+  const probotApp = (app: Probot) => {
+    app.on("installation.deleted", async (context) => {
+      const { payload } = context;
+      const { installation, repositories } = payload;
+      await server.services.secretScanning.handleRepoDeleteEvent(
+        String(installation.id),
+        (repositories || [])?.map(({ id }) => String(id))
+      );
+    });
+
+    app.on("installation", async (context) => {
+      const { payload } = context;
+      logger.info("Installed secret scanner to:", { repositories: payload.repositories });
+    });
+
+    app.on("push", async (context) => {
+      const { payload } = context;
+      await server.services.secretScanning.handleRepoPushEvent(payload);
+    });
+  };
+
+  const appCfg = getConfig();
+  if (appCfg.isSecretScanningConfigured) {
+    const probot = new Probot({
+      appId: appCfg.SECRET_SCANNING_GIT_APP_ID as string,
+      privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY as string,
+      secret: appCfg.SECRET_SCANNING_WEBHOOK_SECRET as string
+    });
+
+    if (appCfg.NODE_ENV === "development") {
+      const smee = new SmeeClient({
+        source: appCfg.SECRET_SCANNING_WEBHOOK_PROXY as string,
+        target: "http://backend:4000/ss-webhook",
+        logger: console
+      });
+      smee.start();
+    }
+
+    await probot.load(probotApp);
+
+    server.route({
+      method: "POST",
+      url: "/",
+      handler: async (req, res) => {
+        const eventName = req.headers["x-github-event"] as any;
+        const signatureSHA256 = req.headers["x-hub-signature-256"] as string;
+        const id = req.headers["x-github-delivery"] as string;
+        await probot.webhooks.verifyAndReceive({
+          id,
+          name: eventName,
+          payload: req.body as string,
+          signature: signatureSHA256
+        });
+        res.send("ok");
+      }
+    });
+  }
+};

backend-pg/src/server/routes/index.ts

@@ -19,6 +19,11 @@ import { secretApprovalRequestServiceFactory } from "@app/ee/services/secret-app
 import { secretRotationDalFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal";
 import { secretRotationQueueFactory } from "@app/ee/services/secret-rotation/secret-rotation-queue";
 import { secretRotationServiceFactory } from "@app/ee/services/secret-rotation/secret-rotation-service";
+import { gitAppDalFactory } from "@app/ee/services/secret-scanning/git-app-dal";
+import { gitAppInstallSessionDalFactory } from "@app/ee/services/secret-scanning/git-app-install-session-dal";
+import { secretScanningDalFactory } from "@app/ee/services/secret-scanning/secret-scanning-dal";
+import { secretScanningQueueFactory } from "@app/ee/services/secret-scanning/secret-scanning-queue";
+import { secretScanningServiceFactory } from "@app/ee/services/secret-scanning/secret-scanning-service";
 import { secretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
 import { snapshotDalFactory } from "@app/ee/services/secret-snapshot/snapshot-dal";
 import { snapshotFolderDalFactory } from "@app/ee/services/secret-snapshot/snapshot-folder-dal";
@@ -90,6 +95,7 @@ import { webhookServiceFactory } from "@app/services/webhook/webhook-service";
 import { injectAuditLogInfo } from "../plugins/audit-log";
 import { injectIdentity } from "../plugins/auth/inject-identity";
 import { injectPermission } from "../plugins/auth/inject-permission";
+import { registerSecretScannerGhApp } from "../plugins/secret-scanner";
 import { registerV1Routes } from "./v1";
 import { registerV2Routes } from "./v2";
 import { registerV3Routes } from "./v3";
@@ -102,6 +108,8 @@ export const registerRoutes = async (
     queue: queueService
   }: { db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory }
 ) => {
+  server.register(registerSecretScannerGhApp, { prefix: "/ss-webhook" });
+
   // db layers
   const userDal = userDalFactory(db);
   const authDal = authDalFactory(db);
@@ -157,6 +165,10 @@ export const registerRoutes = async (
   const snapshotSecretDal = snapshotSecretDalFactory(db);
   const snapshotFolderDal = snapshotFolderDalFactory(db);
 
+  const gitAppInstallSessionDal = gitAppInstallSessionDalFactory(db);
+  const gitAppOrgDal = gitAppDalFactory(db);
+  const secretScanningDal = secretScanningDalFactory(db);
+
   const permissionService = permissionServiceFactory({ permissionDal, orgRoleDal, projectRoleDal });
   const auditLogQueue = auditLogQueueServiceFactory({ auditLogDal, queueService });
   const auditLogService = auditLogServiceFactory({ auditLogDal, permissionService, auditLogQueue });
@@ -210,6 +222,20 @@ export const registerRoutes = async (
   });
   const apiKeyService = apiKeyServiceFactory({ apiKeyDal });
 
+  const secretScanningQueue = secretScanningQueueFactory({
+    userDal,
+    smtpService,
+    secretScanningDal,
+    queueService,
+    orgMembershipDal: orgDal
+  });
+  const secretScanningService = secretScanningServiceFactory({
+    permissionService,
+    gitAppOrgDal,
+    gitAppInstallSessionDal,
+    secretScanningDal,
+    secretScanningQueue
+  });
   const projectService = projectServiceFactory({
     permissionService,
     projectDal,
@@ -391,7 +417,8 @@ export const registerRoutes = async (
     secretRotation: secretRotationService,
     snapshot: snapshotService,
     saml: samlService,
-    auditLog: auditLogService
+    auditLog: auditLogService,
+    secretScanning: secretScanningService
   });
 
   server.decorate<FastifyZodProvider["store"]>("store", {
@@ -418,12 +445,12 @@ export const registerRoutes = async (
       }
     },
     handler: () => {
-      const appCfg = getConfig();
+      const cfg = getConfig();
 
       return {
         date: new Date(),
         message: "Ok" as const,
-        emailConfigured: appCfg.isSmtpConfigured,
+        emailConfigured: cfg.isSmtpConfigured,
         inviteOnlySignup: false,
         redisConfigured: false,
         secretScanningConfigured: false
@@ -437,8 +464,8 @@ export const registerRoutes = async (
       await v1Server.register(registerV1EERoutes);
       await v1Server.register(registerV1Routes);
     },
-    { prefix: "/v1" }
+    { prefix: "/api/v1" }
   );
-  await server.register(registerV2Routes, { prefix: "/v2" });
+  await server.register(registerV2Routes, { prefix: "/api/v2" });
-  await server.register(registerV3Routes, { prefix: "/v3" });
+  await server.register(registerV3Routes, { prefix: "/api/v3" });
 };

frontend/src/pages/api/secret-scanning/createSecretScanningSession.ts

@@ -5,17 +5,20 @@ import SecurityClient from "@app/components/utilities/SecurityClient";
  * @returns
  */
 const createNewIntegrationSession = (organizationId: string) =>
-  SecurityClient.fetchCall(`/api/v1/secret-scanning/create-installation-session/organization/${organizationId}`, {
+  SecurityClient.fetchCall("/api/v1/secret-scanning/create-installation-session/organization", {
     method: "POST",
     headers: {
       "Content-Type": "application/json"
-    }
+    },
+    body: JSON.stringify({
+      organizationId
+    })
   }).then(async (res) => {
     if (res && res.status === 200) {
       return res.json();
     }
     console.log("Failed to create integration session");
-    console.log("response", res)
+    console.log("response", res);
     return undefined;
   });
 

frontend/src/pages/api/secret-scanning/getInstallationStatus.ts

@@ -5,17 +5,20 @@ import SecurityClient from "@app/components/utilities/SecurityClient";
  * @returns
  */
 const getInstallationStatus = (organizationId: string) =>
-  SecurityClient.fetchCall(`/api/v1/secret-scanning/installation-status/organization/${organizationId}`, {
+  SecurityClient.fetchCall(
+    `/api/v1/secret-scanning/installation-status/organization/${organizationId}`,
+    {
       method: "GET",
       headers: {
         "Content-Type": "application/json"
       }
-  }).then(async (res) => {
+    }
+  ).then(async (res) => {
     if (res && res.status === 200) {
-      return (await res.json()).appInstallationComplete;
+      return (await res.json()).appInstallationCompleted;
     }
     console.log("Failed to check installation status");
-    console.log("response", res)
+    console.log("response", res);
     return undefined;
   });
 

frontend/src/pages/api/secret-scanning/getRisksByOrganization.ts

@@ -34,7 +34,7 @@ export type IGitRisks = {
     email: string;
   };
   createdAt: string;
-  organization: string;
+  orgId: string;
 };
 
 /**

frontend/src/pages/org/[id]/secret-scanning/index.tsx

@@ -22,11 +22,11 @@ const SecretScanning = withPermission(
       const linkInstallation = async () => {
         if (
           typeof queryParams.state === "string" &&
-          typeof queryParams.installationid === "string"
+          typeof queryParams.installation_id === "string"
         ) {
           try {
             const isLinked = await linkGitAppInstallationWithOrganization(
-              queryParams.installationid as string,
+              queryParams.installation_id as string,
               queryParams.state as string
             );
             if (isLinked) {
@@ -47,12 +47,12 @@ const SecretScanning = withPermission(
 
       fetchInstallationStatus();
       linkInstallation();
-    }, [queryParams.state, queryParams.installationid]);
+    }, [queryParams.state, queryParams.installation_id]);
 
     const generateNewIntegrationSession = async () => {
       const session = await createNewIntegrationSession(String(localStorage.getItem("orgData.id")));
       router.push(
-        `https://github.com/apps/infisical-radar/installations/new?state=${session.sessionId}`
+        `https://github.com/apps/infisical-test/installations/new?state=${session.sessionId}`
       );
     };