feature: secret scanning pt 3

backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.ts

@@ -8,7 +8,11 @@ import {
   SecretScanningFindingSeverity,
   SecretScanningResource
 } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
-import { cloneRepository } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
+import {
+  cloneRepository,
+  convertPatchLineToFileLineNumber,
+  replaceNonChangesWithNewlines
+} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
 import {
   TSecretScanningFactoryGetDiffScanFindingsPayload,
   TSecretScanningFactoryGetDiffScanResourcePayload,
@@ -121,7 +125,7 @@ export const GitHubSecretScanningFactory = () => {
   const getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<
     TGitHubDataSourceWithConnection,
     TQueueGitHubResourceDiffScan["payload"]
-  > = async ({ dataSource, payload, resourceName }) => {
+  > = async ({ dataSource, payload, resourceName, configPath }) => {
     const appCfg = getConfig();
     const {
       connection: {
@@ -144,33 +148,52 @@ export const GitHubSecretScanningFactory = () => {
     const allFindings: SecretMatch[] = [];
 
     for (const commit of commits) {
-      for (const filepath of [...commit.added, ...commit.modified]) {
-        // eslint-disable-next-line
-        const fileContentsResponse = await octokit.repos.getContent({
-          owner,
-          repo,
-          path: filepath
-        });
+      // eslint-disable-next-line no-await-in-loop
+      const commitData = await octokit.repos.getCommit({
+        owner,
+        repo,
+        ref: commit.id
+      });
 
-        const { data } = fileContentsResponse;
-        const fileContent = Buffer.from((data as { content: string }).content, "base64").toString();
+      // eslint-disable-next-line no-continue
+      if (!commitData.data.files) continue;
 
-        // eslint-disable-next-line
-        const findings = await scanContentAndGetFindings(`\n${fileContent}`); // extra line to count lines correctly
+      for (const file of commitData.data.files) {
+        if ((file.status === "added" || file.status === "modified") && file.patch) {
+          // eslint-disable-next-line
+          const findings = await scanContentAndGetFindings(
+            replaceNonChangesWithNewlines(`\n${file.patch}`),
+            configPath
+          );
 
-        allFindings.push(
-          ...findings.map((finding) => ({
-            ...finding,
-            File: filepath,
-            Commit: commit.id,
-            Author: commit.author.name,
-            Email: commit.author.email ?? "",
-            Message: commit.message,
-            Fingerprint: `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`,
-            Date: commit.timestamp,
-            Link: `https://github.com/${resourceName}/blob/${commit.id}/${filepath}#L${finding.StartLine}`
-          }))
-        );
+          const adjustedFindings = findings.map((finding) => {
+            const startLine = convertPatchLineToFileLineNumber(file.patch!, finding.StartLine);
+            const endLine =
+              finding.StartLine === finding.EndLine
+                ? startLine
+                : convertPatchLineToFileLineNumber(file.patch!, finding.EndLine);
+            const startColumn = finding.StartColumn - 1; // subtract 1 for +
+            const endColumn = finding.EndColumn - 1; // subtract 1 for +
+
+            return {
+              ...finding,
+              StartLine: startLine,
+              EndLine: endLine,
+              StartColumn: startColumn,
+              EndColumn: endColumn,
+              File: file.filename,
+              Commit: commit.id,
+              Author: commit.author.name,
+              Email: commit.author.email ?? "",
+              Message: commit.message,
+              Fingerprint: `${commit.id}:${file.filename}:${finding.RuleID}:${startLine}:${startColumn}`,
+              Date: commit.timestamp,
+              Link: `https://github.com/${resourceName}/blob/${commit.id}/${file.filename}#L${startLine}`
+            };
+          });
+
+          allFindings.push(...adjustedFindings);
+        }
       }
     }
 

backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-schemas.ts

@@ -16,8 +16,9 @@ import { AppConnection } from "@app/services/app-connection/app-connection-enums
 
 export const GitHubDataSourceConfigSchema = z.object({
   includeRepos: z
-    .array(z.string())
+    .array(z.string().min(1).max(256))
     .nonempty("One or more repositories required")
+    .max(25)
     .default(["*"])
     .describe(SecretScanningDataSources.CONFIG.GITHUB.includeRepos)
 });

backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-dal.ts

@@ -111,6 +111,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
   const resourceOrm = ormify(db, TableName.SecretScanningResource);
   const scanOrm = ormify(db, TableName.SecretScanningScan);
   const findingOrm = ormify(db, TableName.SecretScanningFinding);
+  const configOrm = ormify(db, TableName.SecretScanningConfig);
 
   const findDataSource = async (filter: Parameters<(typeof dataSourceOrm)["find"]>[0], tx?: Knex) => {
     try {
@@ -406,7 +407,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
           unresolvedFindings:
             findings?.filter((finding) => finding.status === SecretScanningFindingStatus.Unresolved).length ?? 0,
           resolvedFindings:
-            findings?.filter((finding) => finding.status === SecretScanningFindingStatus.Resolved).length ?? 0,
+            findings?.filter((finding) => finding.status !== SecretScanningFindingStatus.Unresolved).length ?? 0,
           resourceName: resources[0].name
         };
       });
@@ -453,6 +454,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
       findWithDetailsByDataSourceId: findScansWithDetailsByDataSourceId,
       findByDataSourceId: findScansByDataSourceId
     },
-    findings: findingOrm
+    findings: findingOrm,
+    configs: configOrm
   };
 };

backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts

@@ -30,9 +30,9 @@ export const cloneRepository = async ({ cloneUrl, repoPath }: TCloneRepository):
   });
 };
 
-export function scanDirectory(inputPath: string, outputPath: string): Promise<void> {
+export function scanDirectory(inputPath: string, outputPath: string, configPath?: string): Promise<void> {
   return new Promise((resolve, reject) => {
-    const command = `cd ${inputPath} && infisical scan --exit-code=77 -r "${outputPath}"`;
+    const command = `cd ${inputPath} && infisical scan --exit-code=77 -r "${outputPath}" ${configPath ? `-c ${configPath}` : ""}`;
     exec(command, (error) => {
       if (error && error.code !== 77) {
         reject(error);
@@ -43,8 +43,12 @@ export function scanDirectory(inputPath: string, outputPath: string): Promise<vo
   });
 }
 
-export const scanGitRepositoryAndGetFindings = async (scanPath: string, findingsPath: string): TGetFindingsPayload => {
-  await scanDirectory(scanPath, findingsPath);
+export const scanGitRepositoryAndGetFindings = async (
+  scanPath: string,
+  findingsPath: string,
+  configPath?: string
+): TGetFindingsPayload => {
+  await scanDirectory(scanPath, findingsPath, configPath);
 
   const findingsData = JSON.parse(await readFindingsFile(findingsPath)) as SecretMatch[];
 
@@ -56,13 +60,64 @@ export const scanGitRepositoryAndGetFindings = async (scanPath: string, findings
       ...finding
     }) => ({
       details: titleCaseToCamelCase(finding),
-      fingerprint: finding.Fingerprint,
+      fingerprint: `${finding.Fingerprint}:${finding.StartColumn}`,
       severity: SecretScanningFindingSeverity.High,
       rule: finding.RuleID
     })
   );
 };
 
+export const replaceNonChangesWithNewlines = (patch: string) => {
+  return patch
+    .split("\n")
+    .map((line) => {
+      // Keep added lines (remove the + prefix)
+      if (line.startsWith("+") && !line.startsWith("+++")) {
+        return line.substring(1);
+      }
+
+      // Replace everything else with newlines to maintain line positioning
+
+      return "";
+    })
+    .join("\n");
+};
+
+export const convertPatchLineToFileLineNumber = (patch: string, patchLineNumber: number) => {
+  const lines = patch.split("\n");
+  let currentPatchLine = 0;
+  let currentNewLine = 0;
+
+  for (const line of lines) {
+    currentPatchLine += 1;
+
+    // Hunk header: @@ -a,b +c,d @@
+    const hunkHeaderMatch = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/);
+    if (hunkHeaderMatch) {
+      const startLine = parseInt(hunkHeaderMatch[1], 10);
+      currentNewLine = startLine;
+      // eslint-disable-next-line no-continue
+      continue;
+    }
+
+    if (currentPatchLine === patchLineNumber) {
+      return currentNewLine;
+    }
+
+    if (line.startsWith("+++")) {
+      // eslint-disable-next-line no-continue
+      continue; // skip file metadata lines
+    }
+
+    // Advance only if the line exists in the new file
+    if (line.startsWith("+") || line.startsWith(" ")) {
+      currentNewLine += 1;
+    }
+  }
+
+  return currentNewLine;
+};
+
 const MAX_MESSAGE_LENGTH = 1024;
 
 export const parseScanErrorMessage = (err: unknown): string => {

backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts

@@ -1,11 +1,12 @@
 import { join } from "path";
 
-import { ProjectMembershipRole } from "@app/db/schemas";
+import { ProjectMembershipRole, TSecretScanningFindings } from "@app/db/schemas";
 import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service";
 import { EventType } from "@app/ee/services/audit-log/audit-log-types";
 import {
   createTempFolder,
-  deleteTempFolder
+  deleteTempFolder,
+  writeTextToFile
 } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
 import {
   parseScanErrorMessage,
@@ -184,21 +185,33 @@ export const secretScanningV2QueueServiceFactory = async ({
           tempFolder
         });
 
+        const config = await secretScanningV2DAL.configs.findOne({
+          projectId: dataSource.projectId
+        });
+
+        let configPath: string | undefined;
+
+        if (config && config.content) {
+          configPath = join(tempFolder, "infisical-scan.toml");
+          await writeTextToFile(configPath, config.content);
+        }
+
         let findingsPayload: TFindingsPayload;
         switch (resource.type) {
           case SecretScanningResource.Repository:
           case SecretScanningResource.Project:
-            findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath);
+            findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
             break;
           default:
             throw new Error("Unhandled resource type");
         }
 
-        if (findingsPayload.length) {
-          await secretScanningV2DAL.findings.transaction(async (tx) => {
-            await secretScanningV2DAL.findings.upsert(
-              findingsPayload.map((findings) => ({
-                ...findings,
+        const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
+          let findings: TSecretScanningFindings[] = [];
+          if (findingsPayload.length) {
+            findings = await secretScanningV2DAL.findings.upsert(
+              findingsPayload.map((finding) => ({
+                ...finding,
                 projectId: dataSource.projectId,
                 dataSourceName: dataSource.name,
                 dataSourceType: dataSource.type,
@@ -211,22 +224,28 @@ export const secretScanningV2QueueServiceFactory = async ({
               tx,
               ["resourceName", "dataSourceName", "status"]
             );
+          }
 
-            await secretScanningV2DAL.scans.update(
-              { id: scanId },
-              {
-                status: SecretScanningScanStatus.Completed,
-                statusMessage: null
-              }
-            );
-          });
+          await secretScanningV2DAL.scans.update(
+            { id: scanId },
+            {
+              status: SecretScanningScanStatus.Completed,
+              statusMessage: null
+            }
+          );
 
+          return findings;
+        });
+
+        const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
+
+        if (newFindings.length) {
           await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
             status: SecretScanningScanStatus.Completed,
             resourceName: resource.name,
             isDiffScan: false,
             dataSource,
-            numberOfSecrets: findingsPayload.length,
+            numberOfSecrets: newFindings.length,
             scanId
           });
         }
@@ -376,6 +395,8 @@ export const secretScanningV2QueueServiceFactory = async ({
 
       const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]();
 
+      const tempFolder = await createTempFolder();
+
       try {
         await secretScanningV2DAL.scans.update(
           { id: scanId },
@@ -387,20 +408,34 @@ export const secretScanningV2QueueServiceFactory = async ({
         let connection: TAppConnection | null = null;
         if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
 
+        const config = await secretScanningV2DAL.configs.findOne({
+          projectId: dataSource.projectId
+        });
+
+        let configPath: string | undefined;
+
+        if (config && config.content) {
+          configPath = join(tempFolder, "infisical-scan.toml");
+          await writeTextToFile(configPath, config.content);
+        }
+
         const findingsPayload = await factory.getDiffScanFindingsPayload({
           dataSource: {
             ...dataSource,
             connection
           } as TSecretScanningDataSourceWithConnection,
           resourceName: resource.name,
-          payload
+          payload,
+          configPath
         });
 
-        if (findingsPayload.length) {
-          await secretScanningV2DAL.findings.transaction(async (tx) => {
-            await secretScanningV2DAL.findings.upsert(
-              findingsPayload.map((findings) => ({
-                ...findings,
+        const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
+          let findings: TSecretScanningFindings[] = [];
+
+          if (findingsPayload.length) {
+            findings = await secretScanningV2DAL.findings.upsert(
+              findingsPayload.map((finding) => ({
+                ...finding,
                 projectId: dataSource.projectId,
                 dataSourceName: dataSource.name,
                 dataSourceType: dataSource.type,
@@ -413,21 +448,27 @@ export const secretScanningV2QueueServiceFactory = async ({
               tx,
               ["resourceName", "dataSourceName", "status"]
             );
+          }
 
-            await secretScanningV2DAL.scans.update(
-              { id: scanId },
-              {
-                status: SecretScanningScanStatus.Completed
-              }
-            );
-          });
+          await secretScanningV2DAL.scans.update(
+            { id: scanId },
+            {
+              status: SecretScanningScanStatus.Completed
+            }
+          );
 
+          return findings;
+        });
+
+        const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
+
+        if (newFindings.length) {
           await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
             status: SecretScanningScanStatus.Completed,
             resourceName: resource.name,
             isDiffScan: true,
             dataSource,
-            numberOfSecrets: findingsPayload.length,
+            numberOfSecrets: newFindings.length,
             scanId
           });
         }
@@ -496,6 +537,8 @@ export const secretScanningV2QueueServiceFactory = async ({
 
         logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
         throw error;
+      } finally {
+        await deleteTempFolder(tempFolder);
       }
     },
     {

backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.ts

@@ -4,6 +4,7 @@ import { ActionProjectType } from "@app/db/schemas";
 import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import {
+  ProjectPermissionSecretScanningConfigActions,
   ProjectPermissionSecretScanningDataSourceActions,
   ProjectPermissionSecretScanningFindingActions,
   ProjectPermissionSub
@@ -30,7 +31,8 @@ import {
   TSecretScanningScanWithDetails,
   TTriggerSecretScanningDataSourceDTO,
   TUpdateSecretScanningDataSourceDTO,
-  TUpdateSecretScanningFinding
+  TUpdateSecretScanningFindingDTO,
+  TUpsertSecretScanningConfigDTO
 } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
 import { DatabaseErrorCode } from "@app/lib/error-codes";
 import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
@@ -714,7 +716,7 @@ export const secretScanningV2ServiceFactory = ({
   };
 
   const updateSecretScanningFindingById = async (
-    { findingId, remarks, status }: TUpdateSecretScanningFinding,
+    { findingId, remarks, status }: TUpdateSecretScanningFindingDTO,
     actor: OrgServiceActor
   ) => {
     const plan = await licenseService.getPlan(actor.orgId);
@@ -754,6 +756,77 @@ export const secretScanningV2ServiceFactory = ({
     return { finding: updatedFinding as TSecretScanningFinding, projectId: finding.projectId };
   };
 
+  const findSecretScanningConfigByProjectId = async (projectId: string, actor: OrgServiceActor) => {
+    const plan = await licenseService.getPlan(actor.orgId);
+
+    if (!plan.secretScanning)
+      throw new BadRequestError({
+        message:
+          "Failed to access Secret Scanning Configuration due to plan restriction. Upgrade plan to enable Secret Scanning."
+      });
+
+    const { permission } = await permissionService.getProjectPermission({
+      actor: actor.type,
+      actorId: actor.id,
+      actorAuthMethod: actor.authMethod,
+      actorOrgId: actor.orgId,
+      actionProjectType: ActionProjectType.SecretScanning,
+      projectId
+    });
+
+    ForbiddenError.from(permission).throwUnlessCan(
+      ProjectPermissionSecretScanningConfigActions.Read,
+      ProjectPermissionSub.SecretScanningConfigs
+    );
+
+    const config = await secretScanningV2DAL.configs.findOne({
+      projectId
+    });
+
+    return (
+      config ?? { content: null, projectId, updatedAt: null } // using default config
+    );
+  };
+
+  const upsertSecretScanningConfig = async (
+    { projectId, content }: TUpsertSecretScanningConfigDTO,
+    actor: OrgServiceActor
+  ) => {
+    const plan = await licenseService.getPlan(actor.orgId);
+
+    if (!plan.secretScanning)
+      throw new BadRequestError({
+        message:
+          "Failed to access Secret Scanning Configuration due to plan restriction. Upgrade plan to enable Secret Scanning."
+      });
+
+    const { permission } = await permissionService.getProjectPermission({
+      actor: actor.type,
+      actorId: actor.id,
+      actorAuthMethod: actor.authMethod,
+      actorOrgId: actor.orgId,
+      actionProjectType: ActionProjectType.SecretScanning,
+      projectId
+    });
+
+    ForbiddenError.from(permission).throwUnlessCan(
+      ProjectPermissionSecretScanningConfigActions.Read,
+      ProjectPermissionSub.SecretScanningConfigs
+    );
+
+    const [config] = await secretScanningV2DAL.configs.upsert(
+      [
+        {
+          projectId,
+          content
+        }
+      ],
+      "projectId"
+    );
+
+    return config;
+  };
+
   return {
     listSecretScanningDataSourceOptions,
     listSecretScanningDataSourcesByProjectId,
@@ -771,6 +844,8 @@ export const secretScanningV2ServiceFactory = ({
     getSecretScanningUnresolvedFindingsCountByProjectId,
     listSecretScanningFindingsByProjectId,
     updateSecretScanningFindingById,
+    findSecretScanningConfigByProjectId,
+    upsertSecretScanningConfig,
     github: githubSecretScanningService(secretScanningV2DAL, secretScanningV2Queue)
   };
 };

backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts

@@ -131,7 +131,7 @@ export type TSecretScanningFactoryGetFullScanPath<T extends TSecretScanningDataS
 export type TSecretScanningFactoryGetDiffScanFindingsPayload<
   T extends TSecretScanningDataSourceWithConnection,
   P extends TQueueSecretScanningResourceDiffScan["payload"]
-> = (parameters: { dataSource: T; resourceName: string; payload: P }) => Promise<TFindingsPayload>;
+> = (parameters: { dataSource: T; resourceName: string; payload: P; configPath?: string }) => Promise<TFindingsPayload>;
 
 export type TSecretScanningDataSourceRaw = NonNullable<
   Awaited<ReturnType<TSecretScanningV2DALFactory["dataSources"]["findById"]>>
@@ -175,10 +175,15 @@ export type TSecretScanningFactory<
 export type TFindingsPayload = Pick<TSecretScanningFindingsInsert, "details" | "fingerprint" | "severity" | "rule">[];
 export type TGetFindingsPayload = Promise<TFindingsPayload>;
 
-export type TUpdateSecretScanningFinding = {
+export type TUpdateSecretScanningFindingDTO = {
   status?: SecretScanningFindingStatus;
   remarks?: string | null;
   findingId: string;
 };
 
+export type TUpsertSecretScanningConfigDTO = {
+  projectId: string;
+  content: string | null;
+};
+
 export type TSecretScanningDataSourceCredentials = undefined;