feature: secret scanning pt 3

This commit is contained in:
Scott Wilson
2025-05-30 20:19:44 -07:00
parent 4773336a04
commit ce4e35e908
37 changed files with 946 additions and 102 deletions
+8
View File
@@ -324,6 +324,9 @@ import {
TSecretRotationV2SecretMappingsInsert,
TSecretRotationV2SecretMappingsUpdate,
TSecrets,
TSecretScanningConfigs,
TSecretScanningConfigsInsert,
TSecretScanningConfigsUpdate,
TSecretScanningDataSources,
TSecretScanningDataSourcesInsert,
TSecretScanningDataSourcesUpdate,
@@ -1106,5 +1109,10 @@ declare module "knex/types/tables" {
TSecretScanningFindingsInsert,
TSecretScanningFindingsUpdate
>;
[TableName.SecretScanningConfig]: KnexOriginal.CompositeTableType<
TSecretScanningConfigs,
TSecretScanningConfigsInsert,
TSecretScanningConfigsUpdate
>;
}
}
@@ -78,7 +78,16 @@ export async function up(knex: Knex): Promise<void> {
await createOnUpdateTrigger(knex, TableName.SecretScanningFinding);
}
// TODO: Rules
if (!(await knex.schema.hasTable(TableName.SecretScanningConfig))) {
await knex.schema.createTable(TableName.SecretScanningConfig, (t) => {
t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
t.string("projectId").notNullable().unique();
t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
t.string("content", 5000);
t.timestamps(true, true, true);
});
await createOnUpdateTrigger(knex, TableName.SecretScanningConfig);
}
}
export async function down(knex: Knex): Promise<void> {
@@ -92,4 +101,7 @@ export async function down(knex: Knex): Promise<void> {
await knex.schema.dropTableIfExists(TableName.SecretScanningDataSource);
await dropOnUpdateTrigger(knex, TableName.SecretScanningDataSource);
await knex.schema.dropTableIfExists(TableName.SecretScanningConfig);
await dropOnUpdateTrigger(knex, TableName.SecretScanningConfig);
}
+1
View File
@@ -107,6 +107,7 @@ export * from "./secret-rotation-outputs";
export * from "./secret-rotation-v2-secret-mappings";
export * from "./secret-rotations";
export * from "./secret-rotations-v2";
export * from "./secret-scanning-configs";
export * from "./secret-scanning-data-sources";
export * from "./secret-scanning-findings";
export * from "./secret-scanning-git-risks";
+2 -1
View File
@@ -159,7 +159,8 @@ export enum TableName {
SecretScanningDataSource = "secret_scanning_data_sources",
SecretScanningResource = "secret_scanning_resources",
SecretScanningScan = "secret_scanning_scans",
SecretScanningFinding = "secret_scanning_findings"
SecretScanningFinding = "secret_scanning_findings",
SecretScanningConfig = "secret_scanning_configs"
}
export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt";
@@ -0,0 +1,20 @@
// Code generated by automation script, DO NOT EDIT.
// Automated by pulling database and generating zod schema
// To update. Just run npm run generate:schema
// Written by akhilmhdh.
import { z } from "zod";
import { TImmutableDBKeys } from "./models";
export const SecretScanningConfigsSchema = z.object({
id: z.string().uuid(),
projectId: z.string(),
content: z.string().nullable().optional(),
createdAt: z.date(),
updatedAt: z.date()
});
export type TSecretScanningConfigs = z.infer<typeof SecretScanningConfigsSchema>;
export type TSecretScanningConfigsInsert = Omit<z.input<typeof SecretScanningConfigsSchema>, TImmutableDBKeys>;
export type TSecretScanningConfigsUpdate = Partial<Omit<z.input<typeof SecretScanningConfigsSchema>, TImmutableDBKeys>>;
@@ -1,5 +1,6 @@
import { z } from "zod";
import { SecretScanningConfigsSchema } from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { GitHubDataSourceListItemSchema } from "@app/ee/services/secret-scanning-v2/github";
import {
@@ -10,7 +11,12 @@ import {
SecretScanningDataSourceSchema,
SecretScanningFindingSchema
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-union-schemas";
import { ApiDocsTags, SecretScanningDataSources, SecretScanningFindings } from "@app/lib/api-docs";
import {
ApiDocsTags,
SecretScanningConfigs,
SecretScanningDataSources,
SecretScanningFindings
} from "@app/lib/api-docs";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
@@ -181,6 +187,96 @@ export const registerSecretScanningV2Router = async (server: FastifyZodProvider)
}
});
server.route({
method: "GET",
url: "/configs/:projectId",
config: {
rateLimit: readLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: "Get the Secret Scanning Config for the specified project.",
params: z.object({
projectId: z
.string()
.trim()
.min(1, "Project ID required")
.describe(SecretScanningConfigs.GET_BY_PROJECT_ID.projectId)
}),
response: {
200: z.object({
config: z.object({ content: z.string().nullish(), projectId: z.string(), updatedAt: z.date().nullish() })
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const {
params: { projectId },
permission
} = req;
const config = await server.services.secretScanningV2.findSecretScanningConfigByProjectId(projectId, permission);
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_CONFIG_GET
}
});
return { config };
}
});
server.route({
method: "PATCH",
url: "/configs/:projectId",
config: {
rateLimit: writeLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: "Update the specified Secret Scanning Configuration.",
params: z.object({
projectId: z.string().trim().min(1, "Finding ID required").describe(SecretScanningConfigs.UPDATE.projectId)
}),
body: z.object({
content: z.string().nullable().describe(SecretScanningConfigs.UPDATE.content)
}),
response: {
200: z.object({ config: SecretScanningConfigsSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const {
params: { projectId },
body,
permission
} = req;
const config = await server.services.secretScanningV2.upsertSecretScanningConfig(
{ projectId, ...body },
permission
);
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_CONFIG_UPDATE,
metadata: body
}
});
return { config };
}
});
// not exposed, for UI only
server.route({
method: "GET",
@@ -399,7 +399,9 @@ export enum EventType {
SECRET_SCANNING_RESOURCE_LIST = "secret-scanning-resource-list",
SECRET_SCANNING_SCAN_LIST = "secret-scanning-scan-list",
SECRET_SCANNING_FINDING_LIST = "secret-scanning-finding-list",
SECRET_SCANNING_FINDING_UPDATE = "secret-scanning-finding-update"
SECRET_SCANNING_FINDING_UPDATE = "secret-scanning-finding-update",
SECRET_SCANNING_CONFIG_GET = "secret-scanning-config-get",
SECRET_SCANNING_CONFIG_UPDATE = "secret-scanning-config-update"
}
export const filterableSecretEvents: EventType[] = [
@@ -3020,6 +3022,18 @@ interface SecretScanningFindingUpdateEvent {
metadata: TUpdateSecretScanningFinding;
}
interface SecretScanningConfigUpdateEvent {
type: EventType.SECRET_SCANNING_CONFIG_UPDATE;
metadata: {
content: string | null;
};
}
interface SecretScanningConfigReadEvent {
type: EventType.SECRET_SCANNING_CONFIG_GET;
metadata?: Record<string, never>; // not needed, based off projectId
}
export type Event =
| GetSecretsEvent
| GetSecretEvent
@@ -3297,4 +3311,6 @@ export type Event =
| SecretScanningResourceListEvent
| SecretScanningScanListEvent
| SecretScanningFindingListEvent
| SecretScanningFindingUpdateEvent;
| SecretScanningFindingUpdateEvent
| SecretScanningConfigUpdateEvent
| SecretScanningConfigReadEvent;
@@ -26,8 +26,8 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
customRateLimits: false,
customAlerts: false,
secretAccessInsights: false,
auditLogs: false,
auditLogsRetentionDays: 0,
auditLogs: true,
auditLogsRetentionDays: 30,
auditLogStreams: false,
auditLogStreamLimit: 3,
samlSSO: false,
@@ -55,7 +55,7 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
kmip: false,
gateway: false,
sshHostGroups: false,
secretScanning: false
secretScanning: true
});
export const setupLicenseRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => {
@@ -12,6 +12,7 @@ import {
ProjectPermissionPkiSubscriberActions,
ProjectPermissionSecretActions,
ProjectPermissionSecretRotationActions,
ProjectPermissionSecretScanningConfigActions,
ProjectPermissionSecretScanningDataSourceActions,
ProjectPermissionSecretScanningFindingActions,
ProjectPermissionSecretSyncActions,
@@ -218,6 +219,11 @@ const buildAdminPermissionRules = () => {
ProjectPermissionSub.SecretScanningFindings
);
can(
[ProjectPermissionSecretScanningConfigActions.Read, ProjectPermissionSecretScanningConfigActions.Update],
ProjectPermissionSub.SecretScanningConfigs
);
return rules;
};
@@ -413,6 +419,8 @@ const buildMemberPermissionRules = () => {
ProjectPermissionSub.SecretScanningFindings
);
can([ProjectPermissionSecretScanningConfigActions.Read], ProjectPermissionSub.SecretScanningConfigs);
return rules;
};
@@ -459,6 +467,8 @@ const buildViewerPermissionRules = () => {
can([ProjectPermissionSecretScanningFindingActions.Read], ProjectPermissionSub.SecretScanningFindings);
can([ProjectPermissionSecretScanningConfigActions.Read], ProjectPermissionSub.SecretScanningConfigs);
return rules;
};
@@ -138,6 +138,11 @@ export enum ProjectPermissionSecretScanningFindingActions {
Update = "update-findings"
}
export enum ProjectPermissionSecretScanningConfigActions {
Read = "read-configs",
Update = "update-configs"
}
export enum ProjectPermissionSub {
Role = "role",
Member = "member",
@@ -175,7 +180,8 @@ export enum ProjectPermissionSub {
SecretSyncs = "secret-syncs",
Kmip = "kmip",
SecretScanningDataSources = "secret-scanning-data-sources",
SecretScanningFindings = "secret-scanning-findings"
SecretScanningFindings = "secret-scanning-findings",
SecretScanningConfigs = "secret-scanning-configs"
}
export type SecretSubjectFields = {
@@ -300,7 +306,8 @@ export type ProjectPermissionSet =
| [ProjectPermissionActions.Create, ProjectPermissionSub.SecretRollback]
| [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms]
| [ProjectPermissionSecretScanningDataSourceActions, ProjectPermissionSub.SecretScanningDataSources]
| [ProjectPermissionSecretScanningFindingActions, ProjectPermissionSub.SecretScanningFindings];
| [ProjectPermissionSecretScanningFindingActions, ProjectPermissionSub.SecretScanningFindings]
| [ProjectPermissionSecretScanningConfigActions, ProjectPermissionSub.SecretScanningConfigs];
const SECRET_PATH_MISSING_SLASH_ERR_MSG = "Invalid Secret Path; it must start with a '/'";
const SECRET_PATH_PERMISSION_OPERATOR_SCHEMA = z.union([
@@ -635,6 +642,12 @@ const GeneralPermissionSchema = [
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningFindingActions).describe(
"Describe what action an entity can take."
)
}),
z.object({
subject: z.literal(ProjectPermissionSub.SecretScanningConfigs).describe("The entity this permission pertains to."),
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningConfigActions).describe(
"Describe what action an entity can take."
)
})
];
@@ -8,7 +8,11 @@ import {
SecretScanningFindingSeverity,
SecretScanningResource
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { cloneRepository } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
import {
cloneRepository,
convertPatchLineToFileLineNumber,
replaceNonChangesWithNewlines
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
import {
TSecretScanningFactoryGetDiffScanFindingsPayload,
TSecretScanningFactoryGetDiffScanResourcePayload,
@@ -121,7 +125,7 @@ export const GitHubSecretScanningFactory = () => {
const getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<
TGitHubDataSourceWithConnection,
TQueueGitHubResourceDiffScan["payload"]
> = async ({ dataSource, payload, resourceName }) => {
> = async ({ dataSource, payload, resourceName, configPath }) => {
const appCfg = getConfig();
const {
connection: {
@@ -144,33 +148,52 @@ export const GitHubSecretScanningFactory = () => {
const allFindings: SecretMatch[] = [];
for (const commit of commits) {
for (const filepath of [...commit.added, ...commit.modified]) {
// eslint-disable-next-line
const fileContentsResponse = await octokit.repos.getContent({
owner,
repo,
path: filepath
});
// eslint-disable-next-line no-await-in-loop
const commitData = await octokit.repos.getCommit({
owner,
repo,
ref: commit.id
});
const { data } = fileContentsResponse;
const fileContent = Buffer.from((data as { content: string }).content, "base64").toString();
// eslint-disable-next-line no-continue
if (!commitData.data.files) continue;
// eslint-disable-next-line
const findings = await scanContentAndGetFindings(`\n${fileContent}`); // extra line to count lines correctly
for (const file of commitData.data.files) {
if ((file.status === "added" || file.status === "modified") && file.patch) {
// eslint-disable-next-line
const findings = await scanContentAndGetFindings(
replaceNonChangesWithNewlines(`\n${file.patch}`),
configPath
);
allFindings.push(
...findings.map((finding) => ({
...finding,
File: filepath,
Commit: commit.id,
Author: commit.author.name,
Email: commit.author.email ?? "",
Message: commit.message,
Fingerprint: `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`,
Date: commit.timestamp,
Link: `https://github.com/${resourceName}/blob/${commit.id}/${filepath}#L${finding.StartLine}`
}))
);
const adjustedFindings = findings.map((finding) => {
const startLine = convertPatchLineToFileLineNumber(file.patch!, finding.StartLine);
const endLine =
finding.StartLine === finding.EndLine
? startLine
: convertPatchLineToFileLineNumber(file.patch!, finding.EndLine);
const startColumn = finding.StartColumn - 1; // subtract 1 for +
const endColumn = finding.EndColumn - 1; // subtract 1 for +
return {
...finding,
StartLine: startLine,
EndLine: endLine,
StartColumn: startColumn,
EndColumn: endColumn,
File: file.filename,
Commit: commit.id,
Author: commit.author.name,
Email: commit.author.email ?? "",
Message: commit.message,
Fingerprint: `${commit.id}:${file.filename}:${finding.RuleID}:${startLine}:${startColumn}`,
Date: commit.timestamp,
Link: `https://github.com/${resourceName}/blob/${commit.id}/${file.filename}#L${startLine}`
};
});
allFindings.push(...adjustedFindings);
}
}
}
@@ -16,8 +16,9 @@ import { AppConnection } from "@app/services/app-connection/app-connection-enums
export const GitHubDataSourceConfigSchema = z.object({
includeRepos: z
.array(z.string())
.array(z.string().min(1).max(256))
.nonempty("One or more repositories required")
.max(25)
.default(["*"])
.describe(SecretScanningDataSources.CONFIG.GITHUB.includeRepos)
});
@@ -111,6 +111,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
const resourceOrm = ormify(db, TableName.SecretScanningResource);
const scanOrm = ormify(db, TableName.SecretScanningScan);
const findingOrm = ormify(db, TableName.SecretScanningFinding);
const configOrm = ormify(db, TableName.SecretScanningConfig);
const findDataSource = async (filter: Parameters<(typeof dataSourceOrm)["find"]>[0], tx?: Knex) => {
try {
@@ -406,7 +407,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
unresolvedFindings:
findings?.filter((finding) => finding.status === SecretScanningFindingStatus.Unresolved).length ?? 0,
resolvedFindings:
findings?.filter((finding) => finding.status === SecretScanningFindingStatus.Resolved).length ?? 0,
findings?.filter((finding) => finding.status !== SecretScanningFindingStatus.Unresolved).length ?? 0,
resourceName: resources[0].name
};
});
@@ -453,6 +454,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
findWithDetailsByDataSourceId: findScansWithDetailsByDataSourceId,
findByDataSourceId: findScansByDataSourceId
},
findings: findingOrm
findings: findingOrm,
configs: configOrm
};
};
@@ -30,9 +30,9 @@ export const cloneRepository = async ({ cloneUrl, repoPath }: TCloneRepository):
});
};
export function scanDirectory(inputPath: string, outputPath: string): Promise<void> {
export function scanDirectory(inputPath: string, outputPath: string, configPath?: string): Promise<void> {
return new Promise((resolve, reject) => {
const command = `cd ${inputPath} && infisical scan --exit-code=77 -r "${outputPath}"`;
const command = `cd ${inputPath} && infisical scan --exit-code=77 -r "${outputPath}" ${configPath ? `-c ${configPath}` : ""}`;
exec(command, (error) => {
if (error && error.code !== 77) {
reject(error);
@@ -43,8 +43,12 @@ export function scanDirectory(inputPath: string, outputPath: string): Promise<vo
});
}
export const scanGitRepositoryAndGetFindings = async (scanPath: string, findingsPath: string): TGetFindingsPayload => {
await scanDirectory(scanPath, findingsPath);
export const scanGitRepositoryAndGetFindings = async (
scanPath: string,
findingsPath: string,
configPath?: string
): TGetFindingsPayload => {
await scanDirectory(scanPath, findingsPath, configPath);
const findingsData = JSON.parse(await readFindingsFile(findingsPath)) as SecretMatch[];
@@ -56,13 +60,64 @@ export const scanGitRepositoryAndGetFindings = async (scanPath: string, findings
...finding
}) => ({
details: titleCaseToCamelCase(finding),
fingerprint: finding.Fingerprint,
fingerprint: `${finding.Fingerprint}:${finding.StartColumn}`,
severity: SecretScanningFindingSeverity.High,
rule: finding.RuleID
})
);
};
export const replaceNonChangesWithNewlines = (patch: string) => {
return patch
.split("\n")
.map((line) => {
// Keep added lines (remove the + prefix)
if (line.startsWith("+") && !line.startsWith("+++")) {
return line.substring(1);
}
// Replace everything else with newlines to maintain line positioning
return "";
})
.join("\n");
};
export const convertPatchLineToFileLineNumber = (patch: string, patchLineNumber: number) => {
const lines = patch.split("\n");
let currentPatchLine = 0;
let currentNewLine = 0;
for (const line of lines) {
currentPatchLine += 1;
// Hunk header: @@ -a,b +c,d @@
const hunkHeaderMatch = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/);
if (hunkHeaderMatch) {
const startLine = parseInt(hunkHeaderMatch[1], 10);
currentNewLine = startLine;
// eslint-disable-next-line no-continue
continue;
}
if (currentPatchLine === patchLineNumber) {
return currentNewLine;
}
if (line.startsWith("+++")) {
// eslint-disable-next-line no-continue
continue; // skip file metadata lines
}
// Advance only if the line exists in the new file
if (line.startsWith("+") || line.startsWith(" ")) {
currentNewLine += 1;
}
}
return currentNewLine;
};
const MAX_MESSAGE_LENGTH = 1024;
export const parseScanErrorMessage = (err: unknown): string => {
@@ -1,11 +1,12 @@
import { join } from "path";
import { ProjectMembershipRole } from "@app/db/schemas";
import { ProjectMembershipRole, TSecretScanningFindings } from "@app/db/schemas";
import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import {
createTempFolder,
deleteTempFolder
deleteTempFolder,
writeTextToFile
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import {
parseScanErrorMessage,
@@ -184,21 +185,33 @@ export const secretScanningV2QueueServiceFactory = async ({
tempFolder
});
const config = await secretScanningV2DAL.configs.findOne({
projectId: dataSource.projectId
});
let configPath: string | undefined;
if (config && config.content) {
configPath = join(tempFolder, "infisical-scan.toml");
await writeTextToFile(configPath, config.content);
}
let findingsPayload: TFindingsPayload;
switch (resource.type) {
case SecretScanningResource.Repository:
case SecretScanningResource.Project:
findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath);
findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
break;
default:
throw new Error("Unhandled resource type");
}
if (findingsPayload.length) {
await secretScanningV2DAL.findings.transaction(async (tx) => {
await secretScanningV2DAL.findings.upsert(
findingsPayload.map((findings) => ({
...findings,
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
let findings: TSecretScanningFindings[] = [];
if (findingsPayload.length) {
findings = await secretScanningV2DAL.findings.upsert(
findingsPayload.map((finding) => ({
...finding,
projectId: dataSource.projectId,
dataSourceName: dataSource.name,
dataSourceType: dataSource.type,
@@ -211,22 +224,28 @@ export const secretScanningV2QueueServiceFactory = async ({
tx,
["resourceName", "dataSourceName", "status"]
);
}
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Completed,
statusMessage: null
}
);
});
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Completed,
statusMessage: null
}
);
return findings;
});
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
if (newFindings.length) {
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Completed,
resourceName: resource.name,
isDiffScan: false,
dataSource,
numberOfSecrets: findingsPayload.length,
numberOfSecrets: newFindings.length,
scanId
});
}
@@ -376,6 +395,8 @@ export const secretScanningV2QueueServiceFactory = async ({
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]();
const tempFolder = await createTempFolder();
try {
await secretScanningV2DAL.scans.update(
{ id: scanId },
@@ -387,20 +408,34 @@ export const secretScanningV2QueueServiceFactory = async ({
let connection: TAppConnection | null = null;
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
const config = await secretScanningV2DAL.configs.findOne({
projectId: dataSource.projectId
});
let configPath: string | undefined;
if (config && config.content) {
configPath = join(tempFolder, "infisical-scan.toml");
await writeTextToFile(configPath, config.content);
}
const findingsPayload = await factory.getDiffScanFindingsPayload({
dataSource: {
...dataSource,
connection
} as TSecretScanningDataSourceWithConnection,
resourceName: resource.name,
payload
payload,
configPath
});
if (findingsPayload.length) {
await secretScanningV2DAL.findings.transaction(async (tx) => {
await secretScanningV2DAL.findings.upsert(
findingsPayload.map((findings) => ({
...findings,
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
let findings: TSecretScanningFindings[] = [];
if (findingsPayload.length) {
findings = await secretScanningV2DAL.findings.upsert(
findingsPayload.map((finding) => ({
...finding,
projectId: dataSource.projectId,
dataSourceName: dataSource.name,
dataSourceType: dataSource.type,
@@ -413,21 +448,27 @@ export const secretScanningV2QueueServiceFactory = async ({
tx,
["resourceName", "dataSourceName", "status"]
);
}
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Completed
}
);
});
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Completed
}
);
return findings;
});
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
if (newFindings.length) {
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Completed,
resourceName: resource.name,
isDiffScan: true,
dataSource,
numberOfSecrets: findingsPayload.length,
numberOfSecrets: newFindings.length,
scanId
});
}
@@ -496,6 +537,8 @@ export const secretScanningV2QueueServiceFactory = async ({
logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
throw error;
} finally {
await deleteTempFolder(tempFolder);
}
},
{
@@ -4,6 +4,7 @@ import { ActionProjectType } from "@app/db/schemas";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import {
ProjectPermissionSecretScanningConfigActions,
ProjectPermissionSecretScanningDataSourceActions,
ProjectPermissionSecretScanningFindingActions,
ProjectPermissionSub
@@ -30,7 +31,8 @@ import {
TSecretScanningScanWithDetails,
TTriggerSecretScanningDataSourceDTO,
TUpdateSecretScanningDataSourceDTO,
TUpdateSecretScanningFinding
TUpdateSecretScanningFindingDTO,
TUpsertSecretScanningConfigDTO
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { DatabaseErrorCode } from "@app/lib/error-codes";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
@@ -714,7 +716,7 @@ export const secretScanningV2ServiceFactory = ({
};
const updateSecretScanningFindingById = async (
{ findingId, remarks, status }: TUpdateSecretScanningFinding,
{ findingId, remarks, status }: TUpdateSecretScanningFindingDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
@@ -754,6 +756,77 @@ export const secretScanningV2ServiceFactory = ({
return { finding: updatedFinding as TSecretScanningFinding, projectId: finding.projectId };
};
const findSecretScanningConfigByProjectId = async (projectId: string, actor: OrgServiceActor) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Configuration due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningConfigActions.Read,
ProjectPermissionSub.SecretScanningConfigs
);
const config = await secretScanningV2DAL.configs.findOne({
projectId
});
return (
config ?? { content: null, projectId, updatedAt: null } // using default config
);
};
const upsertSecretScanningConfig = async (
{ projectId, content }: TUpsertSecretScanningConfigDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Configuration due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningConfigActions.Read,
ProjectPermissionSub.SecretScanningConfigs
);
const [config] = await secretScanningV2DAL.configs.upsert(
[
{
projectId,
content
}
],
"projectId"
);
return config;
};
return {
listSecretScanningDataSourceOptions,
listSecretScanningDataSourcesByProjectId,
@@ -771,6 +844,8 @@ export const secretScanningV2ServiceFactory = ({
getSecretScanningUnresolvedFindingsCountByProjectId,
listSecretScanningFindingsByProjectId,
updateSecretScanningFindingById,
findSecretScanningConfigByProjectId,
upsertSecretScanningConfig,
github: githubSecretScanningService(secretScanningV2DAL, secretScanningV2Queue)
};
};
@@ -131,7 +131,7 @@ export type TSecretScanningFactoryGetFullScanPath<T extends TSecretScanningDataS
export type TSecretScanningFactoryGetDiffScanFindingsPayload<
T extends TSecretScanningDataSourceWithConnection,
P extends TQueueSecretScanningResourceDiffScan["payload"]
> = (parameters: { dataSource: T; resourceName: string; payload: P }) => Promise<TFindingsPayload>;
> = (parameters: { dataSource: T; resourceName: string; payload: P; configPath?: string }) => Promise<TFindingsPayload>;
export type TSecretScanningDataSourceRaw = NonNullable<
Awaited<ReturnType<TSecretScanningV2DALFactory["dataSources"]["findById"]>>
@@ -175,10 +175,15 @@ export type TSecretScanningFactory<
export type TFindingsPayload = Pick<TSecretScanningFindingsInsert, "details" | "fingerprint" | "severity" | "rule">[];
export type TGetFindingsPayload = Promise<TFindingsPayload>;
export type TUpdateSecretScanningFinding = {
export type TUpdateSecretScanningFindingDTO = {
status?: SecretScanningFindingStatus;
remarks?: string | null;
findingId: string;
};
export type TUpsertSecretScanningConfigDTO = {
projectId: string;
content: string | null;
};
export type TSecretScanningDataSourceCredentials = undefined;
@@ -65,9 +65,9 @@ export function runInfisicalScanOnRepo(repoPath: string, outputPath: string): Pr
});
}
export function runInfisicalScan(inputPath: string, outputPath: string): Promise<void> {
export function runInfisicalScan(inputPath: string, outputPath: string, configPath?: string): Promise<void> {
return new Promise((resolve, reject) => {
const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}"`;
const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}" ${configPath ? `-c "${configPath}"` : ""}`;
exec(command, (error) => {
if (error && error.code !== 77) {
reject(error);
@@ -138,14 +138,14 @@ export async function scanFullRepoContentAndGetFindings(
}
}
export async function scanContentAndGetFindings(textContent: string): Promise<SecretMatch[]> {
export async function scanContentAndGetFindings(textContent: string, configPath?: string): Promise<SecretMatch[]> {
const tempFolder = await createTempFolder();
const filePath = join(tempFolder, "content.txt");
const findingsPath = join(tempFolder, "findings.json");
try {
await writeTextToFile(filePath, textContent);
await runInfisicalScan(filePath, findingsPath);
await runInfisicalScan(filePath, findingsPath, configPath);
const findingsData = await readFindingsFile(findingsPath);
return JSON.parse(findingsData) as SecretMatch[];
} finally {
+10
View File
@@ -2426,3 +2426,13 @@ export const SecretScanningFindings = {
remarks: "Remarks pertaining to the status of this finding."
}
};
export const SecretScanningConfigs = {
GET_BY_PROJECT_ID: {
projectId: `The ID of the project to retrieve the Secret Scanning Configuration for.`
},
UPDATE: {
projectId: "The ID of the project to update the Secret Scanning Configuration for.",
content: "The contents of the Secret Scanning Configuration file."
}
};