mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-10-14 10:29:52 +00:00
feature: secret scanning pt 3
This commit is contained in:
Vendored
+8
@@ -324,6 +324,9 @@ import {
|
||||
TSecretRotationV2SecretMappingsInsert,
|
||||
TSecretRotationV2SecretMappingsUpdate,
|
||||
TSecrets,
|
||||
TSecretScanningConfigs,
|
||||
TSecretScanningConfigsInsert,
|
||||
TSecretScanningConfigsUpdate,
|
||||
TSecretScanningDataSources,
|
||||
TSecretScanningDataSourcesInsert,
|
||||
TSecretScanningDataSourcesUpdate,
|
||||
@@ -1106,5 +1109,10 @@ declare module "knex/types/tables" {
|
||||
TSecretScanningFindingsInsert,
|
||||
TSecretScanningFindingsUpdate
|
||||
>;
|
||||
[TableName.SecretScanningConfig]: KnexOriginal.CompositeTableType<
|
||||
TSecretScanningConfigs,
|
||||
TSecretScanningConfigsInsert,
|
||||
TSecretScanningConfigsUpdate
|
||||
>;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,7 +78,16 @@ export async function up(knex: Knex): Promise<void> {
|
||||
await createOnUpdateTrigger(knex, TableName.SecretScanningFinding);
|
||||
}
|
||||
|
||||
// TODO: Rules
|
||||
if (!(await knex.schema.hasTable(TableName.SecretScanningConfig))) {
|
||||
await knex.schema.createTable(TableName.SecretScanningConfig, (t) => {
|
||||
t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
|
||||
t.string("projectId").notNullable().unique();
|
||||
t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
|
||||
t.string("content", 5000);
|
||||
t.timestamps(true, true, true);
|
||||
});
|
||||
await createOnUpdateTrigger(knex, TableName.SecretScanningConfig);
|
||||
}
|
||||
}
|
||||
|
||||
export async function down(knex: Knex): Promise<void> {
|
||||
@@ -92,4 +101,7 @@ export async function down(knex: Knex): Promise<void> {
|
||||
|
||||
await knex.schema.dropTableIfExists(TableName.SecretScanningDataSource);
|
||||
await dropOnUpdateTrigger(knex, TableName.SecretScanningDataSource);
|
||||
|
||||
await knex.schema.dropTableIfExists(TableName.SecretScanningConfig);
|
||||
await dropOnUpdateTrigger(knex, TableName.SecretScanningConfig);
|
||||
}
|
||||
|
||||
@@ -107,6 +107,7 @@ export * from "./secret-rotation-outputs";
|
||||
export * from "./secret-rotation-v2-secret-mappings";
|
||||
export * from "./secret-rotations";
|
||||
export * from "./secret-rotations-v2";
|
||||
export * from "./secret-scanning-configs";
|
||||
export * from "./secret-scanning-data-sources";
|
||||
export * from "./secret-scanning-findings";
|
||||
export * from "./secret-scanning-git-risks";
|
||||
|
||||
@@ -159,7 +159,8 @@ export enum TableName {
|
||||
SecretScanningDataSource = "secret_scanning_data_sources",
|
||||
SecretScanningResource = "secret_scanning_resources",
|
||||
SecretScanningScan = "secret_scanning_scans",
|
||||
SecretScanningFinding = "secret_scanning_findings"
|
||||
SecretScanningFinding = "secret_scanning_findings",
|
||||
SecretScanningConfig = "secret_scanning_configs"
|
||||
}
|
||||
|
||||
export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt";
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
// Code generated by automation script, DO NOT EDIT.
|
||||
// Automated by pulling database and generating zod schema
|
||||
// To update. Just run npm run generate:schema
|
||||
// Written by akhilmhdh.
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
import { TImmutableDBKeys } from "./models";
|
||||
|
||||
export const SecretScanningConfigsSchema = z.object({
|
||||
id: z.string().uuid(),
|
||||
projectId: z.string(),
|
||||
content: z.string().nullable().optional(),
|
||||
createdAt: z.date(),
|
||||
updatedAt: z.date()
|
||||
});
|
||||
|
||||
export type TSecretScanningConfigs = z.infer<typeof SecretScanningConfigsSchema>;
|
||||
export type TSecretScanningConfigsInsert = Omit<z.input<typeof SecretScanningConfigsSchema>, TImmutableDBKeys>;
|
||||
export type TSecretScanningConfigsUpdate = Partial<Omit<z.input<typeof SecretScanningConfigsSchema>, TImmutableDBKeys>>;
|
||||
@@ -1,5 +1,6 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { SecretScanningConfigsSchema } from "@app/db/schemas";
|
||||
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
|
||||
import { GitHubDataSourceListItemSchema } from "@app/ee/services/secret-scanning-v2/github";
|
||||
import {
|
||||
@@ -10,7 +11,12 @@ import {
|
||||
SecretScanningDataSourceSchema,
|
||||
SecretScanningFindingSchema
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-union-schemas";
|
||||
import { ApiDocsTags, SecretScanningDataSources, SecretScanningFindings } from "@app/lib/api-docs";
|
||||
import {
|
||||
ApiDocsTags,
|
||||
SecretScanningConfigs,
|
||||
SecretScanningDataSources,
|
||||
SecretScanningFindings
|
||||
} from "@app/lib/api-docs";
|
||||
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
|
||||
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
|
||||
import { AuthMode } from "@app/services/auth/auth-type";
|
||||
@@ -181,6 +187,96 @@ export const registerSecretScanningV2Router = async (server: FastifyZodProvider)
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "GET",
|
||||
url: "/configs/:projectId",
|
||||
config: {
|
||||
rateLimit: readLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: "Get the Secret Scanning Config for the specified project.",
|
||||
params: z.object({
|
||||
projectId: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1, "Project ID required")
|
||||
.describe(SecretScanningConfigs.GET_BY_PROJECT_ID.projectId)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({
|
||||
config: z.object({ content: z.string().nullish(), projectId: z.string(), updatedAt: z.date().nullish() })
|
||||
})
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const {
|
||||
params: { projectId },
|
||||
permission
|
||||
} = req;
|
||||
|
||||
const config = await server.services.secretScanningV2.findSecretScanningConfigByProjectId(projectId, permission);
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_CONFIG_GET
|
||||
}
|
||||
});
|
||||
|
||||
return { config };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "PATCH",
|
||||
url: "/configs/:projectId",
|
||||
config: {
|
||||
rateLimit: writeLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: "Update the specified Secret Scanning Configuration.",
|
||||
params: z.object({
|
||||
projectId: z.string().trim().min(1, "Finding ID required").describe(SecretScanningConfigs.UPDATE.projectId)
|
||||
}),
|
||||
body: z.object({
|
||||
content: z.string().nullable().describe(SecretScanningConfigs.UPDATE.content)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({ config: SecretScanningConfigsSchema })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const {
|
||||
params: { projectId },
|
||||
body,
|
||||
permission
|
||||
} = req;
|
||||
|
||||
const config = await server.services.secretScanningV2.upsertSecretScanningConfig(
|
||||
{ projectId, ...body },
|
||||
permission
|
||||
);
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_CONFIG_UPDATE,
|
||||
metadata: body
|
||||
}
|
||||
});
|
||||
|
||||
return { config };
|
||||
}
|
||||
});
|
||||
|
||||
// not exposed, for UI only
|
||||
server.route({
|
||||
method: "GET",
|
||||
|
||||
@@ -399,7 +399,9 @@ export enum EventType {
|
||||
SECRET_SCANNING_RESOURCE_LIST = "secret-scanning-resource-list",
|
||||
SECRET_SCANNING_SCAN_LIST = "secret-scanning-scan-list",
|
||||
SECRET_SCANNING_FINDING_LIST = "secret-scanning-finding-list",
|
||||
SECRET_SCANNING_FINDING_UPDATE = "secret-scanning-finding-update"
|
||||
SECRET_SCANNING_FINDING_UPDATE = "secret-scanning-finding-update",
|
||||
SECRET_SCANNING_CONFIG_GET = "secret-scanning-config-get",
|
||||
SECRET_SCANNING_CONFIG_UPDATE = "secret-scanning-config-update"
|
||||
}
|
||||
|
||||
export const filterableSecretEvents: EventType[] = [
|
||||
@@ -3020,6 +3022,18 @@ interface SecretScanningFindingUpdateEvent {
|
||||
metadata: TUpdateSecretScanningFinding;
|
||||
}
|
||||
|
||||
interface SecretScanningConfigUpdateEvent {
|
||||
type: EventType.SECRET_SCANNING_CONFIG_UPDATE;
|
||||
metadata: {
|
||||
content: string | null;
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningConfigReadEvent {
|
||||
type: EventType.SECRET_SCANNING_CONFIG_GET;
|
||||
metadata?: Record<string, never>; // not needed, based off projectId
|
||||
}
|
||||
|
||||
export type Event =
|
||||
| GetSecretsEvent
|
||||
| GetSecretEvent
|
||||
@@ -3297,4 +3311,6 @@ export type Event =
|
||||
| SecretScanningResourceListEvent
|
||||
| SecretScanningScanListEvent
|
||||
| SecretScanningFindingListEvent
|
||||
| SecretScanningFindingUpdateEvent;
|
||||
| SecretScanningFindingUpdateEvent
|
||||
| SecretScanningConfigUpdateEvent
|
||||
| SecretScanningConfigReadEvent;
|
||||
|
||||
@@ -26,8 +26,8 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
|
||||
customRateLimits: false,
|
||||
customAlerts: false,
|
||||
secretAccessInsights: false,
|
||||
auditLogs: false,
|
||||
auditLogsRetentionDays: 0,
|
||||
auditLogs: true,
|
||||
auditLogsRetentionDays: 30,
|
||||
auditLogStreams: false,
|
||||
auditLogStreamLimit: 3,
|
||||
samlSSO: false,
|
||||
@@ -55,7 +55,7 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
|
||||
kmip: false,
|
||||
gateway: false,
|
||||
sshHostGroups: false,
|
||||
secretScanning: false
|
||||
secretScanning: true
|
||||
});
|
||||
|
||||
export const setupLicenseRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => {
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
ProjectPermissionPkiSubscriberActions,
|
||||
ProjectPermissionSecretActions,
|
||||
ProjectPermissionSecretRotationActions,
|
||||
ProjectPermissionSecretScanningConfigActions,
|
||||
ProjectPermissionSecretScanningDataSourceActions,
|
||||
ProjectPermissionSecretScanningFindingActions,
|
||||
ProjectPermissionSecretSyncActions,
|
||||
@@ -218,6 +219,11 @@ const buildAdminPermissionRules = () => {
|
||||
ProjectPermissionSub.SecretScanningFindings
|
||||
);
|
||||
|
||||
can(
|
||||
[ProjectPermissionSecretScanningConfigActions.Read, ProjectPermissionSecretScanningConfigActions.Update],
|
||||
ProjectPermissionSub.SecretScanningConfigs
|
||||
);
|
||||
|
||||
return rules;
|
||||
};
|
||||
|
||||
@@ -413,6 +419,8 @@ const buildMemberPermissionRules = () => {
|
||||
ProjectPermissionSub.SecretScanningFindings
|
||||
);
|
||||
|
||||
can([ProjectPermissionSecretScanningConfigActions.Read], ProjectPermissionSub.SecretScanningConfigs);
|
||||
|
||||
return rules;
|
||||
};
|
||||
|
||||
@@ -459,6 +467,8 @@ const buildViewerPermissionRules = () => {
|
||||
|
||||
can([ProjectPermissionSecretScanningFindingActions.Read], ProjectPermissionSub.SecretScanningFindings);
|
||||
|
||||
can([ProjectPermissionSecretScanningConfigActions.Read], ProjectPermissionSub.SecretScanningConfigs);
|
||||
|
||||
return rules;
|
||||
};
|
||||
|
||||
|
||||
@@ -138,6 +138,11 @@ export enum ProjectPermissionSecretScanningFindingActions {
|
||||
Update = "update-findings"
|
||||
}
|
||||
|
||||
export enum ProjectPermissionSecretScanningConfigActions {
|
||||
Read = "read-configs",
|
||||
Update = "update-configs"
|
||||
}
|
||||
|
||||
export enum ProjectPermissionSub {
|
||||
Role = "role",
|
||||
Member = "member",
|
||||
@@ -175,7 +180,8 @@ export enum ProjectPermissionSub {
|
||||
SecretSyncs = "secret-syncs",
|
||||
Kmip = "kmip",
|
||||
SecretScanningDataSources = "secret-scanning-data-sources",
|
||||
SecretScanningFindings = "secret-scanning-findings"
|
||||
SecretScanningFindings = "secret-scanning-findings",
|
||||
SecretScanningConfigs = "secret-scanning-configs"
|
||||
}
|
||||
|
||||
export type SecretSubjectFields = {
|
||||
@@ -300,7 +306,8 @@ export type ProjectPermissionSet =
|
||||
| [ProjectPermissionActions.Create, ProjectPermissionSub.SecretRollback]
|
||||
| [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms]
|
||||
| [ProjectPermissionSecretScanningDataSourceActions, ProjectPermissionSub.SecretScanningDataSources]
|
||||
| [ProjectPermissionSecretScanningFindingActions, ProjectPermissionSub.SecretScanningFindings];
|
||||
| [ProjectPermissionSecretScanningFindingActions, ProjectPermissionSub.SecretScanningFindings]
|
||||
| [ProjectPermissionSecretScanningConfigActions, ProjectPermissionSub.SecretScanningConfigs];
|
||||
|
||||
const SECRET_PATH_MISSING_SLASH_ERR_MSG = "Invalid Secret Path; it must start with a '/'";
|
||||
const SECRET_PATH_PERMISSION_OPERATOR_SCHEMA = z.union([
|
||||
@@ -635,6 +642,12 @@ const GeneralPermissionSchema = [
|
||||
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningFindingActions).describe(
|
||||
"Describe what action an entity can take."
|
||||
)
|
||||
}),
|
||||
z.object({
|
||||
subject: z.literal(ProjectPermissionSub.SecretScanningConfigs).describe("The entity this permission pertains to."),
|
||||
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningConfigActions).describe(
|
||||
"Describe what action an entity can take."
|
||||
)
|
||||
})
|
||||
];
|
||||
|
||||
|
||||
+49
-26
@@ -8,7 +8,11 @@ import {
|
||||
SecretScanningFindingSeverity,
|
||||
SecretScanningResource
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { cloneRepository } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
|
||||
import {
|
||||
cloneRepository,
|
||||
convertPatchLineToFileLineNumber,
|
||||
replaceNonChangesWithNewlines
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
|
||||
import {
|
||||
TSecretScanningFactoryGetDiffScanFindingsPayload,
|
||||
TSecretScanningFactoryGetDiffScanResourcePayload,
|
||||
@@ -121,7 +125,7 @@ export const GitHubSecretScanningFactory = () => {
|
||||
const getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<
|
||||
TGitHubDataSourceWithConnection,
|
||||
TQueueGitHubResourceDiffScan["payload"]
|
||||
> = async ({ dataSource, payload, resourceName }) => {
|
||||
> = async ({ dataSource, payload, resourceName, configPath }) => {
|
||||
const appCfg = getConfig();
|
||||
const {
|
||||
connection: {
|
||||
@@ -144,33 +148,52 @@ export const GitHubSecretScanningFactory = () => {
|
||||
const allFindings: SecretMatch[] = [];
|
||||
|
||||
for (const commit of commits) {
|
||||
for (const filepath of [...commit.added, ...commit.modified]) {
|
||||
// eslint-disable-next-line
|
||||
const fileContentsResponse = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: filepath
|
||||
});
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const commitData = await octokit.repos.getCommit({
|
||||
owner,
|
||||
repo,
|
||||
ref: commit.id
|
||||
});
|
||||
|
||||
const { data } = fileContentsResponse;
|
||||
const fileContent = Buffer.from((data as { content: string }).content, "base64").toString();
|
||||
// eslint-disable-next-line no-continue
|
||||
if (!commitData.data.files) continue;
|
||||
|
||||
// eslint-disable-next-line
|
||||
const findings = await scanContentAndGetFindings(`\n${fileContent}`); // extra line to count lines correctly
|
||||
for (const file of commitData.data.files) {
|
||||
if ((file.status === "added" || file.status === "modified") && file.patch) {
|
||||
// eslint-disable-next-line
|
||||
const findings = await scanContentAndGetFindings(
|
||||
replaceNonChangesWithNewlines(`\n${file.patch}`),
|
||||
configPath
|
||||
);
|
||||
|
||||
allFindings.push(
|
||||
...findings.map((finding) => ({
|
||||
...finding,
|
||||
File: filepath,
|
||||
Commit: commit.id,
|
||||
Author: commit.author.name,
|
||||
Email: commit.author.email ?? "",
|
||||
Message: commit.message,
|
||||
Fingerprint: `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`,
|
||||
Date: commit.timestamp,
|
||||
Link: `https://github.com/${resourceName}/blob/${commit.id}/${filepath}#L${finding.StartLine}`
|
||||
}))
|
||||
);
|
||||
const adjustedFindings = findings.map((finding) => {
|
||||
const startLine = convertPatchLineToFileLineNumber(file.patch!, finding.StartLine);
|
||||
const endLine =
|
||||
finding.StartLine === finding.EndLine
|
||||
? startLine
|
||||
: convertPatchLineToFileLineNumber(file.patch!, finding.EndLine);
|
||||
const startColumn = finding.StartColumn - 1; // subtract 1 for +
|
||||
const endColumn = finding.EndColumn - 1; // subtract 1 for +
|
||||
|
||||
return {
|
||||
...finding,
|
||||
StartLine: startLine,
|
||||
EndLine: endLine,
|
||||
StartColumn: startColumn,
|
||||
EndColumn: endColumn,
|
||||
File: file.filename,
|
||||
Commit: commit.id,
|
||||
Author: commit.author.name,
|
||||
Email: commit.author.email ?? "",
|
||||
Message: commit.message,
|
||||
Fingerprint: `${commit.id}:${file.filename}:${finding.RuleID}:${startLine}:${startColumn}`,
|
||||
Date: commit.timestamp,
|
||||
Link: `https://github.com/${resourceName}/blob/${commit.id}/${file.filename}#L${startLine}`
|
||||
};
|
||||
});
|
||||
|
||||
allFindings.push(...adjustedFindings);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,8 +16,9 @@ import { AppConnection } from "@app/services/app-connection/app-connection-enums
|
||||
|
||||
export const GitHubDataSourceConfigSchema = z.object({
|
||||
includeRepos: z
|
||||
.array(z.string())
|
||||
.array(z.string().min(1).max(256))
|
||||
.nonempty("One or more repositories required")
|
||||
.max(25)
|
||||
.default(["*"])
|
||||
.describe(SecretScanningDataSources.CONFIG.GITHUB.includeRepos)
|
||||
});
|
||||
|
||||
@@ -111,6 +111,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
|
||||
const resourceOrm = ormify(db, TableName.SecretScanningResource);
|
||||
const scanOrm = ormify(db, TableName.SecretScanningScan);
|
||||
const findingOrm = ormify(db, TableName.SecretScanningFinding);
|
||||
const configOrm = ormify(db, TableName.SecretScanningConfig);
|
||||
|
||||
const findDataSource = async (filter: Parameters<(typeof dataSourceOrm)["find"]>[0], tx?: Knex) => {
|
||||
try {
|
||||
@@ -406,7 +407,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
|
||||
unresolvedFindings:
|
||||
findings?.filter((finding) => finding.status === SecretScanningFindingStatus.Unresolved).length ?? 0,
|
||||
resolvedFindings:
|
||||
findings?.filter((finding) => finding.status === SecretScanningFindingStatus.Resolved).length ?? 0,
|
||||
findings?.filter((finding) => finding.status !== SecretScanningFindingStatus.Unresolved).length ?? 0,
|
||||
resourceName: resources[0].name
|
||||
};
|
||||
});
|
||||
@@ -453,6 +454,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
|
||||
findWithDetailsByDataSourceId: findScansWithDetailsByDataSourceId,
|
||||
findByDataSourceId: findScansByDataSourceId
|
||||
},
|
||||
findings: findingOrm
|
||||
findings: findingOrm,
|
||||
configs: configOrm
|
||||
};
|
||||
};
|
||||
|
||||
@@ -30,9 +30,9 @@ export const cloneRepository = async ({ cloneUrl, repoPath }: TCloneRepository):
|
||||
});
|
||||
};
|
||||
|
||||
export function scanDirectory(inputPath: string, outputPath: string): Promise<void> {
|
||||
export function scanDirectory(inputPath: string, outputPath: string, configPath?: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const command = `cd ${inputPath} && infisical scan --exit-code=77 -r "${outputPath}"`;
|
||||
const command = `cd ${inputPath} && infisical scan --exit-code=77 -r "${outputPath}" ${configPath ? `-c ${configPath}` : ""}`;
|
||||
exec(command, (error) => {
|
||||
if (error && error.code !== 77) {
|
||||
reject(error);
|
||||
@@ -43,8 +43,12 @@ export function scanDirectory(inputPath: string, outputPath: string): Promise<vo
|
||||
});
|
||||
}
|
||||
|
||||
export const scanGitRepositoryAndGetFindings = async (scanPath: string, findingsPath: string): TGetFindingsPayload => {
|
||||
await scanDirectory(scanPath, findingsPath);
|
||||
export const scanGitRepositoryAndGetFindings = async (
|
||||
scanPath: string,
|
||||
findingsPath: string,
|
||||
configPath?: string
|
||||
): TGetFindingsPayload => {
|
||||
await scanDirectory(scanPath, findingsPath, configPath);
|
||||
|
||||
const findingsData = JSON.parse(await readFindingsFile(findingsPath)) as SecretMatch[];
|
||||
|
||||
@@ -56,13 +60,64 @@ export const scanGitRepositoryAndGetFindings = async (scanPath: string, findings
|
||||
...finding
|
||||
}) => ({
|
||||
details: titleCaseToCamelCase(finding),
|
||||
fingerprint: finding.Fingerprint,
|
||||
fingerprint: `${finding.Fingerprint}:${finding.StartColumn}`,
|
||||
severity: SecretScanningFindingSeverity.High,
|
||||
rule: finding.RuleID
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
export const replaceNonChangesWithNewlines = (patch: string) => {
|
||||
return patch
|
||||
.split("\n")
|
||||
.map((line) => {
|
||||
// Keep added lines (remove the + prefix)
|
||||
if (line.startsWith("+") && !line.startsWith("+++")) {
|
||||
return line.substring(1);
|
||||
}
|
||||
|
||||
// Replace everything else with newlines to maintain line positioning
|
||||
|
||||
return "";
|
||||
})
|
||||
.join("\n");
|
||||
};
|
||||
|
||||
export const convertPatchLineToFileLineNumber = (patch: string, patchLineNumber: number) => {
|
||||
const lines = patch.split("\n");
|
||||
let currentPatchLine = 0;
|
||||
let currentNewLine = 0;
|
||||
|
||||
for (const line of lines) {
|
||||
currentPatchLine += 1;
|
||||
|
||||
// Hunk header: @@ -a,b +c,d @@
|
||||
const hunkHeaderMatch = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/);
|
||||
if (hunkHeaderMatch) {
|
||||
const startLine = parseInt(hunkHeaderMatch[1], 10);
|
||||
currentNewLine = startLine;
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
if (currentPatchLine === patchLineNumber) {
|
||||
return currentNewLine;
|
||||
}
|
||||
|
||||
if (line.startsWith("+++")) {
|
||||
// eslint-disable-next-line no-continue
|
||||
continue; // skip file metadata lines
|
||||
}
|
||||
|
||||
// Advance only if the line exists in the new file
|
||||
if (line.startsWith("+") || line.startsWith(" ")) {
|
||||
currentNewLine += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return currentNewLine;
|
||||
};
|
||||
|
||||
const MAX_MESSAGE_LENGTH = 1024;
|
||||
|
||||
export const parseScanErrorMessage = (err: unknown): string => {
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import { join } from "path";
|
||||
|
||||
import { ProjectMembershipRole } from "@app/db/schemas";
|
||||
import { ProjectMembershipRole, TSecretScanningFindings } from "@app/db/schemas";
|
||||
import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service";
|
||||
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
|
||||
import {
|
||||
createTempFolder,
|
||||
deleteTempFolder
|
||||
deleteTempFolder,
|
||||
writeTextToFile
|
||||
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
|
||||
import {
|
||||
parseScanErrorMessage,
|
||||
@@ -184,21 +185,33 @@ export const secretScanningV2QueueServiceFactory = async ({
|
||||
tempFolder
|
||||
});
|
||||
|
||||
const config = await secretScanningV2DAL.configs.findOne({
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
let configPath: string | undefined;
|
||||
|
||||
if (config && config.content) {
|
||||
configPath = join(tempFolder, "infisical-scan.toml");
|
||||
await writeTextToFile(configPath, config.content);
|
||||
}
|
||||
|
||||
let findingsPayload: TFindingsPayload;
|
||||
switch (resource.type) {
|
||||
case SecretScanningResource.Repository:
|
||||
case SecretScanningResource.Project:
|
||||
findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath);
|
||||
findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
|
||||
break;
|
||||
default:
|
||||
throw new Error("Unhandled resource type");
|
||||
}
|
||||
|
||||
if (findingsPayload.length) {
|
||||
await secretScanningV2DAL.findings.transaction(async (tx) => {
|
||||
await secretScanningV2DAL.findings.upsert(
|
||||
findingsPayload.map((findings) => ({
|
||||
...findings,
|
||||
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
|
||||
let findings: TSecretScanningFindings[] = [];
|
||||
if (findingsPayload.length) {
|
||||
findings = await secretScanningV2DAL.findings.upsert(
|
||||
findingsPayload.map((finding) => ({
|
||||
...finding,
|
||||
projectId: dataSource.projectId,
|
||||
dataSourceName: dataSource.name,
|
||||
dataSourceType: dataSource.type,
|
||||
@@ -211,22 +224,28 @@ export const secretScanningV2QueueServiceFactory = async ({
|
||||
tx,
|
||||
["resourceName", "dataSourceName", "status"]
|
||||
);
|
||||
}
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Completed,
|
||||
statusMessage: null
|
||||
}
|
||||
);
|
||||
});
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Completed,
|
||||
statusMessage: null
|
||||
}
|
||||
);
|
||||
|
||||
return findings;
|
||||
});
|
||||
|
||||
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
|
||||
|
||||
if (newFindings.length) {
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Completed,
|
||||
resourceName: resource.name,
|
||||
isDiffScan: false,
|
||||
dataSource,
|
||||
numberOfSecrets: findingsPayload.length,
|
||||
numberOfSecrets: newFindings.length,
|
||||
scanId
|
||||
});
|
||||
}
|
||||
@@ -376,6 +395,8 @@ export const secretScanningV2QueueServiceFactory = async ({
|
||||
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]();
|
||||
|
||||
const tempFolder = await createTempFolder();
|
||||
|
||||
try {
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
@@ -387,20 +408,34 @@ export const secretScanningV2QueueServiceFactory = async ({
|
||||
let connection: TAppConnection | null = null;
|
||||
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
|
||||
|
||||
const config = await secretScanningV2DAL.configs.findOne({
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
let configPath: string | undefined;
|
||||
|
||||
if (config && config.content) {
|
||||
configPath = join(tempFolder, "infisical-scan.toml");
|
||||
await writeTextToFile(configPath, config.content);
|
||||
}
|
||||
|
||||
const findingsPayload = await factory.getDiffScanFindingsPayload({
|
||||
dataSource: {
|
||||
...dataSource,
|
||||
connection
|
||||
} as TSecretScanningDataSourceWithConnection,
|
||||
resourceName: resource.name,
|
||||
payload
|
||||
payload,
|
||||
configPath
|
||||
});
|
||||
|
||||
if (findingsPayload.length) {
|
||||
await secretScanningV2DAL.findings.transaction(async (tx) => {
|
||||
await secretScanningV2DAL.findings.upsert(
|
||||
findingsPayload.map((findings) => ({
|
||||
...findings,
|
||||
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
|
||||
let findings: TSecretScanningFindings[] = [];
|
||||
|
||||
if (findingsPayload.length) {
|
||||
findings = await secretScanningV2DAL.findings.upsert(
|
||||
findingsPayload.map((finding) => ({
|
||||
...finding,
|
||||
projectId: dataSource.projectId,
|
||||
dataSourceName: dataSource.name,
|
||||
dataSourceType: dataSource.type,
|
||||
@@ -413,21 +448,27 @@ export const secretScanningV2QueueServiceFactory = async ({
|
||||
tx,
|
||||
["resourceName", "dataSourceName", "status"]
|
||||
);
|
||||
}
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Completed
|
||||
}
|
||||
);
|
||||
});
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Completed
|
||||
}
|
||||
);
|
||||
|
||||
return findings;
|
||||
});
|
||||
|
||||
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
|
||||
|
||||
if (newFindings.length) {
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Completed,
|
||||
resourceName: resource.name,
|
||||
isDiffScan: true,
|
||||
dataSource,
|
||||
numberOfSecrets: findingsPayload.length,
|
||||
numberOfSecrets: newFindings.length,
|
||||
scanId
|
||||
});
|
||||
}
|
||||
@@ -496,6 +537,8 @@ export const secretScanningV2QueueServiceFactory = async ({
|
||||
|
||||
logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
|
||||
throw error;
|
||||
} finally {
|
||||
await deleteTempFolder(tempFolder);
|
||||
}
|
||||
},
|
||||
{
|
||||
|
||||
@@ -4,6 +4,7 @@ import { ActionProjectType } from "@app/db/schemas";
|
||||
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
|
||||
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
|
||||
import {
|
||||
ProjectPermissionSecretScanningConfigActions,
|
||||
ProjectPermissionSecretScanningDataSourceActions,
|
||||
ProjectPermissionSecretScanningFindingActions,
|
||||
ProjectPermissionSub
|
||||
@@ -30,7 +31,8 @@ import {
|
||||
TSecretScanningScanWithDetails,
|
||||
TTriggerSecretScanningDataSourceDTO,
|
||||
TUpdateSecretScanningDataSourceDTO,
|
||||
TUpdateSecretScanningFinding
|
||||
TUpdateSecretScanningFindingDTO,
|
||||
TUpsertSecretScanningConfigDTO
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
|
||||
import { DatabaseErrorCode } from "@app/lib/error-codes";
|
||||
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
|
||||
@@ -714,7 +716,7 @@ export const secretScanningV2ServiceFactory = ({
|
||||
};
|
||||
|
||||
const updateSecretScanningFindingById = async (
|
||||
{ findingId, remarks, status }: TUpdateSecretScanningFinding,
|
||||
{ findingId, remarks, status }: TUpdateSecretScanningFindingDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
@@ -754,6 +756,77 @@ export const secretScanningV2ServiceFactory = ({
|
||||
return { finding: updatedFinding as TSecretScanningFinding, projectId: finding.projectId };
|
||||
};
|
||||
|
||||
const findSecretScanningConfigByProjectId = async (projectId: string, actor: OrgServiceActor) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Configuration due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningConfigActions.Read,
|
||||
ProjectPermissionSub.SecretScanningConfigs
|
||||
);
|
||||
|
||||
const config = await secretScanningV2DAL.configs.findOne({
|
||||
projectId
|
||||
});
|
||||
|
||||
return (
|
||||
config ?? { content: null, projectId, updatedAt: null } // using default config
|
||||
);
|
||||
};
|
||||
|
||||
const upsertSecretScanningConfig = async (
|
||||
{ projectId, content }: TUpsertSecretScanningConfigDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Configuration due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningConfigActions.Read,
|
||||
ProjectPermissionSub.SecretScanningConfigs
|
||||
);
|
||||
|
||||
const [config] = await secretScanningV2DAL.configs.upsert(
|
||||
[
|
||||
{
|
||||
projectId,
|
||||
content
|
||||
}
|
||||
],
|
||||
"projectId"
|
||||
);
|
||||
|
||||
return config;
|
||||
};
|
||||
|
||||
return {
|
||||
listSecretScanningDataSourceOptions,
|
||||
listSecretScanningDataSourcesByProjectId,
|
||||
@@ -771,6 +844,8 @@ export const secretScanningV2ServiceFactory = ({
|
||||
getSecretScanningUnresolvedFindingsCountByProjectId,
|
||||
listSecretScanningFindingsByProjectId,
|
||||
updateSecretScanningFindingById,
|
||||
findSecretScanningConfigByProjectId,
|
||||
upsertSecretScanningConfig,
|
||||
github: githubSecretScanningService(secretScanningV2DAL, secretScanningV2Queue)
|
||||
};
|
||||
};
|
||||
|
||||
@@ -131,7 +131,7 @@ export type TSecretScanningFactoryGetFullScanPath<T extends TSecretScanningDataS
|
||||
export type TSecretScanningFactoryGetDiffScanFindingsPayload<
|
||||
T extends TSecretScanningDataSourceWithConnection,
|
||||
P extends TQueueSecretScanningResourceDiffScan["payload"]
|
||||
> = (parameters: { dataSource: T; resourceName: string; payload: P }) => Promise<TFindingsPayload>;
|
||||
> = (parameters: { dataSource: T; resourceName: string; payload: P; configPath?: string }) => Promise<TFindingsPayload>;
|
||||
|
||||
export type TSecretScanningDataSourceRaw = NonNullable<
|
||||
Awaited<ReturnType<TSecretScanningV2DALFactory["dataSources"]["findById"]>>
|
||||
@@ -175,10 +175,15 @@ export type TSecretScanningFactory<
|
||||
export type TFindingsPayload = Pick<TSecretScanningFindingsInsert, "details" | "fingerprint" | "severity" | "rule">[];
|
||||
export type TGetFindingsPayload = Promise<TFindingsPayload>;
|
||||
|
||||
export type TUpdateSecretScanningFinding = {
|
||||
export type TUpdateSecretScanningFindingDTO = {
|
||||
status?: SecretScanningFindingStatus;
|
||||
remarks?: string | null;
|
||||
findingId: string;
|
||||
};
|
||||
|
||||
export type TUpsertSecretScanningConfigDTO = {
|
||||
projectId: string;
|
||||
content: string | null;
|
||||
};
|
||||
|
||||
export type TSecretScanningDataSourceCredentials = undefined;
|
||||
|
||||
+4
-4
@@ -65,9 +65,9 @@ export function runInfisicalScanOnRepo(repoPath: string, outputPath: string): Pr
|
||||
});
|
||||
}
|
||||
|
||||
export function runInfisicalScan(inputPath: string, outputPath: string): Promise<void> {
|
||||
export function runInfisicalScan(inputPath: string, outputPath: string, configPath?: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}"`;
|
||||
const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}" ${configPath ? `-c "${configPath}"` : ""}`;
|
||||
exec(command, (error) => {
|
||||
if (error && error.code !== 77) {
|
||||
reject(error);
|
||||
@@ -138,14 +138,14 @@ export async function scanFullRepoContentAndGetFindings(
|
||||
}
|
||||
}
|
||||
|
||||
export async function scanContentAndGetFindings(textContent: string): Promise<SecretMatch[]> {
|
||||
export async function scanContentAndGetFindings(textContent: string, configPath?: string): Promise<SecretMatch[]> {
|
||||
const tempFolder = await createTempFolder();
|
||||
const filePath = join(tempFolder, "content.txt");
|
||||
const findingsPath = join(tempFolder, "findings.json");
|
||||
|
||||
try {
|
||||
await writeTextToFile(filePath, textContent);
|
||||
await runInfisicalScan(filePath, findingsPath);
|
||||
await runInfisicalScan(filePath, findingsPath, configPath);
|
||||
const findingsData = await readFindingsFile(findingsPath);
|
||||
return JSON.parse(findingsData) as SecretMatch[];
|
||||
} finally {
|
||||
|
||||
@@ -2426,3 +2426,13 @@ export const SecretScanningFindings = {
|
||||
remarks: "Remarks pertaining to the status of this finding."
|
||||
}
|
||||
};
|
||||
|
||||
export const SecretScanningConfigs = {
|
||||
GET_BY_PROJECT_ID: {
|
||||
projectId: `The ID of the project to retrieve the Secret Scanning Configuration for.`
|
||||
},
|
||||
UPDATE: {
|
||||
projectId: "The ID of the project to update the Secret Scanning Configuration for.",
|
||||
content: "The contents of the Secret Scanning Configuration file."
|
||||
}
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user