awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-15 01:52:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

feature: secret scanning pt2 and address initial feedback

Browse Source
This commit is contained in:
Scott Wilson
2025-05-29 20:40:48 -07:00
parent e6c97510ca
commit 4773336a04
78 changed files with 1038 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints.ts
+3 -3
View File
@@ -298,7 +298,7 @@ export const registerSecretScanningEndpoints = <
handler: async (req) => {
const { dataSourceId } = req.params;
const dataSource = (await server.services.secretScanningV2.deleteSecretScanningResource(
const dataSource = (await server.services.secretScanningV2.deleteSecretScanningDataSource(
{ type, dataSourceId },
req.permission
)) as T;
@@ -349,7 +349,7 @@ export const registerSecretScanningEndpoints = <
...req.auditLogInfo,
projectId: dataSource.projectId,
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
type: EventType.SECRET_SCANNING_DATA_SOURCE_TRIGGER_SCAN,
metadata: {
type,
dataSourceId
@@ -392,7 +392,7 @@ export const registerSecretScanningEndpoints = <
...req.auditLogInfo,
projectId: dataSource.projectId,
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
type: EventType.SECRET_SCANNING_DATA_SOURCE_TRIGGER_SCAN,
metadata: {
type,
dataSourceId,
backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-router.ts
+2 -2
View File
@@ -140,12 +140,12 @@ export const registerSecretScanningV2Router = async (server: FastifyZodProvider)
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: "Update the resolve status of the specified Secret Scanning Finding.",
description: "Update the specified Secret Scanning Finding.",
params: z.object({
findingId: z.string().trim().min(1, "Finding ID required").describe(SecretScanningFindings.UPDATE.findingId)
}),
body: z.object({
status: z.nativeEnum(SecretScanningFindingStatus).describe(SecretScanningFindings.UPDATE.status),
status: z.nativeEnum(SecretScanningFindingStatus).optional().describe(SecretScanningFindings.UPDATE.status),
remarks: z.string().nullish().describe(SecretScanningFindings.UPDATE.remarks)
}),
response: {
backend/src/ee/services/audit-log/audit-log-types.ts
+22 -2
View File
@@ -9,7 +9,11 @@ import {
TSecretRotationV2Raw,
TUpdateSecretRotationV2DTO
} from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-types";
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
SecretScanningDataSource,
SecretScanningScanStatus,
SecretScanningScanType
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
TCreateSecretScanningDataSourceDTO,
TDeleteSecretScanningDataSourceDTO,
@@ -390,6 +394,7 @@ export enum EventType {
SECRET_SCANNING_DATA_SOURCE_UPDATE = "secret-scanning-data-source-update",
SECRET_SCANNING_DATA_SOURCE_DELETE = "secret-scanning-data-source-delete",
SECRET_SCANNING_DATA_SOURCE_GET = "secret-scanning-data-source-get",
SECRET_SCANNING_DATA_SOURCE_TRIGGER_SCAN = "secret-scanning-data-source-trigger-scan",
SECRET_SCANNING_DATA_SOURCE_SCAN = "secret-scanning-data-source-scan",
SECRET_SCANNING_RESOURCE_LIST = "secret-scanning-resource-list",
SECRET_SCANNING_SCAN_LIST = "secret-scanning-scan-list",
@@ -2964,9 +2969,23 @@ interface SecretScanningDataSourceDeleteEvent {
metadata: TDeleteSecretScanningDataSourceDTO;
}
interface SecretScanningDataSourceTriggerScanEvent {
type: EventType.SECRET_SCANNING_DATA_SOURCE_TRIGGER_SCAN;
metadata: TTriggerSecretScanningDataSourceDTO;
}
interface SecretScanningDataSourceScanEvent {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN;
metadata: TTriggerSecretScanningDataSourceDTO;
metadata: {
scanId: string;
resourceId: string;
resourceType: string;
dataSourceId: string;
dataSourceType: string;
scanStatus: SecretScanningScanStatus;
scanType: SecretScanningScanType;
numberOfSecretsDetected?: number;
};
}
interface SecretScanningResourceListEvent {
@@ -3273,6 +3292,7 @@ export type Event =
| SecretScanningDataSourceCreateEvent
| SecretScanningDataSourceUpdateEvent
| SecretScanningDataSourceDeleteEvent
| SecretScanningDataSourceTriggerScanEvent
| SecretScanningDataSourceScanEvent
| SecretScanningResourceListEvent
| SecretScanningScanListEvent
backend/src/ee/services/permission/default-roles.ts
+2 -5
View File
@@ -214,7 +214,7 @@ const buildAdminPermissionRules = () => {
);
can(
[ProjectPermissionSecretScanningFindingActions.Read, ProjectPermissionSecretScanningFindingActions.Resolve],
[ProjectPermissionSecretScanningFindingActions.Read, ProjectPermissionSecretScanningFindingActions.Update],
ProjectPermissionSub.SecretScanningFindings
);
@@ -400,9 +400,6 @@ const buildMemberPermissionRules = () => {
can(
[
ProjectPermissionSecretScanningDataSourceActions.Create,
ProjectPermissionSecretScanningDataSourceActions.Edit,
ProjectPermissionSecretScanningDataSourceActions.Delete,
ProjectPermissionSecretScanningDataSourceActions.Read,
ProjectPermissionSecretScanningDataSourceActions.TriggerScans,
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
@@ -412,7 +409,7 @@ const buildMemberPermissionRules = () => {
);
can(
[ProjectPermissionSecretScanningFindingActions.Read, ProjectPermissionSecretScanningFindingActions.Resolve],
[ProjectPermissionSecretScanningFindingActions.Read, ProjectPermissionSecretScanningFindingActions.Update],
ProjectPermissionSub.SecretScanningFindings
);
backend/src/ee/services/permission/project-permission.ts
+1 -1
View File
@@ -135,7 +135,7 @@ export enum ProjectPermissionSecretScanningDataSourceActions {
export enum ProjectPermissionSecretScanningFindingActions {
Read = "read-findings",
Resolve = "resolve-findings"
Update = "update-findings"
}
export enum ProjectPermissionSub {
backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.ts
+5 -2
View File
@@ -4,10 +4,11 @@ import { ProbotOctokit } from "probot";
import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import {
SecretScanningDataSource,
SecretScanningFindingSeverity,
SecretScanningResource
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { cloneRepository, titleCaseToCamelCase } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
import { cloneRepository } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
import {
TSecretScanningFactoryGetDiffScanFindingsPayload,
TSecretScanningFactoryGetDiffScanResourcePayload,
@@ -18,6 +19,7 @@ import {
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { getConfig } from "@app/lib/config/env";
import { BadRequestError } from "@app/lib/errors";
import { titleCaseToCamelCase } from "@app/lib/fn";
import { listGitHubRadarRepositories, TGitHubRadarConnection } from "@app/services/app-connection/github-radar";
import { TGitHubDataSourceWithConnection, TQueueGitHubResourceDiffScan } from "./github-secret-scanning-types";
@@ -30,7 +32,8 @@ export const GitHubSecretScanningFactory = () => {
const externalId = connection.credentials.installationId;
const existingDataSource = await secretScanningV2DAL.dataSources.findOne({
externalId
externalId,
type: SecretScanningDataSource.GitHub
});
if (existingDataSource)
backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-schemas.ts
+6 -1
View File
@@ -11,10 +11,15 @@ import {
BaseUpdateSecretScanningDataSourceSchema,
GitRepositoryScanFindingDetailsSchema
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-schemas";
import { SecretScanningDataSources } from "@app/lib/api-docs";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
export const GitHubDataSourceConfigSchema = z.object({
includeRepos: z.array(z.string()).nonempty("One or more repositories required").default(["*"])
includeRepos: z
.array(z.string())
.nonempty("One or more repositories required")
.default(["*"])
.describe(SecretScanningDataSources.CONFIG.GITHUB.includeRepos)
});
export const GitHubDataSourceSchema = BaseSecretScanningDataSourceSchema({
backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-service.ts
+36 -12
View File
@@ -5,6 +5,8 @@ import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/se
import { TSecretScanningV2QueueServiceFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-queue";
import { logger } from "@app/lib/logger";
import { TGitHubDataSource } from "./github-secret-scanning-types";
export const githubSecretScanningService = (
secretScanningV2DAL: TSecretScanningV2DALFactory,
secretScanningV2Queue: Pick<TSecretScanningV2QueueServiceFactory, "queueResourceDiffScan">
@@ -21,9 +23,12 @@ export const githubSecretScanningService = (
return;
}
// scott: maybe add disabled col instead?
await secretScanningV2DAL.resources.delete({
dataSourceId: dataSource.id
logger.info(
`secretScanningV2RemoveEvent: GitHub - installation deleted [installationId=${installationId}] [dataSourceId=${dataSource.id}]`
);
await secretScanningV2DAL.dataSources.updateById(dataSource.id, {
isDisconnected: true
});
};
@@ -37,9 +42,10 @@ export const githubSecretScanningService = (
return;
}
const dataSource = await secretScanningV2DAL.dataSources.findOne({
externalId: String(installation.id)
});
const dataSource = (await secretScanningV2DAL.dataSources.findOne({
externalId: String(installation.id),
type: SecretScanningDataSource.GitHub
})) as TGitHubDataSource | undefined;
if (!dataSource) {
logger.error(
@@ -48,15 +54,33 @@ export const githubSecretScanningService = (
return;
}
await secretScanningV2Queue.queueResourceDiffScan({
dataSourceType: SecretScanningDataSource.GitHub,
payload,
dataSourceId: dataSource.id
});
const {
isAutoScanEnabled,
config: { includeRepos }
} = dataSource;
if (!isAutoScanEnabled) {
logger.info(
`secretScanningV2PushEvent: GitHub - ignoring due to auto scan disabled [dataSourceId=${dataSource.id}] [installationId=${installation.id}]`
);
return;
}
if (includeRepos.includes("*") || includeRepos.includes(repository.full_name)) {
await secretScanningV2Queue.queueResourceDiffScan({
dataSourceType: SecretScanningDataSource.GitHub,
payload,
dataSourceId: dataSource.id
});
} else {
logger.info(
`secretScanningV2PushEvent: GitHub - ignoring due to repository not being present in config [installationId=${installation.id}] [dataSourceId=${dataSource.id}]`
);
}
};
return {
handlePushEvent,
handleInstallationDeleted: handleInstallationDeletedEvent
handleInstallationDeletedEvent
};
};
backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-types.ts
+2
View File
@@ -27,4 +27,6 @@ export type TQueueGitHubResourceDiffScan = {
dataSourceType: SecretScanningDataSource.GitHub;
payload: PushEvent;
dataSourceId: string;
resourceId: string;
scanId: string;
};
backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-dal.ts
+10 -10
View File
@@ -163,7 +163,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
};
const deleteDataSourceById = async (dataSourceId: string, tx?: Knex) => {
const secretRotation = (await baseSecretScanningDataSourceQuery({
const dataSource = (await baseSecretScanningDataSourceQuery({
filter: { id: dataSourceId },
db,
tx
@@ -171,18 +171,18 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
await dataSourceOrm.deleteById(dataSourceId, tx);
return expandSecretScanningDataSource(secretRotation);
return expandSecretScanningDataSource(dataSource);
};
const findOneDataSource = async (filter: Parameters<(typeof dataSourceOrm)["findOne"]>[0], tx?: Knex) => {
try {
const secretRotation = await baseSecretScanningDataSourceQuery({ filter, db, tx }).first();
const dataSource = await baseSecretScanningDataSourceQuery({ filter, db, tx }).first();
if (secretRotation) {
return expandSecretScanningDataSource(secretRotation);
if (dataSource) {
return expandSecretScanningDataSource(dataSource);
}
} catch (error) {
throw new DatabaseError({ error, name: "Find One - Secret Rotation V2" });
throw new DatabaseError({ error, name: "Find One - Secret Scanning Data Source" });
}
};
@@ -264,7 +264,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
};
});
} catch (error) {
throw new DatabaseError({ error, name: "Find Data Source with Details - Secret Scanning V2" });
throw new DatabaseError({ error, name: "Find with Details - Secret Scanning Data Source" });
}
};
@@ -348,7 +348,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
};
});
} catch (error) {
throw new DatabaseError({ error, name: "Find Resource with Details - Secret Scanning V2" });
throw new DatabaseError({ error, name: "Find with Details - Secret Scanning Resource" });
}
};
@@ -411,7 +411,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
};
});
} catch (error) {
throw new DatabaseError({ error, name: "Find Scan with Details By Data Source ID - Secret Scanning V2" });
throw new DatabaseError({ error, name: "Find with Details By Data Source ID - Secret Scanning Scan" });
}
};
@@ -429,7 +429,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
return scans;
} catch (error) {
throw new DatabaseError({ error, name: "Find Scan By Data Source ID - Secret Scanning V2" });
throw new DatabaseError({ error, name: "Find By Data Source ID - Secret Scanning Scan" });
}
};
backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts
+1 -21
View File
@@ -4,6 +4,7 @@ import { exec } from "child_process";
import { readFindingsFile } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import { GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/github";
import { titleCaseToCamelCase } from "@app/lib/fn";
import { SecretScanningDataSource, SecretScanningFindingSeverity } from "./secret-scanning-v2-enums";
import { TCloneRepository, TGetFindingsPayload, TSecretScanningDataSourceListItem } from "./secret-scanning-v2-types";
@@ -42,27 +43,6 @@ export function scanDirectory(inputPath: string, outputPath: string): Promise<vo
});
}
export const titleCaseToCamelCase = (obj: unknown): unknown => {
if (typeof obj !== "object" || obj === null) {
return obj;
}
if (Array.isArray(obj)) {
return obj.map((item: object) => titleCaseToCamelCase(item));
}
const result: Record<string, unknown> = {};
for (const key in obj) {
if (Object.prototype.hasOwnProperty.call(obj, key)) {
const camelKey = key.charAt(0).toLowerCase() + key.slice(1);
result[camelKey] = titleCaseToCamelCase((obj as Record<string, unknown>)[key]);
}
}
return result;
};
export const scanGitRepositoryAndGetFindings = async (scanPath: string, findingsPath: string): TGetFindingsPayload => {
await scanDirectory(scanPath, findingsPath);
backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
+361 -171
View File
@@ -1,5 +1,8 @@
import { join } from "path";
import { ProjectMembershipRole } from "@app/db/schemas";
import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import {
createTempFolder,
deleteTempFolder
@@ -8,12 +11,18 @@ import {
parseScanErrorMessage,
scanGitRepositoryAndGetFindings
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { BadRequestError, InternalServerError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns";
import { TAppConnection } from "@app/services/app-connection/app-connection-types";
import { ActorType } from "@app/services/auth/auth-type";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal";
import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
import { TSecretScanningV2DALFactory } from "./secret-scanning-v2-dal";
import {
@@ -28,16 +37,19 @@ import {
TFindingsPayload,
TQueueSecretScanningDataSourceFullScan,
TQueueSecretScanningResourceDiffScan,
TQueueSecretScanningSendNotification,
TSecretScanningDataSourceWithConnection
} from "./secret-scanning-v2-types";
type TSecretRotationV2QueueServiceFactoryDep = {
queueService: TQueueServiceFactory;
secretScanningV2DAL: TSecretScanningV2DALFactory;
// smtpService: Pick<TSmtpService, "sendMail">;
// projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
// projectDAL: Pick<TProjectDALFactory, "findById">;
smtpService: Pick<TSmtpService, "sendMail">;
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
projectDAL: Pick<TProjectDALFactory, "findById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
keyStore: Pick<TKeyStoreFactory, "acquireLock" | "getItem">;
};
export type TSecretScanningV2QueueServiceFactory = Awaited<ReturnType<typeof secretScanningV2QueueServiceFactory>>;
@@ -45,10 +57,12 @@ export type TSecretScanningV2QueueServiceFactory = Awaited<ReturnType<typeof sec
export const secretScanningV2QueueServiceFactory = async ({
queueService,
secretScanningV2DAL,
// projectMembershipDAL,
// projectDAL,
// smtpService,
kmsService
projectMembershipDAL,
projectDAL,
smtpService,
kmsService,
auditLogService,
keyStore
}: TSecretRotationV2QueueServiceFactoryDep) => {
const queueDataSourceFullScan = async (
dataSource: TSecretScanningDataSourceWithConnection,
@@ -63,7 +77,7 @@ export const secretScanningV2QueueServiceFactory = async ({
let filteredRawResources = rawResources;
// TODO: should add indivial resource fetch to factory
// TODO: should add individual resource fetch to factory
if (resourceExternalId) {
filteredRawResources = rawResources.filter((resource) => resource.externalId === resourceExternalId);
}
@@ -74,6 +88,13 @@ export const secretScanningV2QueueServiceFactory = async ({
});
}
for (const resource of filteredRawResources) {
// eslint-disable-next-line no-await-in-loop
if (await keyStore.getItem(KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId))) {
throw new BadRequestError({ message: `A scan is already in progress for resource "${resource.name}"` });
}
}
await secretScanningV2DAL.resources.transaction(async (tx) => {
const resources = await secretScanningV2DAL.resources.upsert(
filteredRawResources.map((rawResource) => ({
@@ -110,9 +131,6 @@ export const secretScanningV2QueueServiceFactory = async ({
}
};
const queueResourceDiffScan = async (payload: TQueueSecretScanningResourceDiffScan) =>
queueService.queuePg(QueueJobs.SecretScanningV2DiffScan, payload);
await queueService.startPg<QueueName.SecretScanningV2>(
QueueJobs.SecretScanningV2FullScan,
async ([job]) => {
@@ -123,7 +141,26 @@ export const secretScanningV2QueueServiceFactory = async ({
const tempFolder = await createTempFolder();
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
const resource = await secretScanningV2DAL.resources.findById(resourceId);
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
try {
try {
lock = await keyStore.acquireLock(
[KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
60 * 1000 * 5
);
} catch (e) {
throw new Error("Failed to acquire scanning lock.");
}
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
@@ -131,14 +168,6 @@ export const secretScanningV2QueueServiceFactory = async ({
}
);
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
const resource = await secretScanningV2DAL.resources.findById(resourceId);
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
let connection: TAppConnection | null = null;
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
@@ -165,110 +194,196 @@ export const secretScanningV2QueueServiceFactory = async ({
throw new Error("Unhandled resource type");
}
await secretScanningV2DAL.findings.transaction(async (tx) => {
await secretScanningV2DAL.findings.upsert(
findingsPayload.map((findings) => ({
...findings,
projectId: dataSource.projectId,
dataSourceName: dataSource.name,
if (findingsPayload.length) {
await secretScanningV2DAL.findings.transaction(async (tx) => {
await secretScanningV2DAL.findings.upsert(
findingsPayload.map((findings) => ({
...findings,
projectId: dataSource.projectId,
dataSourceName: dataSource.name,
dataSourceType: dataSource.type,
resourceName: resource.name,
resourceType: resource.type,
scanId,
status: SecretScanningFindingStatus.Unresolved
})),
["projectId", "fingerprint"],
tx,
["resourceName", "dataSourceName", "status"]
);
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Completed,
statusMessage: null
}
);
});
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Completed,
resourceName: resource.name,
isDiffScan: false,
dataSource,
numberOfSecrets: findingsPayload.length,
scanId
});
}
await auditLogService.createAuditLog({
projectId: dataSource.projectId,
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
metadata: {
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceName: resource.name,
resourceId: resource.id,
resourceType: resource.type,
scanId,
status: SecretScanningFindingStatus.Unresolved
})),
["projectId", "fingerprint"],
tx,
["resourceName", "dataSourceName", "status"]
);
scanStatus: SecretScanningScanStatus.Completed,
scanType: SecretScanningScanType.FullScan,
numberOfSecretsDetected: findingsPayload.length
}
}
});
logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`);
} catch (error) {
if (retryCount === retryLimit) {
const errorMessage = parseScanErrorMessage(error);
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Completed
status: SecretScanningScanStatus.Failed,
statusMessage: errorMessage
}
);
});
// TODO: send notification
logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails}`);
} catch (error) {
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Failed,
statusMessage: parseScanErrorMessage(error)
}
);
resourceName: resource.name,
dataSource,
errorMessage
});
// TODO: send error notification
await auditLogService.createAuditLog({
projectId: dataSource.projectId,
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
metadata: {
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId: resource.id,
resourceType: resource.type,
scanId,
scanStatus: SecretScanningScanStatus.Failed,
scanType: SecretScanningScanType.FullScan
}
}
});
}
logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
throw error;
} finally {
await deleteTempFolder(tempFolder);
await lock?.release();
}
},
{
batchSize: 1,
workerCount: 2,
workerCount: 20,
pollingIntervalSeconds: 1
}
);
const queueResourceDiffScan = async ({
payload,
dataSourceId,
dataSourceType
}: Pick<TQueueSecretScanningResourceDiffScan, "payload" | "dataSourceId" | "dataSourceType">) => {
const factory = SECRET_SCANNING_FACTORY_MAP[dataSourceType as SecretScanningDataSource]();
const resourcePayload = factory.getDiffScanResourcePayload(payload);
try {
const { resourceId, scanId } = await secretScanningV2DAL.resources.transaction(async (tx) => {
const [resource] = await secretScanningV2DAL.resources.upsert(
[
{
...resourcePayload,
dataSourceId
}
],
["externalId", "dataSourceId"],
tx
);
const scan = await secretScanningV2DAL.scans.create(
{
resourceId: resource.id,
type: SecretScanningScanType.DiffScan
},
tx
);
return {
resourceId: resource.id,
scanId: scan.id
};
});
await queueService.queuePg(QueueJobs.SecretScanningV2DiffScan, {
payload,
dataSourceId,
dataSourceType,
scanId,
resourceId
});
} catch (error) {
logger.error(
error,
`secretScanningV2Queue: Failed to queue diff scan [dataSourceId=${dataSourceId}] [resourceExternalId=${resourcePayload.externalId}]`
);
}
};
await queueService.startPg<QueueName.SecretScanningV2>(
QueueJobs.SecretScanningV2DiffScan,
async ([job]) => {
const { payload, dataSourceId } = job.data as TQueueSecretScanningResourceDiffScan;
const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan;
const { retryCount, retryLimit } = job;
let scanId: string | undefined;
let logDetails = `[dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
const resource = await secretScanningV2DAL.resources.findById(resourceId);
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]();
try {
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]();
const resourcePayload = factory.getDiffScanResourcePayload(payload);
const { resourceId, resourceName, resourceType } = await secretScanningV2DAL.resources.transaction(
async (tx) => {
const [resource] = await secretScanningV2DAL.resources.upsert(
[
{
...resourcePayload,
dataSourceId
}
],
["externalId", "dataSourceId"],
tx
);
const scan = await secretScanningV2DAL.scans.create(
{
resourceId: resource.id,
type: SecretScanningScanType.DiffScan,
status: SecretScanningScanStatus.Scanning
},
tx
);
scanId = scan.id;
return {
resourceId: resource.id,
resourceName: resource.name,
resourceType: resource.type
};
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Scanning
}
);
logDetails += ` [scanId=${scanId}] [resourceId=${resourceId}]`;
let connection: TAppConnection | null = null;
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
@@ -277,49 +392,107 @@ export const secretScanningV2QueueServiceFactory = async ({
...dataSource,
connection
} as TSecretScanningDataSourceWithConnection,
resourceName,
resourceName: resource.name,
payload
});
await secretScanningV2DAL.findings.transaction(async (tx) => {
await secretScanningV2DAL.findings.upsert(
findingsPayload.map((findings) => ({
...findings,
projectId: dataSource.projectId,
dataSourceName: dataSource.name,
if (findingsPayload.length) {
await secretScanningV2DAL.findings.transaction(async (tx) => {
await secretScanningV2DAL.findings.upsert(
findingsPayload.map((findings) => ({
...findings,
projectId: dataSource.projectId,
dataSourceName: dataSource.name,
dataSourceType: dataSource.type,
resourceName: resource.name,
resourceType: resource.type,
scanId,
status: SecretScanningFindingStatus.Unresolved
})),
["projectId", "fingerprint"],
tx,
["resourceName", "dataSourceName", "status"]
);
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Completed
}
);
});
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Completed,
resourceName: resource.name,
isDiffScan: true,
dataSource,
numberOfSecrets: findingsPayload.length,
scanId
});
}
await auditLogService.createAuditLog({
projectId: dataSource.projectId,
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
metadata: {
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceName,
resourceType,
resourceId,
resourceType: resource.type,
scanId,
status: SecretScanningFindingStatus.Unresolved
})),
["projectId", "fingerprint"],
tx,
["resourceName", "dataSourceName", "status"]
);
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Completed
scanStatus: SecretScanningScanStatus.Completed,
scanType: SecretScanningScanType.DiffScan,
numberOfSecretsDetected: findingsPayload.length
}
);
}
});
// TODO: send notification
logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`);
} catch (error) {
if (scanId)
if (retryCount === retryLimit) {
const errorMessage = parseScanErrorMessage(error);
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Failed,
statusMessage: parseScanErrorMessage(error)
statusMessage: errorMessage
}
);
// TODO: send error notification
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Failed,
resourceName: resource.name,
dataSource,
errorMessage
});
await auditLogService.createAuditLog({
projectId: dataSource.projectId,
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
metadata: {
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId: resource.id,
resourceType: resource.type,
scanId,
scanStatus: SecretScanningScanStatus.Failed,
scanType: SecretScanningScanType.DiffScan
}
}
});
}
logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
throw error;
@@ -327,67 +500,84 @@ export const secretScanningV2QueueServiceFactory = async ({
},
{
batchSize: 1,
workerCount: 2,
workerCount: 20,
pollingIntervalSeconds: 1
}
);
// await queueService.startPg<QueueName.SecretRotationV2>(
// QueueJobs.SecretRotationV2SendNotification,
// async ([job]) => {
// const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload;
// try {
// const {
// name: rotationName,
// type,
// projectId,
// lastRotationAttemptedAt,
// folder,
// environment,
// id: dataSourceId
// } = secretRotation;
//
// logger.info(`secretRotationV2Queue: Sending Status Notification [dataSourceId=${dataSourceId}]`);
//
// const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
// const project = await projectDAL.findById(projectId);
//
// const projectAdmins = projectMembers.filter((member) =>
// member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
// );
//
// const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation];
//
// await smtpService.sendMail({
// recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
// template: SmtpTemplates.SecretRotationFailed,
// subjectLine: `Secret Rotation Failed`,
// substitutions: {
// rotationName,
// rotationType,
// content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date(
// lastRotationAttemptedAt
// ).toISOString()}. Please check the rotation status in Infisical for more details.`,
// secretPath: folder.path,
// environment: environment.name,
// projectName: project.name,
// rotationUrl: encodeURI(`${appCfg.SITE_URL}/secret-manager/${projectId}/secrets/${environment.slug}`)
// }
// });
// } catch (error) {
// logger.error(
// error,
// `secretRotationV2Queue: Failed to Send Status Notification [dataSourceId=${secretRotation.id}]`
// );
// throw error;
// }
// },
// {
// batchSize: 1,
// workerCount: 2,
// pollingIntervalSeconds: 1
// }
// );
await queueService.startPg<QueueName.SecretScanningV2>(
QueueJobs.SecretScanningV2SendNotification,
async ([job]) => {
const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification;
const appCfg = getConfig();
if (!appCfg.isSmtpConfigured) return;
try {
const { projectId } = dataSource;
logger.info(
`secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
);
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
const project = await projectDAL.findById(projectId);
const projectAdmins = projectMembers.filter((member) =>
member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
);
const timestamp = new Date().toISOString();
await smtpService.sendMail({
recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
template:
payload.status === SecretScanningScanStatus.Completed
? SmtpTemplates.SecretScanningV2SecretsDetected
: SmtpTemplates.SecretScanningV2ScanFailed,
subjectLine:
payload.status === SecretScanningScanStatus.Completed
? "Incident Alert: Secret(s) Leaked"
: `Secret Scanning Failed`,
substitutions:
payload.status === SecretScanningScanStatus.Completed
? {
authorName: "Jim",
authorEmail: "[email protected]",
resourceName,
numberOfSecrets: payload.numberOfSecrets,
isDiffScan: payload.isDiffScan,
url: encodeURI(
`${appCfg.SITE_URL}/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
),
timestamp
}
: {
dataSourceName: dataSource.name,
resourceName,
projectName: project.name,
timestamp,
errorMessage: payload.errorMessage,
url: encodeURI(
`${appCfg.SITE_URL}/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
)
}
});
} catch (error) {
logger.error(
error,
`secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
);
throw error;
}
},
{
batchSize: 1,
workerCount: 5,
pollingIntervalSeconds: 1
}
);
return {
queueDataSourceFullScan,
backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.ts
+3 -11
View File
@@ -49,10 +49,6 @@ export type TSecretScanningV2ServiceFactoryDep = {
appConnectionService: Pick<TAppConnectionServiceFactory, "connectAppConnectionById">;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission" | "getOrgPermission">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
// auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
// keyStore: Pick<TKeyStoreFactory, "acquireLock" | "setItemWithExpiry" | "getItem">;
// queueService: Pick<TQueueServiceFactory, "queuePg">;
// appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update" | "updateById">;
secretScanningV2Queue: Pick<
TSecretScanningV2QueueServiceFactory,
"queueDataSourceFullScan" | "queueResourceDiffScan"
@@ -67,10 +63,6 @@ export const secretScanningV2ServiceFactory = ({
permissionService,
appConnectionService,
licenseService,
// auditLogService,
// keyStore,
// queueService,
// appConnectionDAL,
secretScanningV2Queue,
kmsService
}: TSecretScanningV2ServiceFactoryDep) => {
@@ -374,7 +366,7 @@ export const secretScanningV2ServiceFactory = ({
}
};
const deleteSecretScanningResource = async (
const deleteSecretScanningDataSource = async (
{ type, dataSourceId }: TDeleteSecretScanningDataSourceDTO,
actor: OrgServiceActor
) => {
@@ -750,7 +742,7 @@ export const secretScanningV2ServiceFactory = ({
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningFindingActions.Resolve,
ProjectPermissionSecretScanningFindingActions.Update,
ProjectPermissionSub.SecretScanningFindings
);
@@ -770,7 +762,7 @@ export const secretScanningV2ServiceFactory = ({
findSecretScanningDataSourceByName,
createSecretScanningDataSource,
updateSecretScanningDataSource,
deleteSecretScanningResource,
deleteSecretScanningDataSource,
triggerSecretScanningDataSourceScan,
listSecretScanningResourcesByDataSourceId,
listSecretScanningScansByDataSourceId,
backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts
+15 -2
View File
@@ -1,4 +1,9 @@
import { TSecretScanningFindingsInsert, TSecretScanningResources, TSecretScanningScans } from "@app/db/schemas";
import {
TSecretScanningDataSources,
TSecretScanningFindingsInsert,
TSecretScanningResources,
TSecretScanningScans
} from "@app/db/schemas";
import {
TGitHubDataSource,
TGitHubDataSourceInput,
@@ -96,6 +101,14 @@ export type TQueueSecretScanningDataSourceFullScan = {
export type TQueueSecretScanningResourceDiffScan = TQueueGitHubResourceDiffScan;
export type TQueueSecretScanningSendNotification = {
dataSource: TSecretScanningDataSources;
resourceName: string;
} & (
| { status: SecretScanningScanStatus.Failed; errorMessage: string }
| { status: SecretScanningScanStatus.Completed; numberOfSecrets: number; scanId: string; isDiffScan: boolean }
);
export type TCloneRepository = {
cloneUrl: string;
repoPath: string;
@@ -163,7 +176,7 @@ export type TFindingsPayload = Pick<TSecretScanningFindingsInsert, "details" | "
export type TGetFindingsPayload = Promise<TFindingsPayload>;
export type TUpdateSecretScanningFinding = {
status: SecretScanningFindingStatus;
status?: SecretScanningFindingStatus;
remarks?: string | null;
findingId: string;
};