mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-10-14 20:51:50 +00:00
feature: secret scanning architecture and github data source (wip)
This commit is contained in:
@@ -2,6 +2,10 @@ import {
|
||||
registerSecretRotationV2Router,
|
||||
SECRET_ROTATION_REGISTER_ROUTER_MAP
|
||||
} from "@app/ee/routes/v2/secret-rotation-v2-routers";
|
||||
import {
|
||||
registerSecretScanningV2Router,
|
||||
SECRET_SCANNING_REGISTER_ROUTER_MAP
|
||||
} from "@app/ee/routes/v2/secret-scanning-v2-routers";
|
||||
|
||||
import { registerIdentityProjectAdditionalPrivilegeRouter } from "./identity-project-additional-privilege-router";
|
||||
import { registerProjectRoleRouter } from "./project-role-router";
|
||||
@@ -31,4 +35,17 @@ export const registerV2EERoutes = async (server: FastifyZodProvider) => {
|
||||
},
|
||||
{ prefix: "/secret-rotations" }
|
||||
);
|
||||
|
||||
await server.register(
|
||||
async (secretScanningV2Router) => {
|
||||
// register generic secret scanning endpoints
|
||||
await secretScanningV2Router.register(registerSecretScanningV2Router);
|
||||
|
||||
// register service-specific secret scanning endpoints (gitlab/github, etc.)
|
||||
for await (const [type, router] of Object.entries(SECRET_SCANNING_REGISTER_ROUTER_MAP)) {
|
||||
await secretScanningV2Router.register(router, { prefix: `data-sources/${type}` });
|
||||
}
|
||||
},
|
||||
{ prefix: "/secret-scanning" }
|
||||
);
|
||||
};
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { registerSecretScanningEndpoints } from "@app/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints";
|
||||
import {
|
||||
CreateGitHubDataSourceSchema,
|
||||
GitHubDataSourceSchema,
|
||||
UpdateGitHubDataSourceSchema
|
||||
} from "@app/ee/services/secret-scanning-v2/github";
|
||||
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
|
||||
export const registerGitHubSecretScanningRouter = async (server: FastifyZodProvider) =>
|
||||
registerSecretScanningEndpoints({
|
||||
type: SecretScanningDataSource.GitHub,
|
||||
server,
|
||||
responseSchema: GitHubDataSourceSchema,
|
||||
createSchema: CreateGitHubDataSourceSchema,
|
||||
updateSchema: UpdateGitHubDataSourceSchema
|
||||
});
|
||||
@@ -0,0 +1,12 @@
|
||||
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
|
||||
import { registerGitHubSecretScanningRouter } from "./github-secret-scanning-router";
|
||||
|
||||
export * from "./secret-scanning-v2-router";
|
||||
|
||||
export const SECRET_SCANNING_REGISTER_ROUTER_MAP: Record<
|
||||
SecretScanningDataSource,
|
||||
(server: FastifyZodProvider) => Promise<void>
|
||||
> = {
|
||||
[SecretScanningDataSource.GitHub]: registerGitHubSecretScanningRouter
|
||||
};
|
||||
@@ -0,0 +1,593 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { SecretScanningResourcesSchema, SecretScanningScansSchema } from "@app/db/schemas";
|
||||
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
|
||||
import {
|
||||
SecretScanningDataSource,
|
||||
SecretScanningScanStatus
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { SECRET_SCANNING_DATA_SOURCE_NAME_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps";
|
||||
import {
|
||||
TSecretScanningDataSource,
|
||||
TSecretScanningDataSourceInput
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
|
||||
import { ApiDocsTags, SecretScanningDataSources } from "@app/lib/api-docs";
|
||||
import { startsWithVowel } from "@app/lib/fn";
|
||||
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
|
||||
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
|
||||
import { AuthMode } from "@app/services/auth/auth-type";
|
||||
|
||||
export const registerSecretScanningEndpoints = <
|
||||
T extends TSecretScanningDataSource,
|
||||
I extends TSecretScanningDataSourceInput
|
||||
>({
|
||||
server,
|
||||
type,
|
||||
createSchema,
|
||||
updateSchema,
|
||||
responseSchema
|
||||
}: {
|
||||
type: SecretScanningDataSource;
|
||||
server: FastifyZodProvider;
|
||||
createSchema: z.ZodType<{
|
||||
name: string;
|
||||
projectId: string;
|
||||
connectionId?: string;
|
||||
config: Partial<I["config"]>;
|
||||
description?: string | null;
|
||||
isAutoScanEnabled?: boolean;
|
||||
}>;
|
||||
updateSchema: z.ZodType<{
|
||||
name?: string;
|
||||
config?: Partial<I["config"]>;
|
||||
description?: string | null;
|
||||
isAutoScanEnabled?: boolean;
|
||||
}>;
|
||||
responseSchema: z.ZodTypeAny;
|
||||
}) => {
|
||||
const sourceType = SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type];
|
||||
|
||||
server.route({
|
||||
method: "GET",
|
||||
url: `/`,
|
||||
config: {
|
||||
rateLimit: readLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: `List the ${sourceType} Data Sources for the specified project.`,
|
||||
querystring: z.object({
|
||||
projectId: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1, "Project ID required")
|
||||
.describe(SecretScanningDataSources.LIST(type).projectId)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({ dataSources: responseSchema.array() })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const {
|
||||
query: { projectId }
|
||||
} = req;
|
||||
|
||||
const dataSources = (await server.services.secretScanningV2.listSecretScanningDataSourcesByProjectId(
|
||||
{ projectId, type },
|
||||
req.permission
|
||||
)) as T[];
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST,
|
||||
metadata: {
|
||||
type,
|
||||
count: dataSources.length,
|
||||
dataSourceIds: dataSources.map((source) => source.id)
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { dataSources };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "GET",
|
||||
url: "/:dataSourceId",
|
||||
config: {
|
||||
rateLimit: readLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: `Get the specified ${sourceType} Data Source by ID.`,
|
||||
params: z.object({
|
||||
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.GET_BY_ID(type).dataSourceId)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({ dataSource: responseSchema })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const { dataSourceId } = req.params;
|
||||
|
||||
const dataSource = (await server.services.secretScanningV2.findSecretScanningDataSourceById(
|
||||
{ dataSourceId, type },
|
||||
req.permission
|
||||
)) as T;
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId: dataSource.projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_GET,
|
||||
metadata: {
|
||||
dataSourceId,
|
||||
type
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { dataSource };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "GET",
|
||||
url: `/data-source-name/:dataSourceName`,
|
||||
config: {
|
||||
rateLimit: readLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: `Get the specified ${sourceType} Data Source by name and project ID.`,
|
||||
params: z.object({
|
||||
sourceName: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1, "Data Source name required")
|
||||
.describe(SecretScanningDataSources.GET_BY_NAME(type).sourceName)
|
||||
}),
|
||||
querystring: z.object({
|
||||
projectId: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1, "Project ID required")
|
||||
.describe(SecretScanningDataSources.GET_BY_NAME(type).projectId)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({ dataSource: responseSchema })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const { sourceName } = req.params;
|
||||
const { projectId } = req.query;
|
||||
|
||||
const dataSource = (await server.services.secretScanningV2.findSecretScanningDataSourceByName(
|
||||
{ sourceName, projectId, type },
|
||||
req.permission
|
||||
)) as T;
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_GET,
|
||||
metadata: {
|
||||
dataSourceId: dataSource.id,
|
||||
type
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { dataSource };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "POST",
|
||||
url: "/",
|
||||
config: {
|
||||
rateLimit: writeLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: `Create ${
|
||||
startsWithVowel(sourceType) ? "an" : "a"
|
||||
} ${sourceType} Data Source for the specified project.`,
|
||||
body: createSchema,
|
||||
response: {
|
||||
200: z.object({ dataSource: responseSchema })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const dataSource = (await server.services.secretScanningV2.createSecretScanningDataSource(
|
||||
{ ...req.body, type },
|
||||
req.permission
|
||||
)) as T;
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId: dataSource.projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_CREATE,
|
||||
metadata: {
|
||||
dataSourceId: dataSource.id,
|
||||
type,
|
||||
...req.body
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { dataSource };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "PATCH",
|
||||
url: "/:dataSourceId",
|
||||
config: {
|
||||
rateLimit: writeLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: `Update the specified ${sourceType} Data Source.`,
|
||||
params: z.object({
|
||||
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.UPDATE(type).dataSourceId)
|
||||
}),
|
||||
body: updateSchema,
|
||||
response: {
|
||||
200: z.object({ dataSource: responseSchema })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const { dataSourceId } = req.params;
|
||||
|
||||
const dataSource = (await server.services.secretScanningV2.updateSecretScanningDataSource(
|
||||
{ ...req.body, dataSourceId, type },
|
||||
req.permission
|
||||
)) as T;
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId: dataSource.projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_UPDATE,
|
||||
metadata: {
|
||||
dataSourceId,
|
||||
type,
|
||||
...req.body
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { dataSource };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "DELETE",
|
||||
url: `/:dataSourceId`,
|
||||
config: {
|
||||
rateLimit: writeLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: `Delete the specified ${sourceType} Data Source.`,
|
||||
params: z.object({
|
||||
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.DELETE(type).dataSourceId)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({ dataSource: responseSchema })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const { dataSourceId } = req.params;
|
||||
|
||||
const dataSource = (await server.services.secretScanningV2.deleteSecretScanningResource(
|
||||
{ type, dataSourceId },
|
||||
req.permission
|
||||
)) as T;
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId: dataSource.projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_DELETE,
|
||||
metadata: {
|
||||
type,
|
||||
dataSourceId
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { dataSource };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "POST",
|
||||
url: `/:dataSourceId/scan`,
|
||||
config: {
|
||||
rateLimit: writeLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: `Trigger a scan for the specified ${sourceType} Data Source.`,
|
||||
params: z.object({
|
||||
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.SCAN(type).dataSourceId)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({ dataSource: responseSchema })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const { dataSourceId } = req.params;
|
||||
|
||||
const dataSource = (await server.services.secretScanningV2.triggerSecretScanningDataSourceScan(
|
||||
{ type, dataSourceId },
|
||||
req.permission
|
||||
)) as T;
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId: dataSource.projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
|
||||
metadata: {
|
||||
type,
|
||||
dataSourceId
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { dataSource };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "POST",
|
||||
url: `/:dataSourceId/resources/:resourceId/scan`,
|
||||
config: {
|
||||
rateLimit: writeLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: `Trigger a scan for the specified ${sourceType} Data Source resource.`,
|
||||
params: z.object({
|
||||
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.SCAN(type).dataSourceId),
|
||||
resourceId: z.string().uuid().describe(SecretScanningDataSources.SCAN(type).resourceId)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({ dataSource: responseSchema })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const { dataSourceId, resourceId } = req.params;
|
||||
|
||||
const dataSource = (await server.services.secretScanningV2.triggerSecretScanningDataSourceScan(
|
||||
{ type, dataSourceId, resourceId },
|
||||
req.permission
|
||||
)) as T;
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId: dataSource.projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
|
||||
metadata: {
|
||||
type,
|
||||
dataSourceId,
|
||||
resourceId
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { dataSource };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "GET",
|
||||
url: "/:dataSourceId/resources",
|
||||
config: {
|
||||
rateLimit: readLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: `Get the resources associated with the specified ${sourceType} Data Source by ID.`,
|
||||
params: z.object({
|
||||
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.LIST_RESOURCES(type).dataSourceId)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({ resources: SecretScanningResourcesSchema.array() })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const { dataSourceId } = req.params;
|
||||
|
||||
const { resources, projectId } = await server.services.secretScanningV2.listSecretScanningResourcesByDataSourceId(
|
||||
{ dataSourceId, type },
|
||||
req.permission
|
||||
);
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_RESOURCE_LIST,
|
||||
metadata: {
|
||||
dataSourceId,
|
||||
type,
|
||||
resourceIds: resources.map((resource) => resource.id),
|
||||
count: resources.length
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { resources };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "GET",
|
||||
url: "/:dataSourceId/scans",
|
||||
config: {
|
||||
rateLimit: readLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: `Get the scans associated with the specified ${sourceType} Data Source by ID.`,
|
||||
params: z.object({
|
||||
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.LIST_SCANS(type).dataSourceId)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({ scans: SecretScanningScansSchema.array() })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const { dataSourceId } = req.params;
|
||||
|
||||
const { scans, projectId } = await server.services.secretScanningV2.listSecretScanningScansByDataSourceId(
|
||||
{ dataSourceId, type },
|
||||
req.permission
|
||||
);
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_SCAN_LIST,
|
||||
metadata: {
|
||||
dataSourceId,
|
||||
type,
|
||||
count: scans.length
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { scans };
|
||||
}
|
||||
});
|
||||
|
||||
// not exposed, for UI only
|
||||
server.route({
|
||||
method: "GET",
|
||||
url: "/:dataSourceId/resources-dashboard",
|
||||
config: {
|
||||
rateLimit: readLimit
|
||||
},
|
||||
schema: {
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
params: z.object({
|
||||
dataSourceId: z.string().uuid()
|
||||
}),
|
||||
response: {
|
||||
200: z.object({
|
||||
resources: SecretScanningResourcesSchema.extend({
|
||||
lastScannedAt: z.date().nullish(),
|
||||
lastScanStatus: z.nativeEnum(SecretScanningScanStatus).nullish(),
|
||||
lastScanStatusMessage: z.string().nullish(),
|
||||
unresolvedFindings: z.number()
|
||||
}).array()
|
||||
})
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT]),
|
||||
handler: async (req) => {
|
||||
const { dataSourceId } = req.params;
|
||||
|
||||
const { resources, projectId } =
|
||||
await server.services.secretScanningV2.listSecretScanningResourcesWithDetailsByDataSourceId(
|
||||
{ dataSourceId, type },
|
||||
req.permission
|
||||
);
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_RESOURCE_LIST,
|
||||
metadata: {
|
||||
dataSourceId,
|
||||
type,
|
||||
resourceIds: resources.map((resource) => resource.id),
|
||||
count: resources.length
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { resources };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "GET",
|
||||
url: "/:dataSourceId/scans-dashboard",
|
||||
config: {
|
||||
rateLimit: readLimit
|
||||
},
|
||||
schema: {
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
params: z.object({
|
||||
dataSourceId: z.string().uuid()
|
||||
}),
|
||||
response: {
|
||||
200: z.object({
|
||||
scans: SecretScanningScansSchema.extend({
|
||||
unresolvedFindings: z.number(),
|
||||
resolvedFindings: z.number(),
|
||||
resourceName: z.string()
|
||||
}).array()
|
||||
})
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT]),
|
||||
handler: async (req) => {
|
||||
const { dataSourceId } = req.params;
|
||||
|
||||
const { scans, projectId } =
|
||||
await server.services.secretScanningV2.listSecretScanningScansWithDetailsByDataSourceId(
|
||||
{ dataSourceId, type },
|
||||
req.permission
|
||||
);
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_SCAN_LIST,
|
||||
metadata: {
|
||||
dataSourceId,
|
||||
type,
|
||||
count: scans.length
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { scans };
|
||||
}
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,270 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
|
||||
import { GitHubDataSourceListItemSchema } from "@app/ee/services/secret-scanning-v2/github";
|
||||
import {
|
||||
SecretScanningFindingStatus,
|
||||
SecretScanningScanStatus
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import {
|
||||
SecretScanningDataSourceSchema,
|
||||
SecretScanningFindingSchema
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-union-schemas";
|
||||
import { ApiDocsTags, SecretScanningDataSources, SecretScanningFindings } from "@app/lib/api-docs";
|
||||
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
|
||||
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
|
||||
import { AuthMode } from "@app/services/auth/auth-type";
|
||||
|
||||
const SecretScanningDataSourceOptionsSchema = z.discriminatedUnion("type", [GitHubDataSourceListItemSchema]);
|
||||
|
||||
export const registerSecretScanningV2Router = async (server: FastifyZodProvider) => {
|
||||
server.route({
|
||||
method: "GET",
|
||||
url: "/data-sources/options",
|
||||
config: {
|
||||
rateLimit: readLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: "List the available Secret Scanning Data Source Options.",
|
||||
response: {
|
||||
200: z.object({
|
||||
dataSourceOptions: SecretScanningDataSourceOptionsSchema.array()
|
||||
})
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: () => {
|
||||
const dataSourceOptions = server.services.secretScanningV2.listSecretScanningDataSourceOptions();
|
||||
return { dataSourceOptions };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "GET",
|
||||
url: "/data-sources",
|
||||
config: {
|
||||
rateLimit: readLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: "List all the Secret Scanning Data Sources for the specified project.",
|
||||
querystring: z.object({
|
||||
projectId: z.string().trim().min(1, "Project ID required").describe(SecretScanningDataSources.LIST().projectId)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({ dataSources: SecretScanningDataSourceSchema.array() })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const {
|
||||
query: { projectId },
|
||||
permission
|
||||
} = req;
|
||||
|
||||
const dataSources = await server.services.secretScanningV2.listSecretScanningDataSourcesByProjectId(
|
||||
{ projectId },
|
||||
permission
|
||||
);
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST,
|
||||
metadata: {
|
||||
dataSourceIds: dataSources.map((dataSource) => dataSource.id),
|
||||
count: dataSources.length
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { dataSources };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "GET",
|
||||
url: "/findings",
|
||||
config: {
|
||||
rateLimit: readLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: "List all the Secret Scanning Findings for the specified project.",
|
||||
querystring: z.object({
|
||||
projectId: z.string().trim().min(1, "Project ID required").describe(SecretScanningFindings.LIST.projectId)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({ findings: SecretScanningFindingSchema.array() })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const {
|
||||
query: { projectId },
|
||||
permission
|
||||
} = req;
|
||||
|
||||
const findings = await server.services.secretScanningV2.listSecretScanningFindingsByProjectId(
|
||||
projectId,
|
||||
permission
|
||||
);
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_FINDING_LIST,
|
||||
metadata: {
|
||||
findingIds: findings.map((finding) => finding.id),
|
||||
count: findings.length
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { findings };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "PATCH",
|
||||
url: "/findings/:findingId",
|
||||
config: {
|
||||
rateLimit: writeLimit
|
||||
},
|
||||
schema: {
|
||||
hide: false,
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
description: "Update the resolve status of the specified Secret Scanning Finding.",
|
||||
params: z.object({
|
||||
findingId: z.string().trim().min(1, "Finding ID required").describe(SecretScanningFindings.UPDATE.findingId)
|
||||
}),
|
||||
body: z.object({
|
||||
status: z.nativeEnum(SecretScanningFindingStatus).describe(SecretScanningFindings.UPDATE.status),
|
||||
remarks: z.string().nullish().describe(SecretScanningFindings.UPDATE.remarks)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({ finding: SecretScanningFindingSchema })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
|
||||
handler: async (req) => {
|
||||
const {
|
||||
params: { findingId },
|
||||
body,
|
||||
permission
|
||||
} = req;
|
||||
|
||||
const { finding, projectId } = await server.services.secretScanningV2.updateSecretScanningFindingById(
|
||||
{ findingId, ...body },
|
||||
permission
|
||||
);
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_FINDING_UPDATE,
|
||||
metadata: {
|
||||
findingId,
|
||||
...body
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { finding };
|
||||
}
|
||||
});
|
||||
|
||||
// not exposed, for UI only
|
||||
server.route({
|
||||
method: "GET",
|
||||
url: "/data-sources-dashboard",
|
||||
config: {
|
||||
rateLimit: readLimit
|
||||
},
|
||||
schema: {
|
||||
querystring: z.object({
|
||||
projectId: z.string().trim().min(1, "Project ID required")
|
||||
}),
|
||||
response: {
|
||||
200: z.object({
|
||||
dataSources: z
|
||||
.intersection(
|
||||
SecretScanningDataSourceSchema,
|
||||
z.object({
|
||||
lastScannedAt: z.date().nullish(),
|
||||
lastScanStatus: z.nativeEnum(SecretScanningScanStatus).nullish(),
|
||||
lastScanStatusMessage: z.string().nullish(),
|
||||
unresolvedFindings: z.number().nullish()
|
||||
})
|
||||
)
|
||||
.array()
|
||||
})
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT]),
|
||||
handler: async (req) => {
|
||||
const {
|
||||
query: { projectId },
|
||||
permission
|
||||
} = req;
|
||||
|
||||
const dataSources = await server.services.secretScanningV2.listSecretScanningDataSourcesWithDetailsByProjectId(
|
||||
{ projectId },
|
||||
permission
|
||||
);
|
||||
|
||||
await server.services.auditLog.createAuditLog({
|
||||
...req.auditLogInfo,
|
||||
projectId,
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST,
|
||||
metadata: {
|
||||
dataSourceIds: dataSources.map((dataSource) => dataSource.id),
|
||||
count: dataSources.length
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { dataSources };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
method: "GET",
|
||||
url: "/unresolved-findings-count",
|
||||
config: {
|
||||
rateLimit: readLimit
|
||||
},
|
||||
schema: {
|
||||
tags: [ApiDocsTags.SecretScanning],
|
||||
querystring: z.object({
|
||||
projectId: z.string().trim().min(1, "Project ID required").describe(SecretScanningFindings.LIST.projectId)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({ unresolvedFindings: z.number() })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT]),
|
||||
handler: async (req) => {
|
||||
const {
|
||||
query: { projectId },
|
||||
permission
|
||||
} = req;
|
||||
|
||||
const unresolvedFindings =
|
||||
await server.services.secretScanningV2.getSecretScanningUnresolvedFindingsCountByProjectId(
|
||||
projectId,
|
||||
permission
|
||||
);
|
||||
|
||||
return { unresolvedFindings };
|
||||
}
|
||||
});
|
||||
};
|
||||
@@ -9,6 +9,14 @@ import {
|
||||
TSecretRotationV2Raw,
|
||||
TUpdateSecretRotationV2DTO
|
||||
} from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-types";
|
||||
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import {
|
||||
TCreateSecretScanningDataSourceDTO,
|
||||
TDeleteSecretScanningDataSourceDTO,
|
||||
TTriggerSecretScanningDataSourceDTO,
|
||||
TUpdateSecretScanningDataSourceDTO,
|
||||
TUpdateSecretScanningFinding
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
|
||||
import { SshCaStatus, SshCertType } from "@app/ee/services/ssh/ssh-certificate-authority-types";
|
||||
import { SshCertKeyAlgorithm } from "@app/ee/services/ssh-certificate/ssh-certificate-types";
|
||||
import { SshCertTemplateStatus } from "@app/ee/services/ssh-certificate-template/ssh-certificate-template-types";
|
||||
@@ -375,7 +383,18 @@ export enum EventType {
|
||||
MICROSOFT_TEAMS_WORKFLOW_INTEGRATION_LIST = "microsoft-teams-workflow-integration-list",
|
||||
|
||||
PROJECT_ASSUME_PRIVILEGE_SESSION_START = "project-assume-privileges-session-start",
|
||||
PROJECT_ASSUME_PRIVILEGE_SESSION_END = "project-assume-privileges-session-end"
|
||||
PROJECT_ASSUME_PRIVILEGE_SESSION_END = "project-assume-privileges-session-end",
|
||||
|
||||
SECRET_SCANNING_DATA_SOURCE_LIST = "secret-scanning-data-source-list",
|
||||
SECRET_SCANNING_DATA_SOURCE_CREATE = "secret-scanning-data-source-create",
|
||||
SECRET_SCANNING_DATA_SOURCE_UPDATE = "secret-scanning-data-source-update",
|
||||
SECRET_SCANNING_DATA_SOURCE_DELETE = "secret-scanning-data-source-delete",
|
||||
SECRET_SCANNING_DATA_SOURCE_GET = "secret-scanning-data-source-get",
|
||||
SECRET_SCANNING_DATA_SOURCE_SCAN = "secret-scanning-data-source-scan",
|
||||
SECRET_SCANNING_RESOURCE_LIST = "secret-scanning-resource-list",
|
||||
SECRET_SCANNING_SCAN_LIST = "secret-scanning-scan-list",
|
||||
SECRET_SCANNING_FINDING_LIST = "secret-scanning-finding-list",
|
||||
SECRET_SCANNING_FINDING_UPDATE = "secret-scanning-finding-update"
|
||||
}
|
||||
|
||||
export const filterableSecretEvents: EventType[] = [
|
||||
@@ -2913,6 +2932,75 @@ interface MicrosoftTeamsWorkflowIntegrationUpdateEvent {
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningDataSourceListEvent {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST;
|
||||
metadata: {
|
||||
type?: SecretScanningDataSource;
|
||||
count: number;
|
||||
dataSourceIds: string[];
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningDataSourceGetEvent {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_GET;
|
||||
metadata: {
|
||||
type: SecretScanningDataSource;
|
||||
dataSourceId: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningDataSourceCreateEvent {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_CREATE;
|
||||
metadata: Omit<TCreateSecretScanningDataSourceDTO, "projectId"> & { dataSourceId: string };
|
||||
}
|
||||
|
||||
interface SecretScanningDataSourceUpdateEvent {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_UPDATE;
|
||||
metadata: TUpdateSecretScanningDataSourceDTO;
|
||||
}
|
||||
|
||||
interface SecretScanningDataSourceDeleteEvent {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_DELETE;
|
||||
metadata: TDeleteSecretScanningDataSourceDTO;
|
||||
}
|
||||
|
||||
interface SecretScanningDataSourceScanEvent {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN;
|
||||
metadata: TTriggerSecretScanningDataSourceDTO;
|
||||
}
|
||||
|
||||
interface SecretScanningResourceListEvent {
|
||||
type: EventType.SECRET_SCANNING_RESOURCE_LIST;
|
||||
metadata: {
|
||||
type: SecretScanningDataSource;
|
||||
dataSourceId: string;
|
||||
resourceIds: string[];
|
||||
count: number;
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningScanListEvent {
|
||||
type: EventType.SECRET_SCANNING_SCAN_LIST;
|
||||
metadata: {
|
||||
type: SecretScanningDataSource;
|
||||
dataSourceId: string;
|
||||
count: number;
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningFindingListEvent {
|
||||
type: EventType.SECRET_SCANNING_FINDING_LIST;
|
||||
metadata: {
|
||||
findingIds: string[];
|
||||
count: number;
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningFindingUpdateEvent {
|
||||
type: EventType.SECRET_SCANNING_FINDING_UPDATE;
|
||||
metadata: TUpdateSecretScanningFinding;
|
||||
}
|
||||
|
||||
export type Event =
|
||||
| GetSecretsEvent
|
||||
| GetSecretEvent
|
||||
@@ -3179,4 +3267,14 @@ export type Event =
|
||||
| MicrosoftTeamsWorkflowIntegrationGetTeamsEvent
|
||||
| MicrosoftTeamsWorkflowIntegrationGetEvent
|
||||
| MicrosoftTeamsWorkflowIntegrationListEvent
|
||||
| MicrosoftTeamsWorkflowIntegrationUpdateEvent;
|
||||
| MicrosoftTeamsWorkflowIntegrationUpdateEvent
|
||||
| SecretScanningDataSourceListEvent
|
||||
| SecretScanningDataSourceGetEvent
|
||||
| SecretScanningDataSourceCreateEvent
|
||||
| SecretScanningDataSourceUpdateEvent
|
||||
| SecretScanningDataSourceDeleteEvent
|
||||
| SecretScanningDataSourceScanEvent
|
||||
| SecretScanningResourceListEvent
|
||||
| SecretScanningScanListEvent
|
||||
| SecretScanningFindingListEvent
|
||||
| SecretScanningFindingUpdateEvent;
|
||||
|
||||
@@ -54,7 +54,8 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
|
||||
projectTemplates: false,
|
||||
kmip: false,
|
||||
gateway: false,
|
||||
sshHostGroups: false
|
||||
sshHostGroups: false,
|
||||
secretScanning: false
|
||||
});
|
||||
|
||||
export const setupLicenseRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => {
|
||||
|
||||
@@ -72,6 +72,7 @@ export type TFeatureSet = {
|
||||
kmip: false;
|
||||
gateway: false;
|
||||
sshHostGroups: false;
|
||||
secretScanning: false;
|
||||
};
|
||||
|
||||
export type TOrgPlansTableDTO = {
|
||||
|
||||
@@ -12,6 +12,8 @@ import {
|
||||
ProjectPermissionPkiSubscriberActions,
|
||||
ProjectPermissionSecretActions,
|
||||
ProjectPermissionSecretRotationActions,
|
||||
ProjectPermissionSecretScanningDataSourceActions,
|
||||
ProjectPermissionSecretScanningFindingActions,
|
||||
ProjectPermissionSecretSyncActions,
|
||||
ProjectPermissionSet,
|
||||
ProjectPermissionSshHostActions,
|
||||
@@ -198,6 +200,24 @@ const buildAdminPermissionRules = () => {
|
||||
ProjectPermissionSub.SecretRotation
|
||||
);
|
||||
|
||||
can(
|
||||
[
|
||||
ProjectPermissionSecretScanningDataSourceActions.Create,
|
||||
ProjectPermissionSecretScanningDataSourceActions.Edit,
|
||||
ProjectPermissionSecretScanningDataSourceActions.Delete,
|
||||
ProjectPermissionSecretScanningDataSourceActions.Read,
|
||||
ProjectPermissionSecretScanningDataSourceActions.TriggerScans,
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadResources
|
||||
],
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
can(
|
||||
[ProjectPermissionSecretScanningFindingActions.Read, ProjectPermissionSecretScanningFindingActions.Resolve],
|
||||
ProjectPermissionSub.SecretScanningFindings
|
||||
);
|
||||
|
||||
return rules;
|
||||
};
|
||||
|
||||
@@ -378,6 +398,24 @@ const buildMemberPermissionRules = () => {
|
||||
ProjectPermissionSub.SecretSyncs
|
||||
);
|
||||
|
||||
can(
|
||||
[
|
||||
ProjectPermissionSecretScanningDataSourceActions.Create,
|
||||
ProjectPermissionSecretScanningDataSourceActions.Edit,
|
||||
ProjectPermissionSecretScanningDataSourceActions.Delete,
|
||||
ProjectPermissionSecretScanningDataSourceActions.Read,
|
||||
ProjectPermissionSecretScanningDataSourceActions.TriggerScans,
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadResources
|
||||
],
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
can(
|
||||
[ProjectPermissionSecretScanningFindingActions.Read, ProjectPermissionSecretScanningFindingActions.Resolve],
|
||||
ProjectPermissionSub.SecretScanningFindings
|
||||
);
|
||||
|
||||
return rules;
|
||||
};
|
||||
|
||||
@@ -413,6 +451,17 @@ const buildViewerPermissionRules = () => {
|
||||
can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateTemplates);
|
||||
can(ProjectPermissionSecretSyncActions.Read, ProjectPermissionSub.SecretSyncs);
|
||||
|
||||
can(
|
||||
[
|
||||
ProjectPermissionSecretScanningDataSourceActions.Read,
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadResources
|
||||
],
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
can([ProjectPermissionSecretScanningFindingActions.Read], ProjectPermissionSub.SecretScanningFindings);
|
||||
|
||||
return rules;
|
||||
};
|
||||
|
||||
|
||||
@@ -123,6 +123,21 @@ export enum ProjectPermissionKmipActions {
|
||||
GenerateClientCertificates = "generate-client-certificates"
|
||||
}
|
||||
|
||||
export enum ProjectPermissionSecretScanningDataSourceActions {
|
||||
Read = "read-data-sources",
|
||||
Create = "create-data-sources",
|
||||
Edit = "edit-data-sources",
|
||||
Delete = "delete-data-sources",
|
||||
TriggerScans = "trigger-data-source-scans",
|
||||
ReadScans = "read-data-source-scans",
|
||||
ReadResources = "read-data-source-resources"
|
||||
}
|
||||
|
||||
export enum ProjectPermissionSecretScanningFindingActions {
|
||||
Read = "read-findings",
|
||||
Resolve = "resolve-findings"
|
||||
}
|
||||
|
||||
export enum ProjectPermissionSub {
|
||||
Role = "role",
|
||||
Member = "member",
|
||||
@@ -158,7 +173,9 @@ export enum ProjectPermissionSub {
|
||||
Kms = "kms",
|
||||
Cmek = "cmek",
|
||||
SecretSyncs = "secret-syncs",
|
||||
Kmip = "kmip"
|
||||
Kmip = "kmip",
|
||||
SecretScanningDataSources = "secret-scanning-data-sources",
|
||||
SecretScanningFindings = "secret-scanning-findings"
|
||||
}
|
||||
|
||||
export type SecretSubjectFields = {
|
||||
@@ -281,7 +298,9 @@ export type ProjectPermissionSet =
|
||||
| [ProjectPermissionActions.Edit, ProjectPermissionSub.Project]
|
||||
| [ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback]
|
||||
| [ProjectPermissionActions.Create, ProjectPermissionSub.SecretRollback]
|
||||
| [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms];
|
||||
| [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms]
|
||||
| [ProjectPermissionSecretScanningDataSourceActions, ProjectPermissionSub.SecretScanningDataSources]
|
||||
| [ProjectPermissionSecretScanningFindingActions, ProjectPermissionSub.SecretScanningFindings];
|
||||
|
||||
const SECRET_PATH_MISSING_SLASH_ERR_MSG = "Invalid Secret Path; it must start with a '/'";
|
||||
const SECRET_PATH_PERMISSION_OPERATOR_SCHEMA = z.union([
|
||||
@@ -602,6 +621,20 @@ const GeneralPermissionSchema = [
|
||||
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionKmipActions).describe(
|
||||
"Describe what action an entity can take."
|
||||
)
|
||||
}),
|
||||
z.object({
|
||||
subject: z
|
||||
.literal(ProjectPermissionSub.SecretScanningDataSources)
|
||||
.describe("The entity this permission pertains to."),
|
||||
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningDataSourceActions).describe(
|
||||
"Describe what action an entity can take."
|
||||
)
|
||||
}),
|
||||
z.object({
|
||||
subject: z.literal(ProjectPermissionSub.SecretScanningFindings).describe("The entity this permission pertains to."),
|
||||
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningFindingActions).describe(
|
||||
"Describe what action an entity can take."
|
||||
)
|
||||
})
|
||||
];
|
||||
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { TSecretScanningDataSourceListItem } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
|
||||
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
|
||||
|
||||
export const GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION: TSecretScanningDataSourceListItem = {
|
||||
name: "GitHub",
|
||||
type: SecretScanningDataSource.GitHub,
|
||||
connection: AppConnection.GitHubRadar
|
||||
};
|
||||
@@ -0,0 +1,197 @@
|
||||
import { join } from "path";
|
||||
import { ProbotOctokit } from "probot";
|
||||
|
||||
import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
|
||||
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
|
||||
import {
|
||||
SecretScanningFindingSeverity,
|
||||
SecretScanningResource
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { cloneRepository, titleCaseToCamelCase } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
|
||||
import {
|
||||
TSecretScanningFactoryGetDiffScanFindingsPayload,
|
||||
TSecretScanningFactoryGetDiffScanResourcePayload,
|
||||
TSecretScanningFactoryGetFullScanPath,
|
||||
TSecretScanningFactoryInitialize,
|
||||
TSecretScanningFactoryListRawResources,
|
||||
TSecretScanningFactoryPostInitialization
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { BadRequestError } from "@app/lib/errors";
|
||||
import { listGitHubRadarRepositories, TGitHubRadarConnection } from "@app/services/app-connection/github-radar";
|
||||
|
||||
import { TGitHubDataSourceWithConnection, TQueueGitHubResourceDiffScan } from "./github-secret-scanning-types";
|
||||
|
||||
export const GitHubSecretScanningFactory = () => {
|
||||
const initialize: TSecretScanningFactoryInitialize<TGitHubRadarConnection> = async (
|
||||
{ connection, secretScanningV2DAL },
|
||||
callback
|
||||
) => {
|
||||
const externalId = connection.credentials.installationId;
|
||||
|
||||
const existingDataSource = await secretScanningV2DAL.dataSources.findOne({
|
||||
externalId
|
||||
});
|
||||
|
||||
if (existingDataSource)
|
||||
throw new BadRequestError({ message: "A Data Source already exists for this GitHub Radar Connection" });
|
||||
|
||||
return callback({
|
||||
externalId
|
||||
});
|
||||
};
|
||||
|
||||
const postInitialization: TSecretScanningFactoryPostInitialization<TGitHubRadarConnection> = async () => {
|
||||
// no post-initialization required
|
||||
};
|
||||
|
||||
const listRawResources: TSecretScanningFactoryListRawResources<TGitHubDataSourceWithConnection> = async (
|
||||
dataSource
|
||||
) => {
|
||||
const {
|
||||
connection,
|
||||
config: { includeRepos }
|
||||
} = dataSource;
|
||||
|
||||
const repos = await listGitHubRadarRepositories(connection);
|
||||
|
||||
const filteredRepos: typeof repos = [];
|
||||
if (includeRepos.includes("*")) {
|
||||
filteredRepos.push(...repos);
|
||||
} else {
|
||||
filteredRepos.push(...repos.filter((repo) => includeRepos.includes(repo.full_name)));
|
||||
}
|
||||
|
||||
return filteredRepos.map(({ id, full_name }) => ({
|
||||
name: full_name,
|
||||
externalId: id.toString(),
|
||||
type: SecretScanningResource.Repository
|
||||
}));
|
||||
};
|
||||
|
||||
const getFullScanPath: TSecretScanningFactoryGetFullScanPath<TGitHubDataSourceWithConnection> = async ({
|
||||
dataSource,
|
||||
resourceName,
|
||||
tempFolder
|
||||
}) => {
|
||||
const appCfg = getConfig();
|
||||
const {
|
||||
connection: {
|
||||
credentials: { installationId }
|
||||
}
|
||||
} = dataSource;
|
||||
|
||||
const octokit = new ProbotOctokit({
|
||||
auth: {
|
||||
appId: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID,
|
||||
privateKey: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY,
|
||||
installationId
|
||||
}
|
||||
});
|
||||
|
||||
const {
|
||||
data: { token }
|
||||
} = await octokit.apps.createInstallationAccessToken({
|
||||
installation_id: Number(installationId)
|
||||
});
|
||||
|
||||
const repoPath = join(tempFolder, "repo.git");
|
||||
|
||||
await cloneRepository({
|
||||
cloneUrl: `https://x-access-token:${token}@github.com/${resourceName}.git`,
|
||||
repoPath
|
||||
});
|
||||
|
||||
return repoPath;
|
||||
};
|
||||
|
||||
const getDiffScanResourcePayload: TSecretScanningFactoryGetDiffScanResourcePayload<
|
||||
TQueueGitHubResourceDiffScan["payload"]
|
||||
> = ({ repository }) => {
|
||||
return {
|
||||
name: repository.full_name,
|
||||
externalId: repository.id.toString(),
|
||||
type: SecretScanningResource.Repository
|
||||
};
|
||||
};
|
||||
|
||||
const getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<
|
||||
TGitHubDataSourceWithConnection,
|
||||
TQueueGitHubResourceDiffScan["payload"]
|
||||
> = async ({ dataSource, payload, resourceName }) => {
|
||||
const appCfg = getConfig();
|
||||
const {
|
||||
connection: {
|
||||
credentials: { installationId }
|
||||
}
|
||||
} = dataSource;
|
||||
|
||||
const octokit = new ProbotOctokit({
|
||||
auth: {
|
||||
appId: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID,
|
||||
privateKey: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY,
|
||||
installationId
|
||||
}
|
||||
});
|
||||
|
||||
const { commits, repository } = payload;
|
||||
|
||||
const [owner, repo] = repository.full_name.split("/");
|
||||
|
||||
const allFindings: SecretMatch[] = [];
|
||||
|
||||
for (const commit of commits) {
|
||||
for (const filepath of [...commit.added, ...commit.modified]) {
|
||||
// eslint-disable-next-line
|
||||
const fileContentsResponse = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: filepath
|
||||
});
|
||||
|
||||
const { data } = fileContentsResponse;
|
||||
const fileContent = Buffer.from((data as { content: string }).content, "base64").toString();
|
||||
|
||||
// eslint-disable-next-line
|
||||
const findings = await scanContentAndGetFindings(`\n${fileContent}`); // extra line to count lines correctly
|
||||
|
||||
allFindings.push(
|
||||
...findings.map((finding) => ({
|
||||
...finding,
|
||||
File: filepath,
|
||||
Commit: commit.id,
|
||||
Author: commit.author.name,
|
||||
Email: commit.author.email ?? "",
|
||||
Message: commit.message,
|
||||
Fingerprint: `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`,
|
||||
Date: commit.timestamp,
|
||||
Link: `https://github.com/${resourceName}/blob/${commit.id}/${filepath}#L${finding.StartLine}`
|
||||
}))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return allFindings.map(
|
||||
({
|
||||
// discard match and secret as we don't want to store
|
||||
Match,
|
||||
Secret,
|
||||
...finding
|
||||
}) => ({
|
||||
details: titleCaseToCamelCase(finding),
|
||||
fingerprint: finding.Fingerprint,
|
||||
severity: SecretScanningFindingSeverity.High,
|
||||
rule: finding.RuleID
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
return {
|
||||
initialize,
|
||||
postInitialization,
|
||||
listRawResources,
|
||||
getFullScanPath,
|
||||
getDiffScanResourcePayload,
|
||||
getDiffScanFindingsPayload
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,72 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
SecretScanningDataSource,
|
||||
SecretScanningResource
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import {
|
||||
BaseCreateSecretScanningDataSourceSchema,
|
||||
BaseSecretScanningDataSourceSchema,
|
||||
BaseSecretScanningFindingSchema,
|
||||
BaseUpdateSecretScanningDataSourceSchema,
|
||||
GitRepositoryScanFindingDetailsSchema
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-schemas";
|
||||
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
|
||||
|
||||
export const GitHubDataSourceConfigSchema = z.object({
|
||||
includeRepos: z.array(z.string()).nonempty("One or more repositories required").default(["*"])
|
||||
});
|
||||
|
||||
export const GitHubDataSourceSchema = BaseSecretScanningDataSourceSchema({
|
||||
type: SecretScanningDataSource.GitHub,
|
||||
isConnectionRequired: true
|
||||
})
|
||||
.extend({
|
||||
config: GitHubDataSourceConfigSchema
|
||||
})
|
||||
.describe(
|
||||
JSON.stringify({
|
||||
title: "GitHub"
|
||||
})
|
||||
);
|
||||
|
||||
export const CreateGitHubDataSourceSchema = BaseCreateSecretScanningDataSourceSchema({
|
||||
type: SecretScanningDataSource.GitHub,
|
||||
isConnectionRequired: true
|
||||
})
|
||||
.extend({
|
||||
config: GitHubDataSourceConfigSchema
|
||||
})
|
||||
.describe(
|
||||
JSON.stringify({
|
||||
title: "GitHub"
|
||||
})
|
||||
);
|
||||
|
||||
export const UpdateGitHubDataSourceSchema = BaseUpdateSecretScanningDataSourceSchema(SecretScanningDataSource.GitHub)
|
||||
.extend({
|
||||
config: GitHubDataSourceConfigSchema.optional()
|
||||
})
|
||||
.describe(
|
||||
JSON.stringify({
|
||||
title: "GitHub"
|
||||
})
|
||||
);
|
||||
|
||||
export const GitHubDataSourceListItemSchema = z
|
||||
.object({
|
||||
name: z.literal("GitHub"),
|
||||
connection: z.literal(AppConnection.GitHubRadar),
|
||||
type: z.literal(SecretScanningDataSource.GitHub)
|
||||
})
|
||||
.describe(
|
||||
JSON.stringify({
|
||||
title: "GitHub"
|
||||
})
|
||||
);
|
||||
|
||||
export const GitHubFindingSchema = BaseSecretScanningFindingSchema.extend({
|
||||
resourceType: z.literal(SecretScanningResource.Repository),
|
||||
dataSourceType: z.literal(SecretScanningDataSource.GitHub),
|
||||
details: GitRepositoryScanFindingDetailsSchema
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
import { PushEvent } from "@octokit/webhooks-types";
|
||||
|
||||
import { TSecretScanningV2DALFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-dal";
|
||||
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { TSecretScanningV2QueueServiceFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-queue";
|
||||
import { logger } from "@app/lib/logger";
|
||||
|
||||
export const githubSecretScanningService = (
|
||||
secretScanningV2DAL: TSecretScanningV2DALFactory,
|
||||
secretScanningV2Queue: Pick<TSecretScanningV2QueueServiceFactory, "queueResourceDiffScan">
|
||||
) => {
|
||||
const handleInstallationDeletedEvent = async (installationId: number) => {
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findOne({
|
||||
externalId: String(installationId)
|
||||
});
|
||||
|
||||
if (!dataSource) {
|
||||
logger.error(
|
||||
`secretScanningV2RemoveEvent: GitHub - Could not find data source [installationId=${installationId}]`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// scott: maybe add disabled col instead?
|
||||
await secretScanningV2DAL.resources.delete({
|
||||
dataSourceId: dataSource.id
|
||||
});
|
||||
};
|
||||
|
||||
const handlePushEvent = async (payload: PushEvent) => {
|
||||
const { commits, repository, installation } = payload;
|
||||
|
||||
if (!commits || !repository || !installation) {
|
||||
logger.warn(
|
||||
`secretScanningV2PushEvent: GitHub - Insufficient data [commits=${commits?.length ?? 0}] [repository=${repository.name}] [installationId=${installation?.id}]`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findOne({
|
||||
externalId: String(installation.id)
|
||||
});
|
||||
|
||||
if (!dataSource) {
|
||||
logger.error(
|
||||
`secretScanningV2PushEvent: GitHub - Could not find data source [installationId=${installation.id}]`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await secretScanningV2Queue.queueResourceDiffScan({
|
||||
dataSourceType: SecretScanningDataSource.GitHub,
|
||||
payload,
|
||||
dataSourceId: dataSource.id
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
handlePushEvent,
|
||||
handleInstallationDeleted: handleInstallationDeletedEvent
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,30 @@
|
||||
import { PushEvent } from "@octokit/webhooks-types";
|
||||
import { z } from "zod";
|
||||
|
||||
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { TGitHubRadarConnection } from "@app/services/app-connection/github-radar";
|
||||
|
||||
import {
|
||||
CreateGitHubDataSourceSchema,
|
||||
GitHubDataSourceListItemSchema,
|
||||
GitHubDataSourceSchema,
|
||||
GitHubFindingSchema
|
||||
} from "./github-secret-scanning-schemas";
|
||||
|
||||
export type TGitHubDataSource = z.infer<typeof GitHubDataSourceSchema>;
|
||||
|
||||
export type TGitHubDataSourceInput = z.infer<typeof CreateGitHubDataSourceSchema>;
|
||||
|
||||
export type TGitHubDataSourceListItem = z.infer<typeof GitHubDataSourceListItemSchema>;
|
||||
|
||||
export type TGitHubFinding = z.infer<typeof GitHubFindingSchema>;
|
||||
|
||||
export type TGitHubDataSourceWithConnection = TGitHubDataSource & {
|
||||
connection: TGitHubRadarConnection;
|
||||
};
|
||||
|
||||
export type TQueueGitHubResourceDiffScan = {
|
||||
dataSourceType: SecretScanningDataSource.GitHub;
|
||||
payload: PushEvent;
|
||||
dataSourceId: string;
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from "./github-secret-scanning-constants";
|
||||
export * from "./github-secret-scanning-schemas";
|
||||
export * from "./github-secret-scanning-types";
|
||||
@@ -0,0 +1,458 @@
|
||||
import { Knex } from "knex";
|
||||
|
||||
import { TDbClient } from "@app/db";
|
||||
import {
|
||||
SecretScanningResourcesSchema,
|
||||
SecretScanningScansSchema,
|
||||
TableName,
|
||||
TSecretScanningDataSources
|
||||
} from "@app/db/schemas";
|
||||
import { SecretScanningFindingStatus } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { DatabaseError } from "@app/lib/errors";
|
||||
import {
|
||||
buildFindFilter,
|
||||
ormify,
|
||||
prependTableNameToFindFilter,
|
||||
selectAllTableCols,
|
||||
sqlNestRelationships,
|
||||
TFindOpt
|
||||
} from "@app/lib/knex";
|
||||
|
||||
export type TSecretScanningV2DALFactory = ReturnType<typeof secretScanningV2DALFactory>;
|
||||
|
||||
type TSecretScanningDataSourceFindFilter = Parameters<typeof buildFindFilter<TSecretScanningDataSources>>[0];
|
||||
type TSecretScanningDataSourceFindOptions = TFindOpt<TSecretScanningDataSources, true, "name">;
|
||||
|
||||
const baseSecretScanningDataSourceQuery = ({
|
||||
filter = {},
|
||||
db,
|
||||
tx
|
||||
}: {
|
||||
db: TDbClient;
|
||||
filter?: TSecretScanningDataSourceFindFilter;
|
||||
options?: TSecretScanningDataSourceFindOptions;
|
||||
tx?: Knex;
|
||||
}) => {
|
||||
const query = (tx || db.replicaNode())(TableName.SecretScanningDataSource)
|
||||
.join(
|
||||
TableName.AppConnection,
|
||||
`${TableName.SecretScanningDataSource}.connectionId`,
|
||||
`${TableName.AppConnection}.id`
|
||||
)
|
||||
.select(selectAllTableCols(TableName.SecretScanningDataSource))
|
||||
.select(
|
||||
// entire connection
|
||||
db.ref("name").withSchema(TableName.AppConnection).as("connectionName"),
|
||||
db.ref("method").withSchema(TableName.AppConnection).as("connectionMethod"),
|
||||
db.ref("app").withSchema(TableName.AppConnection).as("connectionApp"),
|
||||
db.ref("orgId").withSchema(TableName.AppConnection).as("connectionOrgId"),
|
||||
db.ref("encryptedCredentials").withSchema(TableName.AppConnection).as("connectionEncryptedCredentials"),
|
||||
db.ref("description").withSchema(TableName.AppConnection).as("connectionDescription"),
|
||||
db.ref("version").withSchema(TableName.AppConnection).as("connectionVersion"),
|
||||
db.ref("createdAt").withSchema(TableName.AppConnection).as("connectionCreatedAt"),
|
||||
db.ref("updatedAt").withSchema(TableName.AppConnection).as("connectionUpdatedAt"),
|
||||
db
|
||||
.ref("isPlatformManagedCredentials")
|
||||
.withSchema(TableName.AppConnection)
|
||||
.as("connectionIsPlatformManagedCredentials")
|
||||
);
|
||||
|
||||
if (filter) {
|
||||
/* eslint-disable @typescript-eslint/no-misused-promises */
|
||||
void query.where(buildFindFilter(prependTableNameToFindFilter(TableName.SecretScanningDataSource, filter)));
|
||||
}
|
||||
|
||||
return query;
|
||||
};
|
||||
|
||||
const expandSecretScanningDataSource = <
|
||||
T extends Awaited<ReturnType<typeof baseSecretScanningDataSourceQuery>>[number]
|
||||
>(
|
||||
dataSource: T
|
||||
) => {
|
||||
const {
|
||||
connectionApp,
|
||||
connectionName,
|
||||
connectionId,
|
||||
connectionOrgId,
|
||||
connectionEncryptedCredentials,
|
||||
connectionMethod,
|
||||
connectionDescription,
|
||||
connectionCreatedAt,
|
||||
connectionUpdatedAt,
|
||||
connectionVersion,
|
||||
connectionIsPlatformManagedCredentials,
|
||||
...el
|
||||
} = dataSource;
|
||||
|
||||
return {
|
||||
...el,
|
||||
connectionId,
|
||||
connection: connectionId
|
||||
? {
|
||||
app: connectionApp,
|
||||
id: connectionId,
|
||||
name: connectionName,
|
||||
orgId: connectionOrgId,
|
||||
encryptedCredentials: connectionEncryptedCredentials,
|
||||
method: connectionMethod,
|
||||
description: connectionDescription,
|
||||
createdAt: connectionCreatedAt,
|
||||
updatedAt: connectionUpdatedAt,
|
||||
version: connectionVersion,
|
||||
isPlatformManagedCredentials: connectionIsPlatformManagedCredentials
|
||||
}
|
||||
: undefined
|
||||
};
|
||||
};
|
||||
|
||||
export const secretScanningV2DALFactory = (db: TDbClient) => {
|
||||
const dataSourceOrm = ormify(db, TableName.SecretScanningDataSource);
|
||||
const resourceOrm = ormify(db, TableName.SecretScanningResource);
|
||||
const scanOrm = ormify(db, TableName.SecretScanningScan);
|
||||
const findingOrm = ormify(db, TableName.SecretScanningFinding);
|
||||
|
||||
const findDataSource = async (filter: Parameters<(typeof dataSourceOrm)["find"]>[0], tx?: Knex) => {
|
||||
try {
|
||||
const dataSources = await baseSecretScanningDataSourceQuery({ filter, db, tx });
|
||||
|
||||
if (!dataSources.length) return [];
|
||||
|
||||
return dataSources.map(expandSecretScanningDataSource);
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find - Secret Scanning Data Source" });
|
||||
}
|
||||
};
|
||||
|
||||
const findDataSourceById = async (id: string, tx?: Knex) => {
|
||||
try {
|
||||
const dataSource = await baseSecretScanningDataSourceQuery({ filter: { id }, db, tx }).first();
|
||||
|
||||
if (dataSource) return expandSecretScanningDataSource(dataSource);
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find By ID - Secret Scanning Data Source" });
|
||||
}
|
||||
};
|
||||
|
||||
const createDataSource = async (data: Parameters<(typeof dataSourceOrm)["create"]>[0], tx?: Knex) => {
|
||||
const source = await dataSourceOrm.create(data, tx);
|
||||
|
||||
const dataSource = (await baseSecretScanningDataSourceQuery({
|
||||
filter: { id: source.id },
|
||||
db,
|
||||
tx
|
||||
}).first())!;
|
||||
|
||||
return expandSecretScanningDataSource(dataSource);
|
||||
};
|
||||
|
||||
const updateDataSourceById = async (
|
||||
dataSourceId: string,
|
||||
data: Parameters<(typeof dataSourceOrm)["updateById"]>[1],
|
||||
tx?: Knex
|
||||
) => {
|
||||
const source = await dataSourceOrm.updateById(dataSourceId, data, tx);
|
||||
|
||||
const dataSource = (await baseSecretScanningDataSourceQuery({
|
||||
filter: { id: source.id },
|
||||
db,
|
||||
tx
|
||||
}).first())!;
|
||||
|
||||
return expandSecretScanningDataSource(dataSource);
|
||||
};
|
||||
|
||||
const deleteDataSourceById = async (dataSourceId: string, tx?: Knex) => {
|
||||
const secretRotation = (await baseSecretScanningDataSourceQuery({
|
||||
filter: { id: dataSourceId },
|
||||
db,
|
||||
tx
|
||||
}).first())!;
|
||||
|
||||
await dataSourceOrm.deleteById(dataSourceId, tx);
|
||||
|
||||
return expandSecretScanningDataSource(secretRotation);
|
||||
};
|
||||
|
||||
const findOneDataSource = async (filter: Parameters<(typeof dataSourceOrm)["findOne"]>[0], tx?: Knex) => {
|
||||
try {
|
||||
const secretRotation = await baseSecretScanningDataSourceQuery({ filter, db, tx }).first();
|
||||
|
||||
if (secretRotation) {
|
||||
return expandSecretScanningDataSource(secretRotation);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find One - Secret Rotation V2" });
|
||||
}
|
||||
};
|
||||
|
||||
const findDataSourceWithDetails = async (filter: Parameters<(typeof dataSourceOrm)["find"]>[0], tx?: Knex) => {
|
||||
try {
|
||||
// TODO (scott): this query will probably need to be optimized
|
||||
|
||||
const dataSources = await baseSecretScanningDataSourceQuery({ filter, db, tx })
|
||||
.leftJoin(
|
||||
TableName.SecretScanningResource,
|
||||
`${TableName.SecretScanningResource}.dataSourceId`,
|
||||
`${TableName.SecretScanningDataSource}.id`
|
||||
)
|
||||
.leftJoin(
|
||||
TableName.SecretScanningScan,
|
||||
`${TableName.SecretScanningScan}.resourceId`,
|
||||
`${TableName.SecretScanningResource}.id`
|
||||
)
|
||||
.leftJoin(
|
||||
TableName.SecretScanningFinding,
|
||||
`${TableName.SecretScanningFinding}.scanId`,
|
||||
`${TableName.SecretScanningScan}.id`
|
||||
)
|
||||
.where((qb) => {
|
||||
void qb
|
||||
.where(`${TableName.SecretScanningFinding}.status`, SecretScanningFindingStatus.Unresolved)
|
||||
.orWhereNull(`${TableName.SecretScanningFinding}.status`);
|
||||
})
|
||||
.select(
|
||||
db.ref("id").withSchema(TableName.SecretScanningScan).as("scanId"),
|
||||
db.ref("status").withSchema(TableName.SecretScanningScan).as("scanStatus"),
|
||||
db.ref("statusMessage").withSchema(TableName.SecretScanningScan).as("scanStatusMessage"),
|
||||
db.ref("createdAt").withSchema(TableName.SecretScanningScan).as("scanCreatedAt"),
|
||||
db.ref("status").withSchema(TableName.SecretScanningFinding).as("findingStatus"),
|
||||
db.ref("id").withSchema(TableName.SecretScanningFinding).as("findingId")
|
||||
);
|
||||
|
||||
if (!dataSources.length) return [];
|
||||
|
||||
const results = sqlNestRelationships({
|
||||
data: dataSources,
|
||||
key: "id",
|
||||
parentMapper: (dataSource) => expandSecretScanningDataSource(dataSource),
|
||||
childrenMapper: [
|
||||
{
|
||||
key: "scanId",
|
||||
label: "scans" as const,
|
||||
mapper: ({ scanId, scanCreatedAt, scanStatus, scanStatusMessage }) => ({
|
||||
id: scanId,
|
||||
createdAt: scanCreatedAt,
|
||||
status: scanStatus,
|
||||
statusMessage: scanStatusMessage
|
||||
})
|
||||
},
|
||||
{
|
||||
key: "findingId",
|
||||
label: "findings" as const,
|
||||
mapper: ({ findingId }) => ({
|
||||
id: findingId
|
||||
})
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
return results.map(({ scans, findings, ...dataSource }) => {
|
||||
const lastScan =
|
||||
scans && scans.length
|
||||
? scans.reduce((latest, current) => {
|
||||
return new Date(current.createdAt) > new Date(latest.createdAt) ? current : latest;
|
||||
})
|
||||
: null;
|
||||
|
||||
return {
|
||||
...dataSource,
|
||||
lastScanStatus: lastScan?.status ?? null,
|
||||
lastScanStatusMessage: lastScan?.statusMessage ?? null,
|
||||
lastScannedAt: lastScan?.createdAt ?? null,
|
||||
unresolvedFindings: scans.length ? findings.length : null
|
||||
};
|
||||
});
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find Data Source with Details - Secret Scanning V2" });
|
||||
}
|
||||
};
|
||||
|
||||
const findResourcesWithDetails = async (filter: Parameters<(typeof resourceOrm)["find"]>[0], tx?: Knex) => {
|
||||
try {
|
||||
// TODO (scott): this query will probably need to be optimized
|
||||
|
||||
const resources = await (tx || db.replicaNode())(TableName.SecretScanningResource)
|
||||
.where((qb) => {
|
||||
if (filter)
|
||||
void qb.where(buildFindFilter(prependTableNameToFindFilter(TableName.SecretScanningResource, filter)));
|
||||
})
|
||||
.leftJoin(
|
||||
TableName.SecretScanningScan,
|
||||
`${TableName.SecretScanningScan}.resourceId`,
|
||||
`${TableName.SecretScanningResource}.id`
|
||||
)
|
||||
.leftJoin(
|
||||
TableName.SecretScanningFinding,
|
||||
`${TableName.SecretScanningFinding}.scanId`,
|
||||
`${TableName.SecretScanningScan}.id`
|
||||
)
|
||||
.where((qb) => {
|
||||
void qb
|
||||
.where(`${TableName.SecretScanningFinding}.status`, SecretScanningFindingStatus.Unresolved)
|
||||
.orWhereNull(`${TableName.SecretScanningFinding}.status`);
|
||||
})
|
||||
.select(selectAllTableCols(TableName.SecretScanningResource))
|
||||
.select(
|
||||
db.ref("id").withSchema(TableName.SecretScanningScan).as("scanId"),
|
||||
db.ref("status").withSchema(TableName.SecretScanningScan).as("scanStatus"),
|
||||
db.ref("type").withSchema(TableName.SecretScanningScan).as("scanType"),
|
||||
db.ref("statusMessage").withSchema(TableName.SecretScanningScan).as("scanStatusMessage"),
|
||||
db.ref("createdAt").withSchema(TableName.SecretScanningScan).as("scanCreatedAt"),
|
||||
db.ref("status").withSchema(TableName.SecretScanningFinding).as("findingStatus"),
|
||||
db.ref("id").withSchema(TableName.SecretScanningFinding).as("findingId")
|
||||
);
|
||||
|
||||
if (!resources.length) return [];
|
||||
|
||||
const results = sqlNestRelationships({
|
||||
data: resources,
|
||||
key: "id",
|
||||
parentMapper: (resource) => SecretScanningResourcesSchema.parse(resource),
|
||||
childrenMapper: [
|
||||
{
|
||||
key: "scanId",
|
||||
label: "scans" as const,
|
||||
mapper: ({ scanId, scanCreatedAt, scanStatus, scanStatusMessage, scanType }) => ({
|
||||
id: scanId,
|
||||
type: scanType,
|
||||
createdAt: scanCreatedAt,
|
||||
status: scanStatus,
|
||||
statusMessage: scanStatusMessage
|
||||
})
|
||||
},
|
||||
{
|
||||
key: "findingId",
|
||||
label: "findings" as const,
|
||||
mapper: ({ findingId }) => ({
|
||||
id: findingId
|
||||
})
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
return results.map(({ scans, findings, ...resource }) => {
|
||||
const lastScan =
|
||||
scans && scans.length
|
||||
? scans.reduce((latest, current) => {
|
||||
return new Date(current.createdAt) > new Date(latest.createdAt) ? current : latest;
|
||||
})
|
||||
: null;
|
||||
|
||||
return {
|
||||
...resource,
|
||||
lastScanStatus: lastScan?.status ?? null,
|
||||
lastScanStatusMessage: lastScan?.statusMessage ?? null,
|
||||
lastScannedAt: lastScan?.createdAt ?? null,
|
||||
unresolvedFindings: findings?.length ?? 0
|
||||
};
|
||||
});
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find Resource with Details - Secret Scanning V2" });
|
||||
}
|
||||
};
|
||||
|
||||
const findScansWithDetailsByDataSourceId = async (dataSourceId: string, tx?: Knex) => {
|
||||
try {
|
||||
// TODO (scott): this query will probably need to be optimized
|
||||
|
||||
const scans = await (tx || db.replicaNode())(TableName.SecretScanningScan)
|
||||
.leftJoin(
|
||||
TableName.SecretScanningResource,
|
||||
`${TableName.SecretScanningResource}.id`,
|
||||
`${TableName.SecretScanningScan}.resourceId`
|
||||
)
|
||||
.where(`${TableName.SecretScanningResource}.dataSourceId`, dataSourceId)
|
||||
.leftJoin(
|
||||
TableName.SecretScanningFinding,
|
||||
`${TableName.SecretScanningFinding}.scanId`,
|
||||
`${TableName.SecretScanningScan}.id`
|
||||
)
|
||||
.select(selectAllTableCols(TableName.SecretScanningScan))
|
||||
.select(
|
||||
db.ref("status").withSchema(TableName.SecretScanningFinding).as("findingStatus"),
|
||||
db.ref("id").withSchema(TableName.SecretScanningFinding).as("findingId"),
|
||||
db.ref("name").withSchema(TableName.SecretScanningResource).as("resourceName")
|
||||
);
|
||||
|
||||
if (!scans.length) return [];
|
||||
|
||||
const results = sqlNestRelationships({
|
||||
data: scans,
|
||||
key: "id",
|
||||
parentMapper: (scan) => SecretScanningScansSchema.parse(scan),
|
||||
childrenMapper: [
|
||||
{
|
||||
key: "findingId",
|
||||
label: "findings" as const,
|
||||
mapper: ({ findingId, findingStatus }) => ({
|
||||
id: findingId,
|
||||
status: findingStatus
|
||||
})
|
||||
},
|
||||
{
|
||||
key: "resourceId",
|
||||
label: "resources" as const,
|
||||
mapper: ({ resourceName }) => ({
|
||||
name: resourceName
|
||||
})
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
return results.map(({ findings, resources, ...scan }) => {
|
||||
return {
|
||||
...scan,
|
||||
unresolvedFindings:
|
||||
findings?.filter((finding) => finding.status === SecretScanningFindingStatus.Unresolved).length ?? 0,
|
||||
resolvedFindings:
|
||||
findings?.filter((finding) => finding.status === SecretScanningFindingStatus.Resolved).length ?? 0,
|
||||
resourceName: resources[0].name
|
||||
};
|
||||
});
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find Scan with Details By Data Source ID - Secret Scanning V2" });
|
||||
}
|
||||
};
|
||||
|
||||
const findScansByDataSourceId = async (dataSourceId: string, tx?: Knex) => {
|
||||
try {
|
||||
const scans = await (tx || db.replicaNode())(TableName.SecretScanningScan)
|
||||
.leftJoin(
|
||||
TableName.SecretScanningResource,
|
||||
`${TableName.SecretScanningResource}.id`,
|
||||
`${TableName.SecretScanningScan}.resourceId`
|
||||
)
|
||||
.where(`${TableName.SecretScanningResource}.dataSourceId`, dataSourceId)
|
||||
|
||||
.select(selectAllTableCols(TableName.SecretScanningScan));
|
||||
|
||||
return scans;
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find Scan By Data Source ID - Secret Scanning V2" });
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
dataSources: {
|
||||
...dataSourceOrm,
|
||||
find: findDataSource,
|
||||
findById: findDataSourceById,
|
||||
findOne: findOneDataSource,
|
||||
create: createDataSource,
|
||||
updateById: updateDataSourceById,
|
||||
deleteById: deleteDataSourceById,
|
||||
findWithDetails: findDataSourceWithDetails
|
||||
},
|
||||
resources: {
|
||||
...resourceOrm,
|
||||
findWithDetails: findResourcesWithDetails
|
||||
},
|
||||
scans: {
|
||||
...scanOrm,
|
||||
findWithDetailsByDataSourceId: findScansWithDetailsByDataSourceId,
|
||||
findByDataSourceId: findScansByDataSourceId
|
||||
},
|
||||
findings: findingOrm
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,33 @@
|
||||
export enum SecretScanningDataSource {
|
||||
GitHub = "github"
|
||||
}
|
||||
|
||||
export enum SecretScanningScanStatus {
|
||||
Completed = "completed",
|
||||
Failed = "failed",
|
||||
Queued = "queued",
|
||||
Scanning = "scanning"
|
||||
}
|
||||
|
||||
export enum SecretScanningScanType {
|
||||
FullScan = "full-scan",
|
||||
DiffScan = "diff-scan"
|
||||
}
|
||||
|
||||
export enum SecretScanningFindingStatus {
|
||||
Resolved = "resolved",
|
||||
Unresolved = "unresolved",
|
||||
FalsePositive = "false-positive",
|
||||
Ignore = "ignore"
|
||||
}
|
||||
|
||||
export enum SecretScanningResource {
|
||||
Repository = "repository",
|
||||
Project = "project"
|
||||
}
|
||||
|
||||
export enum SecretScanningFindingSeverity {
|
||||
High = "high",
|
||||
Medium = "medium",
|
||||
Low = "low"
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { GitHubSecretScanningFactory } from "@app/ee/services/secret-scanning-v2/github/github-secret-scanning-factory";
|
||||
|
||||
import { SecretScanningDataSource } from "./secret-scanning-v2-enums";
|
||||
import {
|
||||
TQueueSecretScanningResourceDiffScan,
|
||||
TSecretScanningDataSourceCredentials,
|
||||
TSecretScanningDataSourceWithConnection,
|
||||
TSecretScanningFactory
|
||||
} from "./secret-scanning-v2-types";
|
||||
|
||||
type TSecretScanningFactoryImplementation = TSecretScanningFactory<
|
||||
TSecretScanningDataSourceWithConnection,
|
||||
TSecretScanningDataSourceCredentials,
|
||||
TQueueSecretScanningResourceDiffScan["payload"]
|
||||
>;
|
||||
|
||||
export const SECRET_SCANNING_FACTORY_MAP: Record<SecretScanningDataSource, TSecretScanningFactoryImplementation> = {
|
||||
[SecretScanningDataSource.GitHub]: GitHubSecretScanningFactory as TSecretScanningFactoryImplementation
|
||||
};
|
||||
@@ -0,0 +1,102 @@
|
||||
import { AxiosError } from "axios";
|
||||
import { exec } from "child_process";
|
||||
|
||||
import { readFindingsFile } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
|
||||
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
|
||||
import { GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/github";
|
||||
|
||||
import { SecretScanningDataSource, SecretScanningFindingSeverity } from "./secret-scanning-v2-enums";
|
||||
import { TCloneRepository, TGetFindingsPayload, TSecretScanningDataSourceListItem } from "./secret-scanning-v2-types";
|
||||
|
||||
const SECRET_SCANNING_SOURCE_LIST_OPTIONS: Record<SecretScanningDataSource, TSecretScanningDataSourceListItem> = {
|
||||
[SecretScanningDataSource.GitHub]: GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION
|
||||
};
|
||||
|
||||
export const listSecretScanningDataSourceOptions = () => {
|
||||
return Object.values(SECRET_SCANNING_SOURCE_LIST_OPTIONS).sort((a, b) => a.name.localeCompare(b.name));
|
||||
};
|
||||
|
||||
export const cloneRepository = async ({ cloneUrl, repoPath }: TCloneRepository): Promise<void> => {
|
||||
const command = `git clone ${cloneUrl} ${repoPath} --bare`;
|
||||
return new Promise((resolve, reject) => {
|
||||
exec(command, (error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
export function scanDirectory(inputPath: string, outputPath: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const command = `cd ${inputPath} && infisical scan --exit-code=77 -r "${outputPath}"`;
|
||||
exec(command, (error) => {
|
||||
if (error && error.code !== 77) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export const titleCaseToCamelCase = (obj: unknown): unknown => {
|
||||
if (typeof obj !== "object" || obj === null) {
|
||||
return obj;
|
||||
}
|
||||
|
||||
if (Array.isArray(obj)) {
|
||||
return obj.map((item: object) => titleCaseToCamelCase(item));
|
||||
}
|
||||
|
||||
const result: Record<string, unknown> = {};
|
||||
|
||||
for (const key in obj) {
|
||||
if (Object.prototype.hasOwnProperty.call(obj, key)) {
|
||||
const camelKey = key.charAt(0).toLowerCase() + key.slice(1);
|
||||
result[camelKey] = titleCaseToCamelCase((obj as Record<string, unknown>)[key]);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
export const scanGitRepositoryAndGetFindings = async (scanPath: string, findingsPath: string): TGetFindingsPayload => {
|
||||
await scanDirectory(scanPath, findingsPath);
|
||||
|
||||
const findingsData = JSON.parse(await readFindingsFile(findingsPath)) as SecretMatch[];
|
||||
|
||||
return findingsData.map(
|
||||
({
|
||||
// discard match and secret as we don't want to store
|
||||
Match,
|
||||
Secret,
|
||||
...finding
|
||||
}) => ({
|
||||
details: titleCaseToCamelCase(finding),
|
||||
fingerprint: finding.Fingerprint,
|
||||
severity: SecretScanningFindingSeverity.High,
|
||||
rule: finding.RuleID
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
const MAX_MESSAGE_LENGTH = 1024;
|
||||
|
||||
export const parseScanErrorMessage = (err: unknown): string => {
|
||||
let errorMessage: string;
|
||||
|
||||
if (err instanceof AxiosError) {
|
||||
errorMessage = err?.response?.data
|
||||
? JSON.stringify(err?.response?.data)
|
||||
: (err?.message ?? "An unknown error occurred.");
|
||||
} else {
|
||||
errorMessage = (err as Error)?.message || "An unknown error occurred.";
|
||||
}
|
||||
|
||||
return errorMessage.length <= MAX_MESSAGE_LENGTH
|
||||
? errorMessage
|
||||
: `${errorMessage.substring(0, MAX_MESSAGE_LENGTH - 3)}...`;
|
||||
};
|
||||
@@ -0,0 +1,14 @@
|
||||
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
|
||||
|
||||
export const SECRET_SCANNING_DATA_SOURCE_NAME_MAP: Record<SecretScanningDataSource, string> = {
|
||||
[SecretScanningDataSource.GitHub]: "GitHub"
|
||||
};
|
||||
|
||||
export const SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP: Record<SecretScanningDataSource, AppConnection> = {
|
||||
[SecretScanningDataSource.GitHub]: AppConnection.GitHubRadar
|
||||
};
|
||||
|
||||
export const AUTO_SYNC_DESCRIPTION_HELPER: Record<SecretScanningDataSource, { verb: string; noun: string }> = {
|
||||
[SecretScanningDataSource.GitHub]: { verb: "push", noun: "repositories" }
|
||||
};
|
||||
@@ -0,0 +1,396 @@
|
||||
import { join } from "path";
|
||||
|
||||
import {
|
||||
createTempFolder,
|
||||
deleteTempFolder
|
||||
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
|
||||
import {
|
||||
parseScanErrorMessage,
|
||||
scanGitRepositoryAndGetFindings
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
|
||||
import { BadRequestError, InternalServerError } from "@app/lib/errors";
|
||||
import { logger } from "@app/lib/logger";
|
||||
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
|
||||
import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns";
|
||||
import { TAppConnection } from "@app/services/app-connection/app-connection-types";
|
||||
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
|
||||
|
||||
import { TSecretScanningV2DALFactory } from "./secret-scanning-v2-dal";
|
||||
import {
|
||||
SecretScanningDataSource,
|
||||
SecretScanningFindingStatus,
|
||||
SecretScanningResource,
|
||||
SecretScanningScanStatus,
|
||||
SecretScanningScanType
|
||||
} from "./secret-scanning-v2-enums";
|
||||
import { SECRET_SCANNING_FACTORY_MAP } from "./secret-scanning-v2-factory";
|
||||
import {
|
||||
TFindingsPayload,
|
||||
TQueueSecretScanningDataSourceFullScan,
|
||||
TQueueSecretScanningResourceDiffScan,
|
||||
TSecretScanningDataSourceWithConnection
|
||||
} from "./secret-scanning-v2-types";
|
||||
|
||||
type TSecretRotationV2QueueServiceFactoryDep = {
|
||||
queueService: TQueueServiceFactory;
|
||||
secretScanningV2DAL: TSecretScanningV2DALFactory;
|
||||
// smtpService: Pick<TSmtpService, "sendMail">;
|
||||
// projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
|
||||
// projectDAL: Pick<TProjectDALFactory, "findById">;
|
||||
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
|
||||
};
|
||||
|
||||
export type TSecretScanningV2QueueServiceFactory = Awaited<ReturnType<typeof secretScanningV2QueueServiceFactory>>;
|
||||
|
||||
export const secretScanningV2QueueServiceFactory = async ({
|
||||
queueService,
|
||||
secretScanningV2DAL,
|
||||
// projectMembershipDAL,
|
||||
// projectDAL,
|
||||
// smtpService,
|
||||
kmsService
|
||||
}: TSecretRotationV2QueueServiceFactoryDep) => {
|
||||
const queueDataSourceFullScan = async (
|
||||
dataSource: TSecretScanningDataSourceWithConnection,
|
||||
resourceExternalId?: string
|
||||
) => {
|
||||
try {
|
||||
const { type } = dataSource;
|
||||
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[type]();
|
||||
|
||||
const rawResources = await factory.listRawResources(dataSource);
|
||||
|
||||
let filteredRawResources = rawResources;
|
||||
|
||||
// TODO: should add indivial resource fetch to factory
|
||||
if (resourceExternalId) {
|
||||
filteredRawResources = rawResources.filter((resource) => resource.externalId === resourceExternalId);
|
||||
}
|
||||
|
||||
if (!filteredRawResources.length) {
|
||||
throw new BadRequestError({
|
||||
message: `${resourceExternalId ? `Resource with "ID" ${resourceExternalId} could not be found.` : "Data source has no resources to scan"}. Ensure your data source config is correct and not filtering out scanning resources.`
|
||||
});
|
||||
}
|
||||
|
||||
await secretScanningV2DAL.resources.transaction(async (tx) => {
|
||||
const resources = await secretScanningV2DAL.resources.upsert(
|
||||
filteredRawResources.map((rawResource) => ({
|
||||
...rawResource,
|
||||
dataSourceId: dataSource.id
|
||||
})),
|
||||
["externalId", "dataSourceId"],
|
||||
tx
|
||||
);
|
||||
|
||||
const scans = await secretScanningV2DAL.scans.insertMany(
|
||||
resources.map((resource) => ({
|
||||
resourceId: resource.id,
|
||||
type: SecretScanningScanType.FullScan
|
||||
})),
|
||||
tx
|
||||
);
|
||||
|
||||
for (const scan of scans) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2FullScan, {
|
||||
scanId: scan.id,
|
||||
resourceId: scan.resourceId,
|
||||
dataSourceId: dataSource.id
|
||||
});
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(error, `Failed to queue full-scan for data source with ID "${dataSource.id}"`);
|
||||
|
||||
if (error instanceof BadRequestError) throw error;
|
||||
|
||||
throw new InternalServerError({ message: `Failed to queue scan: ${(error as Error).message}` });
|
||||
}
|
||||
};
|
||||
|
||||
const queueResourceDiffScan = async (payload: TQueueSecretScanningResourceDiffScan) =>
|
||||
queueService.queuePg(QueueJobs.SecretScanningV2DiffScan, payload);
|
||||
|
||||
await queueService.startPg<QueueName.SecretScanningV2>(
|
||||
QueueJobs.SecretScanningV2FullScan,
|
||||
async ([job]) => {
|
||||
const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan;
|
||||
const { retryCount, retryLimit } = job;
|
||||
|
||||
const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
|
||||
|
||||
const tempFolder = await createTempFolder();
|
||||
|
||||
try {
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Scanning
|
||||
}
|
||||
);
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
|
||||
|
||||
const resource = await secretScanningV2DAL.resources.findById(resourceId);
|
||||
|
||||
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
|
||||
|
||||
let connection: TAppConnection | null = null;
|
||||
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
|
||||
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]();
|
||||
|
||||
const findingsPath = join(tempFolder, "findings.json");
|
||||
|
||||
const scanPath = await factory.getFullScanPath({
|
||||
dataSource: {
|
||||
...dataSource,
|
||||
connection
|
||||
} as TSecretScanningDataSourceWithConnection,
|
||||
resourceName: resource.name,
|
||||
tempFolder
|
||||
});
|
||||
|
||||
let findingsPayload: TFindingsPayload;
|
||||
switch (resource.type) {
|
||||
case SecretScanningResource.Repository:
|
||||
case SecretScanningResource.Project:
|
||||
findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath);
|
||||
break;
|
||||
default:
|
||||
throw new Error("Unhandled resource type");
|
||||
}
|
||||
|
||||
await secretScanningV2DAL.findings.transaction(async (tx) => {
|
||||
await secretScanningV2DAL.findings.upsert(
|
||||
findingsPayload.map((findings) => ({
|
||||
...findings,
|
||||
projectId: dataSource.projectId,
|
||||
dataSourceName: dataSource.name,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceName: resource.name,
|
||||
resourceType: resource.type,
|
||||
scanId,
|
||||
status: SecretScanningFindingStatus.Unresolved
|
||||
})),
|
||||
["projectId", "fingerprint"],
|
||||
tx,
|
||||
["resourceName", "dataSourceName", "status"]
|
||||
);
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Completed
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
// TODO: send notification
|
||||
|
||||
logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails}`);
|
||||
} catch (error) {
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
statusMessage: parseScanErrorMessage(error)
|
||||
}
|
||||
);
|
||||
|
||||
// TODO: send error notification
|
||||
|
||||
logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
|
||||
throw error;
|
||||
} finally {
|
||||
await deleteTempFolder(tempFolder);
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
|
||||
await queueService.startPg<QueueName.SecretScanningV2>(
|
||||
QueueJobs.SecretScanningV2DiffScan,
|
||||
async ([job]) => {
|
||||
const { payload, dataSourceId } = job.data as TQueueSecretScanningResourceDiffScan;
|
||||
const { retryCount, retryLimit } = job;
|
||||
|
||||
let scanId: string | undefined;
|
||||
let logDetails = `[dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
|
||||
|
||||
try {
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
|
||||
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]();
|
||||
|
||||
const resourcePayload = factory.getDiffScanResourcePayload(payload);
|
||||
|
||||
const { resourceId, resourceName, resourceType } = await secretScanningV2DAL.resources.transaction(
|
||||
async (tx) => {
|
||||
const [resource] = await secretScanningV2DAL.resources.upsert(
|
||||
[
|
||||
{
|
||||
...resourcePayload,
|
||||
dataSourceId
|
||||
}
|
||||
],
|
||||
["externalId", "dataSourceId"],
|
||||
tx
|
||||
);
|
||||
|
||||
const scan = await secretScanningV2DAL.scans.create(
|
||||
{
|
||||
resourceId: resource.id,
|
||||
type: SecretScanningScanType.DiffScan,
|
||||
status: SecretScanningScanStatus.Scanning
|
||||
},
|
||||
tx
|
||||
);
|
||||
|
||||
scanId = scan.id;
|
||||
|
||||
return {
|
||||
resourceId: resource.id,
|
||||
resourceName: resource.name,
|
||||
resourceType: resource.type
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
logDetails += ` [scanId=${scanId}] [resourceId=${resourceId}]`;
|
||||
|
||||
let connection: TAppConnection | null = null;
|
||||
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
|
||||
|
||||
const findingsPayload = await factory.getDiffScanFindingsPayload({
|
||||
dataSource: {
|
||||
...dataSource,
|
||||
connection
|
||||
} as TSecretScanningDataSourceWithConnection,
|
||||
resourceName,
|
||||
payload
|
||||
});
|
||||
|
||||
await secretScanningV2DAL.findings.transaction(async (tx) => {
|
||||
await secretScanningV2DAL.findings.upsert(
|
||||
findingsPayload.map((findings) => ({
|
||||
...findings,
|
||||
projectId: dataSource.projectId,
|
||||
dataSourceName: dataSource.name,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceName,
|
||||
resourceType,
|
||||
scanId,
|
||||
status: SecretScanningFindingStatus.Unresolved
|
||||
})),
|
||||
["projectId", "fingerprint"],
|
||||
tx,
|
||||
["resourceName", "dataSourceName", "status"]
|
||||
);
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Completed
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
// TODO: send notification
|
||||
|
||||
logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`);
|
||||
} catch (error) {
|
||||
if (scanId)
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
statusMessage: parseScanErrorMessage(error)
|
||||
}
|
||||
);
|
||||
|
||||
// TODO: send error notification
|
||||
|
||||
logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
|
||||
// await queueService.startPg<QueueName.SecretRotationV2>(
|
||||
// QueueJobs.SecretRotationV2SendNotification,
|
||||
// async ([job]) => {
|
||||
// const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload;
|
||||
// try {
|
||||
// const {
|
||||
// name: rotationName,
|
||||
// type,
|
||||
// projectId,
|
||||
// lastRotationAttemptedAt,
|
||||
// folder,
|
||||
// environment,
|
||||
// id: dataSourceId
|
||||
// } = secretRotation;
|
||||
//
|
||||
// logger.info(`secretRotationV2Queue: Sending Status Notification [dataSourceId=${dataSourceId}]`);
|
||||
//
|
||||
// const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
|
||||
// const project = await projectDAL.findById(projectId);
|
||||
//
|
||||
// const projectAdmins = projectMembers.filter((member) =>
|
||||
// member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
|
||||
// );
|
||||
//
|
||||
// const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation];
|
||||
//
|
||||
// await smtpService.sendMail({
|
||||
// recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
|
||||
// template: SmtpTemplates.SecretRotationFailed,
|
||||
// subjectLine: `Secret Rotation Failed`,
|
||||
// substitutions: {
|
||||
// rotationName,
|
||||
// rotationType,
|
||||
// content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date(
|
||||
// lastRotationAttemptedAt
|
||||
// ).toISOString()}. Please check the rotation status in Infisical for more details.`,
|
||||
// secretPath: folder.path,
|
||||
// environment: environment.name,
|
||||
// projectName: project.name,
|
||||
// rotationUrl: encodeURI(`${appCfg.SITE_URL}/secret-manager/${projectId}/secrets/${environment.slug}`)
|
||||
// }
|
||||
// });
|
||||
// } catch (error) {
|
||||
// logger.error(
|
||||
// error,
|
||||
// `secretRotationV2Queue: Failed to Send Status Notification [dataSourceId=${secretRotation.id}]`
|
||||
// );
|
||||
// throw error;
|
||||
// }
|
||||
// },
|
||||
// {
|
||||
// batchSize: 1,
|
||||
// workerCount: 2,
|
||||
// pollingIntervalSeconds: 1
|
||||
// }
|
||||
// );
|
||||
|
||||
return {
|
||||
queueDataSourceFullScan,
|
||||
queueResourceDiffScan
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,99 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { SecretScanningDataSourcesSchema, SecretScanningFindingsSchema } from "@app/db/schemas";
|
||||
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps";
|
||||
import { SecretScanningDataSources } from "@app/lib/api-docs";
|
||||
import { slugSchema } from "@app/server/lib/schemas";
|
||||
|
||||
type SecretScanningDataSourceSchemaOpts = {
|
||||
type: SecretScanningDataSource;
|
||||
isConnectionRequired: boolean;
|
||||
};
|
||||
|
||||
export const BaseSecretScanningDataSourceSchema = ({
|
||||
type,
|
||||
isConnectionRequired
|
||||
}: SecretScanningDataSourceSchemaOpts) =>
|
||||
SecretScanningDataSourcesSchema.omit({
|
||||
// unique to provider
|
||||
type: true,
|
||||
connectionId: true,
|
||||
config: true,
|
||||
encryptedCredentials: true
|
||||
}).extend({
|
||||
type: z.literal(type),
|
||||
connectionId: isConnectionRequired ? z.string().uuid() : z.null(),
|
||||
connection: isConnectionRequired
|
||||
? z.object({
|
||||
app: z.literal(SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[type]),
|
||||
name: z.string(),
|
||||
id: z.string().uuid()
|
||||
})
|
||||
: z.null()
|
||||
});
|
||||
|
||||
export const BaseCreateSecretScanningDataSourceSchema = ({
|
||||
type,
|
||||
isConnectionRequired
|
||||
}: SecretScanningDataSourceSchemaOpts) =>
|
||||
z.object({
|
||||
name: slugSchema({ field: "name" }).describe(SecretScanningDataSources.CREATE(type).name),
|
||||
projectId: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1, "Project ID required")
|
||||
.describe(SecretScanningDataSources.CREATE(type).projectId),
|
||||
description: z
|
||||
.string()
|
||||
.trim()
|
||||
.max(256, "Description cannot exceed 256 characters")
|
||||
.nullish()
|
||||
.describe(SecretScanningDataSources.CREATE(type).description),
|
||||
connectionId: isConnectionRequired
|
||||
? z.string().uuid().describe(SecretScanningDataSources.CREATE(type).connectionId)
|
||||
: z.undefined(),
|
||||
isAutoScanEnabled: z
|
||||
.boolean()
|
||||
.optional()
|
||||
.default(true)
|
||||
.describe(SecretScanningDataSources.CREATE(type).isAutoScanEnabled)
|
||||
});
|
||||
|
||||
export const BaseUpdateSecretScanningDataSourceSchema = (type: SecretScanningDataSource) =>
|
||||
z.object({
|
||||
name: slugSchema({ field: "name" }).describe(SecretScanningDataSources.UPDATE(type).name).optional(),
|
||||
description: z
|
||||
.string()
|
||||
.trim()
|
||||
.max(256, "Description cannot exceed 256 characters")
|
||||
.nullish()
|
||||
.describe(SecretScanningDataSources.UPDATE(type).description),
|
||||
isAutoScanEnabled: z.boolean().optional().describe(SecretScanningDataSources.UPDATE(type).isAutoScanEnabled)
|
||||
});
|
||||
|
||||
export const GitRepositoryScanFindingDetailsSchema = z.object({
|
||||
description: z.string(),
|
||||
startLine: z.number(),
|
||||
endLine: z.number(),
|
||||
startColumn: z.number(),
|
||||
endColumn: z.number(),
|
||||
file: z.string(),
|
||||
link: z.string(),
|
||||
symlinkFile: z.string(),
|
||||
commit: z.string(),
|
||||
entropy: z.number(),
|
||||
author: z.string(),
|
||||
email: z.string(),
|
||||
date: z.string(),
|
||||
message: z.string(),
|
||||
tags: z.string().array(),
|
||||
ruleID: z.string(),
|
||||
fingerprint: z.string()
|
||||
});
|
||||
|
||||
export const BaseSecretScanningFindingSchema = SecretScanningFindingsSchema.omit({
|
||||
dataSourceType: true,
|
||||
resourceType: true,
|
||||
details: true
|
||||
});
|
||||
@@ -0,0 +1,784 @@
|
||||
import { ForbiddenError } from "@casl/ability";
|
||||
|
||||
import { ActionProjectType } from "@app/db/schemas";
|
||||
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
|
||||
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
|
||||
import {
|
||||
ProjectPermissionSecretScanningDataSourceActions,
|
||||
ProjectPermissionSecretScanningFindingActions,
|
||||
ProjectPermissionSub
|
||||
} from "@app/ee/services/permission/project-permission";
|
||||
import { githubSecretScanningService } from "@app/ee/services/secret-scanning-v2/github/github-secret-scanning-service";
|
||||
import { SecretScanningFindingStatus } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { SECRET_SCANNING_FACTORY_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-factory";
|
||||
import { listSecretScanningDataSourceOptions } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
|
||||
import {
|
||||
SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP,
|
||||
SECRET_SCANNING_DATA_SOURCE_NAME_MAP
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps";
|
||||
import {
|
||||
TCreateSecretScanningDataSourceDTO,
|
||||
TDeleteSecretScanningDataSourceDTO,
|
||||
TFindSecretScanningDataSourceByIdDTO,
|
||||
TFindSecretScanningDataSourceByNameDTO,
|
||||
TListSecretScanningDataSourcesByProjectId,
|
||||
TSecretScanningDataSource,
|
||||
TSecretScanningDataSourceWithConnection,
|
||||
TSecretScanningDataSourceWithDetails,
|
||||
TSecretScanningFinding,
|
||||
TSecretScanningResourceWithDetails,
|
||||
TSecretScanningScanWithDetails,
|
||||
TTriggerSecretScanningDataSourceDTO,
|
||||
TUpdateSecretScanningDataSourceDTO,
|
||||
TUpdateSecretScanningFinding
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
|
||||
import { DatabaseErrorCode } from "@app/lib/error-codes";
|
||||
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
|
||||
import { OrgServiceActor } from "@app/lib/types";
|
||||
import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns";
|
||||
import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service";
|
||||
import { TAppConnection } from "@app/services/app-connection/app-connection-types";
|
||||
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
|
||||
import { KmsDataKey } from "@app/services/kms/kms-types";
|
||||
|
||||
import { TSecretScanningV2DALFactory } from "./secret-scanning-v2-dal";
|
||||
import { TSecretScanningV2QueueServiceFactory } from "./secret-scanning-v2-queue";
|
||||
|
||||
export type TSecretScanningV2ServiceFactoryDep = {
|
||||
secretScanningV2DAL: TSecretScanningV2DALFactory;
|
||||
appConnectionService: Pick<TAppConnectionServiceFactory, "connectAppConnectionById">;
|
||||
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission" | "getOrgPermission">;
|
||||
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
|
||||
// auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
|
||||
// keyStore: Pick<TKeyStoreFactory, "acquireLock" | "setItemWithExpiry" | "getItem">;
|
||||
// queueService: Pick<TQueueServiceFactory, "queuePg">;
|
||||
// appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update" | "updateById">;
|
||||
secretScanningV2Queue: Pick<
|
||||
TSecretScanningV2QueueServiceFactory,
|
||||
"queueDataSourceFullScan" | "queueResourceDiffScan"
|
||||
>;
|
||||
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
|
||||
};
|
||||
|
||||
export type TSecretScanningV2ServiceFactory = ReturnType<typeof secretScanningV2ServiceFactory>;
|
||||
|
||||
export const secretScanningV2ServiceFactory = ({
|
||||
secretScanningV2DAL,
|
||||
permissionService,
|
||||
appConnectionService,
|
||||
licenseService,
|
||||
// auditLogService,
|
||||
// keyStore,
|
||||
// queueService,
|
||||
// appConnectionDAL,
|
||||
secretScanningV2Queue,
|
||||
kmsService
|
||||
}: TSecretScanningV2ServiceFactoryDep) => {
|
||||
const $checkListSecretScanningDataSourcesByProjectIdPermissions = async (
|
||||
projectId: string,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Data Sources due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.Read,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
};
|
||||
|
||||
const listSecretScanningDataSourcesByProjectId = async (
|
||||
{ projectId, type }: TListSecretScanningDataSourcesByProjectId,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
await $checkListSecretScanningDataSourcesByProjectIdPermissions(projectId, actor);
|
||||
|
||||
const dataSources = await secretScanningV2DAL.dataSources.find({
|
||||
...(type && { type }),
|
||||
projectId
|
||||
});
|
||||
|
||||
return dataSources as TSecretScanningDataSource[];
|
||||
};
|
||||
|
||||
const listSecretScanningDataSourcesWithDetailsByProjectId = async (
|
||||
{ projectId, type }: TListSecretScanningDataSourcesByProjectId,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
await $checkListSecretScanningDataSourcesByProjectIdPermissions(projectId, actor);
|
||||
|
||||
const dataSources = await secretScanningV2DAL.dataSources.findWithDetails({
|
||||
...(type && { type }),
|
||||
projectId
|
||||
});
|
||||
|
||||
return dataSources as TSecretScanningDataSourceWithDetails[];
|
||||
};
|
||||
|
||||
const findSecretScanningDataSourceById = async (
|
||||
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.Read,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
return dataSource as TSecretScanningDataSource;
|
||||
};
|
||||
|
||||
const findSecretScanningDataSourceByName = async (
|
||||
{ type, sourceName, projectId }: TFindSecretScanningDataSourceByNameDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
// we prevent conflicting names within a folder
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findOne({
|
||||
name: sourceName,
|
||||
projectId
|
||||
});
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with name "${sourceName}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.Read,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSource.id}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
return dataSource as TSecretScanningDataSource;
|
||||
};
|
||||
|
||||
const createSecretScanningDataSource = async (
|
||||
payload: TCreateSecretScanningDataSourceDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to create Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: payload.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.Create,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
let connection: TAppConnection | null = null;
|
||||
if (payload.connectionId) {
|
||||
// validates permission to connect and app is valid for data source
|
||||
connection = await appConnectionService.connectAppConnectionById(
|
||||
SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[payload.type],
|
||||
payload.connectionId,
|
||||
actor
|
||||
);
|
||||
}
|
||||
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[payload.type]();
|
||||
|
||||
try {
|
||||
const createdDataSource = await factory.initialize(
|
||||
{
|
||||
payload,
|
||||
connection: connection as TSecretScanningDataSourceWithConnection["connection"],
|
||||
secretScanningV2DAL
|
||||
},
|
||||
async ({ credentials, externalId }) => {
|
||||
let encryptedCredentials: Buffer | null = null;
|
||||
|
||||
if (credentials) {
|
||||
const { encryptor } = await kmsService.createCipherPairWithDataKey({
|
||||
type: KmsDataKey.SecretManager,
|
||||
projectId: payload.projectId
|
||||
});
|
||||
|
||||
const { cipherTextBlob } = encryptor({
|
||||
plainText: Buffer.from(JSON.stringify(credentials))
|
||||
});
|
||||
|
||||
encryptedCredentials = cipherTextBlob;
|
||||
}
|
||||
|
||||
return secretScanningV2DAL.dataSources.transaction(async (tx) => {
|
||||
const dataSource = await secretScanningV2DAL.dataSources.create(
|
||||
{
|
||||
encryptedCredentials,
|
||||
externalId,
|
||||
...payload
|
||||
},
|
||||
tx
|
||||
);
|
||||
|
||||
await factory.postInitialization({
|
||||
payload,
|
||||
connection: connection as TSecretScanningDataSourceWithConnection["connection"],
|
||||
dataSourceId: dataSource.id,
|
||||
credentials
|
||||
});
|
||||
|
||||
return dataSource;
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
if (payload.isAutoScanEnabled) {
|
||||
try {
|
||||
await secretScanningV2Queue.queueDataSourceFullScan({
|
||||
...createdDataSource,
|
||||
connection
|
||||
} as TSecretScanningDataSourceWithConnection);
|
||||
} catch {
|
||||
// silently fail, don't want to block creation, they'll try scanning when they don't see anything and get the error
|
||||
}
|
||||
}
|
||||
|
||||
return createdDataSource as TSecretScanningDataSource;
|
||||
} catch (err) {
|
||||
if (err instanceof DatabaseError && (err.error as { code: string })?.code === DatabaseErrorCode.UniqueViolation) {
|
||||
throw new BadRequestError({
|
||||
message: `A Secret Scanning Data Source with the name "${payload.name}" already exists for the project with ID "${payload.projectId}"`
|
||||
});
|
||||
}
|
||||
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
const updateSecretScanningDataSource = async (
|
||||
{ type, dataSourceId, ...payload }: TUpdateSecretScanningDataSourceDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to update Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.Edit,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
try {
|
||||
const updatedDataSource = await secretScanningV2DAL.dataSources.updateById(dataSourceId, payload);
|
||||
|
||||
return updatedDataSource as TSecretScanningDataSource;
|
||||
} catch (err) {
|
||||
if (err instanceof DatabaseError && (err.error as { code: string })?.code === DatabaseErrorCode.UniqueViolation) {
|
||||
throw new BadRequestError({
|
||||
message: `A Secret Scanning Data Source with the name "${payload.name}" already exists for the project with ID "${dataSource.projectId}"`
|
||||
});
|
||||
}
|
||||
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
const deleteSecretScanningResource = async (
|
||||
{ type, dataSourceId }: TDeleteSecretScanningDataSourceDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to delete Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.Delete,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
// TODO: clean up webhooks
|
||||
|
||||
await secretScanningV2DAL.dataSources.deleteById(dataSourceId);
|
||||
|
||||
return dataSource as TSecretScanningDataSource;
|
||||
};
|
||||
|
||||
const triggerSecretScanningDataSourceScan = async (
|
||||
{ type, dataSourceId, resourceId }: TTriggerSecretScanningDataSourceDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to trigger scan for Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.TriggerScans,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
let connection: TAppConnection | null = null;
|
||||
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
|
||||
|
||||
let resourceExternalId: string | undefined;
|
||||
|
||||
if (resourceId) {
|
||||
const resource = await secretScanningV2DAL.resources.findOne({ id: resourceId, dataSourceId });
|
||||
if (!resource) {
|
||||
throw new NotFoundError({
|
||||
message: `Could not find Secret Scanning Resource with ID "${resourceId}" for Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
}
|
||||
resourceExternalId = resource.externalId;
|
||||
}
|
||||
|
||||
await secretScanningV2Queue.queueDataSourceFullScan(
|
||||
{
|
||||
...dataSource,
|
||||
connection
|
||||
} as TSecretScanningDataSourceWithConnection,
|
||||
resourceExternalId
|
||||
);
|
||||
|
||||
return dataSource as TSecretScanningDataSource;
|
||||
};
|
||||
|
||||
const listSecretScanningResourcesByDataSourceId = async (
|
||||
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Resources due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadResources,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
const resources = await secretScanningV2DAL.resources.find({
|
||||
dataSourceId
|
||||
});
|
||||
|
||||
return { resources, projectId: dataSource.projectId };
|
||||
};
|
||||
|
||||
const listSecretScanningScansByDataSourceId = async (
|
||||
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Resources due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
const scans = await secretScanningV2DAL.scans.findByDataSourceId(dataSourceId);
|
||||
|
||||
return { scans, projectId: dataSource.projectId };
|
||||
};
|
||||
|
||||
const listSecretScanningResourcesWithDetailsByDataSourceId = async (
|
||||
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Resources due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadResources,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
const resources = await secretScanningV2DAL.resources.findWithDetails({ dataSourceId });
|
||||
|
||||
return { resources: resources as TSecretScanningResourceWithDetails[], projectId: dataSource.projectId };
|
||||
};
|
||||
|
||||
const listSecretScanningScansWithDetailsByDataSourceId = async (
|
||||
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Scans due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
const scans = await secretScanningV2DAL.scans.findWithDetailsByDataSourceId(dataSourceId);
|
||||
|
||||
return { scans: scans as TSecretScanningScanWithDetails[], projectId: dataSource.projectId };
|
||||
};
|
||||
|
||||
const getSecretScanningUnresolvedFindingsCountByProjectId = async (projectId: string, actor: OrgServiceActor) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningFindingActions.Read,
|
||||
ProjectPermissionSub.SecretScanningFindings
|
||||
);
|
||||
|
||||
const [finding] = await secretScanningV2DAL.findings.find(
|
||||
{
|
||||
projectId,
|
||||
status: SecretScanningFindingStatus.Unresolved
|
||||
},
|
||||
{ count: true }
|
||||
);
|
||||
|
||||
return Number(finding?.count ?? 0);
|
||||
};
|
||||
|
||||
const listSecretScanningFindingsByProjectId = async (projectId: string, actor: OrgServiceActor) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningFindingActions.Read,
|
||||
ProjectPermissionSub.SecretScanningFindings
|
||||
);
|
||||
|
||||
const findings = await secretScanningV2DAL.findings.find({
|
||||
projectId
|
||||
});
|
||||
|
||||
return findings as TSecretScanningFinding[];
|
||||
};
|
||||
|
||||
const updateSecretScanningFindingById = async (
|
||||
{ findingId, remarks, status }: TUpdateSecretScanningFinding,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const finding = await secretScanningV2DAL.findings.findById(findingId);
|
||||
|
||||
if (!finding)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find Secret Scanning Finding with ID "${findingId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: finding.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningFindingActions.Resolve,
|
||||
ProjectPermissionSub.SecretScanningFindings
|
||||
);
|
||||
|
||||
const updatedFinding = await secretScanningV2DAL.findings.updateById(findingId, {
|
||||
remarks,
|
||||
status
|
||||
});
|
||||
|
||||
return { finding: updatedFinding as TSecretScanningFinding, projectId: finding.projectId };
|
||||
};
|
||||
|
||||
return {
|
||||
listSecretScanningDataSourceOptions,
|
||||
listSecretScanningDataSourcesByProjectId,
|
||||
listSecretScanningDataSourcesWithDetailsByProjectId,
|
||||
findSecretScanningDataSourceById,
|
||||
findSecretScanningDataSourceByName,
|
||||
createSecretScanningDataSource,
|
||||
updateSecretScanningDataSource,
|
||||
deleteSecretScanningResource,
|
||||
triggerSecretScanningDataSourceScan,
|
||||
listSecretScanningResourcesByDataSourceId,
|
||||
listSecretScanningScansByDataSourceId,
|
||||
listSecretScanningResourcesWithDetailsByDataSourceId,
|
||||
listSecretScanningScansWithDetailsByDataSourceId,
|
||||
getSecretScanningUnresolvedFindingsCountByProjectId,
|
||||
listSecretScanningFindingsByProjectId,
|
||||
updateSecretScanningFindingById,
|
||||
github: githubSecretScanningService(secretScanningV2DAL, secretScanningV2Queue)
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,171 @@
|
||||
import { TSecretScanningFindingsInsert, TSecretScanningResources, TSecretScanningScans } from "@app/db/schemas";
|
||||
import {
|
||||
TGitHubDataSource,
|
||||
TGitHubDataSourceInput,
|
||||
TGitHubDataSourceListItem,
|
||||
TGitHubDataSourceWithConnection,
|
||||
TGitHubFinding,
|
||||
TQueueGitHubResourceDiffScan
|
||||
} from "@app/ee/services/secret-scanning-v2/github";
|
||||
import { TSecretScanningV2DALFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-dal";
|
||||
import {
|
||||
SecretScanningDataSource,
|
||||
SecretScanningFindingStatus,
|
||||
SecretScanningScanStatus
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
|
||||
export type TSecretScanningDataSource = TGitHubDataSource;
|
||||
|
||||
export type TSecretScanningDataSourceWithDetails = TSecretScanningDataSource & {
|
||||
lastScannedAt?: Date | null;
|
||||
lastScanStatus?: SecretScanningScanStatus | null;
|
||||
lastScanStatusMessage?: string | null;
|
||||
unresolvedFindings: number;
|
||||
};
|
||||
|
||||
export type TSecretScanningResourceWithDetails = TSecretScanningResources & {
|
||||
lastScannedAt?: Date | null;
|
||||
lastScanStatus?: SecretScanningScanStatus | null;
|
||||
lastScanStatusMessage?: string | null;
|
||||
unresolvedFindings: number;
|
||||
};
|
||||
|
||||
export type TSecretScanningScanWithDetails = TSecretScanningScans & {
|
||||
unresolvedFindings: number;
|
||||
resolvedFindings: number;
|
||||
resourceName: string;
|
||||
};
|
||||
|
||||
export type TSecretScanningDataSourceWithConnection = TGitHubDataSourceWithConnection;
|
||||
|
||||
export type TSecretScanningDataSourceInput = TGitHubDataSourceInput;
|
||||
|
||||
export type TSecretScanningDataSourceListItem = TGitHubDataSourceListItem;
|
||||
|
||||
export type TSecretScanningFinding = TGitHubFinding;
|
||||
|
||||
export type TListSecretScanningDataSourcesByProjectId = {
|
||||
projectId: string;
|
||||
type?: SecretScanningDataSource;
|
||||
};
|
||||
|
||||
export type TFindSecretScanningDataSourceByIdDTO = {
|
||||
dataSourceId: string;
|
||||
type: SecretScanningDataSource;
|
||||
};
|
||||
|
||||
export type TFindSecretScanningDataSourceByNameDTO = {
|
||||
sourceName: string;
|
||||
projectId: string;
|
||||
type: SecretScanningDataSource;
|
||||
};
|
||||
|
||||
export type TCreateSecretScanningDataSourceDTO = Pick<
|
||||
TSecretScanningDataSource,
|
||||
"description" | "name" | "projectId"
|
||||
> & {
|
||||
connectionId?: string;
|
||||
type: SecretScanningDataSource;
|
||||
isAutoScanEnabled?: boolean;
|
||||
config: Partial<TSecretScanningDataSourceInput["config"]>;
|
||||
};
|
||||
|
||||
export type TUpdateSecretScanningDataSourceDTO = Partial<
|
||||
Omit<TCreateSecretScanningDataSourceDTO, "projectId" | "connectionId">
|
||||
> & {
|
||||
dataSourceId: string;
|
||||
type: SecretScanningDataSource;
|
||||
};
|
||||
|
||||
export type TDeleteSecretScanningDataSourceDTO = {
|
||||
type: SecretScanningDataSource;
|
||||
dataSourceId: string;
|
||||
};
|
||||
|
||||
export type TTriggerSecretScanningDataSourceDTO = {
|
||||
type: SecretScanningDataSource;
|
||||
dataSourceId: string;
|
||||
resourceId?: string;
|
||||
};
|
||||
|
||||
export type TQueueSecretScanningDataSourceFullScan = {
|
||||
dataSourceId: string;
|
||||
resourceId: string;
|
||||
scanId: string;
|
||||
};
|
||||
|
||||
export type TQueueSecretScanningResourceDiffScan = TQueueGitHubResourceDiffScan;
|
||||
|
||||
export type TCloneRepository = {
|
||||
cloneUrl: string;
|
||||
repoPath: string;
|
||||
};
|
||||
|
||||
export type TSecretScanningFactoryListRawResources<T extends TSecretScanningDataSourceWithConnection> = (
|
||||
dataSource: T
|
||||
) => Promise<Pick<TSecretScanningResources, "externalId" | "name" | "type">[]>;
|
||||
|
||||
export type TSecretScanningFactoryGetDiffScanResourcePayload<
|
||||
P extends TQueueSecretScanningResourceDiffScan["payload"]
|
||||
> = (payload: P) => Pick<TSecretScanningResources, "externalId" | "name" | "type">;
|
||||
|
||||
export type TSecretScanningFactoryGetFullScanPath<T extends TSecretScanningDataSourceWithConnection> = (parameters: {
|
||||
dataSource: T;
|
||||
resourceName: string;
|
||||
tempFolder: string;
|
||||
}) => Promise<string>;
|
||||
|
||||
export type TSecretScanningFactoryGetDiffScanFindingsPayload<
|
||||
T extends TSecretScanningDataSourceWithConnection,
|
||||
P extends TQueueSecretScanningResourceDiffScan["payload"]
|
||||
> = (parameters: { dataSource: T; resourceName: string; payload: P }) => Promise<TFindingsPayload>;
|
||||
|
||||
export type TSecretScanningDataSourceRaw = NonNullable<
|
||||
Awaited<ReturnType<TSecretScanningV2DALFactory["dataSources"]["findById"]>>
|
||||
>;
|
||||
|
||||
export type TSecretScanningFactoryInitialize<
|
||||
T extends TSecretScanningDataSourceWithConnection["connection"] | undefined = undefined,
|
||||
C extends TSecretScanningDataSourceCredentials = undefined
|
||||
> = (
|
||||
params: {
|
||||
payload: TCreateSecretScanningDataSourceDTO;
|
||||
connection: T;
|
||||
secretScanningV2DAL: TSecretScanningV2DALFactory;
|
||||
},
|
||||
callback: (parameters: { credentials?: C; externalId?: string }) => Promise<TSecretScanningDataSourceRaw>
|
||||
) => Promise<TSecretScanningDataSourceRaw>;
|
||||
|
||||
export type TSecretScanningFactoryPostInitialization<
|
||||
T extends TSecretScanningDataSourceWithConnection["connection"] | undefined = undefined,
|
||||
C extends TSecretScanningDataSourceCredentials = undefined
|
||||
> = (params: {
|
||||
payload: TCreateSecretScanningDataSourceDTO;
|
||||
connection: T;
|
||||
credentials: C;
|
||||
dataSourceId: string;
|
||||
}) => Promise<void>;
|
||||
|
||||
export type TSecretScanningFactory<
|
||||
T extends TSecretScanningDataSourceWithConnection,
|
||||
C extends TSecretScanningDataSourceCredentials,
|
||||
P extends TQueueSecretScanningResourceDiffScan["payload"]
|
||||
> = () => {
|
||||
listRawResources: TSecretScanningFactoryListRawResources<T>;
|
||||
getFullScanPath: TSecretScanningFactoryGetFullScanPath<T>;
|
||||
initialize: TSecretScanningFactoryInitialize<T["connection"] | undefined, C>;
|
||||
postInitialization: TSecretScanningFactoryPostInitialization<T["connection"] | undefined, C>;
|
||||
getDiffScanResourcePayload: TSecretScanningFactoryGetDiffScanResourcePayload<P>;
|
||||
getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<T, P>;
|
||||
};
|
||||
|
||||
export type TFindingsPayload = Pick<TSecretScanningFindingsInsert, "details" | "fingerprint" | "severity" | "rule">[];
|
||||
export type TGetFindingsPayload = Promise<TFindingsPayload>;
|
||||
|
||||
export type TUpdateSecretScanningFinding = {
|
||||
status: SecretScanningFindingStatus;
|
||||
remarks?: string | null;
|
||||
findingId: string;
|
||||
};
|
||||
|
||||
export type TSecretScanningDataSourceCredentials = undefined;
|
||||
@@ -0,0 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { GitHubDataSourceSchema, GitHubFindingSchema } from "@app/ee/services/secret-scanning-v2/github";
|
||||
|
||||
export const SecretScanningDataSourceSchema = z.discriminatedUnion("type", [GitHubDataSourceSchema]);
|
||||
|
||||
export const SecretScanningFindingSchema = z.discriminatedUnion("resourceType", [GitHubFindingSchema]);
|
||||
+1
@@ -9,6 +9,7 @@ export type SecretMatch = {
|
||||
Match: string;
|
||||
Secret: string;
|
||||
File: string;
|
||||
Link: string;
|
||||
SymlinkFile: string;
|
||||
Commit: string;
|
||||
Entropy: number;
|
||||
|
||||
Reference in New Issue
Block a user