Merge pull request #3677 from Infisical/secret-scanning-v2-pt-1

feature(secret-scanning-v2): secret scanning architecture and github data source
This commit is contained in:
Scott Wilson
2025-06-03 16:34:29 -07:00
committed by GitHub
344 changed files with 13130 additions and 1118 deletions
+17
View File
@@ -2,6 +2,10 @@ import {
registerSecretRotationV2Router,
SECRET_ROTATION_REGISTER_ROUTER_MAP
} from "@app/ee/routes/v2/secret-rotation-v2-routers";
import {
registerSecretScanningV2Router,
SECRET_SCANNING_REGISTER_ROUTER_MAP
} from "@app/ee/routes/v2/secret-scanning-v2-routers";
import { registerIdentityProjectAdditionalPrivilegeRouter } from "./identity-project-additional-privilege-router";
import { registerProjectRoleRouter } from "./project-role-router";
@@ -31,4 +35,17 @@ export const registerV2EERoutes = async (server: FastifyZodProvider) => {
},
{ prefix: "/secret-rotations" }
);
await server.register(
async (secretScanningV2Router) => {
// register generic secret scanning endpoints
await secretScanningV2Router.register(registerSecretScanningV2Router);
// register service-specific secret scanning endpoints (gitlab/github, etc.)
for await (const [type, router] of Object.entries(SECRET_SCANNING_REGISTER_ROUTER_MAP)) {
await secretScanningV2Router.register(router, { prefix: `data-sources/${type}` });
}
},
{ prefix: "/secret-scanning" }
);
};
@@ -0,0 +1,16 @@
import { registerSecretScanningEndpoints } from "@app/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints";
import {
CreateGitHubDataSourceSchema,
GitHubDataSourceSchema,
UpdateGitHubDataSourceSchema
} from "@app/ee/services/secret-scanning-v2/github";
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
export const registerGitHubSecretScanningRouter = async (server: FastifyZodProvider) =>
registerSecretScanningEndpoints({
type: SecretScanningDataSource.GitHub,
server,
responseSchema: GitHubDataSourceSchema,
createSchema: CreateGitHubDataSourceSchema,
updateSchema: UpdateGitHubDataSourceSchema
});
@@ -0,0 +1,12 @@
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { registerGitHubSecretScanningRouter } from "./github-secret-scanning-router";
export * from "./secret-scanning-v2-router";
export const SECRET_SCANNING_REGISTER_ROUTER_MAP: Record<
SecretScanningDataSource,
(server: FastifyZodProvider) => Promise<void>
> = {
[SecretScanningDataSource.GitHub]: registerGitHubSecretScanningRouter
};
@@ -0,0 +1,593 @@
import { z } from "zod";
import { SecretScanningResourcesSchema, SecretScanningScansSchema } from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import {
SecretScanningDataSource,
SecretScanningScanStatus
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { SECRET_SCANNING_DATA_SOURCE_NAME_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps";
import {
TSecretScanningDataSource,
TSecretScanningDataSourceInput
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { ApiDocsTags, SecretScanningDataSources } from "@app/lib/api-docs";
import { startsWithVowel } from "@app/lib/fn";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
export const registerSecretScanningEndpoints = <
T extends TSecretScanningDataSource,
I extends TSecretScanningDataSourceInput
>({
server,
type,
createSchema,
updateSchema,
responseSchema
}: {
type: SecretScanningDataSource;
server: FastifyZodProvider;
createSchema: z.ZodType<{
name: string;
projectId: string;
connectionId?: string;
config: Partial<I["config"]>;
description?: string | null;
isAutoScanEnabled?: boolean;
}>;
updateSchema: z.ZodType<{
name?: string;
config?: Partial<I["config"]>;
description?: string | null;
isAutoScanEnabled?: boolean;
}>;
responseSchema: z.ZodTypeAny;
}) => {
const sourceType = SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type];
server.route({
method: "GET",
url: `/`,
config: {
rateLimit: readLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: `List the ${sourceType} Data Sources for the specified project.`,
querystring: z.object({
projectId: z
.string()
.trim()
.min(1, "Project ID required")
.describe(SecretScanningDataSources.LIST(type).projectId)
}),
response: {
200: z.object({ dataSources: responseSchema.array() })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const {
query: { projectId }
} = req;
const dataSources = (await server.services.secretScanningV2.listSecretScanningDataSourcesByProjectId(
{ projectId, type },
req.permission
)) as T[];
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST,
metadata: {
type,
count: dataSources.length,
dataSourceIds: dataSources.map((source) => source.id)
}
}
});
return { dataSources };
}
});
server.route({
method: "GET",
url: "/:dataSourceId",
config: {
rateLimit: readLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: `Get the specified ${sourceType} Data Source by ID.`,
params: z.object({
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.GET_BY_ID(type).dataSourceId)
}),
response: {
200: z.object({ dataSource: responseSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { dataSourceId } = req.params;
const dataSource = (await server.services.secretScanningV2.findSecretScanningDataSourceById(
{ dataSourceId, type },
req.permission
)) as T;
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId: dataSource.projectId,
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_GET,
metadata: {
dataSourceId,
type
}
}
});
return { dataSource };
}
});
server.route({
method: "GET",
url: `/data-source-name/:dataSourceName`,
config: {
rateLimit: readLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: `Get the specified ${sourceType} Data Source by name and project ID.`,
params: z.object({
sourceName: z
.string()
.trim()
.min(1, "Data Source name required")
.describe(SecretScanningDataSources.GET_BY_NAME(type).sourceName)
}),
querystring: z.object({
projectId: z
.string()
.trim()
.min(1, "Project ID required")
.describe(SecretScanningDataSources.GET_BY_NAME(type).projectId)
}),
response: {
200: z.object({ dataSource: responseSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { sourceName } = req.params;
const { projectId } = req.query;
const dataSource = (await server.services.secretScanningV2.findSecretScanningDataSourceByName(
{ sourceName, projectId, type },
req.permission
)) as T;
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_GET,
metadata: {
dataSourceId: dataSource.id,
type
}
}
});
return { dataSource };
}
});
server.route({
method: "POST",
url: "/",
config: {
rateLimit: writeLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: `Create ${
startsWithVowel(sourceType) ? "an" : "a"
} ${sourceType} Data Source for the specified project.`,
body: createSchema,
response: {
200: z.object({ dataSource: responseSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const dataSource = (await server.services.secretScanningV2.createSecretScanningDataSource(
{ ...req.body, type },
req.permission
)) as T;
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId: dataSource.projectId,
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_CREATE,
metadata: {
dataSourceId: dataSource.id,
type,
...req.body
}
}
});
return { dataSource };
}
});
server.route({
method: "PATCH",
url: "/:dataSourceId",
config: {
rateLimit: writeLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: `Update the specified ${sourceType} Data Source.`,
params: z.object({
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.UPDATE(type).dataSourceId)
}),
body: updateSchema,
response: {
200: z.object({ dataSource: responseSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { dataSourceId } = req.params;
const dataSource = (await server.services.secretScanningV2.updateSecretScanningDataSource(
{ ...req.body, dataSourceId, type },
req.permission
)) as T;
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId: dataSource.projectId,
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_UPDATE,
metadata: {
dataSourceId,
type,
...req.body
}
}
});
return { dataSource };
}
});
server.route({
method: "DELETE",
url: `/:dataSourceId`,
config: {
rateLimit: writeLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: `Delete the specified ${sourceType} Data Source.`,
params: z.object({
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.DELETE(type).dataSourceId)
}),
response: {
200: z.object({ dataSource: responseSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { dataSourceId } = req.params;
const dataSource = (await server.services.secretScanningV2.deleteSecretScanningDataSource(
{ type, dataSourceId },
req.permission
)) as T;
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId: dataSource.projectId,
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_DELETE,
metadata: {
type,
dataSourceId
}
}
});
return { dataSource };
}
});
server.route({
method: "POST",
url: `/:dataSourceId/scan`,
config: {
rateLimit: writeLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: `Trigger a scan for the specified ${sourceType} Data Source.`,
params: z.object({
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.SCAN(type).dataSourceId)
}),
response: {
200: z.object({ dataSource: responseSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { dataSourceId } = req.params;
const dataSource = (await server.services.secretScanningV2.triggerSecretScanningDataSourceScan(
{ type, dataSourceId },
req.permission
)) as T;
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId: dataSource.projectId,
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_TRIGGER_SCAN,
metadata: {
type,
dataSourceId
}
}
});
return { dataSource };
}
});
server.route({
method: "POST",
url: `/:dataSourceId/resources/:resourceId/scan`,
config: {
rateLimit: writeLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: `Trigger a scan for the specified ${sourceType} Data Source resource.`,
params: z.object({
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.SCAN(type).dataSourceId),
resourceId: z.string().uuid().describe(SecretScanningDataSources.SCAN(type).resourceId)
}),
response: {
200: z.object({ dataSource: responseSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { dataSourceId, resourceId } = req.params;
const dataSource = (await server.services.secretScanningV2.triggerSecretScanningDataSourceScan(
{ type, dataSourceId, resourceId },
req.permission
)) as T;
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId: dataSource.projectId,
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_TRIGGER_SCAN,
metadata: {
type,
dataSourceId,
resourceId
}
}
});
return { dataSource };
}
});
server.route({
method: "GET",
url: "/:dataSourceId/resources",
config: {
rateLimit: readLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: `Get the resources associated with the specified ${sourceType} Data Source by ID.`,
params: z.object({
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.LIST_RESOURCES(type).dataSourceId)
}),
response: {
200: z.object({ resources: SecretScanningResourcesSchema.array() })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { dataSourceId } = req.params;
const { resources, projectId } = await server.services.secretScanningV2.listSecretScanningResourcesByDataSourceId(
{ dataSourceId, type },
req.permission
);
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_RESOURCE_LIST,
metadata: {
dataSourceId,
type,
resourceIds: resources.map((resource) => resource.id),
count: resources.length
}
}
});
return { resources };
}
});
server.route({
method: "GET",
url: "/:dataSourceId/scans",
config: {
rateLimit: readLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: `Get the scans associated with the specified ${sourceType} Data Source by ID.`,
params: z.object({
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.LIST_SCANS(type).dataSourceId)
}),
response: {
200: z.object({ scans: SecretScanningScansSchema.array() })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { dataSourceId } = req.params;
const { scans, projectId } = await server.services.secretScanningV2.listSecretScanningScansByDataSourceId(
{ dataSourceId, type },
req.permission
);
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_SCAN_LIST,
metadata: {
dataSourceId,
type,
count: scans.length
}
}
});
return { scans };
}
});
// not exposed, for UI only
server.route({
method: "GET",
url: "/:dataSourceId/resources-dashboard",
config: {
rateLimit: readLimit
},
schema: {
tags: [ApiDocsTags.SecretScanning],
params: z.object({
dataSourceId: z.string().uuid()
}),
response: {
200: z.object({
resources: SecretScanningResourcesSchema.extend({
lastScannedAt: z.date().nullish(),
lastScanStatus: z.nativeEnum(SecretScanningScanStatus).nullish(),
lastScanStatusMessage: z.string().nullish(),
unresolvedFindings: z.number()
}).array()
})
}
},
onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => {
const { dataSourceId } = req.params;
const { resources, projectId } =
await server.services.secretScanningV2.listSecretScanningResourcesWithDetailsByDataSourceId(
{ dataSourceId, type },
req.permission
);
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_RESOURCE_LIST,
metadata: {
dataSourceId,
type,
resourceIds: resources.map((resource) => resource.id),
count: resources.length
}
}
});
return { resources };
}
});
server.route({
method: "GET",
url: "/:dataSourceId/scans-dashboard",
config: {
rateLimit: readLimit
},
schema: {
tags: [ApiDocsTags.SecretScanning],
params: z.object({
dataSourceId: z.string().uuid()
}),
response: {
200: z.object({
scans: SecretScanningScansSchema.extend({
unresolvedFindings: z.number(),
resolvedFindings: z.number(),
resourceName: z.string()
}).array()
})
}
},
onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => {
const { dataSourceId } = req.params;
const { scans, projectId } =
await server.services.secretScanningV2.listSecretScanningScansWithDetailsByDataSourceId(
{ dataSourceId, type },
req.permission
);
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_SCAN_LIST,
metadata: {
dataSourceId,
type,
count: scans.length
}
}
});
return { scans };
}
});
};
@@ -0,0 +1,366 @@
import { z } from "zod";
import { SecretScanningConfigsSchema } from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { GitHubDataSourceListItemSchema } from "@app/ee/services/secret-scanning-v2/github";
import {
SecretScanningFindingStatus,
SecretScanningScanStatus
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
SecretScanningDataSourceSchema,
SecretScanningFindingSchema
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-union-schemas";
import {
ApiDocsTags,
SecretScanningConfigs,
SecretScanningDataSources,
SecretScanningFindings
} from "@app/lib/api-docs";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
const SecretScanningDataSourceOptionsSchema = z.discriminatedUnion("type", [GitHubDataSourceListItemSchema]);
export const registerSecretScanningV2Router = async (server: FastifyZodProvider) => {
server.route({
method: "GET",
url: "/data-sources/options",
config: {
rateLimit: readLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: "List the available Secret Scanning Data Source Options.",
response: {
200: z.object({
dataSourceOptions: SecretScanningDataSourceOptionsSchema.array()
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: () => {
const dataSourceOptions = server.services.secretScanningV2.listSecretScanningDataSourceOptions();
return { dataSourceOptions };
}
});
server.route({
method: "GET",
url: "/data-sources",
config: {
rateLimit: readLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: "List all the Secret Scanning Data Sources for the specified project.",
querystring: z.object({
projectId: z.string().trim().min(1, "Project ID required").describe(SecretScanningDataSources.LIST().projectId)
}),
response: {
200: z.object({ dataSources: SecretScanningDataSourceSchema.array() })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const {
query: { projectId },
permission
} = req;
const dataSources = await server.services.secretScanningV2.listSecretScanningDataSourcesByProjectId(
{ projectId },
permission
);
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST,
metadata: {
dataSourceIds: dataSources.map((dataSource) => dataSource.id),
count: dataSources.length
}
}
});
return { dataSources };
}
});
server.route({
method: "GET",
url: "/findings",
config: {
rateLimit: readLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: "List all the Secret Scanning Findings for the specified project.",
querystring: z.object({
projectId: z.string().trim().min(1, "Project ID required").describe(SecretScanningFindings.LIST.projectId)
}),
response: {
200: z.object({ findings: SecretScanningFindingSchema.array() })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const {
query: { projectId },
permission
} = req;
const findings = await server.services.secretScanningV2.listSecretScanningFindingsByProjectId(
projectId,
permission
);
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_FINDING_LIST,
metadata: {
findingIds: findings.map((finding) => finding.id),
count: findings.length
}
}
});
return { findings };
}
});
server.route({
method: "PATCH",
url: "/findings/:findingId",
config: {
rateLimit: writeLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: "Update the specified Secret Scanning Finding.",
params: z.object({
findingId: z.string().trim().min(1, "Finding ID required").describe(SecretScanningFindings.UPDATE.findingId)
}),
body: z.object({
status: z.nativeEnum(SecretScanningFindingStatus).optional().describe(SecretScanningFindings.UPDATE.status),
remarks: z.string().nullish().describe(SecretScanningFindings.UPDATE.remarks)
}),
response: {
200: z.object({ finding: SecretScanningFindingSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const {
params: { findingId },
body,
permission
} = req;
const { finding, projectId } = await server.services.secretScanningV2.updateSecretScanningFindingById(
{ findingId, ...body },
permission
);
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_FINDING_UPDATE,
metadata: {
findingId,
...body
}
}
});
return { finding };
}
});
server.route({
method: "GET",
url: "/configs",
config: {
rateLimit: readLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: "Get the Secret Scanning Config for the specified project.",
querystring: z.object({
projectId: z
.string()
.trim()
.min(1, "Project ID required")
.describe(SecretScanningConfigs.GET_BY_PROJECT_ID.projectId)
}),
response: {
200: z.object({
config: z.object({ content: z.string().nullish(), projectId: z.string(), updatedAt: z.date().nullish() })
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const {
query: { projectId },
permission
} = req;
const config = await server.services.secretScanningV2.findSecretScanningConfigByProjectId(projectId, permission);
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_CONFIG_GET
}
});
return { config };
}
});
server.route({
method: "PATCH",
url: "/configs",
config: {
rateLimit: writeLimit
},
schema: {
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: "Update the specified Secret Scanning Configuration.",
querystring: z.object({
projectId: z.string().trim().min(1, "Project ID required").describe(SecretScanningConfigs.UPDATE.projectId)
}),
body: z.object({
content: z.string().nullable().describe(SecretScanningConfigs.UPDATE.content)
}),
response: {
200: z.object({ config: SecretScanningConfigsSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const {
query: { projectId },
body,
permission
} = req;
const config = await server.services.secretScanningV2.upsertSecretScanningConfig(
{ projectId, ...body },
permission
);
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_CONFIG_UPDATE,
metadata: body
}
});
return { config };
}
});
// not exposed, for UI only
server.route({
method: "GET",
url: "/data-sources-dashboard",
config: {
rateLimit: readLimit
},
schema: {
querystring: z.object({
projectId: z.string().trim().min(1, "Project ID required")
}),
response: {
200: z.object({
dataSources: z
.intersection(
SecretScanningDataSourceSchema,
z.object({
lastScannedAt: z.date().nullish(),
lastScanStatus: z.nativeEnum(SecretScanningScanStatus).nullish(),
lastScanStatusMessage: z.string().nullish(),
unresolvedFindings: z.number().nullish()
})
)
.array()
})
}
},
onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => {
const {
query: { projectId },
permission
} = req;
const dataSources = await server.services.secretScanningV2.listSecretScanningDataSourcesWithDetailsByProjectId(
{ projectId },
permission
);
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId,
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST,
metadata: {
dataSourceIds: dataSources.map((dataSource) => dataSource.id),
count: dataSources.length
}
}
});
return { dataSources };
}
});
server.route({
method: "GET",
url: "/unresolved-findings-count",
config: {
rateLimit: readLimit
},
schema: {
tags: [ApiDocsTags.SecretScanning],
querystring: z.object({
projectId: z.string().trim().min(1, "Project ID required").describe(SecretScanningFindings.LIST.projectId)
}),
response: {
200: z.object({ unresolvedFindings: z.number() })
}
},
onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => {
const {
query: { projectId },
permission
} = req;
const unresolvedFindings =
await server.services.secretScanningV2.getSecretScanningUnresolvedFindingsCountByProjectId(
projectId,
permission
);
return { unresolvedFindings };
}
});
};
@@ -10,6 +10,18 @@ import {
TSecretRotationV2Raw,
TUpdateSecretRotationV2DTO
} from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-types";
import {
SecretScanningDataSource,
SecretScanningScanStatus,
SecretScanningScanType
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
TCreateSecretScanningDataSourceDTO,
TDeleteSecretScanningDataSourceDTO,
TTriggerSecretScanningDataSourceDTO,
TUpdateSecretScanningDataSourceDTO,
TUpdateSecretScanningFindingDTO
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { SshCaStatus, SshCertType } from "@app/ee/services/ssh/ssh-certificate-authority-types";
import { SshCertKeyAlgorithm } from "@app/ee/services/ssh-certificate/ssh-certificate-types";
import { SshCertTemplateStatus } from "@app/ee/services/ssh-certificate-template/ssh-certificate-template-types";
@@ -381,6 +393,20 @@ export enum EventType {
PROJECT_ASSUME_PRIVILEGE_SESSION_START = "project-assume-privileges-session-start",
PROJECT_ASSUME_PRIVILEGE_SESSION_END = "project-assume-privileges-session-end",
SECRET_SCANNING_DATA_SOURCE_LIST = "secret-scanning-data-source-list",
SECRET_SCANNING_DATA_SOURCE_CREATE = "secret-scanning-data-source-create",
SECRET_SCANNING_DATA_SOURCE_UPDATE = "secret-scanning-data-source-update",
SECRET_SCANNING_DATA_SOURCE_DELETE = "secret-scanning-data-source-delete",
SECRET_SCANNING_DATA_SOURCE_GET = "secret-scanning-data-source-get",
SECRET_SCANNING_DATA_SOURCE_TRIGGER_SCAN = "secret-scanning-data-source-trigger-scan",
SECRET_SCANNING_DATA_SOURCE_SCAN = "secret-scanning-data-source-scan",
SECRET_SCANNING_RESOURCE_LIST = "secret-scanning-resource-list",
SECRET_SCANNING_SCAN_LIST = "secret-scanning-scan-list",
SECRET_SCANNING_FINDING_LIST = "secret-scanning-finding-list",
SECRET_SCANNING_FINDING_UPDATE = "secret-scanning-finding-update",
SECRET_SCANNING_CONFIG_GET = "secret-scanning-config-get",
SECRET_SCANNING_CONFIG_UPDATE = "secret-scanning-config-update",
UPDATE_ORG = "update-org",
CREATE_PROJECT = "create-project",
@@ -2953,6 +2979,101 @@ interface MicrosoftTeamsWorkflowIntegrationUpdateEvent {
};
}
interface SecretScanningDataSourceListEvent {
type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST;
metadata: {
type?: SecretScanningDataSource;
count: number;
dataSourceIds: string[];
};
}
interface SecretScanningDataSourceGetEvent {
type: EventType.SECRET_SCANNING_DATA_SOURCE_GET;
metadata: {
type: SecretScanningDataSource;
dataSourceId: string;
};
}
interface SecretScanningDataSourceCreateEvent {
type: EventType.SECRET_SCANNING_DATA_SOURCE_CREATE;
metadata: Omit<TCreateSecretScanningDataSourceDTO, "projectId"> & { dataSourceId: string };
}
interface SecretScanningDataSourceUpdateEvent {
type: EventType.SECRET_SCANNING_DATA_SOURCE_UPDATE;
metadata: TUpdateSecretScanningDataSourceDTO;
}
interface SecretScanningDataSourceDeleteEvent {
type: EventType.SECRET_SCANNING_DATA_SOURCE_DELETE;
metadata: TDeleteSecretScanningDataSourceDTO;
}
interface SecretScanningDataSourceTriggerScanEvent {
type: EventType.SECRET_SCANNING_DATA_SOURCE_TRIGGER_SCAN;
metadata: TTriggerSecretScanningDataSourceDTO;
}
interface SecretScanningDataSourceScanEvent {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN;
metadata: {
scanId: string;
resourceId: string;
resourceType: string;
dataSourceId: string;
dataSourceType: string;
scanStatus: SecretScanningScanStatus;
scanType: SecretScanningScanType;
numberOfSecretsDetected?: number;
};
}
interface SecretScanningResourceListEvent {
type: EventType.SECRET_SCANNING_RESOURCE_LIST;
metadata: {
type: SecretScanningDataSource;
dataSourceId: string;
resourceIds: string[];
count: number;
};
}
interface SecretScanningScanListEvent {
type: EventType.SECRET_SCANNING_SCAN_LIST;
metadata: {
type: SecretScanningDataSource;
dataSourceId: string;
count: number;
};
}
interface SecretScanningFindingListEvent {
type: EventType.SECRET_SCANNING_FINDING_LIST;
metadata: {
findingIds: string[];
count: number;
};
}
interface SecretScanningFindingUpdateEvent {
type: EventType.SECRET_SCANNING_FINDING_UPDATE;
metadata: TUpdateSecretScanningFindingDTO;
}
interface SecretScanningConfigUpdateEvent {
type: EventType.SECRET_SCANNING_CONFIG_UPDATE;
metadata: {
content: string | null;
};
}
interface SecretScanningConfigReadEvent {
type: EventType.SECRET_SCANNING_CONFIG_GET;
metadata?: Record<string, never>; // not needed, based off projectId
}
interface OrgUpdateEvent {
type: EventType.UPDATE_ORG;
metadata: {
@@ -3276,6 +3397,19 @@ export type Event =
| MicrosoftTeamsWorkflowIntegrationGetEvent
| MicrosoftTeamsWorkflowIntegrationListEvent
| MicrosoftTeamsWorkflowIntegrationUpdateEvent
| SecretScanningDataSourceListEvent
| SecretScanningDataSourceGetEvent
| SecretScanningDataSourceCreateEvent
| SecretScanningDataSourceUpdateEvent
| SecretScanningDataSourceDeleteEvent
| SecretScanningDataSourceTriggerScanEvent
| SecretScanningDataSourceScanEvent
| SecretScanningResourceListEvent
| SecretScanningScanListEvent
| SecretScanningFindingListEvent
| SecretScanningFindingUpdateEvent
| SecretScanningConfigUpdateEvent
| SecretScanningConfigReadEvent
| OrgUpdateEvent
| ProjectCreateEvent
| ProjectUpdateEvent
@@ -56,6 +56,7 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
kmip: false,
gateway: false,
sshHostGroups: false,
secretScanning: false,
enterpriseSecretSyncs: false,
enterpriseAppConnections: false
});
@@ -72,6 +72,7 @@ export type TFeatureSet = {
kmip: false;
gateway: false;
sshHostGroups: false;
secretScanning: false;
enterpriseSecretSyncs: false;
enterpriseAppConnections: false;
};
@@ -13,6 +13,9 @@ import {
ProjectPermissionPkiTemplateActions,
ProjectPermissionSecretActions,
ProjectPermissionSecretRotationActions,
ProjectPermissionSecretScanningConfigActions,
ProjectPermissionSecretScanningDataSourceActions,
ProjectPermissionSecretScanningFindingActions,
ProjectPermissionSecretSyncActions,
ProjectPermissionSet,
ProjectPermissionSshHostActions,
@@ -220,6 +223,29 @@ const buildAdminPermissionRules = () => {
ProjectPermissionSub.SecretRotation
);
can(
[
ProjectPermissionSecretScanningDataSourceActions.Create,
ProjectPermissionSecretScanningDataSourceActions.Edit,
ProjectPermissionSecretScanningDataSourceActions.Delete,
ProjectPermissionSecretScanningDataSourceActions.Read,
ProjectPermissionSecretScanningDataSourceActions.TriggerScans,
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
ProjectPermissionSecretScanningDataSourceActions.ReadResources
],
ProjectPermissionSub.SecretScanningDataSources
);
can(
[ProjectPermissionSecretScanningFindingActions.Read, ProjectPermissionSecretScanningFindingActions.Update],
ProjectPermissionSub.SecretScanningFindings
);
can(
[ProjectPermissionSecretScanningConfigActions.Read, ProjectPermissionSecretScanningConfigActions.Update],
ProjectPermissionSub.SecretScanningConfigs
);
return rules;
};
@@ -401,6 +427,23 @@ const buildMemberPermissionRules = () => {
ProjectPermissionSub.SecretSyncs
);
can(
[
ProjectPermissionSecretScanningDataSourceActions.Read,
ProjectPermissionSecretScanningDataSourceActions.TriggerScans,
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
ProjectPermissionSecretScanningDataSourceActions.ReadResources
],
ProjectPermissionSub.SecretScanningDataSources
);
can(
[ProjectPermissionSecretScanningFindingActions.Read, ProjectPermissionSecretScanningFindingActions.Update],
ProjectPermissionSub.SecretScanningFindings
);
can([ProjectPermissionSecretScanningConfigActions.Read], ProjectPermissionSub.SecretScanningConfigs);
return rules;
};
@@ -437,6 +480,19 @@ const buildViewerPermissionRules = () => {
can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateTemplates);
can(ProjectPermissionSecretSyncActions.Read, ProjectPermissionSub.SecretSyncs);
can(
[
ProjectPermissionSecretScanningDataSourceActions.Read,
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
ProjectPermissionSecretScanningDataSourceActions.ReadResources
],
ProjectPermissionSub.SecretScanningDataSources
);
can([ProjectPermissionSecretScanningFindingActions.Read], ProjectPermissionSub.SecretScanningFindings);
can([ProjectPermissionSecretScanningConfigActions.Read], ProjectPermissionSub.SecretScanningConfigs);
return rules;
};
@@ -132,6 +132,26 @@ export enum ProjectPermissionKmipActions {
GenerateClientCertificates = "generate-client-certificates"
}
export enum ProjectPermissionSecretScanningDataSourceActions {
Read = "read-data-sources",
Create = "create-data-sources",
Edit = "edit-data-sources",
Delete = "delete-data-sources",
TriggerScans = "trigger-data-source-scans",
ReadScans = "read-data-source-scans",
ReadResources = "read-data-source-resources"
}
export enum ProjectPermissionSecretScanningFindingActions {
Read = "read-findings",
Update = "update-findings"
}
export enum ProjectPermissionSecretScanningConfigActions {
Read = "read-configs",
Update = "update-configs"
}
export enum ProjectPermissionSub {
Role = "role",
Member = "member",
@@ -167,7 +187,10 @@ export enum ProjectPermissionSub {
Kms = "kms",
Cmek = "cmek",
SecretSyncs = "secret-syncs",
Kmip = "kmip"
Kmip = "kmip",
SecretScanningDataSources = "secret-scanning-data-sources",
SecretScanningFindings = "secret-scanning-findings",
SecretScanningConfigs = "secret-scanning-configs"
}
export type SecretSubjectFields = {
@@ -301,7 +324,10 @@ export type ProjectPermissionSet =
| [ProjectPermissionActions.Edit, ProjectPermissionSub.Project]
| [ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback]
| [ProjectPermissionActions.Create, ProjectPermissionSub.SecretRollback]
| [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms];
| [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms]
| [ProjectPermissionSecretScanningDataSourceActions, ProjectPermissionSub.SecretScanningDataSources]
| [ProjectPermissionSecretScanningFindingActions, ProjectPermissionSub.SecretScanningFindings]
| [ProjectPermissionSecretScanningConfigActions, ProjectPermissionSub.SecretScanningConfigs];
const SECRET_PATH_MISSING_SLASH_ERR_MSG = "Invalid Secret Path; it must start with a '/'";
const SECRET_PATH_PERMISSION_OPERATOR_SCHEMA = z.union([
@@ -631,6 +657,26 @@ const GeneralPermissionSchema = [
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionKmipActions).describe(
"Describe what action an entity can take."
)
}),
z.object({
subject: z
.literal(ProjectPermissionSub.SecretScanningDataSources)
.describe("The entity this permission pertains to."),
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningDataSourceActions).describe(
"Describe what action an entity can take."
)
}),
z.object({
subject: z.literal(ProjectPermissionSub.SecretScanningFindings).describe("The entity this permission pertains to."),
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningFindingActions).describe(
"Describe what action an entity can take."
)
}),
z.object({
subject: z.literal(ProjectPermissionSub.SecretScanningConfigs).describe("The entity this permission pertains to."),
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningConfigActions).describe(
"Describe what action an entity can take."
)
})
];
@@ -0,0 +1,9 @@
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { TSecretScanningDataSourceListItem } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
export const GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION: TSecretScanningDataSourceListItem = {
name: "GitHub",
type: SecretScanningDataSource.GitHub,
connection: AppConnection.GitHubRadar
};
@@ -0,0 +1,230 @@
import { join } from "path";
import { ProbotOctokit } from "probot";
import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import {
SecretScanningDataSource,
SecretScanningFindingSeverity,
SecretScanningResource
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
cloneRepository,
convertPatchLineToFileLineNumber,
replaceNonChangesWithNewlines
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
import {
TSecretScanningFactoryGetDiffScanFindingsPayload,
TSecretScanningFactoryGetDiffScanResourcePayload,
TSecretScanningFactoryGetFullScanPath,
TSecretScanningFactoryInitialize,
TSecretScanningFactoryListRawResources,
TSecretScanningFactoryPostInitialization
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { getConfig } from "@app/lib/config/env";
import { BadRequestError } from "@app/lib/errors";
import { titleCaseToCamelCase } from "@app/lib/fn";
import { GitHubRepositoryRegex } from "@app/lib/regex";
import { listGitHubRadarRepositories, TGitHubRadarConnection } from "@app/services/app-connection/github-radar";
import { TGitHubDataSourceWithConnection, TQueueGitHubResourceDiffScan } from "./github-secret-scanning-types";
export const GitHubSecretScanningFactory = () => {
const initialize: TSecretScanningFactoryInitialize<TGitHubRadarConnection> = async (
{ connection, secretScanningV2DAL },
callback
) => {
const externalId = connection.credentials.installationId;
const existingDataSource = await secretScanningV2DAL.dataSources.findOne({
externalId,
type: SecretScanningDataSource.GitHub
});
if (existingDataSource)
throw new BadRequestError({
message: `A Data Source already exists for this GitHub Radar Connection in the Project with ID "${existingDataSource.projectId}"`
});
return callback({
externalId
});
};
const postInitialization: TSecretScanningFactoryPostInitialization<TGitHubRadarConnection> = async () => {
// no post-initialization required
};
const listRawResources: TSecretScanningFactoryListRawResources<TGitHubDataSourceWithConnection> = async (
dataSource
) => {
const {
connection,
config: { includeRepos }
} = dataSource;
const repos = await listGitHubRadarRepositories(connection);
const filteredRepos: typeof repos = [];
if (includeRepos.includes("*")) {
filteredRepos.push(...repos);
} else {
filteredRepos.push(...repos.filter((repo) => includeRepos.includes(repo.full_name)));
}
return filteredRepos.map(({ id, full_name }) => ({
name: full_name,
externalId: id.toString(),
type: SecretScanningResource.Repository
}));
};
const getFullScanPath: TSecretScanningFactoryGetFullScanPath<TGitHubDataSourceWithConnection> = async ({
dataSource,
resourceName,
tempFolder
}) => {
const appCfg = getConfig();
const {
connection: {
credentials: { installationId }
}
} = dataSource;
const octokit = new ProbotOctokit({
auth: {
appId: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID,
privateKey: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY,
installationId
}
});
const {
data: { token }
} = await octokit.apps.createInstallationAccessToken({
installation_id: Number(installationId)
});
const repoPath = join(tempFolder, "repo.git");
if (!GitHubRepositoryRegex.test(resourceName)) {
throw new Error("Invalid GitHub repository name");
}
await cloneRepository({
cloneUrl: `https://x-access-token:${token}@github.com/${resourceName}.git`,
repoPath
});
return repoPath;
};
const getDiffScanResourcePayload: TSecretScanningFactoryGetDiffScanResourcePayload<
TQueueGitHubResourceDiffScan["payload"]
> = ({ repository }) => {
return {
name: repository.full_name,
externalId: repository.id.toString(),
type: SecretScanningResource.Repository
};
};
const getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<
TGitHubDataSourceWithConnection,
TQueueGitHubResourceDiffScan["payload"]
> = async ({ dataSource, payload, resourceName, configPath }) => {
const appCfg = getConfig();
const {
connection: {
credentials: { installationId }
}
} = dataSource;
const octokit = new ProbotOctokit({
auth: {
appId: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID,
privateKey: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY,
installationId
}
});
const { commits, repository } = payload;
const [owner, repo] = repository.full_name.split("/");
const allFindings: SecretMatch[] = [];
for (const commit of commits) {
// eslint-disable-next-line no-await-in-loop
const commitData = await octokit.repos.getCommit({
owner,
repo,
ref: commit.id
});
// eslint-disable-next-line no-continue
if (!commitData.data.files) continue;
for (const file of commitData.data.files) {
if ((file.status === "added" || file.status === "modified") && file.patch) {
// eslint-disable-next-line
const findings = await scanContentAndGetFindings(
replaceNonChangesWithNewlines(`\n${file.patch}`),
configPath
);
const adjustedFindings = findings.map((finding) => {
const startLine = convertPatchLineToFileLineNumber(file.patch!, finding.StartLine);
const endLine =
finding.StartLine === finding.EndLine
? startLine
: convertPatchLineToFileLineNumber(file.patch!, finding.EndLine);
const startColumn = finding.StartColumn - 1; // subtract 1 for +
const endColumn = finding.EndColumn - 1; // subtract 1 for +
return {
...finding,
StartLine: startLine,
EndLine: endLine,
StartColumn: startColumn,
EndColumn: endColumn,
File: file.filename,
Commit: commit.id,
Author: commit.author.name,
Email: commit.author.email ?? "",
Message: commit.message,
Fingerprint: `${commit.id}:${file.filename}:${finding.RuleID}:${startLine}:${startColumn}`,
Date: commit.timestamp,
Link: `https://github.com/${resourceName}/blob/${commit.id}/${file.filename}#L${startLine}`
};
});
allFindings.push(...adjustedFindings);
}
}
}
return allFindings.map(
({
// discard match and secret as we don't want to store
Match,
Secret,
...finding
}) => ({
details: titleCaseToCamelCase(finding),
fingerprint: finding.Fingerprint,
severity: SecretScanningFindingSeverity.High,
rule: finding.RuleID
})
);
};
return {
initialize,
postInitialization,
listRawResources,
getFullScanPath,
getDiffScanResourcePayload,
getDiffScanFindingsPayload
};
};
@@ -0,0 +1,85 @@
import { z } from "zod";
import {
SecretScanningDataSource,
SecretScanningResource
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
BaseCreateSecretScanningDataSourceSchema,
BaseSecretScanningDataSourceSchema,
BaseSecretScanningFindingSchema,
BaseUpdateSecretScanningDataSourceSchema,
GitRepositoryScanFindingDetailsSchema
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-schemas";
import { SecretScanningDataSources } from "@app/lib/api-docs";
import { GitHubRepositoryRegex } from "@app/lib/regex";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
export const GitHubDataSourceConfigSchema = z.object({
includeRepos: z
.array(
z
.string()
.min(1)
.max(256)
.refine((value) => value === "*" || GitHubRepositoryRegex.test(value), "Invalid repository name format")
)
.nonempty("One or more repositories required")
.max(100, "Cannot configure more than 100 repositories")
.default(["*"])
.describe(SecretScanningDataSources.CONFIG.GITHUB.includeRepos)
});
export const GitHubDataSourceSchema = BaseSecretScanningDataSourceSchema({
type: SecretScanningDataSource.GitHub,
isConnectionRequired: true
})
.extend({
config: GitHubDataSourceConfigSchema
})
.describe(
JSON.stringify({
title: "GitHub"
})
);
export const CreateGitHubDataSourceSchema = BaseCreateSecretScanningDataSourceSchema({
type: SecretScanningDataSource.GitHub,
isConnectionRequired: true
})
.extend({
config: GitHubDataSourceConfigSchema
})
.describe(
JSON.stringify({
title: "GitHub"
})
);
export const UpdateGitHubDataSourceSchema = BaseUpdateSecretScanningDataSourceSchema(SecretScanningDataSource.GitHub)
.extend({
config: GitHubDataSourceConfigSchema.optional()
})
.describe(
JSON.stringify({
title: "GitHub"
})
);
export const GitHubDataSourceListItemSchema = z
.object({
name: z.literal("GitHub"),
connection: z.literal(AppConnection.GitHubRadar),
type: z.literal(SecretScanningDataSource.GitHub)
})
.describe(
JSON.stringify({
title: "GitHub"
})
);
export const GitHubFindingSchema = BaseSecretScanningFindingSchema.extend({
resourceType: z.literal(SecretScanningResource.Repository),
dataSourceType: z.literal(SecretScanningDataSource.GitHub),
details: GitRepositoryScanFindingDetailsSchema
});
@@ -0,0 +1,87 @@
import { PushEvent } from "@octokit/webhooks-types";
import { TSecretScanningV2DALFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-dal";
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { TSecretScanningV2QueueServiceFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-queue";
import { logger } from "@app/lib/logger";
import { TGitHubDataSource } from "./github-secret-scanning-types";
export const githubSecretScanningService = (
secretScanningV2DAL: TSecretScanningV2DALFactory,
secretScanningV2Queue: Pick<TSecretScanningV2QueueServiceFactory, "queueResourceDiffScan">
) => {
const handleInstallationDeletedEvent = async (installationId: number) => {
const dataSource = await secretScanningV2DAL.dataSources.findOne({
externalId: String(installationId),
type: SecretScanningDataSource.GitHub
});
if (!dataSource) {
logger.error(
`secretScanningV2RemoveEvent: GitHub - Could not find data source [installationId=${installationId}]`
);
return;
}
logger.info(
`secretScanningV2RemoveEvent: GitHub - installation deleted [installationId=${installationId}] [dataSourceId=${dataSource.id}]`
);
await secretScanningV2DAL.dataSources.updateById(dataSource.id, {
isDisconnected: true
});
};
const handlePushEvent = async (payload: PushEvent) => {
const { commits, repository, installation } = payload;
if (!commits || !repository || !installation) {
logger.warn(
`secretScanningV2PushEvent: GitHub - Insufficient data [commits=${commits?.length ?? 0}] [repository=${repository.name}] [installationId=${installation?.id}]`
);
return;
}
const dataSource = (await secretScanningV2DAL.dataSources.findOne({
externalId: String(installation.id),
type: SecretScanningDataSource.GitHub
})) as TGitHubDataSource | undefined;
if (!dataSource) {
logger.error(
`secretScanningV2PushEvent: GitHub - Could not find data source [installationId=${installation.id}]`
);
return;
}
const {
isAutoScanEnabled,
config: { includeRepos }
} = dataSource;
if (!isAutoScanEnabled) {
logger.info(
`secretScanningV2PushEvent: GitHub - ignoring due to auto scan disabled [dataSourceId=${dataSource.id}] [installationId=${installation.id}]`
);
return;
}
if (includeRepos.includes("*") || includeRepos.includes(repository.full_name)) {
await secretScanningV2Queue.queueResourceDiffScan({
dataSourceType: SecretScanningDataSource.GitHub,
payload,
dataSourceId: dataSource.id
});
} else {
logger.info(
`secretScanningV2PushEvent: GitHub - ignoring due to repository not being present in config [installationId=${installation.id}] [dataSourceId=${dataSource.id}]`
);
}
};
return {
handlePushEvent,
handleInstallationDeletedEvent
};
};
@@ -0,0 +1,32 @@
import { PushEvent } from "@octokit/webhooks-types";
import { z } from "zod";
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { TGitHubRadarConnection } from "@app/services/app-connection/github-radar";
import {
CreateGitHubDataSourceSchema,
GitHubDataSourceListItemSchema,
GitHubDataSourceSchema,
GitHubFindingSchema
} from "./github-secret-scanning-schemas";
export type TGitHubDataSource = z.infer<typeof GitHubDataSourceSchema>;
export type TGitHubDataSourceInput = z.infer<typeof CreateGitHubDataSourceSchema>;
export type TGitHubDataSourceListItem = z.infer<typeof GitHubDataSourceListItemSchema>;
export type TGitHubFinding = z.infer<typeof GitHubFindingSchema>;
export type TGitHubDataSourceWithConnection = TGitHubDataSource & {
connection: TGitHubRadarConnection;
};
export type TQueueGitHubResourceDiffScan = {
dataSourceType: SecretScanningDataSource.GitHub;
payload: PushEvent;
dataSourceId: string;
resourceId: string;
scanId: string;
};
@@ -0,0 +1,3 @@
export * from "./github-secret-scanning-constants";
export * from "./github-secret-scanning-schemas";
export * from "./github-secret-scanning-types";
@@ -0,0 +1,460 @@
import { Knex } from "knex";
import { TDbClient } from "@app/db";
import {
SecretScanningResourcesSchema,
SecretScanningScansSchema,
TableName,
TSecretScanningDataSources
} from "@app/db/schemas";
import { SecretScanningFindingStatus } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { DatabaseError } from "@app/lib/errors";
import {
buildFindFilter,
ormify,
prependTableNameToFindFilter,
selectAllTableCols,
sqlNestRelationships,
TFindOpt
} from "@app/lib/knex";
export type TSecretScanningV2DALFactory = ReturnType<typeof secretScanningV2DALFactory>;
type TSecretScanningDataSourceFindFilter = Parameters<typeof buildFindFilter<TSecretScanningDataSources>>[0];
type TSecretScanningDataSourceFindOptions = TFindOpt<TSecretScanningDataSources, true, "name">;
const baseSecretScanningDataSourceQuery = ({
filter = {},
db,
tx
}: {
db: TDbClient;
filter?: TSecretScanningDataSourceFindFilter;
options?: TSecretScanningDataSourceFindOptions;
tx?: Knex;
}) => {
const query = (tx || db.replicaNode())(TableName.SecretScanningDataSource)
.join(
TableName.AppConnection,
`${TableName.SecretScanningDataSource}.connectionId`,
`${TableName.AppConnection}.id`
)
.select(selectAllTableCols(TableName.SecretScanningDataSource))
.select(
// entire connection
db.ref("name").withSchema(TableName.AppConnection).as("connectionName"),
db.ref("method").withSchema(TableName.AppConnection).as("connectionMethod"),
db.ref("app").withSchema(TableName.AppConnection).as("connectionApp"),
db.ref("orgId").withSchema(TableName.AppConnection).as("connectionOrgId"),
db.ref("encryptedCredentials").withSchema(TableName.AppConnection).as("connectionEncryptedCredentials"),
db.ref("description").withSchema(TableName.AppConnection).as("connectionDescription"),
db.ref("version").withSchema(TableName.AppConnection).as("connectionVersion"),
db.ref("createdAt").withSchema(TableName.AppConnection).as("connectionCreatedAt"),
db.ref("updatedAt").withSchema(TableName.AppConnection).as("connectionUpdatedAt"),
db
.ref("isPlatformManagedCredentials")
.withSchema(TableName.AppConnection)
.as("connectionIsPlatformManagedCredentials")
);
if (filter) {
/* eslint-disable @typescript-eslint/no-misused-promises */
void query.where(buildFindFilter(prependTableNameToFindFilter(TableName.SecretScanningDataSource, filter)));
}
return query;
};
const expandSecretScanningDataSource = <
T extends Awaited<ReturnType<typeof baseSecretScanningDataSourceQuery>>[number]
>(
dataSource: T
) => {
const {
connectionApp,
connectionName,
connectionId,
connectionOrgId,
connectionEncryptedCredentials,
connectionMethod,
connectionDescription,
connectionCreatedAt,
connectionUpdatedAt,
connectionVersion,
connectionIsPlatformManagedCredentials,
...el
} = dataSource;
return {
...el,
connectionId,
connection: connectionId
? {
app: connectionApp,
id: connectionId,
name: connectionName,
orgId: connectionOrgId,
encryptedCredentials: connectionEncryptedCredentials,
method: connectionMethod,
description: connectionDescription,
createdAt: connectionCreatedAt,
updatedAt: connectionUpdatedAt,
version: connectionVersion,
isPlatformManagedCredentials: connectionIsPlatformManagedCredentials
}
: undefined
};
};
export const secretScanningV2DALFactory = (db: TDbClient) => {
const dataSourceOrm = ormify(db, TableName.SecretScanningDataSource);
const resourceOrm = ormify(db, TableName.SecretScanningResource);
const scanOrm = ormify(db, TableName.SecretScanningScan);
const findingOrm = ormify(db, TableName.SecretScanningFinding);
const configOrm = ormify(db, TableName.SecretScanningConfig);
const findDataSource = async (filter: Parameters<(typeof dataSourceOrm)["find"]>[0], tx?: Knex) => {
try {
const dataSources = await baseSecretScanningDataSourceQuery({ filter, db, tx });
if (!dataSources.length) return [];
return dataSources.map(expandSecretScanningDataSource);
} catch (error) {
throw new DatabaseError({ error, name: "Find - Secret Scanning Data Source" });
}
};
const findDataSourceById = async (id: string, tx?: Knex) => {
try {
const dataSource = await baseSecretScanningDataSourceQuery({ filter: { id }, db, tx }).first();
if (dataSource) return expandSecretScanningDataSource(dataSource);
} catch (error) {
throw new DatabaseError({ error, name: "Find By ID - Secret Scanning Data Source" });
}
};
const createDataSource = async (data: Parameters<(typeof dataSourceOrm)["create"]>[0], tx?: Knex) => {
const source = await dataSourceOrm.create(data, tx);
const dataSource = (await baseSecretScanningDataSourceQuery({
filter: { id: source.id },
db,
tx
}).first())!;
return expandSecretScanningDataSource(dataSource);
};
const updateDataSourceById = async (
dataSourceId: string,
data: Parameters<(typeof dataSourceOrm)["updateById"]>[1],
tx?: Knex
) => {
const source = await dataSourceOrm.updateById(dataSourceId, data, tx);
const dataSource = (await baseSecretScanningDataSourceQuery({
filter: { id: source.id },
db,
tx
}).first())!;
return expandSecretScanningDataSource(dataSource);
};
const deleteDataSourceById = async (dataSourceId: string, tx?: Knex) => {
const dataSource = (await baseSecretScanningDataSourceQuery({
filter: { id: dataSourceId },
db,
tx
}).first())!;
await dataSourceOrm.deleteById(dataSourceId, tx);
return expandSecretScanningDataSource(dataSource);
};
const findOneDataSource = async (filter: Parameters<(typeof dataSourceOrm)["findOne"]>[0], tx?: Knex) => {
try {
const dataSource = await baseSecretScanningDataSourceQuery({ filter, db, tx }).first();
if (dataSource) {
return expandSecretScanningDataSource(dataSource);
}
} catch (error) {
throw new DatabaseError({ error, name: "Find One - Secret Scanning Data Source" });
}
};
const findDataSourceWithDetails = async (filter: Parameters<(typeof dataSourceOrm)["find"]>[0], tx?: Knex) => {
try {
// TODO (scott): this query will probably need to be optimized
const dataSources = await baseSecretScanningDataSourceQuery({ filter, db, tx })
.leftJoin(
TableName.SecretScanningResource,
`${TableName.SecretScanningResource}.dataSourceId`,
`${TableName.SecretScanningDataSource}.id`
)
.leftJoin(
TableName.SecretScanningScan,
`${TableName.SecretScanningScan}.resourceId`,
`${TableName.SecretScanningResource}.id`
)
.leftJoin(
TableName.SecretScanningFinding,
`${TableName.SecretScanningFinding}.scanId`,
`${TableName.SecretScanningScan}.id`
)
.where((qb) => {
void qb
.where(`${TableName.SecretScanningFinding}.status`, SecretScanningFindingStatus.Unresolved)
.orWhereNull(`${TableName.SecretScanningFinding}.status`);
})
.select(
db.ref("id").withSchema(TableName.SecretScanningScan).as("scanId"),
db.ref("status").withSchema(TableName.SecretScanningScan).as("scanStatus"),
db.ref("statusMessage").withSchema(TableName.SecretScanningScan).as("scanStatusMessage"),
db.ref("createdAt").withSchema(TableName.SecretScanningScan).as("scanCreatedAt"),
db.ref("status").withSchema(TableName.SecretScanningFinding).as("findingStatus"),
db.ref("id").withSchema(TableName.SecretScanningFinding).as("findingId")
);
if (!dataSources.length) return [];
const results = sqlNestRelationships({
data: dataSources,
key: "id",
parentMapper: (dataSource) => expandSecretScanningDataSource(dataSource),
childrenMapper: [
{
key: "scanId",
label: "scans" as const,
mapper: ({ scanId, scanCreatedAt, scanStatus, scanStatusMessage }) => ({
id: scanId,
createdAt: scanCreatedAt,
status: scanStatus,
statusMessage: scanStatusMessage
})
},
{
key: "findingId",
label: "findings" as const,
mapper: ({ findingId }) => ({
id: findingId
})
}
]
});
return results.map(({ scans, findings, ...dataSource }) => {
const lastScan =
scans && scans.length
? scans.reduce((latest, current) => {
return new Date(current.createdAt) > new Date(latest.createdAt) ? current : latest;
})
: null;
return {
...dataSource,
lastScanStatus: lastScan?.status ?? null,
lastScanStatusMessage: lastScan?.statusMessage ?? null,
lastScannedAt: lastScan?.createdAt ?? null,
unresolvedFindings: scans.length ? findings.length : null
};
});
} catch (error) {
throw new DatabaseError({ error, name: "Find with Details - Secret Scanning Data Source" });
}
};
const findResourcesWithDetails = async (filter: Parameters<(typeof resourceOrm)["find"]>[0], tx?: Knex) => {
try {
// TODO (scott): this query will probably need to be optimized
const resources = await (tx || db.replicaNode())(TableName.SecretScanningResource)
.where((qb) => {
if (filter)
void qb.where(buildFindFilter(prependTableNameToFindFilter(TableName.SecretScanningResource, filter)));
})
.leftJoin(
TableName.SecretScanningScan,
`${TableName.SecretScanningScan}.resourceId`,
`${TableName.SecretScanningResource}.id`
)
.leftJoin(
TableName.SecretScanningFinding,
`${TableName.SecretScanningFinding}.scanId`,
`${TableName.SecretScanningScan}.id`
)
.where((qb) => {
void qb
.where(`${TableName.SecretScanningFinding}.status`, SecretScanningFindingStatus.Unresolved)
.orWhereNull(`${TableName.SecretScanningFinding}.status`);
})
.select(selectAllTableCols(TableName.SecretScanningResource))
.select(
db.ref("id").withSchema(TableName.SecretScanningScan).as("scanId"),
db.ref("status").withSchema(TableName.SecretScanningScan).as("scanStatus"),
db.ref("type").withSchema(TableName.SecretScanningScan).as("scanType"),
db.ref("statusMessage").withSchema(TableName.SecretScanningScan).as("scanStatusMessage"),
db.ref("createdAt").withSchema(TableName.SecretScanningScan).as("scanCreatedAt"),
db.ref("status").withSchema(TableName.SecretScanningFinding).as("findingStatus"),
db.ref("id").withSchema(TableName.SecretScanningFinding).as("findingId")
);
if (!resources.length) return [];
const results = sqlNestRelationships({
data: resources,
key: "id",
parentMapper: (resource) => SecretScanningResourcesSchema.parse(resource),
childrenMapper: [
{
key: "scanId",
label: "scans" as const,
mapper: ({ scanId, scanCreatedAt, scanStatus, scanStatusMessage, scanType }) => ({
id: scanId,
type: scanType,
createdAt: scanCreatedAt,
status: scanStatus,
statusMessage: scanStatusMessage
})
},
{
key: "findingId",
label: "findings" as const,
mapper: ({ findingId }) => ({
id: findingId
})
}
]
});
return results.map(({ scans, findings, ...resource }) => {
const lastScan =
scans && scans.length
? scans.reduce((latest, current) => {
return new Date(current.createdAt) > new Date(latest.createdAt) ? current : latest;
})
: null;
return {
...resource,
lastScanStatus: lastScan?.status ?? null,
lastScanStatusMessage: lastScan?.statusMessage ?? null,
lastScannedAt: lastScan?.createdAt ?? null,
unresolvedFindings: findings?.length ?? 0
};
});
} catch (error) {
throw new DatabaseError({ error, name: "Find with Details - Secret Scanning Resource" });
}
};
const findScansWithDetailsByDataSourceId = async (dataSourceId: string, tx?: Knex) => {
try {
// TODO (scott): this query will probably need to be optimized
const scans = await (tx || db.replicaNode())(TableName.SecretScanningScan)
.leftJoin(
TableName.SecretScanningResource,
`${TableName.SecretScanningResource}.id`,
`${TableName.SecretScanningScan}.resourceId`
)
.where(`${TableName.SecretScanningResource}.dataSourceId`, dataSourceId)
.leftJoin(
TableName.SecretScanningFinding,
`${TableName.SecretScanningFinding}.scanId`,
`${TableName.SecretScanningScan}.id`
)
.select(selectAllTableCols(TableName.SecretScanningScan))
.select(
db.ref("status").withSchema(TableName.SecretScanningFinding).as("findingStatus"),
db.ref("id").withSchema(TableName.SecretScanningFinding).as("findingId"),
db.ref("name").withSchema(TableName.SecretScanningResource).as("resourceName")
);
if (!scans.length) return [];
const results = sqlNestRelationships({
data: scans,
key: "id",
parentMapper: (scan) => SecretScanningScansSchema.parse(scan),
childrenMapper: [
{
key: "findingId",
label: "findings" as const,
mapper: ({ findingId, findingStatus }) => ({
id: findingId,
status: findingStatus
})
},
{
key: "resourceId",
label: "resources" as const,
mapper: ({ resourceName }) => ({
name: resourceName
})
}
]
});
return results.map(({ findings, resources, ...scan }) => {
return {
...scan,
unresolvedFindings:
findings?.filter((finding) => finding.status === SecretScanningFindingStatus.Unresolved).length ?? 0,
resolvedFindings:
findings?.filter((finding) => finding.status !== SecretScanningFindingStatus.Unresolved).length ?? 0,
resourceName: resources[0].name
};
});
} catch (error) {
throw new DatabaseError({ error, name: "Find with Details By Data Source ID - Secret Scanning Scan" });
}
};
const findScansByDataSourceId = async (dataSourceId: string, tx?: Knex) => {
try {
const scans = await (tx || db.replicaNode())(TableName.SecretScanningScan)
.leftJoin(
TableName.SecretScanningResource,
`${TableName.SecretScanningResource}.id`,
`${TableName.SecretScanningScan}.resourceId`
)
.where(`${TableName.SecretScanningResource}.dataSourceId`, dataSourceId)
.select(selectAllTableCols(TableName.SecretScanningScan));
return scans;
} catch (error) {
throw new DatabaseError({ error, name: "Find By Data Source ID - Secret Scanning Scan" });
}
};
return {
dataSources: {
...dataSourceOrm,
find: findDataSource,
findById: findDataSourceById,
findOne: findOneDataSource,
create: createDataSource,
updateById: updateDataSourceById,
deleteById: deleteDataSourceById,
findWithDetails: findDataSourceWithDetails
},
resources: {
...resourceOrm,
findWithDetails: findResourcesWithDetails
},
scans: {
...scanOrm,
findWithDetailsByDataSourceId: findScansWithDetailsByDataSourceId,
findByDataSourceId: findScansByDataSourceId
},
findings: findingOrm,
configs: configOrm
};
};
@@ -0,0 +1,33 @@
export enum SecretScanningDataSource {
GitHub = "github"
}
export enum SecretScanningScanStatus {
Completed = "completed",
Failed = "failed",
Queued = "queued",
Scanning = "scanning"
}
export enum SecretScanningScanType {
FullScan = "full-scan",
DiffScan = "diff-scan"
}
export enum SecretScanningFindingStatus {
Resolved = "resolved",
Unresolved = "unresolved",
FalsePositive = "false-positive",
Ignore = "ignore"
}
export enum SecretScanningResource {
Repository = "repository",
Project = "project"
}
export enum SecretScanningFindingSeverity {
High = "high",
Medium = "medium",
Low = "low"
}
@@ -0,0 +1,19 @@
import { GitHubSecretScanningFactory } from "@app/ee/services/secret-scanning-v2/github/github-secret-scanning-factory";
import { SecretScanningDataSource } from "./secret-scanning-v2-enums";
import {
TQueueSecretScanningResourceDiffScan,
TSecretScanningDataSourceCredentials,
TSecretScanningDataSourceWithConnection,
TSecretScanningFactory
} from "./secret-scanning-v2-types";
type TSecretScanningFactoryImplementation = TSecretScanningFactory<
TSecretScanningDataSourceWithConnection,
TSecretScanningDataSourceCredentials,
TQueueSecretScanningResourceDiffScan["payload"]
>;
export const SECRET_SCANNING_FACTORY_MAP: Record<SecretScanningDataSource, TSecretScanningFactoryImplementation> = {
[SecretScanningDataSource.GitHub]: GitHubSecretScanningFactory as TSecretScanningFactoryImplementation
};
@@ -0,0 +1,140 @@
import { AxiosError } from "axios";
import { exec } from "child_process";
import RE2 from "re2";
import { readFindingsFile } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import { GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/github";
import { titleCaseToCamelCase } from "@app/lib/fn";
import { SecretScanningDataSource, SecretScanningFindingSeverity } from "./secret-scanning-v2-enums";
import { TCloneRepository, TGetFindingsPayload, TSecretScanningDataSourceListItem } from "./secret-scanning-v2-types";
const SECRET_SCANNING_SOURCE_LIST_OPTIONS: Record<SecretScanningDataSource, TSecretScanningDataSourceListItem> = {
[SecretScanningDataSource.GitHub]: GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION
};
export const listSecretScanningDataSourceOptions = () => {
return Object.values(SECRET_SCANNING_SOURCE_LIST_OPTIONS).sort((a, b) => a.name.localeCompare(b.name));
};
export const cloneRepository = async ({ cloneUrl, repoPath }: TCloneRepository): Promise<void> => {
const command = `git clone ${cloneUrl} ${repoPath} --bare`;
return new Promise((resolve, reject) => {
exec(command, (error) => {
if (error) {
reject(error);
} else {
resolve();
}
});
});
};
export function scanDirectory(inputPath: string, outputPath: string, configPath?: string): Promise<void> {
return new Promise((resolve, reject) => {
const command = `cd ${inputPath} && infisical scan --exit-code=77 -r "${outputPath}" ${configPath ? `-c ${configPath}` : ""}`;
exec(command, (error) => {
if (error && error.code !== 77) {
reject(error);
} else {
resolve();
}
});
});
}
export const scanGitRepositoryAndGetFindings = async (
scanPath: string,
findingsPath: string,
configPath?: string
): TGetFindingsPayload => {
await scanDirectory(scanPath, findingsPath, configPath);
const findingsData = JSON.parse(await readFindingsFile(findingsPath)) as SecretMatch[];
return findingsData.map(
({
// discard match and secret as we don't want to store
Match,
Secret,
...finding
}) => ({
details: titleCaseToCamelCase(finding),
fingerprint: `${finding.Fingerprint}:${finding.StartColumn}`,
severity: SecretScanningFindingSeverity.High,
rule: finding.RuleID
})
);
};
export const replaceNonChangesWithNewlines = (patch: string) => {
return patch
.split("\n")
.map((line) => {
// Keep added lines (remove the + prefix)
if (line.startsWith("+") && !line.startsWith("+++")) {
return line.substring(1);
}
// Replace everything else with newlines to maintain line positioning
return "";
})
.join("\n");
};
const HunkHeaderRegex = new RE2(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/);
export const convertPatchLineToFileLineNumber = (patch: string, patchLineNumber: number) => {
const lines = patch.split("\n");
let currentPatchLine = 0;
let currentNewLine = 0;
for (const line of lines) {
currentPatchLine += 1;
// Hunk header: @@ -a,b +c,d @@
const hunkHeaderMatch = HunkHeaderRegex.match(line);
if (hunkHeaderMatch) {
const startLine = parseInt(hunkHeaderMatch[1], 10);
currentNewLine = startLine;
// eslint-disable-next-line no-continue
continue;
}
if (currentPatchLine === patchLineNumber) {
return currentNewLine;
}
if (line.startsWith("+++")) {
// eslint-disable-next-line no-continue
continue; // skip file metadata lines
}
// Advance only if the line exists in the new file
if (line.startsWith("+") || line.startsWith(" ")) {
currentNewLine += 1;
}
}
return currentNewLine;
};
const MAX_MESSAGE_LENGTH = 1024;
export const parseScanErrorMessage = (err: unknown): string => {
let errorMessage: string;
if (err instanceof AxiosError) {
errorMessage = err?.response?.data
? JSON.stringify(err?.response?.data)
: (err?.message ?? "An unknown error occurred.");
} else {
errorMessage = (err as Error)?.message || "An unknown error occurred.";
}
return errorMessage.length <= MAX_MESSAGE_LENGTH
? errorMessage
: `${errorMessage.substring(0, MAX_MESSAGE_LENGTH - 3)}...`;
};
@@ -0,0 +1,14 @@
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
export const SECRET_SCANNING_DATA_SOURCE_NAME_MAP: Record<SecretScanningDataSource, string> = {
[SecretScanningDataSource.GitHub]: "GitHub"
};
export const SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP: Record<SecretScanningDataSource, AppConnection> = {
[SecretScanningDataSource.GitHub]: AppConnection.GitHubRadar
};
export const AUTO_SYNC_DESCRIPTION_HELPER: Record<SecretScanningDataSource, { verb: string; noun: string }> = {
[SecretScanningDataSource.GitHub]: { verb: "push", noun: "repositories" }
};
@@ -0,0 +1,626 @@
import { join } from "path";
import { ProjectMembershipRole, TSecretScanningFindings } from "@app/db/schemas";
import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import {
createTempFolder,
deleteTempFolder,
writeTextToFile
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import {
parseScanErrorMessage,
scanGitRepositoryAndGetFindings
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { BadRequestError, InternalServerError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns";
import { TAppConnection } from "@app/services/app-connection/app-connection-types";
import { ActorType } from "@app/services/auth/auth-type";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal";
import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
import { TSecretScanningV2DALFactory } from "./secret-scanning-v2-dal";
import {
SecretScanningDataSource,
SecretScanningResource,
SecretScanningScanStatus,
SecretScanningScanType
} from "./secret-scanning-v2-enums";
import { SECRET_SCANNING_FACTORY_MAP } from "./secret-scanning-v2-factory";
import {
TFindingsPayload,
TQueueSecretScanningDataSourceFullScan,
TQueueSecretScanningResourceDiffScan,
TQueueSecretScanningSendNotification,
TSecretScanningDataSourceWithConnection
} from "./secret-scanning-v2-types";
type TSecretRotationV2QueueServiceFactoryDep = {
queueService: TQueueServiceFactory;
secretScanningV2DAL: TSecretScanningV2DALFactory;
smtpService: Pick<TSmtpService, "sendMail">;
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
projectDAL: Pick<TProjectDALFactory, "findById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
keyStore: Pick<TKeyStoreFactory, "acquireLock" | "getItem">;
};
export type TSecretScanningV2QueueServiceFactory = Awaited<ReturnType<typeof secretScanningV2QueueServiceFactory>>;
export const secretScanningV2QueueServiceFactory = async ({
queueService,
secretScanningV2DAL,
projectMembershipDAL,
projectDAL,
smtpService,
kmsService,
auditLogService,
keyStore
}: TSecretRotationV2QueueServiceFactoryDep) => {
const queueDataSourceFullScan = async (
dataSource: TSecretScanningDataSourceWithConnection,
resourceExternalId?: string
) => {
try {
const { type } = dataSource;
const factory = SECRET_SCANNING_FACTORY_MAP[type]();
const rawResources = await factory.listRawResources(dataSource);
let filteredRawResources = rawResources;
// TODO: should add individual resource fetch to factory
if (resourceExternalId) {
filteredRawResources = rawResources.filter((resource) => resource.externalId === resourceExternalId);
}
if (!filteredRawResources.length) {
throw new BadRequestError({
message: `${resourceExternalId ? `Resource with "ID" ${resourceExternalId} could not be found.` : "Data source has no resources to scan"}. Ensure your data source config is correct and not filtering out scanning resources.`
});
}
for (const resource of filteredRawResources) {
// eslint-disable-next-line no-await-in-loop
if (await keyStore.getItem(KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId))) {
throw new BadRequestError({ message: `A scan is already in progress for resource "${resource.name}"` });
}
}
await secretScanningV2DAL.resources.transaction(async (tx) => {
const resources = await secretScanningV2DAL.resources.upsert(
filteredRawResources.map((rawResource) => ({
...rawResource,
dataSourceId: dataSource.id
})),
["externalId", "dataSourceId"],
tx
);
const scans = await secretScanningV2DAL.scans.insertMany(
resources.map((resource) => ({
resourceId: resource.id,
type: SecretScanningScanType.FullScan
})),
tx
);
for (const scan of scans) {
// eslint-disable-next-line no-await-in-loop
await queueService.queuePg(QueueJobs.SecretScanningV2FullScan, {
scanId: scan.id,
resourceId: scan.resourceId,
dataSourceId: dataSource.id
});
}
});
} catch (error) {
logger.error(error, `Failed to queue full-scan for data source with ID "${dataSource.id}"`);
if (error instanceof BadRequestError) throw error;
throw new InternalServerError({ message: `Failed to queue scan: ${(error as Error).message}` });
}
};
await queueService.startPg<QueueName.SecretScanningV2>(
QueueJobs.SecretScanningV2FullScan,
async ([job]) => {
const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan;
const { retryCount, retryLimit } = job;
const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
const tempFolder = await createTempFolder();
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
const resource = await secretScanningV2DAL.resources.findById(resourceId);
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
try {
try {
lock = await keyStore.acquireLock(
[KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
60 * 1000 * 5
);
} catch (e) {
throw new Error("Failed to acquire scanning lock.");
}
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Scanning
}
);
let connection: TAppConnection | null = null;
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]();
const findingsPath = join(tempFolder, "findings.json");
const scanPath = await factory.getFullScanPath({
dataSource: {
...dataSource,
connection
} as TSecretScanningDataSourceWithConnection,
resourceName: resource.name,
tempFolder
});
const config = await secretScanningV2DAL.configs.findOne({
projectId: dataSource.projectId
});
let configPath: string | undefined;
if (config && config.content) {
configPath = join(tempFolder, "infisical-scan.toml");
await writeTextToFile(configPath, config.content);
}
let findingsPayload: TFindingsPayload;
switch (resource.type) {
case SecretScanningResource.Repository:
case SecretScanningResource.Project:
findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
break;
default:
throw new Error("Unhandled resource type");
}
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
let findings: TSecretScanningFindings[] = [];
if (findingsPayload.length) {
findings = await secretScanningV2DAL.findings.upsert(
findingsPayload.map((finding) => ({
...finding,
projectId: dataSource.projectId,
dataSourceName: dataSource.name,
dataSourceType: dataSource.type,
resourceName: resource.name,
resourceType: resource.type,
scanId
})),
["projectId", "fingerprint"],
tx,
["resourceName", "dataSourceName"]
);
}
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Completed,
statusMessage: null
}
);
return findings;
});
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
if (newFindings.length) {
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Completed,
resourceName: resource.name,
isDiffScan: false,
dataSource,
numberOfSecrets: newFindings.length,
scanId
});
}
await auditLogService.createAuditLog({
projectId: dataSource.projectId,
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
metadata: {
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId: resource.id,
resourceType: resource.type,
scanId,
scanStatus: SecretScanningScanStatus.Completed,
scanType: SecretScanningScanType.FullScan,
numberOfSecretsDetected: findingsPayload.length
}
}
});
logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`);
} catch (error) {
if (retryCount === retryLimit) {
const errorMessage = parseScanErrorMessage(error);
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Failed,
statusMessage: errorMessage
}
);
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Failed,
resourceName: resource.name,
dataSource,
errorMessage
});
await auditLogService.createAuditLog({
projectId: dataSource.projectId,
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
metadata: {
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId: resource.id,
resourceType: resource.type,
scanId,
scanStatus: SecretScanningScanStatus.Failed,
scanType: SecretScanningScanType.FullScan
}
}
});
}
logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
throw error;
} finally {
await deleteTempFolder(tempFolder);
await lock?.release();
}
},
{
batchSize: 1,
workerCount: 20,
pollingIntervalSeconds: 1
}
);
const queueResourceDiffScan = async ({
payload,
dataSourceId,
dataSourceType
}: Pick<TQueueSecretScanningResourceDiffScan, "payload" | "dataSourceId" | "dataSourceType">) => {
const factory = SECRET_SCANNING_FACTORY_MAP[dataSourceType as SecretScanningDataSource]();
const resourcePayload = factory.getDiffScanResourcePayload(payload);
try {
const { resourceId, scanId } = await secretScanningV2DAL.resources.transaction(async (tx) => {
const [resource] = await secretScanningV2DAL.resources.upsert(
[
{
...resourcePayload,
dataSourceId
}
],
["externalId", "dataSourceId"],
tx
);
const scan = await secretScanningV2DAL.scans.create(
{
resourceId: resource.id,
type: SecretScanningScanType.DiffScan
},
tx
);
return {
resourceId: resource.id,
scanId: scan.id
};
});
await queueService.queuePg(QueueJobs.SecretScanningV2DiffScan, {
payload,
dataSourceId,
dataSourceType,
scanId,
resourceId
});
} catch (error) {
logger.error(
error,
`secretScanningV2Queue: Failed to queue diff scan [dataSourceId=${dataSourceId}] [resourceExternalId=${resourcePayload.externalId}]`
);
}
};
await queueService.startPg<QueueName.SecretScanningV2>(
QueueJobs.SecretScanningV2DiffScan,
async ([job]) => {
const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan;
const { retryCount, retryLimit } = job;
const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
const resource = await secretScanningV2DAL.resources.findById(resourceId);
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]();
const tempFolder = await createTempFolder();
try {
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Scanning
}
);
let connection: TAppConnection | null = null;
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
const config = await secretScanningV2DAL.configs.findOne({
projectId: dataSource.projectId
});
let configPath: string | undefined;
if (config && config.content) {
configPath = join(tempFolder, "infisical-scan.toml");
await writeTextToFile(configPath, config.content);
}
const findingsPayload = await factory.getDiffScanFindingsPayload({
dataSource: {
...dataSource,
connection
} as TSecretScanningDataSourceWithConnection,
resourceName: resource.name,
payload,
configPath
});
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
let findings: TSecretScanningFindings[] = [];
if (findingsPayload.length) {
findings = await secretScanningV2DAL.findings.upsert(
findingsPayload.map((finding) => ({
...finding,
projectId: dataSource.projectId,
dataSourceName: dataSource.name,
dataSourceType: dataSource.type,
resourceName: resource.name,
resourceType: resource.type,
scanId
})),
["projectId", "fingerprint"],
tx,
["resourceName", "dataSourceName"]
);
}
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Completed
}
);
return findings;
});
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
if (newFindings.length) {
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Completed,
resourceName: resource.name,
isDiffScan: true,
dataSource,
numberOfSecrets: newFindings.length,
scanId
});
}
await auditLogService.createAuditLog({
projectId: dataSource.projectId,
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
metadata: {
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId,
resourceType: resource.type,
scanId,
scanStatus: SecretScanningScanStatus.Completed,
scanType: SecretScanningScanType.DiffScan,
numberOfSecretsDetected: findingsPayload.length
}
}
});
logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`);
} catch (error) {
if (retryCount === retryLimit) {
const errorMessage = parseScanErrorMessage(error);
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Failed,
statusMessage: errorMessage
}
);
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Failed,
resourceName: resource.name,
dataSource,
errorMessage
});
await auditLogService.createAuditLog({
projectId: dataSource.projectId,
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
metadata: {
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId: resource.id,
resourceType: resource.type,
scanId,
scanStatus: SecretScanningScanStatus.Failed,
scanType: SecretScanningScanType.DiffScan
}
}
});
}
logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
throw error;
} finally {
await deleteTempFolder(tempFolder);
}
},
{
batchSize: 1,
workerCount: 20,
pollingIntervalSeconds: 1
}
);
await queueService.startPg<QueueName.SecretScanningV2>(
QueueJobs.SecretScanningV2SendNotification,
async ([job]) => {
const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification;
const appCfg = getConfig();
if (!appCfg.isSmtpConfigured) return;
try {
const { projectId } = dataSource;
logger.info(
`secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
);
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
const project = await projectDAL.findById(projectId);
const projectAdmins = projectMembers.filter((member) =>
member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
);
const timestamp = new Date().toISOString();
await smtpService.sendMail({
recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
template:
payload.status === SecretScanningScanStatus.Completed
? SmtpTemplates.SecretScanningV2SecretsDetected
: SmtpTemplates.SecretScanningV2ScanFailed,
subjectLine:
payload.status === SecretScanningScanStatus.Completed
? "Incident Alert: Secret(s) Leaked"
: `Secret Scanning Failed`,
substitutions:
payload.status === SecretScanningScanStatus.Completed
? {
authorName: "Jim",
authorEmail: "[email protected]",
resourceName,
numberOfSecrets: payload.numberOfSecrets,
isDiffScan: payload.isDiffScan,
url: encodeURI(
`${appCfg.SITE_URL}/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
),
timestamp
}
: {
dataSourceName: dataSource.name,
resourceName,
projectName: project.name,
timestamp,
errorMessage: payload.errorMessage,
url: encodeURI(
`${appCfg.SITE_URL}/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
)
}
});
} catch (error) {
logger.error(
error,
`secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
);
throw error;
}
},
{
batchSize: 1,
workerCount: 5,
pollingIntervalSeconds: 1
}
);
return {
queueDataSourceFullScan,
queueResourceDiffScan
};
};
@@ -0,0 +1,99 @@
import { z } from "zod";
import { SecretScanningDataSourcesSchema, SecretScanningFindingsSchema } from "@app/db/schemas";
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps";
import { SecretScanningDataSources } from "@app/lib/api-docs";
import { slugSchema } from "@app/server/lib/schemas";
type SecretScanningDataSourceSchemaOpts = {
type: SecretScanningDataSource;
isConnectionRequired: boolean;
};
export const BaseSecretScanningDataSourceSchema = ({
type,
isConnectionRequired
}: SecretScanningDataSourceSchemaOpts) =>
SecretScanningDataSourcesSchema.omit({
// unique to provider
type: true,
connectionId: true,
config: true,
encryptedCredentials: true
}).extend({
type: z.literal(type),
connectionId: isConnectionRequired ? z.string().uuid() : z.null(),
connection: isConnectionRequired
? z.object({
app: z.literal(SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[type]),
name: z.string(),
id: z.string().uuid()
})
: z.null()
});
export const BaseCreateSecretScanningDataSourceSchema = ({
type,
isConnectionRequired
}: SecretScanningDataSourceSchemaOpts) =>
z.object({
name: slugSchema({ field: "name" }).describe(SecretScanningDataSources.CREATE(type).name),
projectId: z
.string()
.trim()
.min(1, "Project ID required")
.describe(SecretScanningDataSources.CREATE(type).projectId),
description: z
.string()
.trim()
.max(256, "Description cannot exceed 256 characters")
.nullish()
.describe(SecretScanningDataSources.CREATE(type).description),
connectionId: isConnectionRequired
? z.string().uuid().describe(SecretScanningDataSources.CREATE(type).connectionId)
: z.undefined(),
isAutoScanEnabled: z
.boolean()
.optional()
.default(true)
.describe(SecretScanningDataSources.CREATE(type).isAutoScanEnabled)
});
export const BaseUpdateSecretScanningDataSourceSchema = (type: SecretScanningDataSource) =>
z.object({
name: slugSchema({ field: "name" }).describe(SecretScanningDataSources.UPDATE(type).name).optional(),
description: z
.string()
.trim()
.max(256, "Description cannot exceed 256 characters")
.nullish()
.describe(SecretScanningDataSources.UPDATE(type).description),
isAutoScanEnabled: z.boolean().optional().describe(SecretScanningDataSources.UPDATE(type).isAutoScanEnabled)
});
export const GitRepositoryScanFindingDetailsSchema = z.object({
description: z.string(),
startLine: z.number(),
endLine: z.number(),
startColumn: z.number(),
endColumn: z.number(),
file: z.string(),
link: z.string(),
symlinkFile: z.string(),
commit: z.string(),
entropy: z.number(),
author: z.string(),
email: z.string(),
date: z.string(),
message: z.string(),
tags: z.string().array(),
ruleID: z.string(),
fingerprint: z.string()
});
export const BaseSecretScanningFindingSchema = SecretScanningFindingsSchema.omit({
dataSourceType: true,
resourceType: true,
details: true
});
@@ -0,0 +1,875 @@
import { ForbiddenError } from "@casl/ability";
import { join } from "path";
import { ActionProjectType } from "@app/db/schemas";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import {
ProjectPermissionSecretScanningConfigActions,
ProjectPermissionSecretScanningDataSourceActions,
ProjectPermissionSecretScanningFindingActions,
ProjectPermissionSub
} from "@app/ee/services/permission/project-permission";
import {
createTempFolder,
deleteTempFolder,
scanContentAndGetFindings,
writeTextToFile
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { githubSecretScanningService } from "@app/ee/services/secret-scanning-v2/github/github-secret-scanning-service";
import { SecretScanningFindingStatus } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { SECRET_SCANNING_FACTORY_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-factory";
import { listSecretScanningDataSourceOptions } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
import {
SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP,
SECRET_SCANNING_DATA_SOURCE_NAME_MAP
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps";
import {
TCreateSecretScanningDataSourceDTO,
TDeleteSecretScanningDataSourceDTO,
TFindSecretScanningDataSourceByIdDTO,
TFindSecretScanningDataSourceByNameDTO,
TListSecretScanningDataSourcesByProjectId,
TSecretScanningDataSource,
TSecretScanningDataSourceWithConnection,
TSecretScanningDataSourceWithDetails,
TSecretScanningFinding,
TSecretScanningResourceWithDetails,
TSecretScanningScanWithDetails,
TTriggerSecretScanningDataSourceDTO,
TUpdateSecretScanningDataSourceDTO,
TUpdateSecretScanningFindingDTO,
TUpsertSecretScanningConfigDTO
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { DatabaseErrorCode } from "@app/lib/error-codes";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
import { OrgServiceActor } from "@app/lib/types";
import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns";
import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service";
import { TAppConnection } from "@app/services/app-connection/app-connection-types";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
import { TSecretScanningV2DALFactory } from "./secret-scanning-v2-dal";
import { TSecretScanningV2QueueServiceFactory } from "./secret-scanning-v2-queue";
export type TSecretScanningV2ServiceFactoryDep = {
secretScanningV2DAL: TSecretScanningV2DALFactory;
appConnectionService: Pick<TAppConnectionServiceFactory, "connectAppConnectionById">;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission" | "getOrgPermission">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
secretScanningV2Queue: Pick<
TSecretScanningV2QueueServiceFactory,
"queueDataSourceFullScan" | "queueResourceDiffScan"
>;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
};
export type TSecretScanningV2ServiceFactory = ReturnType<typeof secretScanningV2ServiceFactory>;
export const secretScanningV2ServiceFactory = ({
secretScanningV2DAL,
permissionService,
appConnectionService,
licenseService,
secretScanningV2Queue,
kmsService
}: TSecretScanningV2ServiceFactoryDep) => {
const $checkListSecretScanningDataSourcesByProjectIdPermissions = async (
projectId: string,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Data Sources due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningDataSourceActions.Read,
ProjectPermissionSub.SecretScanningDataSources
);
};
const listSecretScanningDataSourcesByProjectId = async (
{ projectId, type }: TListSecretScanningDataSourcesByProjectId,
actor: OrgServiceActor
) => {
await $checkListSecretScanningDataSourcesByProjectIdPermissions(projectId, actor);
const dataSources = await secretScanningV2DAL.dataSources.find({
...(type && { type }),
projectId
});
return dataSources as TSecretScanningDataSource[];
};
const listSecretScanningDataSourcesWithDetailsByProjectId = async (
{ projectId, type }: TListSecretScanningDataSourcesByProjectId,
actor: OrgServiceActor
) => {
await $checkListSecretScanningDataSourcesByProjectIdPermissions(projectId, actor);
const dataSources = await secretScanningV2DAL.dataSources.findWithDetails({
...(type && { type }),
projectId
});
return dataSources as TSecretScanningDataSourceWithDetails[];
};
const findSecretScanningDataSourceById = async (
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource)
throw new NotFoundError({
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId: dataSource.projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningDataSourceActions.Read,
ProjectPermissionSub.SecretScanningDataSources
);
if (type !== dataSource.type)
throw new BadRequestError({
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
});
return dataSource as TSecretScanningDataSource;
};
const findSecretScanningDataSourceByName = async (
{ type, sourceName, projectId }: TFindSecretScanningDataSourceByNameDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
});
// we prevent conflicting names within a folder
const dataSource = await secretScanningV2DAL.dataSources.findOne({
name: sourceName,
projectId
});
if (!dataSource)
throw new NotFoundError({
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with name "${sourceName}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningDataSourceActions.Read,
ProjectPermissionSub.SecretScanningDataSources
);
if (type !== dataSource.type)
throw new BadRequestError({
message: `Secret Scanning Data Source with ID "${dataSource.id}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
});
return dataSource as TSecretScanningDataSource;
};
const createSecretScanningDataSource = async (
payload: TCreateSecretScanningDataSourceDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to create Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId: payload.projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningDataSourceActions.Create,
ProjectPermissionSub.SecretScanningDataSources
);
let connection: TAppConnection | null = null;
if (payload.connectionId) {
// validates permission to connect and app is valid for data source
connection = await appConnectionService.connectAppConnectionById(
SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[payload.type],
payload.connectionId,
actor
);
}
const factory = SECRET_SCANNING_FACTORY_MAP[payload.type]();
try {
const createdDataSource = await factory.initialize(
{
payload,
connection: connection as TSecretScanningDataSourceWithConnection["connection"],
secretScanningV2DAL
},
async ({ credentials, externalId }) => {
let encryptedCredentials: Buffer | null = null;
if (credentials) {
const { encryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId: payload.projectId
});
const { cipherTextBlob } = encryptor({
plainText: Buffer.from(JSON.stringify(credentials))
});
encryptedCredentials = cipherTextBlob;
}
return secretScanningV2DAL.dataSources.transaction(async (tx) => {
const dataSource = await secretScanningV2DAL.dataSources.create(
{
encryptedCredentials,
externalId,
...payload
},
tx
);
await factory.postInitialization({
payload,
connection: connection as TSecretScanningDataSourceWithConnection["connection"],
dataSourceId: dataSource.id,
credentials
});
return dataSource;
});
}
);
if (payload.isAutoScanEnabled) {
try {
await secretScanningV2Queue.queueDataSourceFullScan({
...createdDataSource,
connection
} as TSecretScanningDataSourceWithConnection);
} catch {
// silently fail, don't want to block creation, they'll try scanning when they don't see anything and get the error
}
}
return createdDataSource as TSecretScanningDataSource;
} catch (err) {
if (err instanceof DatabaseError && (err.error as { code: string })?.code === DatabaseErrorCode.UniqueViolation) {
throw new BadRequestError({
message: `A Secret Scanning Data Source with the name "${payload.name}" already exists for the project with ID "${payload.projectId}"`
});
}
throw err;
}
};
const updateSecretScanningDataSource = async (
{ type, dataSourceId, ...payload }: TUpdateSecretScanningDataSourceDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to update Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource)
throw new NotFoundError({
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId: dataSource.projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningDataSourceActions.Edit,
ProjectPermissionSub.SecretScanningDataSources
);
if (type !== dataSource.type)
throw new BadRequestError({
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
});
try {
const updatedDataSource = await secretScanningV2DAL.dataSources.updateById(dataSourceId, payload);
return updatedDataSource as TSecretScanningDataSource;
} catch (err) {
if (err instanceof DatabaseError && (err.error as { code: string })?.code === DatabaseErrorCode.UniqueViolation) {
throw new BadRequestError({
message: `A Secret Scanning Data Source with the name "${payload.name}" already exists for the project with ID "${dataSource.projectId}"`
});
}
throw err;
}
};
const deleteSecretScanningDataSource = async (
{ type, dataSourceId }: TDeleteSecretScanningDataSourceDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to delete Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource)
throw new NotFoundError({
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId: dataSource.projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningDataSourceActions.Delete,
ProjectPermissionSub.SecretScanningDataSources
);
if (type !== dataSource.type)
throw new BadRequestError({
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
});
// TODO: clean up webhooks
await secretScanningV2DAL.dataSources.deleteById(dataSourceId);
return dataSource as TSecretScanningDataSource;
};
const triggerSecretScanningDataSourceScan = async (
{ type, dataSourceId, resourceId }: TTriggerSecretScanningDataSourceDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to trigger scan for Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource)
throw new NotFoundError({
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId: dataSource.projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningDataSourceActions.TriggerScans,
ProjectPermissionSub.SecretScanningDataSources
);
if (type !== dataSource.type)
throw new BadRequestError({
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
});
let connection: TAppConnection | null = null;
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
let resourceExternalId: string | undefined;
if (resourceId) {
const resource = await secretScanningV2DAL.resources.findOne({ id: resourceId, dataSourceId });
if (!resource) {
throw new NotFoundError({
message: `Could not find Secret Scanning Resource with ID "${resourceId}" for Data Source with ID "${dataSourceId}"`
});
}
resourceExternalId = resource.externalId;
}
await secretScanningV2Queue.queueDataSourceFullScan(
{
...dataSource,
connection
} as TSecretScanningDataSourceWithConnection,
resourceExternalId
);
return dataSource as TSecretScanningDataSource;
};
const listSecretScanningResourcesByDataSourceId = async (
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Resources due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource)
throw new NotFoundError({
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId: dataSource.projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningDataSourceActions.ReadResources,
ProjectPermissionSub.SecretScanningDataSources
);
if (type !== dataSource.type)
throw new BadRequestError({
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
});
const resources = await secretScanningV2DAL.resources.find({
dataSourceId
});
return { resources, projectId: dataSource.projectId };
};
const listSecretScanningScansByDataSourceId = async (
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Resources due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource)
throw new NotFoundError({
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId: dataSource.projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
ProjectPermissionSub.SecretScanningDataSources
);
if (type !== dataSource.type)
throw new BadRequestError({
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
});
const scans = await secretScanningV2DAL.scans.findByDataSourceId(dataSourceId);
return { scans, projectId: dataSource.projectId };
};
const listSecretScanningResourcesWithDetailsByDataSourceId = async (
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Resources due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource)
throw new NotFoundError({
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId: dataSource.projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningDataSourceActions.ReadResources,
ProjectPermissionSub.SecretScanningDataSources
);
if (type !== dataSource.type)
throw new BadRequestError({
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
});
const resources = await secretScanningV2DAL.resources.findWithDetails({ dataSourceId });
return { resources: resources as TSecretScanningResourceWithDetails[], projectId: dataSource.projectId };
};
const listSecretScanningScansWithDetailsByDataSourceId = async (
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Scans due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource)
throw new NotFoundError({
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId: dataSource.projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
ProjectPermissionSub.SecretScanningDataSources
);
if (type !== dataSource.type)
throw new BadRequestError({
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
});
const scans = await secretScanningV2DAL.scans.findWithDetailsByDataSourceId(dataSourceId);
return { scans: scans as TSecretScanningScanWithDetails[], projectId: dataSource.projectId };
};
const getSecretScanningUnresolvedFindingsCountByProjectId = async (projectId: string, actor: OrgServiceActor) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningFindingActions.Read,
ProjectPermissionSub.SecretScanningFindings
);
const [finding] = await secretScanningV2DAL.findings.find(
{
projectId,
status: SecretScanningFindingStatus.Unresolved
},
{ count: true }
);
return Number(finding?.count ?? 0);
};
const listSecretScanningFindingsByProjectId = async (projectId: string, actor: OrgServiceActor) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningFindingActions.Read,
ProjectPermissionSub.SecretScanningFindings
);
const findings = await secretScanningV2DAL.findings.find({
projectId
});
return findings as TSecretScanningFinding[];
};
const updateSecretScanningFindingById = async (
{ findingId, remarks, status }: TUpdateSecretScanningFindingDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const finding = await secretScanningV2DAL.findings.findById(findingId);
if (!finding)
throw new NotFoundError({
message: `Could not find Secret Scanning Finding with ID "${findingId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId: finding.projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningFindingActions.Update,
ProjectPermissionSub.SecretScanningFindings
);
const updatedFinding = await secretScanningV2DAL.findings.updateById(findingId, {
remarks,
status
});
return { finding: updatedFinding as TSecretScanningFinding, projectId: finding.projectId };
};
const findSecretScanningConfigByProjectId = async (projectId: string, actor: OrgServiceActor) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Configuration due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningConfigActions.Read,
ProjectPermissionSub.SecretScanningConfigs
);
const config = await secretScanningV2DAL.configs.findOne({
projectId
});
return (
config ?? { content: null, projectId, updatedAt: null } // using default config
);
};
const upsertSecretScanningConfig = async (
{ projectId, content }: TUpsertSecretScanningConfigDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
if (!plan.secretScanning)
throw new BadRequestError({
message:
"Failed to access Secret Scanning Configuration due to plan restriction. Upgrade plan to enable Secret Scanning."
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretScanning,
projectId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningConfigActions.Update,
ProjectPermissionSub.SecretScanningConfigs
);
if (content) {
const tempFolder = await createTempFolder();
try {
const configPath = join(tempFolder, "infisical-scan.toml");
await writeTextToFile(configPath, content);
// just checking if config parses
await scanContentAndGetFindings("", configPath);
} catch (e) {
throw new BadRequestError({
message: "Unable to parse configuration: Check syntax and formatting."
});
} finally {
await deleteTempFolder(tempFolder);
}
}
const [config] = await secretScanningV2DAL.configs.upsert(
[
{
projectId,
content
}
],
"projectId"
);
return config;
};
return {
listSecretScanningDataSourceOptions,
listSecretScanningDataSourcesByProjectId,
listSecretScanningDataSourcesWithDetailsByProjectId,
findSecretScanningDataSourceById,
findSecretScanningDataSourceByName,
createSecretScanningDataSource,
updateSecretScanningDataSource,
deleteSecretScanningDataSource,
triggerSecretScanningDataSourceScan,
listSecretScanningResourcesByDataSourceId,
listSecretScanningScansByDataSourceId,
listSecretScanningResourcesWithDetailsByDataSourceId,
listSecretScanningScansWithDetailsByDataSourceId,
getSecretScanningUnresolvedFindingsCountByProjectId,
listSecretScanningFindingsByProjectId,
updateSecretScanningFindingById,
findSecretScanningConfigByProjectId,
upsertSecretScanningConfig,
github: githubSecretScanningService(secretScanningV2DAL, secretScanningV2Queue)
};
};
@@ -0,0 +1,189 @@
import {
TSecretScanningDataSources,
TSecretScanningFindingsInsert,
TSecretScanningResources,
TSecretScanningScans
} from "@app/db/schemas";
import {
TGitHubDataSource,
TGitHubDataSourceInput,
TGitHubDataSourceListItem,
TGitHubDataSourceWithConnection,
TGitHubFinding,
TQueueGitHubResourceDiffScan
} from "@app/ee/services/secret-scanning-v2/github";
import { TSecretScanningV2DALFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-dal";
import {
SecretScanningDataSource,
SecretScanningFindingStatus,
SecretScanningScanStatus
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
export type TSecretScanningDataSource = TGitHubDataSource;
export type TSecretScanningDataSourceWithDetails = TSecretScanningDataSource & {
lastScannedAt?: Date | null;
lastScanStatus?: SecretScanningScanStatus | null;
lastScanStatusMessage?: string | null;
unresolvedFindings: number;
};
export type TSecretScanningResourceWithDetails = TSecretScanningResources & {
lastScannedAt?: Date | null;
lastScanStatus?: SecretScanningScanStatus | null;
lastScanStatusMessage?: string | null;
unresolvedFindings: number;
};
export type TSecretScanningScanWithDetails = TSecretScanningScans & {
unresolvedFindings: number;
resolvedFindings: number;
resourceName: string;
};
export type TSecretScanningDataSourceWithConnection = TGitHubDataSourceWithConnection;
export type TSecretScanningDataSourceInput = TGitHubDataSourceInput;
export type TSecretScanningDataSourceListItem = TGitHubDataSourceListItem;
export type TSecretScanningFinding = TGitHubFinding;
export type TListSecretScanningDataSourcesByProjectId = {
projectId: string;
type?: SecretScanningDataSource;
};
export type TFindSecretScanningDataSourceByIdDTO = {
dataSourceId: string;
type: SecretScanningDataSource;
};
export type TFindSecretScanningDataSourceByNameDTO = {
sourceName: string;
projectId: string;
type: SecretScanningDataSource;
};
export type TCreateSecretScanningDataSourceDTO = Pick<
TSecretScanningDataSource,
"description" | "name" | "projectId"
> & {
connectionId?: string;
type: SecretScanningDataSource;
isAutoScanEnabled?: boolean;
config: Partial<TSecretScanningDataSourceInput["config"]>;
};
export type TUpdateSecretScanningDataSourceDTO = Partial<
Omit<TCreateSecretScanningDataSourceDTO, "projectId" | "connectionId">
> & {
dataSourceId: string;
type: SecretScanningDataSource;
};
export type TDeleteSecretScanningDataSourceDTO = {
type: SecretScanningDataSource;
dataSourceId: string;
};
export type TTriggerSecretScanningDataSourceDTO = {
type: SecretScanningDataSource;
dataSourceId: string;
resourceId?: string;
};
export type TQueueSecretScanningDataSourceFullScan = {
dataSourceId: string;
resourceId: string;
scanId: string;
};
export type TQueueSecretScanningResourceDiffScan = TQueueGitHubResourceDiffScan;
export type TQueueSecretScanningSendNotification = {
dataSource: TSecretScanningDataSources;
resourceName: string;
} & (
| { status: SecretScanningScanStatus.Failed; errorMessage: string }
| { status: SecretScanningScanStatus.Completed; numberOfSecrets: number; scanId: string; isDiffScan: boolean }
);
export type TCloneRepository = {
cloneUrl: string;
repoPath: string;
};
export type TSecretScanningFactoryListRawResources<T extends TSecretScanningDataSourceWithConnection> = (
dataSource: T
) => Promise<Pick<TSecretScanningResources, "externalId" | "name" | "type">[]>;
export type TSecretScanningFactoryGetDiffScanResourcePayload<
P extends TQueueSecretScanningResourceDiffScan["payload"]
> = (payload: P) => Pick<TSecretScanningResources, "externalId" | "name" | "type">;
export type TSecretScanningFactoryGetFullScanPath<T extends TSecretScanningDataSourceWithConnection> = (parameters: {
dataSource: T;
resourceName: string;
tempFolder: string;
}) => Promise<string>;
export type TSecretScanningFactoryGetDiffScanFindingsPayload<
T extends TSecretScanningDataSourceWithConnection,
P extends TQueueSecretScanningResourceDiffScan["payload"]
> = (parameters: { dataSource: T; resourceName: string; payload: P; configPath?: string }) => Promise<TFindingsPayload>;
export type TSecretScanningDataSourceRaw = NonNullable<
Awaited<ReturnType<TSecretScanningV2DALFactory["dataSources"]["findById"]>>
>;
export type TSecretScanningFactoryInitialize<
T extends TSecretScanningDataSourceWithConnection["connection"] | undefined = undefined,
C extends TSecretScanningDataSourceCredentials = undefined
> = (
params: {
payload: TCreateSecretScanningDataSourceDTO;
connection: T;
secretScanningV2DAL: TSecretScanningV2DALFactory;
},
callback: (parameters: { credentials?: C; externalId?: string }) => Promise<TSecretScanningDataSourceRaw>
) => Promise<TSecretScanningDataSourceRaw>;
export type TSecretScanningFactoryPostInitialization<
T extends TSecretScanningDataSourceWithConnection["connection"] | undefined = undefined,
C extends TSecretScanningDataSourceCredentials = undefined
> = (params: {
payload: TCreateSecretScanningDataSourceDTO;
connection: T;
credentials: C;
dataSourceId: string;
}) => Promise<void>;
export type TSecretScanningFactory<
T extends TSecretScanningDataSourceWithConnection,
C extends TSecretScanningDataSourceCredentials,
P extends TQueueSecretScanningResourceDiffScan["payload"]
> = () => {
listRawResources: TSecretScanningFactoryListRawResources<T>;
getFullScanPath: TSecretScanningFactoryGetFullScanPath<T>;
initialize: TSecretScanningFactoryInitialize<T["connection"] | undefined, C>;
postInitialization: TSecretScanningFactoryPostInitialization<T["connection"] | undefined, C>;
getDiffScanResourcePayload: TSecretScanningFactoryGetDiffScanResourcePayload<P>;
getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<T, P>;
};
export type TFindingsPayload = Pick<TSecretScanningFindingsInsert, "details" | "fingerprint" | "severity" | "rule">[];
export type TGetFindingsPayload = Promise<TFindingsPayload>;
export type TUpdateSecretScanningFindingDTO = {
status?: SecretScanningFindingStatus;
remarks?: string | null;
findingId: string;
};
export type TUpsertSecretScanningConfigDTO = {
projectId: string;
content: string | null;
};
export type TSecretScanningDataSourceCredentials = undefined;
@@ -0,0 +1,7 @@
import { z } from "zod";
import { GitHubDataSourceSchema, GitHubFindingSchema } from "@app/ee/services/secret-scanning-v2/github";
export const SecretScanningDataSourceSchema = z.discriminatedUnion("type", [GitHubDataSourceSchema]);
export const SecretScanningFindingSchema = z.discriminatedUnion("resourceType", [GitHubFindingSchema]);
@@ -65,9 +65,9 @@ export function runInfisicalScanOnRepo(repoPath: string, outputPath: string): Pr
});
}
export function runInfisicalScan(inputPath: string, outputPath: string): Promise<void> {
export function runInfisicalScan(inputPath: string, outputPath: string, configPath?: string): Promise<void> {
return new Promise((resolve, reject) => {
const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}"`;
const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}" ${configPath ? `-c "${configPath}"` : ""}`;
exec(command, (error) => {
if (error && error.code !== 77) {
reject(error);
@@ -138,14 +138,14 @@ export async function scanFullRepoContentAndGetFindings(
}
}
export async function scanContentAndGetFindings(textContent: string): Promise<SecretMatch[]> {
export async function scanContentAndGetFindings(textContent: string, configPath?: string): Promise<SecretMatch[]> {
const tempFolder = await createTempFolder();
const filePath = join(tempFolder, "content.txt");
const findingsPath = join(tempFolder, "findings.json");
try {
await writeTextToFile(filePath, textContent);
await runInfisicalScan(filePath, findingsPath);
await runInfisicalScan(filePath, findingsPath, configPath);
const findingsData = await readFindingsFile(findingsPath);
return JSON.parse(findingsData) as SecretMatch[];
} finally {
@@ -9,6 +9,7 @@ export type SecretMatch = {
Match: string;
Secret: string;
File: string;
Link: string;
SymlinkFile: string;
Commit: string;
Entropy: number;