diff --git a/.env.example b/.env.example index 8ee3d2001..23f845b71 100644 --- a/.env.example +++ b/.env.example @@ -107,6 +107,14 @@ INF_APP_CONNECTION_GITHUB_APP_PRIVATE_KEY= INF_APP_CONNECTION_GITHUB_APP_SLUG= INF_APP_CONNECTION_GITHUB_APP_ID= +#github radar app connection +INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_ID= +INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_SECRET= +INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY= +INF_APP_CONNECTION_GITHUB_RADAR_APP_SLUG= +INF_APP_CONNECTION_GITHUB_RADAR_APP_ID= +INF_APP_CONNECTION_GITHUB_RADAR_APP_WEBHOOK_SECRET= + #gcp app connection INF_APP_CONNECTION_GCP_SERVICE_ACCOUNT_CREDENTIAL= diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index b098368c4..c967355c5 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -37,6 +37,7 @@ import { TSecretApprovalRequestServiceFactory } from "@app/ee/services/secret-ap import { TSecretRotationServiceFactory } from "@app/ee/services/secret-rotation/secret-rotation-service"; import { TSecretRotationV2ServiceFactory } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-service"; import { TSecretScanningServiceFactory } from "@app/ee/services/secret-scanning/secret-scanning-service"; +import { TSecretScanningV2ServiceFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-service"; import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; import { TSshCertificateAuthorityServiceFactory } from "@app/ee/services/ssh/ssh-certificate-authority-service"; import { TSshCertificateTemplateServiceFactory } from "@app/ee/services/ssh-certificate-template/ssh-certificate-template-service"; @@ -268,6 +269,7 @@ declare module "fastify" { microsoftTeams: TMicrosoftTeamsServiceFactory; assumePrivileges: TAssumePrivilegeServiceFactory; githubOrgSync: TGithubOrgSyncServiceFactory; + secretScanningV2: TSecretScanningV2ServiceFactory; }; // this is exclusive use for middlewares in which we need to inject data // everywhere else access using service layer diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 276669b2a..d1f33cbc7 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -324,9 +324,21 @@ import { TSecretRotationV2SecretMappingsInsert, TSecretRotationV2SecretMappingsUpdate, TSecrets, + TSecretScanningDataSources, + TSecretScanningDataSourcesInsert, + TSecretScanningDataSourcesUpdate, + TSecretScanningFindings, + TSecretScanningFindingsInsert, + TSecretScanningFindingsUpdate, TSecretScanningGitRisks, TSecretScanningGitRisksInsert, TSecretScanningGitRisksUpdate, + TSecretScanningResources, + TSecretScanningResourcesInsert, + TSecretScanningResourcesUpdate, + TSecretScanningScans, + TSecretScanningScansInsert, + TSecretScanningScansUpdate, TSecretSharing, TSecretSharingInsert, TSecretSharingUpdate, @@ -1074,5 +1086,25 @@ declare module "knex/types/tables" { TGithubOrgSyncConfigsInsert, TGithubOrgSyncConfigsUpdate >; + [TableName.SecretScanningDataSource]: KnexOriginal.CompositeTableType< + TSecretScanningDataSources, + TSecretScanningDataSourcesInsert, + TSecretScanningDataSourcesUpdate + >; + [TableName.SecretScanningResource]: KnexOriginal.CompositeTableType< + TSecretScanningResources, + TSecretScanningResourcesInsert, + TSecretScanningResourcesUpdate + >; + [TableName.SecretScanningScan]: KnexOriginal.CompositeTableType< + TSecretScanningScans, + TSecretScanningScansInsert, + TSecretScanningScansUpdate + >; + [TableName.SecretScanningFinding]: KnexOriginal.CompositeTableType< + TSecretScanningFindings, + TSecretScanningFindingsInsert, + TSecretScanningFindingsUpdate + >; } } diff --git a/backend/src/db/migrations/20250517002225_secret-scanning-v2.ts b/backend/src/db/migrations/20250517002225_secret-scanning-v2.ts new file mode 100644 index 000000000..660c623e2 --- /dev/null +++ b/backend/src/db/migrations/20250517002225_secret-scanning-v2.ts @@ -0,0 +1,94 @@ +import { Knex } from "knex"; + +import { TableName } from "@app/db/schemas"; +import { createOnUpdateTrigger, dropOnUpdateTrigger } from "@app/db/utils"; +import { + SecretScanningFindingStatus, + SecretScanningScanStatus +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasTable(TableName.SecretScanningDataSource))) { + await knex.schema.createTable(TableName.SecretScanningDataSource, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.string("externalId").index(); // if we need a unique way of identifying this data source from an external resource + t.string("name", 48).notNullable(); + t.string("description"); + t.string("type").notNullable(); + t.jsonb("config").notNullable(); + t.binary("encryptedCredentials"); // webhook credentials, etc. + t.uuid("connectionId"); + t.boolean("isAutoScanEnabled").defaultTo(true); + t.foreign("connectionId").references("id").inTable(TableName.AppConnection); + t.string("projectId").notNullable(); + t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); + t.timestamps(true, true, true); + t.unique(["projectId", "name"]); + }); + await createOnUpdateTrigger(knex, TableName.SecretScanningDataSource); + } + + if (!(await knex.schema.hasTable(TableName.SecretScanningResource))) { + await knex.schema.createTable(TableName.SecretScanningResource, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.string("externalId").notNullable(); + t.string("name").notNullable(); + t.string("type").notNullable(); + t.uuid("dataSourceId").notNullable(); + t.foreign("dataSourceId").references("id").inTable(TableName.SecretScanningDataSource).onDelete("CASCADE"); + t.timestamps(true, true, true); + t.unique(["dataSourceId", "externalId"]); + }); + await createOnUpdateTrigger(knex, TableName.SecretScanningResource); + } + + if (!(await knex.schema.hasTable(TableName.SecretScanningScan))) { + await knex.schema.createTable(TableName.SecretScanningScan, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.string("status").notNullable().defaultTo(SecretScanningScanStatus.Queued); + t.string("statusMessage", 1024); + t.string("type").notNullable(); + t.uuid("resourceId").notNullable(); + t.foreign("resourceId").references("id").inTable(TableName.SecretScanningResource).onDelete("CASCADE"); + t.timestamp("createdAt").defaultTo(knex.fn.now()); + }); + } + + if (!(await knex.schema.hasTable(TableName.SecretScanningFinding))) { + await knex.schema.createTable(TableName.SecretScanningFinding, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.string("dataSourceName").notNullable(); + t.string("dataSourceType").notNullable(); + t.string("resourceName").notNullable(); + t.string("resourceType").notNullable(); + t.string("rule").notNullable(); + t.string("severity").notNullable(); + t.string("status").notNullable().defaultTo(SecretScanningFindingStatus.Unresolved); + t.string("remarks"); + t.string("fingerprint").notNullable(); + t.jsonb("details").notNullable(); + t.string("projectId").notNullable(); + t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); + t.uuid("scanId"); + t.foreign("scanId").references("id").inTable(TableName.SecretScanningScan).onDelete("SET NULL"); + t.timestamps(true, true, true); + t.unique(["projectId", "fingerprint"]); + }); + await createOnUpdateTrigger(knex, TableName.SecretScanningFinding); + } + + // TODO: Rules +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists(TableName.SecretScanningFinding); + + await dropOnUpdateTrigger(knex, TableName.SecretScanningFinding); + await knex.schema.dropTableIfExists(TableName.SecretScanningScan); + + await knex.schema.dropTableIfExists(TableName.SecretScanningResource); + await dropOnUpdateTrigger(knex, TableName.SecretScanningResource); + + await knex.schema.dropTableIfExists(TableName.SecretScanningDataSource); + await dropOnUpdateTrigger(knex, TableName.SecretScanningDataSource); +} diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index 0bf44c413..985b4bd72 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -107,7 +107,11 @@ export * from "./secret-rotation-outputs"; export * from "./secret-rotation-v2-secret-mappings"; export * from "./secret-rotations"; export * from "./secret-rotations-v2"; +export * from "./secret-scanning-data-sources"; +export * from "./secret-scanning-findings"; export * from "./secret-scanning-git-risks"; +export * from "./secret-scanning-resources"; +export * from "./secret-scanning-scans"; export * from "./secret-sharing"; export * from "./secret-snapshot-folders"; export * from "./secret-snapshot-secrets"; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 730474ad1..5aacee351 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -155,7 +155,11 @@ export enum TableName { MicrosoftTeamsIntegrations = "microsoft_teams_integrations", ProjectMicrosoftTeamsConfigs = "project_microsoft_teams_configs", SecretReminderRecipients = "secret_reminder_recipients", - GithubOrgSyncConfig = "github_org_sync_configs" + GithubOrgSyncConfig = "github_org_sync_configs", + SecretScanningDataSource = "secret_scanning_data_sources", + SecretScanningResource = "secret_scanning_resources", + SecretScanningScan = "secret_scanning_scans", + SecretScanningFinding = "secret_scanning_findings" } export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt"; @@ -244,7 +248,8 @@ export enum ProjectType { SecretManager = "secret-manager", CertificateManager = "cert-manager", KMS = "kms", - SSH = "ssh" + SSH = "ssh", + SecretScanning = "secret-scanning" } export enum ActionProjectType { @@ -252,6 +257,7 @@ export enum ActionProjectType { CertificateManager = ProjectType.CertificateManager, KMS = ProjectType.KMS, SSH = ProjectType.SSH, + SecretScanning = ProjectType.SecretScanning, // project operations that happen on all types Any = "any" } diff --git a/backend/src/db/schemas/secret-scanning-data-sources.ts b/backend/src/db/schemas/secret-scanning-data-sources.ts new file mode 100644 index 000000000..90f997216 --- /dev/null +++ b/backend/src/db/schemas/secret-scanning-data-sources.ts @@ -0,0 +1,31 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { zodBuffer } from "@app/lib/zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretScanningDataSourcesSchema = z.object({ + id: z.string().uuid(), + externalId: z.string().nullable().optional(), + name: z.string(), + description: z.string().nullable().optional(), + type: z.string(), + config: z.unknown(), + encryptedCredentials: zodBuffer.nullable().optional(), + connectionId: z.string().uuid().nullable().optional(), + isAutoScanEnabled: z.boolean().default(true).nullable().optional(), + projectId: z.string(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TSecretScanningDataSources = z.infer; +export type TSecretScanningDataSourcesInsert = Omit, TImmutableDBKeys>; +export type TSecretScanningDataSourcesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-scanning-findings.ts b/backend/src/db/schemas/secret-scanning-findings.ts new file mode 100644 index 000000000..c36f229f8 --- /dev/null +++ b/backend/src/db/schemas/secret-scanning-findings.ts @@ -0,0 +1,32 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretScanningFindingsSchema = z.object({ + id: z.string().uuid(), + dataSourceName: z.string(), + dataSourceType: z.string(), + resourceName: z.string(), + resourceType: z.string(), + rule: z.string(), + severity: z.string(), + status: z.string().default("unresolved"), + remarks: z.string().nullable().optional(), + fingerprint: z.string(), + details: z.unknown(), + projectId: z.string(), + scanId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TSecretScanningFindings = z.infer; +export type TSecretScanningFindingsInsert = Omit, TImmutableDBKeys>; +export type TSecretScanningFindingsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-scanning-resources.ts b/backend/src/db/schemas/secret-scanning-resources.ts new file mode 100644 index 000000000..e791e1cd6 --- /dev/null +++ b/backend/src/db/schemas/secret-scanning-resources.ts @@ -0,0 +1,24 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretScanningResourcesSchema = z.object({ + id: z.string().uuid(), + externalId: z.string(), + name: z.string(), + type: z.string(), + dataSourceId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TSecretScanningResources = z.infer; +export type TSecretScanningResourcesInsert = Omit, TImmutableDBKeys>; +export type TSecretScanningResourcesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-scanning-scans.ts b/backend/src/db/schemas/secret-scanning-scans.ts new file mode 100644 index 000000000..88e676b5b --- /dev/null +++ b/backend/src/db/schemas/secret-scanning-scans.ts @@ -0,0 +1,21 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretScanningScansSchema = z.object({ + id: z.string().uuid(), + status: z.string().default("queued"), + statusMessage: z.string().nullable().optional(), + type: z.string(), + resourceId: z.string().uuid(), + createdAt: z.date().nullable().optional() +}); + +export type TSecretScanningScans = z.infer; +export type TSecretScanningScansInsert = Omit, TImmutableDBKeys>; +export type TSecretScanningScansUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/ee/routes/v2/index.ts b/backend/src/ee/routes/v2/index.ts index 70e5005a4..e364f4949 100644 --- a/backend/src/ee/routes/v2/index.ts +++ b/backend/src/ee/routes/v2/index.ts @@ -2,6 +2,10 @@ import { registerSecretRotationV2Router, SECRET_ROTATION_REGISTER_ROUTER_MAP } from "@app/ee/routes/v2/secret-rotation-v2-routers"; +import { + registerSecretScanningV2Router, + SECRET_SCANNING_REGISTER_ROUTER_MAP +} from "@app/ee/routes/v2/secret-scanning-v2-routers"; import { registerIdentityProjectAdditionalPrivilegeRouter } from "./identity-project-additional-privilege-router"; import { registerProjectRoleRouter } from "./project-role-router"; @@ -31,4 +35,17 @@ export const registerV2EERoutes = async (server: FastifyZodProvider) => { }, { prefix: "/secret-rotations" } ); + + await server.register( + async (secretScanningV2Router) => { + // register generic secret scanning endpoints + await secretScanningV2Router.register(registerSecretScanningV2Router); + + // register service-specific secret scanning endpoints (gitlab/github, etc.) + for await (const [type, router] of Object.entries(SECRET_SCANNING_REGISTER_ROUTER_MAP)) { + await secretScanningV2Router.register(router, { prefix: `data-sources/${type}` }); + } + }, + { prefix: "/secret-scanning" } + ); }; diff --git a/backend/src/ee/routes/v2/secret-scanning-v2-routers/github-secret-scanning-router.ts b/backend/src/ee/routes/v2/secret-scanning-v2-routers/github-secret-scanning-router.ts new file mode 100644 index 000000000..3961e7cbd --- /dev/null +++ b/backend/src/ee/routes/v2/secret-scanning-v2-routers/github-secret-scanning-router.ts @@ -0,0 +1,16 @@ +import { registerSecretScanningEndpoints } from "@app/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints"; +import { + CreateGitHubDataSourceSchema, + GitHubDataSourceSchema, + UpdateGitHubDataSourceSchema +} from "@app/ee/services/secret-scanning-v2/github"; +import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; + +export const registerGitHubSecretScanningRouter = async (server: FastifyZodProvider) => + registerSecretScanningEndpoints({ + type: SecretScanningDataSource.GitHub, + server, + responseSchema: GitHubDataSourceSchema, + createSchema: CreateGitHubDataSourceSchema, + updateSchema: UpdateGitHubDataSourceSchema + }); diff --git a/backend/src/ee/routes/v2/secret-scanning-v2-routers/index.ts b/backend/src/ee/routes/v2/secret-scanning-v2-routers/index.ts new file mode 100644 index 000000000..703529947 --- /dev/null +++ b/backend/src/ee/routes/v2/secret-scanning-v2-routers/index.ts @@ -0,0 +1,12 @@ +import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; + +import { registerGitHubSecretScanningRouter } from "./github-secret-scanning-router"; + +export * from "./secret-scanning-v2-router"; + +export const SECRET_SCANNING_REGISTER_ROUTER_MAP: Record< + SecretScanningDataSource, + (server: FastifyZodProvider) => Promise +> = { + [SecretScanningDataSource.GitHub]: registerGitHubSecretScanningRouter +}; diff --git a/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints.ts b/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints.ts new file mode 100644 index 000000000..2a606e034 --- /dev/null +++ b/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints.ts @@ -0,0 +1,593 @@ +import { z } from "zod"; + +import { SecretScanningResourcesSchema, SecretScanningScansSchema } from "@app/db/schemas"; +import { EventType } from "@app/ee/services/audit-log/audit-log-types"; +import { + SecretScanningDataSource, + SecretScanningScanStatus +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; +import { SECRET_SCANNING_DATA_SOURCE_NAME_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps"; +import { + TSecretScanningDataSource, + TSecretScanningDataSourceInput +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types"; +import { ApiDocsTags, SecretScanningDataSources } from "@app/lib/api-docs"; +import { startsWithVowel } from "@app/lib/fn"; +import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AuthMode } from "@app/services/auth/auth-type"; + +export const registerSecretScanningEndpoints = < + T extends TSecretScanningDataSource, + I extends TSecretScanningDataSourceInput +>({ + server, + type, + createSchema, + updateSchema, + responseSchema +}: { + type: SecretScanningDataSource; + server: FastifyZodProvider; + createSchema: z.ZodType<{ + name: string; + projectId: string; + connectionId?: string; + config: Partial; + description?: string | null; + isAutoScanEnabled?: boolean; + }>; + updateSchema: z.ZodType<{ + name?: string; + config?: Partial; + description?: string | null; + isAutoScanEnabled?: boolean; + }>; + responseSchema: z.ZodTypeAny; +}) => { + const sourceType = SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]; + + server.route({ + method: "GET", + url: `/`, + config: { + rateLimit: readLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: `List the ${sourceType} Data Sources for the specified project.`, + querystring: z.object({ + projectId: z + .string() + .trim() + .min(1, "Project ID required") + .describe(SecretScanningDataSources.LIST(type).projectId) + }), + response: { + 200: z.object({ dataSources: responseSchema.array() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { + query: { projectId } + } = req; + + const dataSources = (await server.services.secretScanningV2.listSecretScanningDataSourcesByProjectId( + { projectId, type }, + req.permission + )) as T[]; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId, + event: { + type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST, + metadata: { + type, + count: dataSources.length, + dataSourceIds: dataSources.map((source) => source.id) + } + } + }); + + return { dataSources }; + } + }); + + server.route({ + method: "GET", + url: "/:dataSourceId", + config: { + rateLimit: readLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: `Get the specified ${sourceType} Data Source by ID.`, + params: z.object({ + dataSourceId: z.string().uuid().describe(SecretScanningDataSources.GET_BY_ID(type).dataSourceId) + }), + response: { + 200: z.object({ dataSource: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { dataSourceId } = req.params; + + const dataSource = (await server.services.secretScanningV2.findSecretScanningDataSourceById( + { dataSourceId, type }, + req.permission + )) as T; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: dataSource.projectId, + event: { + type: EventType.SECRET_SCANNING_DATA_SOURCE_GET, + metadata: { + dataSourceId, + type + } + } + }); + + return { dataSource }; + } + }); + + server.route({ + method: "GET", + url: `/data-source-name/:dataSourceName`, + config: { + rateLimit: readLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: `Get the specified ${sourceType} Data Source by name and project ID.`, + params: z.object({ + sourceName: z + .string() + .trim() + .min(1, "Data Source name required") + .describe(SecretScanningDataSources.GET_BY_NAME(type).sourceName) + }), + querystring: z.object({ + projectId: z + .string() + .trim() + .min(1, "Project ID required") + .describe(SecretScanningDataSources.GET_BY_NAME(type).projectId) + }), + response: { + 200: z.object({ dataSource: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { sourceName } = req.params; + const { projectId } = req.query; + + const dataSource = (await server.services.secretScanningV2.findSecretScanningDataSourceByName( + { sourceName, projectId, type }, + req.permission + )) as T; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId, + event: { + type: EventType.SECRET_SCANNING_DATA_SOURCE_GET, + metadata: { + dataSourceId: dataSource.id, + type + } + } + }); + + return { dataSource }; + } + }); + + server.route({ + method: "POST", + url: "/", + config: { + rateLimit: writeLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: `Create ${ + startsWithVowel(sourceType) ? "an" : "a" + } ${sourceType} Data Source for the specified project.`, + body: createSchema, + response: { + 200: z.object({ dataSource: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const dataSource = (await server.services.secretScanningV2.createSecretScanningDataSource( + { ...req.body, type }, + req.permission + )) as T; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: dataSource.projectId, + event: { + type: EventType.SECRET_SCANNING_DATA_SOURCE_CREATE, + metadata: { + dataSourceId: dataSource.id, + type, + ...req.body + } + } + }); + + return { dataSource }; + } + }); + + server.route({ + method: "PATCH", + url: "/:dataSourceId", + config: { + rateLimit: writeLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: `Update the specified ${sourceType} Data Source.`, + params: z.object({ + dataSourceId: z.string().uuid().describe(SecretScanningDataSources.UPDATE(type).dataSourceId) + }), + body: updateSchema, + response: { + 200: z.object({ dataSource: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { dataSourceId } = req.params; + + const dataSource = (await server.services.secretScanningV2.updateSecretScanningDataSource( + { ...req.body, dataSourceId, type }, + req.permission + )) as T; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: dataSource.projectId, + event: { + type: EventType.SECRET_SCANNING_DATA_SOURCE_UPDATE, + metadata: { + dataSourceId, + type, + ...req.body + } + } + }); + + return { dataSource }; + } + }); + + server.route({ + method: "DELETE", + url: `/:dataSourceId`, + config: { + rateLimit: writeLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: `Delete the specified ${sourceType} Data Source.`, + params: z.object({ + dataSourceId: z.string().uuid().describe(SecretScanningDataSources.DELETE(type).dataSourceId) + }), + response: { + 200: z.object({ dataSource: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { dataSourceId } = req.params; + + const dataSource = (await server.services.secretScanningV2.deleteSecretScanningResource( + { type, dataSourceId }, + req.permission + )) as T; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: dataSource.projectId, + event: { + type: EventType.SECRET_SCANNING_DATA_SOURCE_DELETE, + metadata: { + type, + dataSourceId + } + } + }); + + return { dataSource }; + } + }); + + server.route({ + method: "POST", + url: `/:dataSourceId/scan`, + config: { + rateLimit: writeLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: `Trigger a scan for the specified ${sourceType} Data Source.`, + params: z.object({ + dataSourceId: z.string().uuid().describe(SecretScanningDataSources.SCAN(type).dataSourceId) + }), + response: { + 200: z.object({ dataSource: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { dataSourceId } = req.params; + + const dataSource = (await server.services.secretScanningV2.triggerSecretScanningDataSourceScan( + { type, dataSourceId }, + req.permission + )) as T; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: dataSource.projectId, + event: { + type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN, + metadata: { + type, + dataSourceId + } + } + }); + + return { dataSource }; + } + }); + + server.route({ + method: "POST", + url: `/:dataSourceId/resources/:resourceId/scan`, + config: { + rateLimit: writeLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: `Trigger a scan for the specified ${sourceType} Data Source resource.`, + params: z.object({ + dataSourceId: z.string().uuid().describe(SecretScanningDataSources.SCAN(type).dataSourceId), + resourceId: z.string().uuid().describe(SecretScanningDataSources.SCAN(type).resourceId) + }), + response: { + 200: z.object({ dataSource: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { dataSourceId, resourceId } = req.params; + + const dataSource = (await server.services.secretScanningV2.triggerSecretScanningDataSourceScan( + { type, dataSourceId, resourceId }, + req.permission + )) as T; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: dataSource.projectId, + event: { + type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN, + metadata: { + type, + dataSourceId, + resourceId + } + } + }); + + return { dataSource }; + } + }); + + server.route({ + method: "GET", + url: "/:dataSourceId/resources", + config: { + rateLimit: readLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: `Get the resources associated with the specified ${sourceType} Data Source by ID.`, + params: z.object({ + dataSourceId: z.string().uuid().describe(SecretScanningDataSources.LIST_RESOURCES(type).dataSourceId) + }), + response: { + 200: z.object({ resources: SecretScanningResourcesSchema.array() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { dataSourceId } = req.params; + + const { resources, projectId } = await server.services.secretScanningV2.listSecretScanningResourcesByDataSourceId( + { dataSourceId, type }, + req.permission + ); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId, + event: { + type: EventType.SECRET_SCANNING_RESOURCE_LIST, + metadata: { + dataSourceId, + type, + resourceIds: resources.map((resource) => resource.id), + count: resources.length + } + } + }); + + return { resources }; + } + }); + + server.route({ + method: "GET", + url: "/:dataSourceId/scans", + config: { + rateLimit: readLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: `Get the scans associated with the specified ${sourceType} Data Source by ID.`, + params: z.object({ + dataSourceId: z.string().uuid().describe(SecretScanningDataSources.LIST_SCANS(type).dataSourceId) + }), + response: { + 200: z.object({ scans: SecretScanningScansSchema.array() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { dataSourceId } = req.params; + + const { scans, projectId } = await server.services.secretScanningV2.listSecretScanningScansByDataSourceId( + { dataSourceId, type }, + req.permission + ); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId, + event: { + type: EventType.SECRET_SCANNING_SCAN_LIST, + metadata: { + dataSourceId, + type, + count: scans.length + } + } + }); + + return { scans }; + } + }); + + // not exposed, for UI only + server.route({ + method: "GET", + url: "/:dataSourceId/resources-dashboard", + config: { + rateLimit: readLimit + }, + schema: { + tags: [ApiDocsTags.SecretScanning], + params: z.object({ + dataSourceId: z.string().uuid() + }), + response: { + 200: z.object({ + resources: SecretScanningResourcesSchema.extend({ + lastScannedAt: z.date().nullish(), + lastScanStatus: z.nativeEnum(SecretScanningScanStatus).nullish(), + lastScanStatusMessage: z.string().nullish(), + unresolvedFindings: z.number() + }).array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const { dataSourceId } = req.params; + + const { resources, projectId } = + await server.services.secretScanningV2.listSecretScanningResourcesWithDetailsByDataSourceId( + { dataSourceId, type }, + req.permission + ); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId, + event: { + type: EventType.SECRET_SCANNING_RESOURCE_LIST, + metadata: { + dataSourceId, + type, + resourceIds: resources.map((resource) => resource.id), + count: resources.length + } + } + }); + + return { resources }; + } + }); + + server.route({ + method: "GET", + url: "/:dataSourceId/scans-dashboard", + config: { + rateLimit: readLimit + }, + schema: { + tags: [ApiDocsTags.SecretScanning], + params: z.object({ + dataSourceId: z.string().uuid() + }), + response: { + 200: z.object({ + scans: SecretScanningScansSchema.extend({ + unresolvedFindings: z.number(), + resolvedFindings: z.number(), + resourceName: z.string() + }).array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const { dataSourceId } = req.params; + + const { scans, projectId } = + await server.services.secretScanningV2.listSecretScanningScansWithDetailsByDataSourceId( + { dataSourceId, type }, + req.permission + ); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId, + event: { + type: EventType.SECRET_SCANNING_SCAN_LIST, + metadata: { + dataSourceId, + type, + count: scans.length + } + } + }); + + return { scans }; + } + }); +}; diff --git a/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-router.ts b/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-router.ts new file mode 100644 index 000000000..a7222875a --- /dev/null +++ b/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-router.ts @@ -0,0 +1,270 @@ +import { z } from "zod"; + +import { EventType } from "@app/ee/services/audit-log/audit-log-types"; +import { GitHubDataSourceListItemSchema } from "@app/ee/services/secret-scanning-v2/github"; +import { + SecretScanningFindingStatus, + SecretScanningScanStatus +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; +import { + SecretScanningDataSourceSchema, + SecretScanningFindingSchema +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-union-schemas"; +import { ApiDocsTags, SecretScanningDataSources, SecretScanningFindings } from "@app/lib/api-docs"; +import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AuthMode } from "@app/services/auth/auth-type"; + +const SecretScanningDataSourceOptionsSchema = z.discriminatedUnion("type", [GitHubDataSourceListItemSchema]); + +export const registerSecretScanningV2Router = async (server: FastifyZodProvider) => { + server.route({ + method: "GET", + url: "/data-sources/options", + config: { + rateLimit: readLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: "List the available Secret Scanning Data Source Options.", + response: { + 200: z.object({ + dataSourceOptions: SecretScanningDataSourceOptionsSchema.array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: () => { + const dataSourceOptions = server.services.secretScanningV2.listSecretScanningDataSourceOptions(); + return { dataSourceOptions }; + } + }); + + server.route({ + method: "GET", + url: "/data-sources", + config: { + rateLimit: readLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: "List all the Secret Scanning Data Sources for the specified project.", + querystring: z.object({ + projectId: z.string().trim().min(1, "Project ID required").describe(SecretScanningDataSources.LIST().projectId) + }), + response: { + 200: z.object({ dataSources: SecretScanningDataSourceSchema.array() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { + query: { projectId }, + permission + } = req; + + const dataSources = await server.services.secretScanningV2.listSecretScanningDataSourcesByProjectId( + { projectId }, + permission + ); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId, + event: { + type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST, + metadata: { + dataSourceIds: dataSources.map((dataSource) => dataSource.id), + count: dataSources.length + } + } + }); + + return { dataSources }; + } + }); + + server.route({ + method: "GET", + url: "/findings", + config: { + rateLimit: readLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: "List all the Secret Scanning Findings for the specified project.", + querystring: z.object({ + projectId: z.string().trim().min(1, "Project ID required").describe(SecretScanningFindings.LIST.projectId) + }), + response: { + 200: z.object({ findings: SecretScanningFindingSchema.array() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { + query: { projectId }, + permission + } = req; + + const findings = await server.services.secretScanningV2.listSecretScanningFindingsByProjectId( + projectId, + permission + ); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId, + event: { + type: EventType.SECRET_SCANNING_FINDING_LIST, + metadata: { + findingIds: findings.map((finding) => finding.id), + count: findings.length + } + } + }); + + return { findings }; + } + }); + + server.route({ + method: "PATCH", + url: "/findings/:findingId", + config: { + rateLimit: writeLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.SecretScanning], + description: "Update the resolve status of the specified Secret Scanning Finding.", + params: z.object({ + findingId: z.string().trim().min(1, "Finding ID required").describe(SecretScanningFindings.UPDATE.findingId) + }), + body: z.object({ + status: z.nativeEnum(SecretScanningFindingStatus).describe(SecretScanningFindings.UPDATE.status), + remarks: z.string().nullish().describe(SecretScanningFindings.UPDATE.remarks) + }), + response: { + 200: z.object({ finding: SecretScanningFindingSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { + params: { findingId }, + body, + permission + } = req; + + const { finding, projectId } = await server.services.secretScanningV2.updateSecretScanningFindingById( + { findingId, ...body }, + permission + ); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId, + event: { + type: EventType.SECRET_SCANNING_FINDING_UPDATE, + metadata: { + findingId, + ...body + } + } + }); + + return { finding }; + } + }); + + // not exposed, for UI only + server.route({ + method: "GET", + url: "/data-sources-dashboard", + config: { + rateLimit: readLimit + }, + schema: { + querystring: z.object({ + projectId: z.string().trim().min(1, "Project ID required") + }), + response: { + 200: z.object({ + dataSources: z + .intersection( + SecretScanningDataSourceSchema, + z.object({ + lastScannedAt: z.date().nullish(), + lastScanStatus: z.nativeEnum(SecretScanningScanStatus).nullish(), + lastScanStatusMessage: z.string().nullish(), + unresolvedFindings: z.number().nullish() + }) + ) + .array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const { + query: { projectId }, + permission + } = req; + + const dataSources = await server.services.secretScanningV2.listSecretScanningDataSourcesWithDetailsByProjectId( + { projectId }, + permission + ); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId, + event: { + type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST, + metadata: { + dataSourceIds: dataSources.map((dataSource) => dataSource.id), + count: dataSources.length + } + } + }); + + return { dataSources }; + } + }); + + server.route({ + method: "GET", + url: "/unresolved-findings-count", + config: { + rateLimit: readLimit + }, + schema: { + tags: [ApiDocsTags.SecretScanning], + querystring: z.object({ + projectId: z.string().trim().min(1, "Project ID required").describe(SecretScanningFindings.LIST.projectId) + }), + response: { + 200: z.object({ unresolvedFindings: z.number() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const { + query: { projectId }, + permission + } = req; + + const unresolvedFindings = + await server.services.secretScanningV2.getSecretScanningUnresolvedFindingsCountByProjectId( + projectId, + permission + ); + + return { unresolvedFindings }; + } + }); +}; diff --git a/backend/src/ee/services/audit-log/audit-log-types.ts b/backend/src/ee/services/audit-log/audit-log-types.ts index 365ada987..cdebf78ff 100644 --- a/backend/src/ee/services/audit-log/audit-log-types.ts +++ b/backend/src/ee/services/audit-log/audit-log-types.ts @@ -9,6 +9,14 @@ import { TSecretRotationV2Raw, TUpdateSecretRotationV2DTO } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-types"; +import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; +import { + TCreateSecretScanningDataSourceDTO, + TDeleteSecretScanningDataSourceDTO, + TTriggerSecretScanningDataSourceDTO, + TUpdateSecretScanningDataSourceDTO, + TUpdateSecretScanningFinding +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types"; import { SshCaStatus, SshCertType } from "@app/ee/services/ssh/ssh-certificate-authority-types"; import { SshCertKeyAlgorithm } from "@app/ee/services/ssh-certificate/ssh-certificate-types"; import { SshCertTemplateStatus } from "@app/ee/services/ssh-certificate-template/ssh-certificate-template-types"; @@ -375,7 +383,18 @@ export enum EventType { MICROSOFT_TEAMS_WORKFLOW_INTEGRATION_LIST = "microsoft-teams-workflow-integration-list", PROJECT_ASSUME_PRIVILEGE_SESSION_START = "project-assume-privileges-session-start", - PROJECT_ASSUME_PRIVILEGE_SESSION_END = "project-assume-privileges-session-end" + PROJECT_ASSUME_PRIVILEGE_SESSION_END = "project-assume-privileges-session-end", + + SECRET_SCANNING_DATA_SOURCE_LIST = "secret-scanning-data-source-list", + SECRET_SCANNING_DATA_SOURCE_CREATE = "secret-scanning-data-source-create", + SECRET_SCANNING_DATA_SOURCE_UPDATE = "secret-scanning-data-source-update", + SECRET_SCANNING_DATA_SOURCE_DELETE = "secret-scanning-data-source-delete", + SECRET_SCANNING_DATA_SOURCE_GET = "secret-scanning-data-source-get", + SECRET_SCANNING_DATA_SOURCE_SCAN = "secret-scanning-data-source-scan", + SECRET_SCANNING_RESOURCE_LIST = "secret-scanning-resource-list", + SECRET_SCANNING_SCAN_LIST = "secret-scanning-scan-list", + SECRET_SCANNING_FINDING_LIST = "secret-scanning-finding-list", + SECRET_SCANNING_FINDING_UPDATE = "secret-scanning-finding-update" } export const filterableSecretEvents: EventType[] = [ @@ -2913,6 +2932,75 @@ interface MicrosoftTeamsWorkflowIntegrationUpdateEvent { }; } +interface SecretScanningDataSourceListEvent { + type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST; + metadata: { + type?: SecretScanningDataSource; + count: number; + dataSourceIds: string[]; + }; +} + +interface SecretScanningDataSourceGetEvent { + type: EventType.SECRET_SCANNING_DATA_SOURCE_GET; + metadata: { + type: SecretScanningDataSource; + dataSourceId: string; + }; +} + +interface SecretScanningDataSourceCreateEvent { + type: EventType.SECRET_SCANNING_DATA_SOURCE_CREATE; + metadata: Omit & { dataSourceId: string }; +} + +interface SecretScanningDataSourceUpdateEvent { + type: EventType.SECRET_SCANNING_DATA_SOURCE_UPDATE; + metadata: TUpdateSecretScanningDataSourceDTO; +} + +interface SecretScanningDataSourceDeleteEvent { + type: EventType.SECRET_SCANNING_DATA_SOURCE_DELETE; + metadata: TDeleteSecretScanningDataSourceDTO; +} + +interface SecretScanningDataSourceScanEvent { + type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN; + metadata: TTriggerSecretScanningDataSourceDTO; +} + +interface SecretScanningResourceListEvent { + type: EventType.SECRET_SCANNING_RESOURCE_LIST; + metadata: { + type: SecretScanningDataSource; + dataSourceId: string; + resourceIds: string[]; + count: number; + }; +} + +interface SecretScanningScanListEvent { + type: EventType.SECRET_SCANNING_SCAN_LIST; + metadata: { + type: SecretScanningDataSource; + dataSourceId: string; + count: number; + }; +} + +interface SecretScanningFindingListEvent { + type: EventType.SECRET_SCANNING_FINDING_LIST; + metadata: { + findingIds: string[]; + count: number; + }; +} + +interface SecretScanningFindingUpdateEvent { + type: EventType.SECRET_SCANNING_FINDING_UPDATE; + metadata: TUpdateSecretScanningFinding; +} + export type Event = | GetSecretsEvent | GetSecretEvent @@ -3179,4 +3267,14 @@ export type Event = | MicrosoftTeamsWorkflowIntegrationGetTeamsEvent | MicrosoftTeamsWorkflowIntegrationGetEvent | MicrosoftTeamsWorkflowIntegrationListEvent - | MicrosoftTeamsWorkflowIntegrationUpdateEvent; + | MicrosoftTeamsWorkflowIntegrationUpdateEvent + | SecretScanningDataSourceListEvent + | SecretScanningDataSourceGetEvent + | SecretScanningDataSourceCreateEvent + | SecretScanningDataSourceUpdateEvent + | SecretScanningDataSourceDeleteEvent + | SecretScanningDataSourceScanEvent + | SecretScanningResourceListEvent + | SecretScanningScanListEvent + | SecretScanningFindingListEvent + | SecretScanningFindingUpdateEvent; diff --git a/backend/src/ee/services/license/license-fns.ts b/backend/src/ee/services/license/license-fns.ts index b7ae6f7ee..a9ca8d4cc 100644 --- a/backend/src/ee/services/license/license-fns.ts +++ b/backend/src/ee/services/license/license-fns.ts @@ -54,7 +54,8 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ projectTemplates: false, kmip: false, gateway: false, - sshHostGroups: false + sshHostGroups: false, + secretScanning: false }); export const setupLicenseRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => { diff --git a/backend/src/ee/services/license/license-types.ts b/backend/src/ee/services/license/license-types.ts index 358849fb2..1771a007a 100644 --- a/backend/src/ee/services/license/license-types.ts +++ b/backend/src/ee/services/license/license-types.ts @@ -72,6 +72,7 @@ export type TFeatureSet = { kmip: false; gateway: false; sshHostGroups: false; + secretScanning: false; }; export type TOrgPlansTableDTO = { diff --git a/backend/src/ee/services/permission/default-roles.ts b/backend/src/ee/services/permission/default-roles.ts index a018ffe81..f95687aed 100644 --- a/backend/src/ee/services/permission/default-roles.ts +++ b/backend/src/ee/services/permission/default-roles.ts @@ -12,6 +12,8 @@ import { ProjectPermissionPkiSubscriberActions, ProjectPermissionSecretActions, ProjectPermissionSecretRotationActions, + ProjectPermissionSecretScanningDataSourceActions, + ProjectPermissionSecretScanningFindingActions, ProjectPermissionSecretSyncActions, ProjectPermissionSet, ProjectPermissionSshHostActions, @@ -198,6 +200,24 @@ const buildAdminPermissionRules = () => { ProjectPermissionSub.SecretRotation ); + can( + [ + ProjectPermissionSecretScanningDataSourceActions.Create, + ProjectPermissionSecretScanningDataSourceActions.Edit, + ProjectPermissionSecretScanningDataSourceActions.Delete, + ProjectPermissionSecretScanningDataSourceActions.Read, + ProjectPermissionSecretScanningDataSourceActions.TriggerScans, + ProjectPermissionSecretScanningDataSourceActions.ReadScans, + ProjectPermissionSecretScanningDataSourceActions.ReadResources + ], + ProjectPermissionSub.SecretScanningDataSources + ); + + can( + [ProjectPermissionSecretScanningFindingActions.Read, ProjectPermissionSecretScanningFindingActions.Resolve], + ProjectPermissionSub.SecretScanningFindings + ); + return rules; }; @@ -378,6 +398,24 @@ const buildMemberPermissionRules = () => { ProjectPermissionSub.SecretSyncs ); + can( + [ + ProjectPermissionSecretScanningDataSourceActions.Create, + ProjectPermissionSecretScanningDataSourceActions.Edit, + ProjectPermissionSecretScanningDataSourceActions.Delete, + ProjectPermissionSecretScanningDataSourceActions.Read, + ProjectPermissionSecretScanningDataSourceActions.TriggerScans, + ProjectPermissionSecretScanningDataSourceActions.ReadScans, + ProjectPermissionSecretScanningDataSourceActions.ReadResources + ], + ProjectPermissionSub.SecretScanningDataSources + ); + + can( + [ProjectPermissionSecretScanningFindingActions.Read, ProjectPermissionSecretScanningFindingActions.Resolve], + ProjectPermissionSub.SecretScanningFindings + ); + return rules; }; @@ -413,6 +451,17 @@ const buildViewerPermissionRules = () => { can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateTemplates); can(ProjectPermissionSecretSyncActions.Read, ProjectPermissionSub.SecretSyncs); + can( + [ + ProjectPermissionSecretScanningDataSourceActions.Read, + ProjectPermissionSecretScanningDataSourceActions.ReadScans, + ProjectPermissionSecretScanningDataSourceActions.ReadResources + ], + ProjectPermissionSub.SecretScanningDataSources + ); + + can([ProjectPermissionSecretScanningFindingActions.Read], ProjectPermissionSub.SecretScanningFindings); + return rules; }; diff --git a/backend/src/ee/services/permission/project-permission.ts b/backend/src/ee/services/permission/project-permission.ts index 5474facf6..683e55000 100644 --- a/backend/src/ee/services/permission/project-permission.ts +++ b/backend/src/ee/services/permission/project-permission.ts @@ -123,6 +123,21 @@ export enum ProjectPermissionKmipActions { GenerateClientCertificates = "generate-client-certificates" } +export enum ProjectPermissionSecretScanningDataSourceActions { + Read = "read-data-sources", + Create = "create-data-sources", + Edit = "edit-data-sources", + Delete = "delete-data-sources", + TriggerScans = "trigger-data-source-scans", + ReadScans = "read-data-source-scans", + ReadResources = "read-data-source-resources" +} + +export enum ProjectPermissionSecretScanningFindingActions { + Read = "read-findings", + Resolve = "resolve-findings" +} + export enum ProjectPermissionSub { Role = "role", Member = "member", @@ -158,7 +173,9 @@ export enum ProjectPermissionSub { Kms = "kms", Cmek = "cmek", SecretSyncs = "secret-syncs", - Kmip = "kmip" + Kmip = "kmip", + SecretScanningDataSources = "secret-scanning-data-sources", + SecretScanningFindings = "secret-scanning-findings" } export type SecretSubjectFields = { @@ -281,7 +298,9 @@ export type ProjectPermissionSet = | [ProjectPermissionActions.Edit, ProjectPermissionSub.Project] | [ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback] | [ProjectPermissionActions.Create, ProjectPermissionSub.SecretRollback] - | [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms]; + | [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms] + | [ProjectPermissionSecretScanningDataSourceActions, ProjectPermissionSub.SecretScanningDataSources] + | [ProjectPermissionSecretScanningFindingActions, ProjectPermissionSub.SecretScanningFindings]; const SECRET_PATH_MISSING_SLASH_ERR_MSG = "Invalid Secret Path; it must start with a '/'"; const SECRET_PATH_PERMISSION_OPERATOR_SCHEMA = z.union([ @@ -602,6 +621,20 @@ const GeneralPermissionSchema = [ action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionKmipActions).describe( "Describe what action an entity can take." ) + }), + z.object({ + subject: z + .literal(ProjectPermissionSub.SecretScanningDataSources) + .describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningDataSourceActions).describe( + "Describe what action an entity can take." + ) + }), + z.object({ + subject: z.literal(ProjectPermissionSub.SecretScanningFindings).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningFindingActions).describe( + "Describe what action an entity can take." + ) }) ]; diff --git a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-constants.ts b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-constants.ts new file mode 100644 index 000000000..a8dd2eb42 --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-constants.ts @@ -0,0 +1,9 @@ +import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; +import { TSecretScanningDataSourceListItem } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types"; +import { AppConnection } from "@app/services/app-connection/app-connection-enums"; + +export const GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION: TSecretScanningDataSourceListItem = { + name: "GitHub", + type: SecretScanningDataSource.GitHub, + connection: AppConnection.GitHubRadar +}; diff --git a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.ts b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.ts new file mode 100644 index 000000000..70777ddce --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.ts @@ -0,0 +1,197 @@ +import { join } from "path"; +import { ProbotOctokit } from "probot"; + +import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns"; +import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types"; +import { + SecretScanningFindingSeverity, + SecretScanningResource +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; +import { cloneRepository, titleCaseToCamelCase } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns"; +import { + TSecretScanningFactoryGetDiffScanFindingsPayload, + TSecretScanningFactoryGetDiffScanResourcePayload, + TSecretScanningFactoryGetFullScanPath, + TSecretScanningFactoryInitialize, + TSecretScanningFactoryListRawResources, + TSecretScanningFactoryPostInitialization +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types"; +import { getConfig } from "@app/lib/config/env"; +import { BadRequestError } from "@app/lib/errors"; +import { listGitHubRadarRepositories, TGitHubRadarConnection } from "@app/services/app-connection/github-radar"; + +import { TGitHubDataSourceWithConnection, TQueueGitHubResourceDiffScan } from "./github-secret-scanning-types"; + +export const GitHubSecretScanningFactory = () => { + const initialize: TSecretScanningFactoryInitialize = async ( + { connection, secretScanningV2DAL }, + callback + ) => { + const externalId = connection.credentials.installationId; + + const existingDataSource = await secretScanningV2DAL.dataSources.findOne({ + externalId + }); + + if (existingDataSource) + throw new BadRequestError({ message: "A Data Source already exists for this GitHub Radar Connection" }); + + return callback({ + externalId + }); + }; + + const postInitialization: TSecretScanningFactoryPostInitialization = async () => { + // no post-initialization required + }; + + const listRawResources: TSecretScanningFactoryListRawResources = async ( + dataSource + ) => { + const { + connection, + config: { includeRepos } + } = dataSource; + + const repos = await listGitHubRadarRepositories(connection); + + const filteredRepos: typeof repos = []; + if (includeRepos.includes("*")) { + filteredRepos.push(...repos); + } else { + filteredRepos.push(...repos.filter((repo) => includeRepos.includes(repo.full_name))); + } + + return filteredRepos.map(({ id, full_name }) => ({ + name: full_name, + externalId: id.toString(), + type: SecretScanningResource.Repository + })); + }; + + const getFullScanPath: TSecretScanningFactoryGetFullScanPath = async ({ + dataSource, + resourceName, + tempFolder + }) => { + const appCfg = getConfig(); + const { + connection: { + credentials: { installationId } + } + } = dataSource; + + const octokit = new ProbotOctokit({ + auth: { + appId: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID, + privateKey: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY, + installationId + } + }); + + const { + data: { token } + } = await octokit.apps.createInstallationAccessToken({ + installation_id: Number(installationId) + }); + + const repoPath = join(tempFolder, "repo.git"); + + await cloneRepository({ + cloneUrl: `https://x-access-token:${token}@github.com/${resourceName}.git`, + repoPath + }); + + return repoPath; + }; + + const getDiffScanResourcePayload: TSecretScanningFactoryGetDiffScanResourcePayload< + TQueueGitHubResourceDiffScan["payload"] + > = ({ repository }) => { + return { + name: repository.full_name, + externalId: repository.id.toString(), + type: SecretScanningResource.Repository + }; + }; + + const getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload< + TGitHubDataSourceWithConnection, + TQueueGitHubResourceDiffScan["payload"] + > = async ({ dataSource, payload, resourceName }) => { + const appCfg = getConfig(); + const { + connection: { + credentials: { installationId } + } + } = dataSource; + + const octokit = new ProbotOctokit({ + auth: { + appId: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID, + privateKey: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY, + installationId + } + }); + + const { commits, repository } = payload; + + const [owner, repo] = repository.full_name.split("/"); + + const allFindings: SecretMatch[] = []; + + for (const commit of commits) { + for (const filepath of [...commit.added, ...commit.modified]) { + // eslint-disable-next-line + const fileContentsResponse = await octokit.repos.getContent({ + owner, + repo, + path: filepath + }); + + const { data } = fileContentsResponse; + const fileContent = Buffer.from((data as { content: string }).content, "base64").toString(); + + // eslint-disable-next-line + const findings = await scanContentAndGetFindings(`\n${fileContent}`); // extra line to count lines correctly + + allFindings.push( + ...findings.map((finding) => ({ + ...finding, + File: filepath, + Commit: commit.id, + Author: commit.author.name, + Email: commit.author.email ?? "", + Message: commit.message, + Fingerprint: `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`, + Date: commit.timestamp, + Link: `https://github.com/${resourceName}/blob/${commit.id}/${filepath}#L${finding.StartLine}` + })) + ); + } + } + + return allFindings.map( + ({ + // discard match and secret as we don't want to store + Match, + Secret, + ...finding + }) => ({ + details: titleCaseToCamelCase(finding), + fingerprint: finding.Fingerprint, + severity: SecretScanningFindingSeverity.High, + rule: finding.RuleID + }) + ); + }; + + return { + initialize, + postInitialization, + listRawResources, + getFullScanPath, + getDiffScanResourcePayload, + getDiffScanFindingsPayload + }; +}; diff --git a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-schemas.ts b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-schemas.ts new file mode 100644 index 000000000..fc2fbd241 --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-schemas.ts @@ -0,0 +1,72 @@ +import { z } from "zod"; + +import { + SecretScanningDataSource, + SecretScanningResource +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; +import { + BaseCreateSecretScanningDataSourceSchema, + BaseSecretScanningDataSourceSchema, + BaseSecretScanningFindingSchema, + BaseUpdateSecretScanningDataSourceSchema, + GitRepositoryScanFindingDetailsSchema +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-schemas"; +import { AppConnection } from "@app/services/app-connection/app-connection-enums"; + +export const GitHubDataSourceConfigSchema = z.object({ + includeRepos: z.array(z.string()).nonempty("One or more repositories required").default(["*"]) +}); + +export const GitHubDataSourceSchema = BaseSecretScanningDataSourceSchema({ + type: SecretScanningDataSource.GitHub, + isConnectionRequired: true +}) + .extend({ + config: GitHubDataSourceConfigSchema + }) + .describe( + JSON.stringify({ + title: "GitHub" + }) + ); + +export const CreateGitHubDataSourceSchema = BaseCreateSecretScanningDataSourceSchema({ + type: SecretScanningDataSource.GitHub, + isConnectionRequired: true +}) + .extend({ + config: GitHubDataSourceConfigSchema + }) + .describe( + JSON.stringify({ + title: "GitHub" + }) + ); + +export const UpdateGitHubDataSourceSchema = BaseUpdateSecretScanningDataSourceSchema(SecretScanningDataSource.GitHub) + .extend({ + config: GitHubDataSourceConfigSchema.optional() + }) + .describe( + JSON.stringify({ + title: "GitHub" + }) + ); + +export const GitHubDataSourceListItemSchema = z + .object({ + name: z.literal("GitHub"), + connection: z.literal(AppConnection.GitHubRadar), + type: z.literal(SecretScanningDataSource.GitHub) + }) + .describe( + JSON.stringify({ + title: "GitHub" + }) + ); + +export const GitHubFindingSchema = BaseSecretScanningFindingSchema.extend({ + resourceType: z.literal(SecretScanningResource.Repository), + dataSourceType: z.literal(SecretScanningDataSource.GitHub), + details: GitRepositoryScanFindingDetailsSchema +}); diff --git a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-service.ts b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-service.ts new file mode 100644 index 000000000..178edbd51 --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-service.ts @@ -0,0 +1,62 @@ +import { PushEvent } from "@octokit/webhooks-types"; + +import { TSecretScanningV2DALFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-dal"; +import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; +import { TSecretScanningV2QueueServiceFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-queue"; +import { logger } from "@app/lib/logger"; + +export const githubSecretScanningService = ( + secretScanningV2DAL: TSecretScanningV2DALFactory, + secretScanningV2Queue: Pick +) => { + const handleInstallationDeletedEvent = async (installationId: number) => { + const dataSource = await secretScanningV2DAL.dataSources.findOne({ + externalId: String(installationId) + }); + + if (!dataSource) { + logger.error( + `secretScanningV2RemoveEvent: GitHub - Could not find data source [installationId=${installationId}]` + ); + return; + } + + // scott: maybe add disabled col instead? + await secretScanningV2DAL.resources.delete({ + dataSourceId: dataSource.id + }); + }; + + const handlePushEvent = async (payload: PushEvent) => { + const { commits, repository, installation } = payload; + + if (!commits || !repository || !installation) { + logger.warn( + `secretScanningV2PushEvent: GitHub - Insufficient data [commits=${commits?.length ?? 0}] [repository=${repository.name}] [installationId=${installation?.id}]` + ); + return; + } + + const dataSource = await secretScanningV2DAL.dataSources.findOne({ + externalId: String(installation.id) + }); + + if (!dataSource) { + logger.error( + `secretScanningV2PushEvent: GitHub - Could not find data source [installationId=${installation.id}]` + ); + return; + } + + await secretScanningV2Queue.queueResourceDiffScan({ + dataSourceType: SecretScanningDataSource.GitHub, + payload, + dataSourceId: dataSource.id + }); + }; + + return { + handlePushEvent, + handleInstallationDeleted: handleInstallationDeletedEvent + }; +}; diff --git a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-types.ts b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-types.ts new file mode 100644 index 000000000..e5e481fc8 --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-types.ts @@ -0,0 +1,30 @@ +import { PushEvent } from "@octokit/webhooks-types"; +import { z } from "zod"; + +import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; +import { TGitHubRadarConnection } from "@app/services/app-connection/github-radar"; + +import { + CreateGitHubDataSourceSchema, + GitHubDataSourceListItemSchema, + GitHubDataSourceSchema, + GitHubFindingSchema +} from "./github-secret-scanning-schemas"; + +export type TGitHubDataSource = z.infer; + +export type TGitHubDataSourceInput = z.infer; + +export type TGitHubDataSourceListItem = z.infer; + +export type TGitHubFinding = z.infer; + +export type TGitHubDataSourceWithConnection = TGitHubDataSource & { + connection: TGitHubRadarConnection; +}; + +export type TQueueGitHubResourceDiffScan = { + dataSourceType: SecretScanningDataSource.GitHub; + payload: PushEvent; + dataSourceId: string; +}; diff --git a/backend/src/ee/services/secret-scanning-v2/github/index.ts b/backend/src/ee/services/secret-scanning-v2/github/index.ts new file mode 100644 index 000000000..b8bc755a6 --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/github/index.ts @@ -0,0 +1,3 @@ +export * from "./github-secret-scanning-constants"; +export * from "./github-secret-scanning-schemas"; +export * from "./github-secret-scanning-types"; diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-dal.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-dal.ts new file mode 100644 index 000000000..ac6bcd5b8 --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-dal.ts @@ -0,0 +1,458 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { + SecretScanningResourcesSchema, + SecretScanningScansSchema, + TableName, + TSecretScanningDataSources +} from "@app/db/schemas"; +import { SecretScanningFindingStatus } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; +import { DatabaseError } from "@app/lib/errors"; +import { + buildFindFilter, + ormify, + prependTableNameToFindFilter, + selectAllTableCols, + sqlNestRelationships, + TFindOpt +} from "@app/lib/knex"; + +export type TSecretScanningV2DALFactory = ReturnType; + +type TSecretScanningDataSourceFindFilter = Parameters>[0]; +type TSecretScanningDataSourceFindOptions = TFindOpt; + +const baseSecretScanningDataSourceQuery = ({ + filter = {}, + db, + tx +}: { + db: TDbClient; + filter?: TSecretScanningDataSourceFindFilter; + options?: TSecretScanningDataSourceFindOptions; + tx?: Knex; +}) => { + const query = (tx || db.replicaNode())(TableName.SecretScanningDataSource) + .join( + TableName.AppConnection, + `${TableName.SecretScanningDataSource}.connectionId`, + `${TableName.AppConnection}.id` + ) + .select(selectAllTableCols(TableName.SecretScanningDataSource)) + .select( + // entire connection + db.ref("name").withSchema(TableName.AppConnection).as("connectionName"), + db.ref("method").withSchema(TableName.AppConnection).as("connectionMethod"), + db.ref("app").withSchema(TableName.AppConnection).as("connectionApp"), + db.ref("orgId").withSchema(TableName.AppConnection).as("connectionOrgId"), + db.ref("encryptedCredentials").withSchema(TableName.AppConnection).as("connectionEncryptedCredentials"), + db.ref("description").withSchema(TableName.AppConnection).as("connectionDescription"), + db.ref("version").withSchema(TableName.AppConnection).as("connectionVersion"), + db.ref("createdAt").withSchema(TableName.AppConnection).as("connectionCreatedAt"), + db.ref("updatedAt").withSchema(TableName.AppConnection).as("connectionUpdatedAt"), + db + .ref("isPlatformManagedCredentials") + .withSchema(TableName.AppConnection) + .as("connectionIsPlatformManagedCredentials") + ); + + if (filter) { + /* eslint-disable @typescript-eslint/no-misused-promises */ + void query.where(buildFindFilter(prependTableNameToFindFilter(TableName.SecretScanningDataSource, filter))); + } + + return query; +}; + +const expandSecretScanningDataSource = < + T extends Awaited>[number] +>( + dataSource: T +) => { + const { + connectionApp, + connectionName, + connectionId, + connectionOrgId, + connectionEncryptedCredentials, + connectionMethod, + connectionDescription, + connectionCreatedAt, + connectionUpdatedAt, + connectionVersion, + connectionIsPlatformManagedCredentials, + ...el + } = dataSource; + + return { + ...el, + connectionId, + connection: connectionId + ? { + app: connectionApp, + id: connectionId, + name: connectionName, + orgId: connectionOrgId, + encryptedCredentials: connectionEncryptedCredentials, + method: connectionMethod, + description: connectionDescription, + createdAt: connectionCreatedAt, + updatedAt: connectionUpdatedAt, + version: connectionVersion, + isPlatformManagedCredentials: connectionIsPlatformManagedCredentials + } + : undefined + }; +}; + +export const secretScanningV2DALFactory = (db: TDbClient) => { + const dataSourceOrm = ormify(db, TableName.SecretScanningDataSource); + const resourceOrm = ormify(db, TableName.SecretScanningResource); + const scanOrm = ormify(db, TableName.SecretScanningScan); + const findingOrm = ormify(db, TableName.SecretScanningFinding); + + const findDataSource = async (filter: Parameters<(typeof dataSourceOrm)["find"]>[0], tx?: Knex) => { + try { + const dataSources = await baseSecretScanningDataSourceQuery({ filter, db, tx }); + + if (!dataSources.length) return []; + + return dataSources.map(expandSecretScanningDataSource); + } catch (error) { + throw new DatabaseError({ error, name: "Find - Secret Scanning Data Source" }); + } + }; + + const findDataSourceById = async (id: string, tx?: Knex) => { + try { + const dataSource = await baseSecretScanningDataSourceQuery({ filter: { id }, db, tx }).first(); + + if (dataSource) return expandSecretScanningDataSource(dataSource); + } catch (error) { + throw new DatabaseError({ error, name: "Find By ID - Secret Scanning Data Source" }); + } + }; + + const createDataSource = async (data: Parameters<(typeof dataSourceOrm)["create"]>[0], tx?: Knex) => { + const source = await dataSourceOrm.create(data, tx); + + const dataSource = (await baseSecretScanningDataSourceQuery({ + filter: { id: source.id }, + db, + tx + }).first())!; + + return expandSecretScanningDataSource(dataSource); + }; + + const updateDataSourceById = async ( + dataSourceId: string, + data: Parameters<(typeof dataSourceOrm)["updateById"]>[1], + tx?: Knex + ) => { + const source = await dataSourceOrm.updateById(dataSourceId, data, tx); + + const dataSource = (await baseSecretScanningDataSourceQuery({ + filter: { id: source.id }, + db, + tx + }).first())!; + + return expandSecretScanningDataSource(dataSource); + }; + + const deleteDataSourceById = async (dataSourceId: string, tx?: Knex) => { + const secretRotation = (await baseSecretScanningDataSourceQuery({ + filter: { id: dataSourceId }, + db, + tx + }).first())!; + + await dataSourceOrm.deleteById(dataSourceId, tx); + + return expandSecretScanningDataSource(secretRotation); + }; + + const findOneDataSource = async (filter: Parameters<(typeof dataSourceOrm)["findOne"]>[0], tx?: Knex) => { + try { + const secretRotation = await baseSecretScanningDataSourceQuery({ filter, db, tx }).first(); + + if (secretRotation) { + return expandSecretScanningDataSource(secretRotation); + } + } catch (error) { + throw new DatabaseError({ error, name: "Find One - Secret Rotation V2" }); + } + }; + + const findDataSourceWithDetails = async (filter: Parameters<(typeof dataSourceOrm)["find"]>[0], tx?: Knex) => { + try { + // TODO (scott): this query will probably need to be optimized + + const dataSources = await baseSecretScanningDataSourceQuery({ filter, db, tx }) + .leftJoin( + TableName.SecretScanningResource, + `${TableName.SecretScanningResource}.dataSourceId`, + `${TableName.SecretScanningDataSource}.id` + ) + .leftJoin( + TableName.SecretScanningScan, + `${TableName.SecretScanningScan}.resourceId`, + `${TableName.SecretScanningResource}.id` + ) + .leftJoin( + TableName.SecretScanningFinding, + `${TableName.SecretScanningFinding}.scanId`, + `${TableName.SecretScanningScan}.id` + ) + .where((qb) => { + void qb + .where(`${TableName.SecretScanningFinding}.status`, SecretScanningFindingStatus.Unresolved) + .orWhereNull(`${TableName.SecretScanningFinding}.status`); + }) + .select( + db.ref("id").withSchema(TableName.SecretScanningScan).as("scanId"), + db.ref("status").withSchema(TableName.SecretScanningScan).as("scanStatus"), + db.ref("statusMessage").withSchema(TableName.SecretScanningScan).as("scanStatusMessage"), + db.ref("createdAt").withSchema(TableName.SecretScanningScan).as("scanCreatedAt"), + db.ref("status").withSchema(TableName.SecretScanningFinding).as("findingStatus"), + db.ref("id").withSchema(TableName.SecretScanningFinding).as("findingId") + ); + + if (!dataSources.length) return []; + + const results = sqlNestRelationships({ + data: dataSources, + key: "id", + parentMapper: (dataSource) => expandSecretScanningDataSource(dataSource), + childrenMapper: [ + { + key: "scanId", + label: "scans" as const, + mapper: ({ scanId, scanCreatedAt, scanStatus, scanStatusMessage }) => ({ + id: scanId, + createdAt: scanCreatedAt, + status: scanStatus, + statusMessage: scanStatusMessage + }) + }, + { + key: "findingId", + label: "findings" as const, + mapper: ({ findingId }) => ({ + id: findingId + }) + } + ] + }); + + return results.map(({ scans, findings, ...dataSource }) => { + const lastScan = + scans && scans.length + ? scans.reduce((latest, current) => { + return new Date(current.createdAt) > new Date(latest.createdAt) ? current : latest; + }) + : null; + + return { + ...dataSource, + lastScanStatus: lastScan?.status ?? null, + lastScanStatusMessage: lastScan?.statusMessage ?? null, + lastScannedAt: lastScan?.createdAt ?? null, + unresolvedFindings: scans.length ? findings.length : null + }; + }); + } catch (error) { + throw new DatabaseError({ error, name: "Find Data Source with Details - Secret Scanning V2" }); + } + }; + + const findResourcesWithDetails = async (filter: Parameters<(typeof resourceOrm)["find"]>[0], tx?: Knex) => { + try { + // TODO (scott): this query will probably need to be optimized + + const resources = await (tx || db.replicaNode())(TableName.SecretScanningResource) + .where((qb) => { + if (filter) + void qb.where(buildFindFilter(prependTableNameToFindFilter(TableName.SecretScanningResource, filter))); + }) + .leftJoin( + TableName.SecretScanningScan, + `${TableName.SecretScanningScan}.resourceId`, + `${TableName.SecretScanningResource}.id` + ) + .leftJoin( + TableName.SecretScanningFinding, + `${TableName.SecretScanningFinding}.scanId`, + `${TableName.SecretScanningScan}.id` + ) + .where((qb) => { + void qb + .where(`${TableName.SecretScanningFinding}.status`, SecretScanningFindingStatus.Unresolved) + .orWhereNull(`${TableName.SecretScanningFinding}.status`); + }) + .select(selectAllTableCols(TableName.SecretScanningResource)) + .select( + db.ref("id").withSchema(TableName.SecretScanningScan).as("scanId"), + db.ref("status").withSchema(TableName.SecretScanningScan).as("scanStatus"), + db.ref("type").withSchema(TableName.SecretScanningScan).as("scanType"), + db.ref("statusMessage").withSchema(TableName.SecretScanningScan).as("scanStatusMessage"), + db.ref("createdAt").withSchema(TableName.SecretScanningScan).as("scanCreatedAt"), + db.ref("status").withSchema(TableName.SecretScanningFinding).as("findingStatus"), + db.ref("id").withSchema(TableName.SecretScanningFinding).as("findingId") + ); + + if (!resources.length) return []; + + const results = sqlNestRelationships({ + data: resources, + key: "id", + parentMapper: (resource) => SecretScanningResourcesSchema.parse(resource), + childrenMapper: [ + { + key: "scanId", + label: "scans" as const, + mapper: ({ scanId, scanCreatedAt, scanStatus, scanStatusMessage, scanType }) => ({ + id: scanId, + type: scanType, + createdAt: scanCreatedAt, + status: scanStatus, + statusMessage: scanStatusMessage + }) + }, + { + key: "findingId", + label: "findings" as const, + mapper: ({ findingId }) => ({ + id: findingId + }) + } + ] + }); + + return results.map(({ scans, findings, ...resource }) => { + const lastScan = + scans && scans.length + ? scans.reduce((latest, current) => { + return new Date(current.createdAt) > new Date(latest.createdAt) ? current : latest; + }) + : null; + + return { + ...resource, + lastScanStatus: lastScan?.status ?? null, + lastScanStatusMessage: lastScan?.statusMessage ?? null, + lastScannedAt: lastScan?.createdAt ?? null, + unresolvedFindings: findings?.length ?? 0 + }; + }); + } catch (error) { + throw new DatabaseError({ error, name: "Find Resource with Details - Secret Scanning V2" }); + } + }; + + const findScansWithDetailsByDataSourceId = async (dataSourceId: string, tx?: Knex) => { + try { + // TODO (scott): this query will probably need to be optimized + + const scans = await (tx || db.replicaNode())(TableName.SecretScanningScan) + .leftJoin( + TableName.SecretScanningResource, + `${TableName.SecretScanningResource}.id`, + `${TableName.SecretScanningScan}.resourceId` + ) + .where(`${TableName.SecretScanningResource}.dataSourceId`, dataSourceId) + .leftJoin( + TableName.SecretScanningFinding, + `${TableName.SecretScanningFinding}.scanId`, + `${TableName.SecretScanningScan}.id` + ) + .select(selectAllTableCols(TableName.SecretScanningScan)) + .select( + db.ref("status").withSchema(TableName.SecretScanningFinding).as("findingStatus"), + db.ref("id").withSchema(TableName.SecretScanningFinding).as("findingId"), + db.ref("name").withSchema(TableName.SecretScanningResource).as("resourceName") + ); + + if (!scans.length) return []; + + const results = sqlNestRelationships({ + data: scans, + key: "id", + parentMapper: (scan) => SecretScanningScansSchema.parse(scan), + childrenMapper: [ + { + key: "findingId", + label: "findings" as const, + mapper: ({ findingId, findingStatus }) => ({ + id: findingId, + status: findingStatus + }) + }, + { + key: "resourceId", + label: "resources" as const, + mapper: ({ resourceName }) => ({ + name: resourceName + }) + } + ] + }); + + return results.map(({ findings, resources, ...scan }) => { + return { + ...scan, + unresolvedFindings: + findings?.filter((finding) => finding.status === SecretScanningFindingStatus.Unresolved).length ?? 0, + resolvedFindings: + findings?.filter((finding) => finding.status === SecretScanningFindingStatus.Resolved).length ?? 0, + resourceName: resources[0].name + }; + }); + } catch (error) { + throw new DatabaseError({ error, name: "Find Scan with Details By Data Source ID - Secret Scanning V2" }); + } + }; + + const findScansByDataSourceId = async (dataSourceId: string, tx?: Knex) => { + try { + const scans = await (tx || db.replicaNode())(TableName.SecretScanningScan) + .leftJoin( + TableName.SecretScanningResource, + `${TableName.SecretScanningResource}.id`, + `${TableName.SecretScanningScan}.resourceId` + ) + .where(`${TableName.SecretScanningResource}.dataSourceId`, dataSourceId) + + .select(selectAllTableCols(TableName.SecretScanningScan)); + + return scans; + } catch (error) { + throw new DatabaseError({ error, name: "Find Scan By Data Source ID - Secret Scanning V2" }); + } + }; + + return { + dataSources: { + ...dataSourceOrm, + find: findDataSource, + findById: findDataSourceById, + findOne: findOneDataSource, + create: createDataSource, + updateById: updateDataSourceById, + deleteById: deleteDataSourceById, + findWithDetails: findDataSourceWithDetails + }, + resources: { + ...resourceOrm, + findWithDetails: findResourcesWithDetails + }, + scans: { + ...scanOrm, + findWithDetailsByDataSourceId: findScansWithDetailsByDataSourceId, + findByDataSourceId: findScansByDataSourceId + }, + findings: findingOrm + }; +}; diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-enums.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-enums.ts new file mode 100644 index 000000000..082f3d760 --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-enums.ts @@ -0,0 +1,33 @@ +export enum SecretScanningDataSource { + GitHub = "github" +} + +export enum SecretScanningScanStatus { + Completed = "completed", + Failed = "failed", + Queued = "queued", + Scanning = "scanning" +} + +export enum SecretScanningScanType { + FullScan = "full-scan", + DiffScan = "diff-scan" +} + +export enum SecretScanningFindingStatus { + Resolved = "resolved", + Unresolved = "unresolved", + FalsePositive = "false-positive", + Ignore = "ignore" +} + +export enum SecretScanningResource { + Repository = "repository", + Project = "project" +} + +export enum SecretScanningFindingSeverity { + High = "high", + Medium = "medium", + Low = "low" +} diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-factory.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-factory.ts new file mode 100644 index 000000000..109afe5f3 --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-factory.ts @@ -0,0 +1,19 @@ +import { GitHubSecretScanningFactory } from "@app/ee/services/secret-scanning-v2/github/github-secret-scanning-factory"; + +import { SecretScanningDataSource } from "./secret-scanning-v2-enums"; +import { + TQueueSecretScanningResourceDiffScan, + TSecretScanningDataSourceCredentials, + TSecretScanningDataSourceWithConnection, + TSecretScanningFactory +} from "./secret-scanning-v2-types"; + +type TSecretScanningFactoryImplementation = TSecretScanningFactory< + TSecretScanningDataSourceWithConnection, + TSecretScanningDataSourceCredentials, + TQueueSecretScanningResourceDiffScan["payload"] +>; + +export const SECRET_SCANNING_FACTORY_MAP: Record = { + [SecretScanningDataSource.GitHub]: GitHubSecretScanningFactory as TSecretScanningFactoryImplementation +}; diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts new file mode 100644 index 000000000..5f0dc7286 --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts @@ -0,0 +1,102 @@ +import { AxiosError } from "axios"; +import { exec } from "child_process"; + +import { readFindingsFile } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns"; +import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types"; +import { GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/github"; + +import { SecretScanningDataSource, SecretScanningFindingSeverity } from "./secret-scanning-v2-enums"; +import { TCloneRepository, TGetFindingsPayload, TSecretScanningDataSourceListItem } from "./secret-scanning-v2-types"; + +const SECRET_SCANNING_SOURCE_LIST_OPTIONS: Record = { + [SecretScanningDataSource.GitHub]: GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION +}; + +export const listSecretScanningDataSourceOptions = () => { + return Object.values(SECRET_SCANNING_SOURCE_LIST_OPTIONS).sort((a, b) => a.name.localeCompare(b.name)); +}; + +export const cloneRepository = async ({ cloneUrl, repoPath }: TCloneRepository): Promise => { + const command = `git clone ${cloneUrl} ${repoPath} --bare`; + return new Promise((resolve, reject) => { + exec(command, (error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); +}; + +export function scanDirectory(inputPath: string, outputPath: string): Promise { + return new Promise((resolve, reject) => { + const command = `cd ${inputPath} && infisical scan --exit-code=77 -r "${outputPath}"`; + exec(command, (error) => { + if (error && error.code !== 77) { + reject(error); + } else { + resolve(); + } + }); + }); +} + +export const titleCaseToCamelCase = (obj: unknown): unknown => { + if (typeof obj !== "object" || obj === null) { + return obj; + } + + if (Array.isArray(obj)) { + return obj.map((item: object) => titleCaseToCamelCase(item)); + } + + const result: Record = {}; + + for (const key in obj) { + if (Object.prototype.hasOwnProperty.call(obj, key)) { + const camelKey = key.charAt(0).toLowerCase() + key.slice(1); + result[camelKey] = titleCaseToCamelCase((obj as Record)[key]); + } + } + + return result; +}; + +export const scanGitRepositoryAndGetFindings = async (scanPath: string, findingsPath: string): TGetFindingsPayload => { + await scanDirectory(scanPath, findingsPath); + + const findingsData = JSON.parse(await readFindingsFile(findingsPath)) as SecretMatch[]; + + return findingsData.map( + ({ + // discard match and secret as we don't want to store + Match, + Secret, + ...finding + }) => ({ + details: titleCaseToCamelCase(finding), + fingerprint: finding.Fingerprint, + severity: SecretScanningFindingSeverity.High, + rule: finding.RuleID + }) + ); +}; + +const MAX_MESSAGE_LENGTH = 1024; + +export const parseScanErrorMessage = (err: unknown): string => { + let errorMessage: string; + + if (err instanceof AxiosError) { + errorMessage = err?.response?.data + ? JSON.stringify(err?.response?.data) + : (err?.message ?? "An unknown error occurred."); + } else { + errorMessage = (err as Error)?.message || "An unknown error occurred."; + } + + return errorMessage.length <= MAX_MESSAGE_LENGTH + ? errorMessage + : `${errorMessage.substring(0, MAX_MESSAGE_LENGTH - 3)}...`; +}; diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-maps.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-maps.ts new file mode 100644 index 000000000..f41a2b5c2 --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-maps.ts @@ -0,0 +1,14 @@ +import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; +import { AppConnection } from "@app/services/app-connection/app-connection-enums"; + +export const SECRET_SCANNING_DATA_SOURCE_NAME_MAP: Record = { + [SecretScanningDataSource.GitHub]: "GitHub" +}; + +export const SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP: Record = { + [SecretScanningDataSource.GitHub]: AppConnection.GitHubRadar +}; + +export const AUTO_SYNC_DESCRIPTION_HELPER: Record = { + [SecretScanningDataSource.GitHub]: { verb: "push", noun: "repositories" } +}; diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts new file mode 100644 index 000000000..45920ecc7 --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts @@ -0,0 +1,396 @@ +import { join } from "path"; + +import { + createTempFolder, + deleteTempFolder +} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns"; +import { + parseScanErrorMessage, + scanGitRepositoryAndGetFindings +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns"; +import { BadRequestError, InternalServerError } from "@app/lib/errors"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns"; +import { TAppConnection } from "@app/services/app-connection/app-connection-types"; +import { TKmsServiceFactory } from "@app/services/kms/kms-service"; + +import { TSecretScanningV2DALFactory } from "./secret-scanning-v2-dal"; +import { + SecretScanningDataSource, + SecretScanningFindingStatus, + SecretScanningResource, + SecretScanningScanStatus, + SecretScanningScanType +} from "./secret-scanning-v2-enums"; +import { SECRET_SCANNING_FACTORY_MAP } from "./secret-scanning-v2-factory"; +import { + TFindingsPayload, + TQueueSecretScanningDataSourceFullScan, + TQueueSecretScanningResourceDiffScan, + TSecretScanningDataSourceWithConnection +} from "./secret-scanning-v2-types"; + +type TSecretRotationV2QueueServiceFactoryDep = { + queueService: TQueueServiceFactory; + secretScanningV2DAL: TSecretScanningV2DALFactory; + // smtpService: Pick; + // projectMembershipDAL: Pick; + // projectDAL: Pick; + kmsService: Pick; +}; + +export type TSecretScanningV2QueueServiceFactory = Awaited>; + +export const secretScanningV2QueueServiceFactory = async ({ + queueService, + secretScanningV2DAL, + // projectMembershipDAL, + // projectDAL, + // smtpService, + kmsService +}: TSecretRotationV2QueueServiceFactoryDep) => { + const queueDataSourceFullScan = async ( + dataSource: TSecretScanningDataSourceWithConnection, + resourceExternalId?: string + ) => { + try { + const { type } = dataSource; + + const factory = SECRET_SCANNING_FACTORY_MAP[type](); + + const rawResources = await factory.listRawResources(dataSource); + + let filteredRawResources = rawResources; + + // TODO: should add indivial resource fetch to factory + if (resourceExternalId) { + filteredRawResources = rawResources.filter((resource) => resource.externalId === resourceExternalId); + } + + if (!filteredRawResources.length) { + throw new BadRequestError({ + message: `${resourceExternalId ? `Resource with "ID" ${resourceExternalId} could not be found.` : "Data source has no resources to scan"}. Ensure your data source config is correct and not filtering out scanning resources.` + }); + } + + await secretScanningV2DAL.resources.transaction(async (tx) => { + const resources = await secretScanningV2DAL.resources.upsert( + filteredRawResources.map((rawResource) => ({ + ...rawResource, + dataSourceId: dataSource.id + })), + ["externalId", "dataSourceId"], + tx + ); + + const scans = await secretScanningV2DAL.scans.insertMany( + resources.map((resource) => ({ + resourceId: resource.id, + type: SecretScanningScanType.FullScan + })), + tx + ); + + for (const scan of scans) { + // eslint-disable-next-line no-await-in-loop + await queueService.queuePg(QueueJobs.SecretScanningV2FullScan, { + scanId: scan.id, + resourceId: scan.resourceId, + dataSourceId: dataSource.id + }); + } + }); + } catch (error) { + logger.error(error, `Failed to queue full-scan for data source with ID "${dataSource.id}"`); + + if (error instanceof BadRequestError) throw error; + + throw new InternalServerError({ message: `Failed to queue scan: ${(error as Error).message}` }); + } + }; + + const queueResourceDiffScan = async (payload: TQueueSecretScanningResourceDiffScan) => + queueService.queuePg(QueueJobs.SecretScanningV2DiffScan, payload); + + await queueService.startPg( + QueueJobs.SecretScanningV2FullScan, + async ([job]) => { + const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan; + const { retryCount, retryLimit } = job; + + const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`; + + const tempFolder = await createTempFolder(); + + try { + await secretScanningV2DAL.scans.update( + { id: scanId }, + { + status: SecretScanningScanStatus.Scanning + } + ); + + const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); + + if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`); + + const resource = await secretScanningV2DAL.resources.findById(resourceId); + + if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`); + + let connection: TAppConnection | null = null; + if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService); + + const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource](); + + const findingsPath = join(tempFolder, "findings.json"); + + const scanPath = await factory.getFullScanPath({ + dataSource: { + ...dataSource, + connection + } as TSecretScanningDataSourceWithConnection, + resourceName: resource.name, + tempFolder + }); + + let findingsPayload: TFindingsPayload; + switch (resource.type) { + case SecretScanningResource.Repository: + case SecretScanningResource.Project: + findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath); + break; + default: + throw new Error("Unhandled resource type"); + } + + await secretScanningV2DAL.findings.transaction(async (tx) => { + await secretScanningV2DAL.findings.upsert( + findingsPayload.map((findings) => ({ + ...findings, + projectId: dataSource.projectId, + dataSourceName: dataSource.name, + dataSourceType: dataSource.type, + resourceName: resource.name, + resourceType: resource.type, + scanId, + status: SecretScanningFindingStatus.Unresolved + })), + ["projectId", "fingerprint"], + tx, + ["resourceName", "dataSourceName", "status"] + ); + + await secretScanningV2DAL.scans.update( + { id: scanId }, + { + status: SecretScanningScanStatus.Completed + } + ); + }); + + // TODO: send notification + + logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails}`); + } catch (error) { + await secretScanningV2DAL.scans.update( + { id: scanId }, + { + status: SecretScanningScanStatus.Failed, + statusMessage: parseScanErrorMessage(error) + } + ); + + // TODO: send error notification + + logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`); + throw error; + } finally { + await deleteTempFolder(tempFolder); + } + }, + { + batchSize: 1, + workerCount: 2, + pollingIntervalSeconds: 1 + } + ); + + await queueService.startPg( + QueueJobs.SecretScanningV2DiffScan, + async ([job]) => { + const { payload, dataSourceId } = job.data as TQueueSecretScanningResourceDiffScan; + const { retryCount, retryLimit } = job; + + let scanId: string | undefined; + let logDetails = `[dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`; + + try { + const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); + + if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`); + + const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource](); + + const resourcePayload = factory.getDiffScanResourcePayload(payload); + + const { resourceId, resourceName, resourceType } = await secretScanningV2DAL.resources.transaction( + async (tx) => { + const [resource] = await secretScanningV2DAL.resources.upsert( + [ + { + ...resourcePayload, + dataSourceId + } + ], + ["externalId", "dataSourceId"], + tx + ); + + const scan = await secretScanningV2DAL.scans.create( + { + resourceId: resource.id, + type: SecretScanningScanType.DiffScan, + status: SecretScanningScanStatus.Scanning + }, + tx + ); + + scanId = scan.id; + + return { + resourceId: resource.id, + resourceName: resource.name, + resourceType: resource.type + }; + } + ); + + logDetails += ` [scanId=${scanId}] [resourceId=${resourceId}]`; + + let connection: TAppConnection | null = null; + if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService); + + const findingsPayload = await factory.getDiffScanFindingsPayload({ + dataSource: { + ...dataSource, + connection + } as TSecretScanningDataSourceWithConnection, + resourceName, + payload + }); + + await secretScanningV2DAL.findings.transaction(async (tx) => { + await secretScanningV2DAL.findings.upsert( + findingsPayload.map((findings) => ({ + ...findings, + projectId: dataSource.projectId, + dataSourceName: dataSource.name, + dataSourceType: dataSource.type, + resourceName, + resourceType, + scanId, + status: SecretScanningFindingStatus.Unresolved + })), + ["projectId", "fingerprint"], + tx, + ["resourceName", "dataSourceName", "status"] + ); + + await secretScanningV2DAL.scans.update( + { id: scanId }, + { + status: SecretScanningScanStatus.Completed + } + ); + }); + + // TODO: send notification + + logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`); + } catch (error) { + if (scanId) + await secretScanningV2DAL.scans.update( + { id: scanId }, + { + status: SecretScanningScanStatus.Failed, + statusMessage: parseScanErrorMessage(error) + } + ); + + // TODO: send error notification + + logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`); + throw error; + } + }, + { + batchSize: 1, + workerCount: 2, + pollingIntervalSeconds: 1 + } + ); + + // await queueService.startPg( + // QueueJobs.SecretRotationV2SendNotification, + // async ([job]) => { + // const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload; + // try { + // const { + // name: rotationName, + // type, + // projectId, + // lastRotationAttemptedAt, + // folder, + // environment, + // id: dataSourceId + // } = secretRotation; + // + // logger.info(`secretRotationV2Queue: Sending Status Notification [dataSourceId=${dataSourceId}]`); + // + // const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId); + // const project = await projectDAL.findById(projectId); + // + // const projectAdmins = projectMembers.filter((member) => + // member.roles.some((role) => role.role === ProjectMembershipRole.Admin) + // ); + // + // const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation]; + // + // await smtpService.sendMail({ + // recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean), + // template: SmtpTemplates.SecretRotationFailed, + // subjectLine: `Secret Rotation Failed`, + // substitutions: { + // rotationName, + // rotationType, + // content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date( + // lastRotationAttemptedAt + // ).toISOString()}. Please check the rotation status in Infisical for more details.`, + // secretPath: folder.path, + // environment: environment.name, + // projectName: project.name, + // rotationUrl: encodeURI(`${appCfg.SITE_URL}/secret-manager/${projectId}/secrets/${environment.slug}`) + // } + // }); + // } catch (error) { + // logger.error( + // error, + // `secretRotationV2Queue: Failed to Send Status Notification [dataSourceId=${secretRotation.id}]` + // ); + // throw error; + // } + // }, + // { + // batchSize: 1, + // workerCount: 2, + // pollingIntervalSeconds: 1 + // } + // ); + + return { + queueDataSourceFullScan, + queueResourceDiffScan + }; +}; diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-schemas.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-schemas.ts new file mode 100644 index 000000000..832b73bda --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-schemas.ts @@ -0,0 +1,99 @@ +import { z } from "zod"; + +import { SecretScanningDataSourcesSchema, SecretScanningFindingsSchema } from "@app/db/schemas"; +import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; +import { SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps"; +import { SecretScanningDataSources } from "@app/lib/api-docs"; +import { slugSchema } from "@app/server/lib/schemas"; + +type SecretScanningDataSourceSchemaOpts = { + type: SecretScanningDataSource; + isConnectionRequired: boolean; +}; + +export const BaseSecretScanningDataSourceSchema = ({ + type, + isConnectionRequired +}: SecretScanningDataSourceSchemaOpts) => + SecretScanningDataSourcesSchema.omit({ + // unique to provider + type: true, + connectionId: true, + config: true, + encryptedCredentials: true + }).extend({ + type: z.literal(type), + connectionId: isConnectionRequired ? z.string().uuid() : z.null(), + connection: isConnectionRequired + ? z.object({ + app: z.literal(SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[type]), + name: z.string(), + id: z.string().uuid() + }) + : z.null() + }); + +export const BaseCreateSecretScanningDataSourceSchema = ({ + type, + isConnectionRequired +}: SecretScanningDataSourceSchemaOpts) => + z.object({ + name: slugSchema({ field: "name" }).describe(SecretScanningDataSources.CREATE(type).name), + projectId: z + .string() + .trim() + .min(1, "Project ID required") + .describe(SecretScanningDataSources.CREATE(type).projectId), + description: z + .string() + .trim() + .max(256, "Description cannot exceed 256 characters") + .nullish() + .describe(SecretScanningDataSources.CREATE(type).description), + connectionId: isConnectionRequired + ? z.string().uuid().describe(SecretScanningDataSources.CREATE(type).connectionId) + : z.undefined(), + isAutoScanEnabled: z + .boolean() + .optional() + .default(true) + .describe(SecretScanningDataSources.CREATE(type).isAutoScanEnabled) + }); + +export const BaseUpdateSecretScanningDataSourceSchema = (type: SecretScanningDataSource) => + z.object({ + name: slugSchema({ field: "name" }).describe(SecretScanningDataSources.UPDATE(type).name).optional(), + description: z + .string() + .trim() + .max(256, "Description cannot exceed 256 characters") + .nullish() + .describe(SecretScanningDataSources.UPDATE(type).description), + isAutoScanEnabled: z.boolean().optional().describe(SecretScanningDataSources.UPDATE(type).isAutoScanEnabled) + }); + +export const GitRepositoryScanFindingDetailsSchema = z.object({ + description: z.string(), + startLine: z.number(), + endLine: z.number(), + startColumn: z.number(), + endColumn: z.number(), + file: z.string(), + link: z.string(), + symlinkFile: z.string(), + commit: z.string(), + entropy: z.number(), + author: z.string(), + email: z.string(), + date: z.string(), + message: z.string(), + tags: z.string().array(), + ruleID: z.string(), + fingerprint: z.string() +}); + +export const BaseSecretScanningFindingSchema = SecretScanningFindingsSchema.omit({ + dataSourceType: true, + resourceType: true, + details: true +}); diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.ts new file mode 100644 index 000000000..d6a9de25e --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.ts @@ -0,0 +1,784 @@ +import { ForbiddenError } from "@casl/ability"; + +import { ActionProjectType } from "@app/db/schemas"; +import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; +import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { + ProjectPermissionSecretScanningDataSourceActions, + ProjectPermissionSecretScanningFindingActions, + ProjectPermissionSub +} from "@app/ee/services/permission/project-permission"; +import { githubSecretScanningService } from "@app/ee/services/secret-scanning-v2/github/github-secret-scanning-service"; +import { SecretScanningFindingStatus } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; +import { SECRET_SCANNING_FACTORY_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-factory"; +import { listSecretScanningDataSourceOptions } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns"; +import { + SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP, + SECRET_SCANNING_DATA_SOURCE_NAME_MAP +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps"; +import { + TCreateSecretScanningDataSourceDTO, + TDeleteSecretScanningDataSourceDTO, + TFindSecretScanningDataSourceByIdDTO, + TFindSecretScanningDataSourceByNameDTO, + TListSecretScanningDataSourcesByProjectId, + TSecretScanningDataSource, + TSecretScanningDataSourceWithConnection, + TSecretScanningDataSourceWithDetails, + TSecretScanningFinding, + TSecretScanningResourceWithDetails, + TSecretScanningScanWithDetails, + TTriggerSecretScanningDataSourceDTO, + TUpdateSecretScanningDataSourceDTO, + TUpdateSecretScanningFinding +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types"; +import { DatabaseErrorCode } from "@app/lib/error-codes"; +import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; +import { OrgServiceActor } from "@app/lib/types"; +import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns"; +import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service"; +import { TAppConnection } from "@app/services/app-connection/app-connection-types"; +import { TKmsServiceFactory } from "@app/services/kms/kms-service"; +import { KmsDataKey } from "@app/services/kms/kms-types"; + +import { TSecretScanningV2DALFactory } from "./secret-scanning-v2-dal"; +import { TSecretScanningV2QueueServiceFactory } from "./secret-scanning-v2-queue"; + +export type TSecretScanningV2ServiceFactoryDep = { + secretScanningV2DAL: TSecretScanningV2DALFactory; + appConnectionService: Pick; + permissionService: Pick; + licenseService: Pick; + // auditLogService: Pick; + // keyStore: Pick; + // queueService: Pick; + // appConnectionDAL: Pick; + secretScanningV2Queue: Pick< + TSecretScanningV2QueueServiceFactory, + "queueDataSourceFullScan" | "queueResourceDiffScan" + >; + kmsService: Pick; +}; + +export type TSecretScanningV2ServiceFactory = ReturnType; + +export const secretScanningV2ServiceFactory = ({ + secretScanningV2DAL, + permissionService, + appConnectionService, + licenseService, + // auditLogService, + // keyStore, + // queueService, + // appConnectionDAL, + secretScanningV2Queue, + kmsService +}: TSecretScanningV2ServiceFactoryDep) => { + const $checkListSecretScanningDataSourcesByProjectIdPermissions = async ( + projectId: string, + actor: OrgServiceActor + ) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to access Secret Scanning Data Sources due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningDataSourceActions.Read, + ProjectPermissionSub.SecretScanningDataSources + ); + }; + + const listSecretScanningDataSourcesByProjectId = async ( + { projectId, type }: TListSecretScanningDataSourcesByProjectId, + actor: OrgServiceActor + ) => { + await $checkListSecretScanningDataSourcesByProjectIdPermissions(projectId, actor); + + const dataSources = await secretScanningV2DAL.dataSources.find({ + ...(type && { type }), + projectId + }); + + return dataSources as TSecretScanningDataSource[]; + }; + + const listSecretScanningDataSourcesWithDetailsByProjectId = async ( + { projectId, type }: TListSecretScanningDataSourcesByProjectId, + actor: OrgServiceActor + ) => { + await $checkListSecretScanningDataSourcesByProjectIdPermissions(projectId, actor); + + const dataSources = await secretScanningV2DAL.dataSources.findWithDetails({ + ...(type && { type }), + projectId + }); + + return dataSources as TSecretScanningDataSourceWithDetails[]; + }; + + const findSecretScanningDataSourceById = async ( + { type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO, + actor: OrgServiceActor + ) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to access Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); + + if (!dataSource) + throw new NotFoundError({ + message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId: dataSource.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningDataSourceActions.Read, + ProjectPermissionSub.SecretScanningDataSources + ); + + if (type !== dataSource.type) + throw new BadRequestError({ + message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}` + }); + + return dataSource as TSecretScanningDataSource; + }; + + const findSecretScanningDataSourceByName = async ( + { type, sourceName, projectId }: TFindSecretScanningDataSourceByNameDTO, + actor: OrgServiceActor + ) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to access Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + // we prevent conflicting names within a folder + const dataSource = await secretScanningV2DAL.dataSources.findOne({ + name: sourceName, + projectId + }); + + if (!dataSource) + throw new NotFoundError({ + message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with name "${sourceName}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningDataSourceActions.Read, + ProjectPermissionSub.SecretScanningDataSources + ); + + if (type !== dataSource.type) + throw new BadRequestError({ + message: `Secret Scanning Data Source with ID "${dataSource.id}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}` + }); + + return dataSource as TSecretScanningDataSource; + }; + + const createSecretScanningDataSource = async ( + payload: TCreateSecretScanningDataSourceDTO, + actor: OrgServiceActor + ) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to create Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId: payload.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningDataSourceActions.Create, + ProjectPermissionSub.SecretScanningDataSources + ); + + let connection: TAppConnection | null = null; + if (payload.connectionId) { + // validates permission to connect and app is valid for data source + connection = await appConnectionService.connectAppConnectionById( + SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[payload.type], + payload.connectionId, + actor + ); + } + + const factory = SECRET_SCANNING_FACTORY_MAP[payload.type](); + + try { + const createdDataSource = await factory.initialize( + { + payload, + connection: connection as TSecretScanningDataSourceWithConnection["connection"], + secretScanningV2DAL + }, + async ({ credentials, externalId }) => { + let encryptedCredentials: Buffer | null = null; + + if (credentials) { + const { encryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId: payload.projectId + }); + + const { cipherTextBlob } = encryptor({ + plainText: Buffer.from(JSON.stringify(credentials)) + }); + + encryptedCredentials = cipherTextBlob; + } + + return secretScanningV2DAL.dataSources.transaction(async (tx) => { + const dataSource = await secretScanningV2DAL.dataSources.create( + { + encryptedCredentials, + externalId, + ...payload + }, + tx + ); + + await factory.postInitialization({ + payload, + connection: connection as TSecretScanningDataSourceWithConnection["connection"], + dataSourceId: dataSource.id, + credentials + }); + + return dataSource; + }); + } + ); + + if (payload.isAutoScanEnabled) { + try { + await secretScanningV2Queue.queueDataSourceFullScan({ + ...createdDataSource, + connection + } as TSecretScanningDataSourceWithConnection); + } catch { + // silently fail, don't want to block creation, they'll try scanning when they don't see anything and get the error + } + } + + return createdDataSource as TSecretScanningDataSource; + } catch (err) { + if (err instanceof DatabaseError && (err.error as { code: string })?.code === DatabaseErrorCode.UniqueViolation) { + throw new BadRequestError({ + message: `A Secret Scanning Data Source with the name "${payload.name}" already exists for the project with ID "${payload.projectId}"` + }); + } + + throw err; + } + }; + + const updateSecretScanningDataSource = async ( + { type, dataSourceId, ...payload }: TUpdateSecretScanningDataSourceDTO, + actor: OrgServiceActor + ) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to update Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); + + if (!dataSource) + throw new NotFoundError({ + message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId: dataSource.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningDataSourceActions.Edit, + ProjectPermissionSub.SecretScanningDataSources + ); + + if (type !== dataSource.type) + throw new BadRequestError({ + message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}` + }); + + try { + const updatedDataSource = await secretScanningV2DAL.dataSources.updateById(dataSourceId, payload); + + return updatedDataSource as TSecretScanningDataSource; + } catch (err) { + if (err instanceof DatabaseError && (err.error as { code: string })?.code === DatabaseErrorCode.UniqueViolation) { + throw new BadRequestError({ + message: `A Secret Scanning Data Source with the name "${payload.name}" already exists for the project with ID "${dataSource.projectId}"` + }); + } + + throw err; + } + }; + + const deleteSecretScanningResource = async ( + { type, dataSourceId }: TDeleteSecretScanningDataSourceDTO, + actor: OrgServiceActor + ) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to delete Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); + + if (!dataSource) + throw new NotFoundError({ + message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId: dataSource.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningDataSourceActions.Delete, + ProjectPermissionSub.SecretScanningDataSources + ); + + if (type !== dataSource.type) + throw new BadRequestError({ + message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}` + }); + + // TODO: clean up webhooks + + await secretScanningV2DAL.dataSources.deleteById(dataSourceId); + + return dataSource as TSecretScanningDataSource; + }; + + const triggerSecretScanningDataSourceScan = async ( + { type, dataSourceId, resourceId }: TTriggerSecretScanningDataSourceDTO, + actor: OrgServiceActor + ) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to trigger scan for Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); + + if (!dataSource) + throw new NotFoundError({ + message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId: dataSource.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningDataSourceActions.TriggerScans, + ProjectPermissionSub.SecretScanningDataSources + ); + + if (type !== dataSource.type) + throw new BadRequestError({ + message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}` + }); + + let connection: TAppConnection | null = null; + if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService); + + let resourceExternalId: string | undefined; + + if (resourceId) { + const resource = await secretScanningV2DAL.resources.findOne({ id: resourceId, dataSourceId }); + if (!resource) { + throw new NotFoundError({ + message: `Could not find Secret Scanning Resource with ID "${resourceId}" for Data Source with ID "${dataSourceId}"` + }); + } + resourceExternalId = resource.externalId; + } + + await secretScanningV2Queue.queueDataSourceFullScan( + { + ...dataSource, + connection + } as TSecretScanningDataSourceWithConnection, + resourceExternalId + ); + + return dataSource as TSecretScanningDataSource; + }; + + const listSecretScanningResourcesByDataSourceId = async ( + { type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO, + actor: OrgServiceActor + ) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to access Secret Scanning Resources due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); + + if (!dataSource) + throw new NotFoundError({ + message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId: dataSource.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningDataSourceActions.ReadResources, + ProjectPermissionSub.SecretScanningDataSources + ); + + if (type !== dataSource.type) + throw new BadRequestError({ + message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}` + }); + + const resources = await secretScanningV2DAL.resources.find({ + dataSourceId + }); + + return { resources, projectId: dataSource.projectId }; + }; + + const listSecretScanningScansByDataSourceId = async ( + { type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO, + actor: OrgServiceActor + ) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to access Secret Scanning Resources due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); + + if (!dataSource) + throw new NotFoundError({ + message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId: dataSource.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningDataSourceActions.ReadScans, + ProjectPermissionSub.SecretScanningDataSources + ); + + if (type !== dataSource.type) + throw new BadRequestError({ + message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}` + }); + + const scans = await secretScanningV2DAL.scans.findByDataSourceId(dataSourceId); + + return { scans, projectId: dataSource.projectId }; + }; + + const listSecretScanningResourcesWithDetailsByDataSourceId = async ( + { type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO, + actor: OrgServiceActor + ) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to access Secret Scanning Resources due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); + + if (!dataSource) + throw new NotFoundError({ + message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId: dataSource.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningDataSourceActions.ReadResources, + ProjectPermissionSub.SecretScanningDataSources + ); + + if (type !== dataSource.type) + throw new BadRequestError({ + message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}` + }); + + const resources = await secretScanningV2DAL.resources.findWithDetails({ dataSourceId }); + + return { resources: resources as TSecretScanningResourceWithDetails[], projectId: dataSource.projectId }; + }; + + const listSecretScanningScansWithDetailsByDataSourceId = async ( + { type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO, + actor: OrgServiceActor + ) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to access Secret Scanning Scans due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); + + if (!dataSource) + throw new NotFoundError({ + message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId: dataSource.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningDataSourceActions.ReadScans, + ProjectPermissionSub.SecretScanningDataSources + ); + + if (type !== dataSource.type) + throw new BadRequestError({ + message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}` + }); + + const scans = await secretScanningV2DAL.scans.findWithDetailsByDataSourceId(dataSourceId); + + return { scans: scans as TSecretScanningScanWithDetails[], projectId: dataSource.projectId }; + }; + + const getSecretScanningUnresolvedFindingsCountByProjectId = async (projectId: string, actor: OrgServiceActor) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningFindingActions.Read, + ProjectPermissionSub.SecretScanningFindings + ); + + const [finding] = await secretScanningV2DAL.findings.find( + { + projectId, + status: SecretScanningFindingStatus.Unresolved + }, + { count: true } + ); + + return Number(finding?.count ?? 0); + }; + + const listSecretScanningFindingsByProjectId = async (projectId: string, actor: OrgServiceActor) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningFindingActions.Read, + ProjectPermissionSub.SecretScanningFindings + ); + + const findings = await secretScanningV2DAL.findings.find({ + projectId + }); + + return findings as TSecretScanningFinding[]; + }; + + const updateSecretScanningFindingById = async ( + { findingId, remarks, status }: TUpdateSecretScanningFinding, + actor: OrgServiceActor + ) => { + const plan = await licenseService.getPlan(actor.orgId); + + if (!plan.secretScanning) + throw new BadRequestError({ + message: + "Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning." + }); + + const finding = await secretScanningV2DAL.findings.findById(findingId); + + if (!finding) + throw new NotFoundError({ + message: `Could not find Secret Scanning Finding with ID "${findingId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretScanning, + projectId: finding.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretScanningFindingActions.Resolve, + ProjectPermissionSub.SecretScanningFindings + ); + + const updatedFinding = await secretScanningV2DAL.findings.updateById(findingId, { + remarks, + status + }); + + return { finding: updatedFinding as TSecretScanningFinding, projectId: finding.projectId }; + }; + + return { + listSecretScanningDataSourceOptions, + listSecretScanningDataSourcesByProjectId, + listSecretScanningDataSourcesWithDetailsByProjectId, + findSecretScanningDataSourceById, + findSecretScanningDataSourceByName, + createSecretScanningDataSource, + updateSecretScanningDataSource, + deleteSecretScanningResource, + triggerSecretScanningDataSourceScan, + listSecretScanningResourcesByDataSourceId, + listSecretScanningScansByDataSourceId, + listSecretScanningResourcesWithDetailsByDataSourceId, + listSecretScanningScansWithDetailsByDataSourceId, + getSecretScanningUnresolvedFindingsCountByProjectId, + listSecretScanningFindingsByProjectId, + updateSecretScanningFindingById, + github: githubSecretScanningService(secretScanningV2DAL, secretScanningV2Queue) + }; +}; diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts new file mode 100644 index 000000000..6480a3899 --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts @@ -0,0 +1,171 @@ +import { TSecretScanningFindingsInsert, TSecretScanningResources, TSecretScanningScans } from "@app/db/schemas"; +import { + TGitHubDataSource, + TGitHubDataSourceInput, + TGitHubDataSourceListItem, + TGitHubDataSourceWithConnection, + TGitHubFinding, + TQueueGitHubResourceDiffScan +} from "@app/ee/services/secret-scanning-v2/github"; +import { TSecretScanningV2DALFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-dal"; +import { + SecretScanningDataSource, + SecretScanningFindingStatus, + SecretScanningScanStatus +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; + +export type TSecretScanningDataSource = TGitHubDataSource; + +export type TSecretScanningDataSourceWithDetails = TSecretScanningDataSource & { + lastScannedAt?: Date | null; + lastScanStatus?: SecretScanningScanStatus | null; + lastScanStatusMessage?: string | null; + unresolvedFindings: number; +}; + +export type TSecretScanningResourceWithDetails = TSecretScanningResources & { + lastScannedAt?: Date | null; + lastScanStatus?: SecretScanningScanStatus | null; + lastScanStatusMessage?: string | null; + unresolvedFindings: number; +}; + +export type TSecretScanningScanWithDetails = TSecretScanningScans & { + unresolvedFindings: number; + resolvedFindings: number; + resourceName: string; +}; + +export type TSecretScanningDataSourceWithConnection = TGitHubDataSourceWithConnection; + +export type TSecretScanningDataSourceInput = TGitHubDataSourceInput; + +export type TSecretScanningDataSourceListItem = TGitHubDataSourceListItem; + +export type TSecretScanningFinding = TGitHubFinding; + +export type TListSecretScanningDataSourcesByProjectId = { + projectId: string; + type?: SecretScanningDataSource; +}; + +export type TFindSecretScanningDataSourceByIdDTO = { + dataSourceId: string; + type: SecretScanningDataSource; +}; + +export type TFindSecretScanningDataSourceByNameDTO = { + sourceName: string; + projectId: string; + type: SecretScanningDataSource; +}; + +export type TCreateSecretScanningDataSourceDTO = Pick< + TSecretScanningDataSource, + "description" | "name" | "projectId" +> & { + connectionId?: string; + type: SecretScanningDataSource; + isAutoScanEnabled?: boolean; + config: Partial; +}; + +export type TUpdateSecretScanningDataSourceDTO = Partial< + Omit +> & { + dataSourceId: string; + type: SecretScanningDataSource; +}; + +export type TDeleteSecretScanningDataSourceDTO = { + type: SecretScanningDataSource; + dataSourceId: string; +}; + +export type TTriggerSecretScanningDataSourceDTO = { + type: SecretScanningDataSource; + dataSourceId: string; + resourceId?: string; +}; + +export type TQueueSecretScanningDataSourceFullScan = { + dataSourceId: string; + resourceId: string; + scanId: string; +}; + +export type TQueueSecretScanningResourceDiffScan = TQueueGitHubResourceDiffScan; + +export type TCloneRepository = { + cloneUrl: string; + repoPath: string; +}; + +export type TSecretScanningFactoryListRawResources = ( + dataSource: T +) => Promise[]>; + +export type TSecretScanningFactoryGetDiffScanResourcePayload< + P extends TQueueSecretScanningResourceDiffScan["payload"] +> = (payload: P) => Pick; + +export type TSecretScanningFactoryGetFullScanPath = (parameters: { + dataSource: T; + resourceName: string; + tempFolder: string; +}) => Promise; + +export type TSecretScanningFactoryGetDiffScanFindingsPayload< + T extends TSecretScanningDataSourceWithConnection, + P extends TQueueSecretScanningResourceDiffScan["payload"] +> = (parameters: { dataSource: T; resourceName: string; payload: P }) => Promise; + +export type TSecretScanningDataSourceRaw = NonNullable< + Awaited> +>; + +export type TSecretScanningFactoryInitialize< + T extends TSecretScanningDataSourceWithConnection["connection"] | undefined = undefined, + C extends TSecretScanningDataSourceCredentials = undefined +> = ( + params: { + payload: TCreateSecretScanningDataSourceDTO; + connection: T; + secretScanningV2DAL: TSecretScanningV2DALFactory; + }, + callback: (parameters: { credentials?: C; externalId?: string }) => Promise +) => Promise; + +export type TSecretScanningFactoryPostInitialization< + T extends TSecretScanningDataSourceWithConnection["connection"] | undefined = undefined, + C extends TSecretScanningDataSourceCredentials = undefined +> = (params: { + payload: TCreateSecretScanningDataSourceDTO; + connection: T; + credentials: C; + dataSourceId: string; +}) => Promise; + +export type TSecretScanningFactory< + T extends TSecretScanningDataSourceWithConnection, + C extends TSecretScanningDataSourceCredentials, + P extends TQueueSecretScanningResourceDiffScan["payload"] +> = () => { + listRawResources: TSecretScanningFactoryListRawResources; + getFullScanPath: TSecretScanningFactoryGetFullScanPath; + initialize: TSecretScanningFactoryInitialize; + postInitialization: TSecretScanningFactoryPostInitialization; + getDiffScanResourcePayload: TSecretScanningFactoryGetDiffScanResourcePayload

; + getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload; +}; + +export type TFindingsPayload = Pick[]; +export type TGetFindingsPayload = Promise; + +export type TUpdateSecretScanningFinding = { + status: SecretScanningFindingStatus; + remarks?: string | null; + findingId: string; +}; + +export type TSecretScanningDataSourceCredentials = undefined; diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-union-schemas.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-union-schemas.ts new file mode 100644 index 000000000..4f34791f8 --- /dev/null +++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-union-schemas.ts @@ -0,0 +1,7 @@ +import { z } from "zod"; + +import { GitHubDataSourceSchema, GitHubFindingSchema } from "@app/ee/services/secret-scanning-v2/github"; + +export const SecretScanningDataSourceSchema = z.discriminatedUnion("type", [GitHubDataSourceSchema]); + +export const SecretScanningFindingSchema = z.discriminatedUnion("resourceType", [GitHubFindingSchema]); diff --git a/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types.ts b/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types.ts index 3f14a41e3..6990febc3 100644 --- a/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types.ts +++ b/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types.ts @@ -9,6 +9,7 @@ export type SecretMatch = { Match: string; Secret: string; File: string; + Link: string; SymlinkFile: string; Commit: string; Entropy: number; diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index 3fad36fac..e2fa70f3c 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -3,6 +3,12 @@ import { SECRET_ROTATION_CONNECTION_MAP, SECRET_ROTATION_NAME_MAP } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-maps"; +import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums"; +import { + AUTO_SYNC_DESCRIPTION_HELPER, + SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP, + SECRET_SCANNING_DATA_SOURCE_NAME_MAP +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps"; import { AppConnection } from "@app/services/app-connection/app-connection-enums"; import { APP_CONNECTION_NAME_MAP } from "@app/services/app-connection/app-connection-maps"; import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; @@ -55,7 +61,8 @@ export enum ApiDocsTags { SshHostGroups = "SSH Host Groups", KmsKeys = "KMS Keys", KmsEncryption = "KMS Encryption", - KmsSigning = "KMS Signing" + KmsSigning = "KMS Signing", + SecretScanning = "Secret Scanning" } export const GROUPS = { @@ -2351,3 +2358,66 @@ export const SecretRotations = { } } }; + +export const SecretScanningDataSources = { + LIST: (type?: SecretScanningDataSource) => ({ + projectId: `The ID of the project to list ${type ? SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type] : "Scanning"} Data Sources from.` + }), + GET_BY_ID: (type: SecretScanningDataSource) => ({ + dataSourceId: `The ID of the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source to retrieve.` + }), + GET_BY_NAME: (type: SecretScanningDataSource) => ({ + sourceName: `The name of the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source to retrieve.`, + projectId: `The ID of the project the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source is located in.` + }), + CREATE: (type: SecretScanningDataSource) => { + const sourceType = SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]; + const autoScanDescription = AUTO_SYNC_DESCRIPTION_HELPER[type]; + return { + name: `The name of the ${sourceType} Data Source to create. Must be slug-friendly.`, + description: `An optional description for the ${sourceType} Data Source.`, + projectId: `The ID of the project to create the ${sourceType} Data Source in.`, + connectionId: `The ID of the ${ + APP_CONNECTION_NAME_MAP[SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[type]] + } Connection to use for this Data Source.`, + isAutoScanEnabled: `Whether scans should be automatically performed when a ${autoScanDescription.verb} occurs to ${autoScanDescription.noun} associated with this Data Source.`, + config: `The configuration parameters to use for this Data Source.` + }; + }, + UPDATE: (type: SecretScanningDataSource) => { + const typeName = SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]; + const autoScanDescription = AUTO_SYNC_DESCRIPTION_HELPER[type]; + + return { + dataSourceId: `The ID of the ${typeName} Data Source to be updated.`, + name: `The updated name of the ${typeName} Data Source. Must be slug-friendly.`, + description: `The updated description of the ${typeName} Data Source.`, + isAutoScanEnabled: `Whether scans should be automatically performed when a ${autoScanDescription.verb} occurs to ${autoScanDescription.noun} associated with this Data Source.`, + config: `The updated configuration parameters to use for this Data Source.` + }; + }, + DELETE: (type: SecretScanningDataSource) => ({ + dataSourceId: `The ID of the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source to be deleted.` + }), + SCAN: (type: SecretScanningDataSource) => ({ + dataSourceId: `The ID of the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source to trigger a scan for.`, + resourceId: `The ID of the individual Data Source resource to trigger a scan for.` + }), + LIST_RESOURCES: (type: SecretScanningDataSource) => ({ + dataSourceId: `The ID of the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source to list resources from.` + }), + LIST_SCANS: (type: SecretScanningDataSource) => ({ + dataSourceId: `The ID of the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source to list scans for.` + }) +}; + +export const SecretScanningFindings = { + LIST: { + projectId: `The ID of the project to list Secret Scanning Findings from.` + }, + UPDATE: { + findingId: "The ID of the Secret Scanning Finding to update the resolve status for.", + status: "The updated status of the specified Secret Scanning Finding.", + remarks: "Remarks pertaining to the resolve status of this finding." + } +}; diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index e38dbcfb5..161fef9db 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -215,6 +215,14 @@ const envSchema = z INF_APP_CONNECTION_GITHUB_APP_SLUG: zpStr(z.string().optional()), INF_APP_CONNECTION_GITHUB_APP_ID: zpStr(z.string().optional()), + // github radar app + INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_ID: zpStr(z.string().optional()), + INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_SECRET: zpStr(z.string().optional()), + INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY: zpStr(z.string().optional()), + INF_APP_CONNECTION_GITHUB_RADAR_APP_SLUG: zpStr(z.string().optional()), + INF_APP_CONNECTION_GITHUB_RADAR_APP_ID: zpStr(z.string().optional()), + INF_APP_CONNECTION_GITHUB_RADAR_APP_WEBHOOK_SECRET: zpStr(z.string().optional()), + // gcp app INF_APP_CONNECTION_GCP_SERVICE_ACCOUNT_CREDENTIAL: zpStr(z.string().optional()), diff --git a/backend/src/lib/knex/index.ts b/backend/src/lib/knex/index.ts index 2e17bff20..5949afe33 100644 --- a/backend/src/lib/knex/index.ts +++ b/backend/src/lib/knex/index.ts @@ -179,13 +179,18 @@ export const ormify = (db: Kne throw new DatabaseError({ error, name: "batchInsert" }); } }, - upsert: async (data: readonly Tables[Tname]["insert"][], onConflictField: keyof Tables[Tname]["base"], tx?: Knex) => { + upsert: async ( + data: readonly Tables[Tname]["insert"][], + onConflictField: keyof Tables[Tname]["base"] | Array, + tx?: Knex, + mergeColumns?: (keyof Knex.ResolveTableType, "update">)[] | undefined + ) => { try { if (!data.length) return []; const res = await (tx || db)(tableName) .insert(data as never) .onConflict(onConflictField as never) - .merge() + .merge(mergeColumns) .returning("*"); return res; } catch (error) { diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 807d6c286..bf70571af 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -12,6 +12,10 @@ import { TScanFullRepoEventPayload, TScanPushEventPayload } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types"; +import { + TQueueSecretScanningDataSourceFullScan, + TQueueSecretScanningResourceDiffScan +} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types"; import { getConfig } from "@app/lib/config/env"; import { logger } from "@app/lib/logger"; import { @@ -51,7 +55,8 @@ export enum QueueName { ImportSecretsFromExternalSource = "import-secrets-from-external-source", AppConnectionSecretSync = "app-connection-secret-sync", SecretRotationV2 = "secret-rotation-v2", - InvalidateCache = "invalidate-cache" + InvalidateCache = "invalidate-cache", + SecretScanningV2 = "secret-scanning-v2" } export enum QueueJobs { @@ -84,7 +89,9 @@ export enum QueueJobs { SecretRotationV2QueueRotations = "secret-rotation-v2-queue-rotations", SecretRotationV2RotateSecrets = "secret-rotation-v2-rotate-secrets", SecretRotationV2SendNotification = "secret-rotation-v2-send-notification", - InvalidateCache = "invalidate-cache" + InvalidateCache = "invalidate-cache", + SecretScanningV2FullScan = "secret-scanning-v2-full-scan", + SecretScanningV2DiffScan = "secret-scanning-v2-diff-scan" } export type TQueueJobTypes = { @@ -245,6 +252,15 @@ export type TQueueJobTypes = { }; }; }; + [QueueName.SecretScanningV2]: + | { + name: QueueJobs.SecretScanningV2FullScan; + payload: TQueueSecretScanningDataSourceFullScan; + } + | { + name: QueueJobs.SecretScanningV2DiffScan; + payload: TQueueSecretScanningResourceDiffScan; + }; }; export type TQueueServiceFactory = ReturnType; diff --git a/backend/src/server/plugins/secret-scanner-v2.ts b/backend/src/server/plugins/secret-scanner-v2.ts new file mode 100644 index 000000000..6835309b6 --- /dev/null +++ b/backend/src/server/plugins/secret-scanner-v2.ts @@ -0,0 +1,60 @@ +import type { EmitterWebhookEventName } from "@octokit/webhooks/dist-types/types"; +import { PushEvent } from "@octokit/webhooks-types"; +import { Probot } from "probot"; + +import { getConfig } from "@app/lib/config/env"; +import { logger } from "@app/lib/logger"; +import { writeLimit } from "@app/server/config/rateLimiter"; + +export const registerSecretScanningV2Webhooks = async (server: FastifyZodProvider) => { + const probotApp = (app: Probot) => { + app.on("installation.deleted", async (context) => { + const { payload } = context; + const { installation } = payload; + + await server.services.secretScanningV2.github.handleInstallationDeleted(installation.id); + }); + + app.on("installation", async (context) => { + const { payload } = context; + logger.info({ repositories: payload.repositories }, "Installed secret scanner to"); + }); + + app.on("push", async (context) => { + const { payload } = context; + await server.services.secretScanningV2.github.handlePushEvent(payload as PushEvent); + }); + }; + + const appCfg = getConfig(); + + const probot = new Probot({ + appId: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID as string, + privateKey: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY as string, + secret: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_WEBHOOK_SECRET as string + }); + + await probot.load(probotApp); + + server.route({ + method: "POST", + url: "/github", + config: { + rateLimit: writeLimit + }, + handler: async (req, res) => { + const eventName = req.headers["x-github-event"] as EmitterWebhookEventName; + const signatureSHA256 = req.headers["x-hub-signature-256"] as string; + const id = req.headers["x-github-delivery"] as string; + + await probot.webhooks.verifyAndReceive({ + id, + name: eventName, + payload: JSON.stringify(req.body), + signature: signatureSHA256 + }); + + return res.send("ok"); + } + }); +}; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 5196971f9..caa0587b5 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -86,6 +86,9 @@ import { gitAppInstallSessionDALFactory } from "@app/ee/services/secret-scanning import { secretScanningDALFactory } from "@app/ee/services/secret-scanning/secret-scanning-dal"; import { secretScanningQueueFactory } from "@app/ee/services/secret-scanning/secret-scanning-queue"; import { secretScanningServiceFactory } from "@app/ee/services/secret-scanning/secret-scanning-service"; +import { secretScanningV2DALFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-dal"; +import { secretScanningV2QueueServiceFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-queue"; +import { secretScanningV2ServiceFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-service"; import { secretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; import { snapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal"; import { snapshotFolderDALFactory } from "@app/ee/services/secret-snapshot/snapshot-folder-dal"; @@ -112,6 +115,7 @@ import { getConfig, TEnvConfig } from "@app/lib/config/env"; import { logger } from "@app/lib/logger"; import { TQueueServiceFactory } from "@app/queue"; import { readLimit } from "@app/server/config/rateLimiter"; +import { registerSecretScanningV2Webhooks } from "@app/server/plugins/secret-scanner-v2"; import { accessTokenQueueServiceFactory } from "@app/services/access-token-queue/access-token-queue"; import { apiKeyDALFactory } from "@app/services/api-key/api-key-dal"; import { apiKeyServiceFactory } from "@app/services/api-key/api-key-service"; @@ -299,6 +303,9 @@ export const registerRoutes = async ( ) => { const appCfg = getConfig(); await server.register(registerSecretScannerGhApp, { prefix: "/ss-webhook" }); + await server.register(registerSecretScanningV2Webhooks, { + prefix: "/secret-scanning/webhooks" + }); // db layers const userDAL = userDALFactory(db); @@ -444,6 +451,7 @@ export const registerRoutes = async ( const secretRotationV2DAL = secretRotationV2DALFactory(db, folderDAL); const microsoftTeamsIntegrationDAL = microsoftTeamsIntegrationDALFactory(db); const projectMicrosoftTeamsConfigDAL = projectMicrosoftTeamsConfigDALFactory(db); + const secretScanningV2DAL = secretScanningV2DALFactory(db); const permissionService = permissionServiceFactory({ permissionDAL, @@ -1694,6 +1702,28 @@ export const registerRoutes = async ( smtpService }); + const secretScanningV2Queue = await secretScanningV2QueueServiceFactory({ + secretScanningV2DAL, + queueService, + // projectDAL, + // projectMembershipDAL, + // smtpService, + kmsService + }); + + const secretScanningV2Service = secretScanningV2ServiceFactory({ + // appConnectionDAL, + permissionService, + appConnectionService, + licenseService, + // auditLogService, + // keyStore, + // queueService, + secretScanningV2DAL, + secretScanningV2Queue, + kmsService + }); + await superAdminService.initServerCfg(); // setup the communication with license key server @@ -1805,7 +1835,8 @@ export const registerRoutes = async ( secretRotationV2: secretRotationV2Service, microsoftTeams: microsoftTeamsService, assumePrivileges: assumePrivilegeService, - githubOrgSync: githubOrgSyncConfigService + githubOrgSync: githubOrgSyncConfigService, + secretScanningV2: secretScanningV2Service }); const cronJobs: CronJob[] = []; diff --git a/backend/src/server/routes/v1/app-connection-routers/app-connection-router.ts b/backend/src/server/routes/v1/app-connection-routers/app-connection-router.ts index b9ce3deb8..1fd8218f9 100644 --- a/backend/src/server/routes/v1/app-connection-routers/app-connection-router.ts +++ b/backend/src/server/routes/v1/app-connection-routers/app-connection-router.ts @@ -28,6 +28,10 @@ import { } from "@app/services/app-connection/databricks"; import { GcpConnectionListItemSchema, SanitizedGcpConnectionSchema } from "@app/services/app-connection/gcp"; import { GitHubConnectionListItemSchema, SanitizedGitHubConnectionSchema } from "@app/services/app-connection/github"; +import { + GitHubRadarConnectionListItemSchema, + SanitizedGitHubRadarConnectionSchema +} from "@app/services/app-connection/github-radar"; import { HCVaultConnectionListItemSchema, SanitizedHCVaultConnectionSchema @@ -62,6 +66,7 @@ import { AuthMode } from "@app/services/auth/auth-type"; const SanitizedAppConnectionSchema = z.union([ ...SanitizedAwsConnectionSchema.options, ...SanitizedGitHubConnectionSchema.options, + ...SanitizedGitHubRadarConnectionSchema.options, ...SanitizedGcpConnectionSchema.options, ...SanitizedAzureKeyVaultConnectionSchema.options, ...SanitizedAzureAppConfigurationConnectionSchema.options, @@ -84,6 +89,7 @@ const SanitizedAppConnectionSchema = z.union([ const AppConnectionOptionsSchema = z.discriminatedUnion("app", [ AwsConnectionListItemSchema, GitHubConnectionListItemSchema, + GitHubRadarConnectionListItemSchema, GcpConnectionListItemSchema, AzureKeyVaultConnectionListItemSchema, AzureAppConfigurationConnectionListItemSchema, diff --git a/backend/src/server/routes/v1/app-connection-routers/github-radar-connection-router.ts b/backend/src/server/routes/v1/app-connection-routers/github-radar-connection-router.ts new file mode 100644 index 000000000..086d986b7 --- /dev/null +++ b/backend/src/server/routes/v1/app-connection-routers/github-radar-connection-router.ts @@ -0,0 +1,54 @@ +import { z } from "zod"; + +import { readLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AppConnection } from "@app/services/app-connection/app-connection-enums"; +import { + CreateGitHubRadarConnectionSchema, + SanitizedGitHubRadarConnectionSchema, + UpdateGitHubRadarConnectionSchema +} from "@app/services/app-connection/github-radar"; +import { AuthMode } from "@app/services/auth/auth-type"; + +import { registerAppConnectionEndpoints } from "./app-connection-endpoints"; + +export const registerGitHubRadarConnectionRouter = async (server: FastifyZodProvider) => { + registerAppConnectionEndpoints({ + app: AppConnection.GitHubRadar, + server, + sanitizedResponseSchema: SanitizedGitHubRadarConnectionSchema, + createSchema: CreateGitHubRadarConnectionSchema, + updateSchema: UpdateGitHubRadarConnectionSchema + }); + + // The below endpoints are not exposed and for Infisical App use + + server.route({ + method: "GET", + url: `/:connectionId/repositories`, + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + connectionId: z.string().uuid() + }), + response: { + 200: z.object({ + repositories: z.object({ id: z.number(), name: z.string() }).array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const { connectionId } = req.params; + + const repositories = await server.services.appConnection.githubRadar.listRepositories( + connectionId, + req.permission + ); + + return { repositories }; + } + }); +}; diff --git a/backend/src/server/routes/v1/app-connection-routers/index.ts b/backend/src/server/routes/v1/app-connection-routers/index.ts index 6f6fa1991..43f7968e7 100644 --- a/backend/src/server/routes/v1/app-connection-routers/index.ts +++ b/backend/src/server/routes/v1/app-connection-routers/index.ts @@ -9,6 +9,7 @@ import { registerCamundaConnectionRouter } from "./camunda-connection-router"; import { registerDatabricksConnectionRouter } from "./databricks-connection-router"; import { registerGcpConnectionRouter } from "./gcp-connection-router"; import { registerGitHubConnectionRouter } from "./github-connection-router"; +import { registerGitHubRadarConnectionRouter } from "./github-radar-connection-router"; import { registerHCVaultConnectionRouter } from "./hc-vault-connection-router"; import { registerHumanitecConnectionRouter } from "./humanitec-connection-router"; import { registerLdapConnectionRouter } from "./ldap-connection-router"; @@ -26,6 +27,7 @@ export const APP_CONNECTION_REGISTER_ROUTER_MAP: Record { .default("false") .transform((value) => value === "true"), type: z - .enum([ProjectType.SecretManager, ProjectType.KMS, ProjectType.CertificateManager, ProjectType.SSH, "all"]) + .enum([ + ProjectType.SecretManager, + ProjectType.KMS, + ProjectType.CertificateManager, + ProjectType.SSH, + ProjectType.SecretScanning, + "all" + ]) .optional() }), response: { diff --git a/backend/src/services/app-connection/app-connection-enums.ts b/backend/src/services/app-connection/app-connection-enums.ts index 6e09f1293..654eaedab 100644 --- a/backend/src/services/app-connection/app-connection-enums.ts +++ b/backend/src/services/app-connection/app-connection-enums.ts @@ -1,5 +1,6 @@ export enum AppConnection { GitHub = "github", + GitHubRadar = "github-radar", AWS = "aws", Databricks = "databricks", GCP = "gcp", diff --git a/backend/src/services/app-connection/app-connection-fns.ts b/backend/src/services/app-connection/app-connection-fns.ts index f6fd894a6..d66dabf23 100644 --- a/backend/src/services/app-connection/app-connection-fns.ts +++ b/backend/src/services/app-connection/app-connection-fns.ts @@ -41,6 +41,11 @@ import { } from "./databricks"; import { GcpConnectionMethod, getGcpConnectionListItem, validateGcpConnectionCredentials } from "./gcp"; import { getGitHubConnectionListItem, GitHubConnectionMethod, validateGitHubConnectionCredentials } from "./github"; +import { + getGitHubRadarConnectionListItem, + GitHubRadarConnectionMethod, + validateGitHubRadarConnectionCredentials +} from "./github-radar"; import { getHCVaultConnectionListItem, HCVaultConnectionMethod, @@ -77,6 +82,7 @@ export const listAppConnectionOptions = () => { return [ getAwsConnectionListItem(), getGitHubConnectionListItem(), + getGitHubRadarConnectionListItem(), getGcpConnectionListItem(), getAzureKeyVaultConnectionListItem(), getAzureAppConfigurationConnectionListItem(), @@ -146,6 +152,7 @@ export const validateAppConnectionCredentials = async ( [AppConnection.AWS]: validateAwsConnectionCredentials as TAppConnectionCredentialsValidator, [AppConnection.Databricks]: validateDatabricksConnectionCredentials as TAppConnectionCredentialsValidator, [AppConnection.GitHub]: validateGitHubConnectionCredentials as TAppConnectionCredentialsValidator, + [AppConnection.GitHubRadar]: validateGitHubRadarConnectionCredentials as TAppConnectionCredentialsValidator, [AppConnection.GCP]: validateGcpConnectionCredentials as TAppConnectionCredentialsValidator, [AppConnection.AzureKeyVault]: validateAzureKeyVaultConnectionCredentials as TAppConnectionCredentialsValidator, [AppConnection.AzureAppConfiguration]: @@ -173,6 +180,8 @@ export const getAppConnectionMethodName = (method: TAppConnection["method"]) => switch (method) { case GitHubConnectionMethod.App: return "GitHub App"; + case GitHubRadarConnectionMethod.App: + return "GitHub App"; case AzureKeyVaultConnectionMethod.OAuth: case AzureAppConfigurationConnectionMethod.OAuth: case AzureClientSecretsConnectionMethod.OAuth: @@ -240,6 +249,7 @@ export const TRANSITION_CONNECTION_CREDENTIALS_TO_PLATFORM: Record< [AppConnection.AWS]: platformManagedCredentialsNotSupported, [AppConnection.Databricks]: platformManagedCredentialsNotSupported, [AppConnection.GitHub]: platformManagedCredentialsNotSupported, + [AppConnection.GitHubRadar]: platformManagedCredentialsNotSupported, [AppConnection.GCP]: platformManagedCredentialsNotSupported, [AppConnection.AzureKeyVault]: platformManagedCredentialsNotSupported, [AppConnection.AzureAppConfiguration]: platformManagedCredentialsNotSupported, diff --git a/backend/src/services/app-connection/app-connection-maps.ts b/backend/src/services/app-connection/app-connection-maps.ts index c32336453..a30b82345 100644 --- a/backend/src/services/app-connection/app-connection-maps.ts +++ b/backend/src/services/app-connection/app-connection-maps.ts @@ -3,6 +3,7 @@ import { AppConnection } from "./app-connection-enums"; export const APP_CONNECTION_NAME_MAP: Record = { [AppConnection.AWS]: "AWS", [AppConnection.GitHub]: "GitHub", + [AppConnection.GitHubRadar]: "GitHub Radar", [AppConnection.GCP]: "GCP", [AppConnection.AzureKeyVault]: "Azure Key Vault", [AppConnection.AzureAppConfiguration]: "Azure App Configuration", diff --git a/backend/src/services/app-connection/app-connection-service.ts b/backend/src/services/app-connection/app-connection-service.ts index 85b63138a..cbee29d78 100644 --- a/backend/src/services/app-connection/app-connection-service.ts +++ b/backend/src/services/app-connection/app-connection-service.ts @@ -15,6 +15,7 @@ import { validateAppConnectionCredentials } from "@app/services/app-connection/app-connection-fns"; import { auth0ConnectionService } from "@app/services/app-connection/auth0/auth0-connection-service"; +import { githubRadarConnectionService } from "@app/services/app-connection/github-radar/github-radar-connection-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { TAppConnectionDALFactory } from "./app-connection-dal"; @@ -43,6 +44,7 @@ import { ValidateGcpConnectionCredentialsSchema } from "./gcp"; import { gcpConnectionService } from "./gcp/gcp-connection-service"; import { ValidateGitHubConnectionCredentialsSchema } from "./github"; import { githubConnectionService } from "./github/github-connection-service"; +import { ValidateGitHubRadarConnectionCredentialsSchema } from "./github-radar"; import { ValidateHCVaultConnectionCredentialsSchema } from "./hc-vault"; import { hcVaultConnectionService } from "./hc-vault/hc-vault-connection-service"; import { ValidateHumanitecConnectionCredentialsSchema } from "./humanitec"; @@ -72,6 +74,7 @@ export type TAppConnectionServiceFactory = ReturnType = { [AppConnection.AWS]: ValidateAwsConnectionCredentialsSchema, [AppConnection.GitHub]: ValidateGitHubConnectionCredentialsSchema, + [AppConnection.GitHubRadar]: ValidateGitHubRadarConnectionCredentialsSchema, [AppConnection.GCP]: ValidateGcpConnectionCredentialsSchema, [AppConnection.AzureKeyVault]: ValidateAzureKeyVaultConnectionCredentialsSchema, [AppConnection.AzureAppConfiguration]: ValidateAzureAppConfigurationConnectionCredentialsSchema, @@ -456,6 +459,7 @@ export const appConnectionServiceFactory = ({ connectAppConnectionById, listAvailableAppConnectionsForUser, github: githubConnectionService(connectAppConnectionById), + githubRadar: githubRadarConnectionService(connectAppConnectionById), gcp: gcpConnectionService(connectAppConnectionById), databricks: databricksConnectionService(connectAppConnectionById, appConnectionDAL, kmsService), aws: awsConnectionService(connectAppConnectionById), diff --git a/backend/src/services/app-connection/app-connection-types.ts b/backend/src/services/app-connection/app-connection-types.ts index 42ccfc84e..c5c071e9a 100644 --- a/backend/src/services/app-connection/app-connection-types.ts +++ b/backend/src/services/app-connection/app-connection-types.ts @@ -57,6 +57,12 @@ import { TGitHubConnectionInput, TValidateGitHubConnectionCredentialsSchema } from "./github"; +import { + TGitHubRadarConnection, + TGitHubRadarConnectionConfig, + TGitHubRadarConnectionInput, + TValidateGitHubRadarConnectionCredentialsSchema +} from "./github-radar"; import { THCVaultConnection, THCVaultConnectionConfig, @@ -115,6 +121,7 @@ import { export type TAppConnection = { id: string } & ( | TAwsConnection | TGitHubConnection + | TGitHubRadarConnection | TGcpConnection | TAzureKeyVaultConnection | TAzureAppConfigurationConnection @@ -141,6 +148,7 @@ export type TSqlConnection = TPostgresConnection | TMsSqlConnection; export type TAppConnectionInput = { id: string } & ( | TAwsConnectionInput | TGitHubConnectionInput + | TGitHubRadarConnectionInput | TGcpConnectionInput | TAzureKeyVaultConnectionInput | TAzureAppConfigurationConnectionInput @@ -174,6 +182,7 @@ export type TUpdateAppConnectionDTO = Partial { + const { INF_APP_CONNECTION_GITHUB_RADAR_APP_SLUG } = getConfig(); + + return { + name: "GitHub Radar" as const, + app: AppConnection.GitHubRadar as const, + methods: Object.values(GitHubRadarConnectionMethod) as [GitHubRadarConnectionMethod.App], + appClientSlug: INF_APP_CONNECTION_GITHUB_RADAR_APP_SLUG + }; +}; + +export const getGitHubRadarClient = (appConnection: TGitHubRadarConnection) => { + const appCfg = getConfig(); + + const { method, credentials } = appConnection; + + let client: Octokit; + + switch (method) { + case GitHubRadarConnectionMethod.App: + if (!appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID || !appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY) { + throw new InternalServerError({ + message: `GitHub ${getAppConnectionMethodName(method).replace( + "GitHub", + "" + )} environment variables have not been configured` + }); + } + + client = new Octokit({ + authStrategy: createAppAuth, + auth: { + appId: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID, + privateKey: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY, + installationId: credentials.installationId + } + }); + break; + default: + throw new InternalServerError({ + message: `Unhandled GitHub Radar connection method: ${method as GitHubRadarConnectionMethod}` + }); + } + + return client; +}; + +export const listGitHubRadarRepositories = async (appConnection: TGitHubRadarConnection) => { + const client = getGitHubRadarClient(appConnection); + + const repositories: TGitHubRadarRepository[] = await client.paginate("GET /installation/repositories"); + + return repositories; +}; + +type TokenRespData = { + access_token: string; + scope: string; + token_type: string; + error?: string; +}; + +export const validateGitHubRadarConnectionCredentials = async (config: TGitHubRadarConnectionConfig) => { + const { credentials, method } = config; + + const { INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_ID, INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_SECRET, SITE_URL } = + getConfig(); + + if (!INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_ID || !INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_SECRET) { + throw new InternalServerError({ + message: `GitHub ${getAppConnectionMethodName(method).replace( + "GitHub", + "" + )} environment variables have not been configured` + }); + } + + let tokenResp: AxiosResponse; + + try { + tokenResp = await request.get("https://github.com/login/oauth/access_token", { + params: { + client_id: INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_ID, + client_secret: INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_SECRET, + code: credentials.code, + redirect_uri: `${SITE_URL}/organization/app-connections/github-radar/oauth/callback` + }, + headers: { + Accept: "application/json", + "Accept-Encoding": "application/json" + } + }); + } catch (e: unknown) { + throw new BadRequestError({ + message: `Unable to validate connection: verify credentials` + }); + } + + if (tokenResp.status !== 200) { + throw new BadRequestError({ + message: `Unable to validate credentials: GitHub responded with a status code of ${tokenResp.status} (${tokenResp.statusText}). Verify credentials and try again.` + }); + } + + if (method === GitHubRadarConnectionMethod.App) { + const installationsResp = await request.get<{ + installations: { + id: number; + account: { + login: string; + type: string; + id: number; + }; + }[]; + }>(IntegrationUrls.GITHUB_USER_INSTALLATIONS, { + headers: { + Accept: "application/json", + Authorization: `Bearer ${tokenResp.data.access_token}`, + "Accept-Encoding": "application/json" + } + }); + + const matchingInstallation = installationsResp.data.installations.find( + (installation) => installation.id === +credentials.installationId + ); + + if (!matchingInstallation) { + throw new ForbiddenRequestError({ + message: "User does not have access to the provided installation" + }); + } + } + + if (!tokenResp.data.access_token) { + throw new InternalServerError({ message: `Missing access token: ${tokenResp.data.error}` }); + } + + switch (method) { + case GitHubRadarConnectionMethod.App: + return { + installationId: credentials.installationId + }; + default: + throw new InternalServerError({ + message: `Unhandled GitHub connection method: ${method as GitHubRadarConnectionMethod}` + }); + } +}; diff --git a/backend/src/services/app-connection/github-radar/github-radar-connection-schemas.ts b/backend/src/services/app-connection/github-radar/github-radar-connection-schemas.ts new file mode 100644 index 000000000..ebdfa45e1 --- /dev/null +++ b/backend/src/services/app-connection/github-radar/github-radar-connection-schemas.ts @@ -0,0 +1,66 @@ +import { z } from "zod"; + +import { AppConnections } from "@app/lib/api-docs"; +import { AppConnection } from "@app/services/app-connection/app-connection-enums"; +import { + BaseAppConnectionSchema, + GenericCreateAppConnectionFieldsSchema, + GenericUpdateAppConnectionFieldsSchema +} from "@app/services/app-connection/app-connection-schemas"; + +import { GitHubRadarConnectionMethod } from "./github-radar-connection-enums"; + +export const GitHubRadarConnectionInputCredentialsSchema = z.object({ + code: z.string().trim().min(1, "GitHub Radar App code required"), + installationId: z.string().min(1, "GitHub Radar App Installation ID required") +}); + +export const GitHubRadarConnectionOutputCredentialsSchema = z.object({ + installationId: z.string() +}); + +export const ValidateGitHubRadarConnectionCredentialsSchema = z.discriminatedUnion("method", [ + z.object({ + method: z + .literal(GitHubRadarConnectionMethod.App) + .describe(AppConnections.CREATE(AppConnection.GitHubRadar).method), + credentials: GitHubRadarConnectionInputCredentialsSchema.describe( + AppConnections.CREATE(AppConnection.GitHubRadar).credentials + ) + }) +]); + +export const CreateGitHubRadarConnectionSchema = ValidateGitHubRadarConnectionCredentialsSchema.and( + GenericCreateAppConnectionFieldsSchema(AppConnection.GitHubRadar) +); + +export const UpdateGitHubRadarConnectionSchema = z + .object({ + credentials: GitHubRadarConnectionInputCredentialsSchema.optional().describe( + AppConnections.UPDATE(AppConnection.GitHubRadar).credentials + ) + }) + .and(GenericUpdateAppConnectionFieldsSchema(AppConnection.GitHubRadar)); + +const BaseGitHubRadarConnectionSchema = BaseAppConnectionSchema.extend({ app: z.literal(AppConnection.GitHubRadar) }); + +export const GitHubRadarConnectionSchema = BaseGitHubRadarConnectionSchema.extend({ + method: z.literal(GitHubRadarConnectionMethod.App), + credentials: GitHubRadarConnectionOutputCredentialsSchema +}); + +export const SanitizedGitHubRadarConnectionSchema = z.discriminatedUnion("method", [ + BaseGitHubRadarConnectionSchema.extend({ + method: z.literal(GitHubRadarConnectionMethod.App), + credentials: GitHubRadarConnectionOutputCredentialsSchema.pick({}) + }) +]); + +export const GitHubRadarConnectionListItemSchema = z.object({ + name: z.literal("GitHub Radar"), + app: z.literal(AppConnection.GitHubRadar), + // the below is preferable but currently breaks with our zod to json schema parser + // methods: z.tuple([z.literal(GitHubConnectionMethod.App), z.literal(GitHubConnectionMethod.OAuth)]), + methods: z.nativeEnum(GitHubRadarConnectionMethod).array(), + appClientSlug: z.string().optional() +}); diff --git a/backend/src/services/app-connection/github-radar/github-radar-connection-service.ts b/backend/src/services/app-connection/github-radar/github-radar-connection-service.ts new file mode 100644 index 000000000..583c43952 --- /dev/null +++ b/backend/src/services/app-connection/github-radar/github-radar-connection-service.ts @@ -0,0 +1,24 @@ +import { OrgServiceActor } from "@app/lib/types"; +import { AppConnection } from "@app/services/app-connection/app-connection-enums"; +import { listGitHubRadarRepositories } from "@app/services/app-connection/github-radar/github-radar-connection-fns"; +import { TGitHubRadarConnection } from "@app/services/app-connection/github-radar/github-radar-connection-types"; + +type TGetAppConnectionFunc = ( + app: AppConnection, + connectionId: string, + actor: OrgServiceActor +) => Promise; + +export const githubRadarConnectionService = (getAppConnection: TGetAppConnectionFunc) => { + const listRepositories = async (connectionId: string, actor: OrgServiceActor) => { + const appConnection = await getAppConnection(AppConnection.GitHubRadar, connectionId, actor); + + const repositories = await listGitHubRadarRepositories(appConnection); + + return repositories.map((repo) => ({ id: repo.id, name: repo.full_name })); + }; + + return { + listRepositories + }; +}; diff --git a/backend/src/services/app-connection/github-radar/github-radar-connection-types.ts b/backend/src/services/app-connection/github-radar/github-radar-connection-types.ts new file mode 100644 index 000000000..c9e7e5aa8 --- /dev/null +++ b/backend/src/services/app-connection/github-radar/github-radar-connection-types.ts @@ -0,0 +1,28 @@ +import { z } from "zod"; + +import { DiscriminativePick } from "@app/lib/types"; + +import { AppConnection } from "../app-connection-enums"; +import { + CreateGitHubRadarConnectionSchema, + GitHubRadarConnectionSchema, + ValidateGitHubRadarConnectionCredentialsSchema +} from "./github-radar-connection-schemas"; + +export type TGitHubRadarConnection = z.infer; + +export type TGitHubRadarConnectionInput = z.infer & { + app: AppConnection.GitHubRadar; +}; + +export type TValidateGitHubRadarConnectionCredentialsSchema = typeof ValidateGitHubRadarConnectionCredentialsSchema; + +export type TGitHubRadarConnectionConfig = DiscriminativePick< + TGitHubRadarConnectionInput, + "method" | "app" | "credentials" +>; + +export type TGitHubRadarRepository = { + id: number; + full_name: string; +}; diff --git a/backend/src/services/app-connection/github-radar/index.ts b/backend/src/services/app-connection/github-radar/index.ts new file mode 100644 index 000000000..3f2a5f663 --- /dev/null +++ b/backend/src/services/app-connection/github-radar/index.ts @@ -0,0 +1,4 @@ +export * from "./github-radar-connection-enums"; +export * from "./github-radar-connection-fns"; +export * from "./github-radar-connection-schemas"; +export * from "./github-radar-connection-types"; diff --git a/backend/src/services/certificate/certificate-service.ts b/backend/src/services/certificate/certificate-service.ts index 292b5f109..be3f8677e 100644 --- a/backend/src/services/certificate/certificate-service.ts +++ b/backend/src/services/certificate/certificate-service.ts @@ -8,6 +8,7 @@ import { ProjectPermissionCertificateActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { NotFoundError } from "@app/lib/errors"; import { TCertificateBodyDALFactory } from "@app/services/certificate/certificate-body-dal"; import { TCertificateDALFactory } from "@app/services/certificate/certificate-dal"; import { TCertificateAuthorityCertDALFactory } from "@app/services/certificate-authority/certificate-authority-cert-dal"; @@ -29,7 +30,6 @@ import { TGetCertPrivateKeyDTO, TRevokeCertDTO } from "./certificate-types"; -import { NotFoundError } from "@app/lib/errors"; type TCertificateServiceFactoryDep = { certificateDAL: Pick; diff --git a/docs/api-reference/endpoints/app-connections/github-radar/available.mdx b/docs/api-reference/endpoints/app-connections/github-radar/available.mdx new file mode 100644 index 000000000..6cfc0758c --- /dev/null +++ b/docs/api-reference/endpoints/app-connections/github-radar/available.mdx @@ -0,0 +1,4 @@ +--- +title: "Available" +openapi: "GET /api/v1/app-connections/github-radar/available" +--- diff --git a/docs/api-reference/endpoints/app-connections/github-radar/create.mdx b/docs/api-reference/endpoints/app-connections/github-radar/create.mdx new file mode 100644 index 000000000..0cd66a49b --- /dev/null +++ b/docs/api-reference/endpoints/app-connections/github-radar/create.mdx @@ -0,0 +1,10 @@ +--- +title: "Create" +openapi: "POST /api/v1/app-connections/github-radar" +--- + + + GitHub Radar Connections must be created through the Infisical UI. + Check out the configuration docs for [GitHub Radar Connections](/integrations/app-connections/github-radar) for a step-by-step + guide. + \ No newline at end of file diff --git a/docs/api-reference/endpoints/app-connections/github-radar/delete.mdx b/docs/api-reference/endpoints/app-connections/github-radar/delete.mdx new file mode 100644 index 000000000..64b252538 --- /dev/null +++ b/docs/api-reference/endpoints/app-connections/github-radar/delete.mdx @@ -0,0 +1,4 @@ +--- +title: "Delete" +openapi: "DELETE /api/v1/app-connections/github-radar/{connectionId}" +--- diff --git a/docs/api-reference/endpoints/app-connections/github-radar/get-by-id.mdx b/docs/api-reference/endpoints/app-connections/github-radar/get-by-id.mdx new file mode 100644 index 000000000..1ffc291c9 --- /dev/null +++ b/docs/api-reference/endpoints/app-connections/github-radar/get-by-id.mdx @@ -0,0 +1,4 @@ +--- +title: "Get by ID" +openapi: "GET /api/v1/app-connections/github-radar/{connectionId}" +--- diff --git a/docs/api-reference/endpoints/app-connections/github-radar/get-by-name.mdx b/docs/api-reference/endpoints/app-connections/github-radar/get-by-name.mdx new file mode 100644 index 000000000..a5ca5e3f5 --- /dev/null +++ b/docs/api-reference/endpoints/app-connections/github-radar/get-by-name.mdx @@ -0,0 +1,4 @@ +--- +title: "Get by Name" +openapi: "GET /api/v1/app-connections/github-radar/connection-name/{connectionName}" +--- diff --git a/docs/api-reference/endpoints/app-connections/github-radar/list.mdx b/docs/api-reference/endpoints/app-connections/github-radar/list.mdx new file mode 100644 index 000000000..2bd832941 --- /dev/null +++ b/docs/api-reference/endpoints/app-connections/github-radar/list.mdx @@ -0,0 +1,4 @@ +--- +title: "List" +openapi: "GET /api/v1/app-connections/github-radar" +--- diff --git a/docs/api-reference/endpoints/app-connections/github-radar/update.mdx b/docs/api-reference/endpoints/app-connections/github-radar/update.mdx new file mode 100644 index 000000000..4ffb88dc5 --- /dev/null +++ b/docs/api-reference/endpoints/app-connections/github-radar/update.mdx @@ -0,0 +1,10 @@ +--- +title: "Update" +openapi: "PATCH /api/v1/app-connections/github-radar/{connectionId}" +--- + + + GitHub Radar Connections must be updated through the Infisical UI. + Check out the configuration docs for [GitHub Radar Connections](/integrations/app-connections/github-radar) for a step-by-step + guide. + diff --git a/docs/api-reference/endpoints/secret-scanning/data-sources/github/create.mdx b/docs/api-reference/endpoints/secret-scanning/data-sources/github/create.mdx new file mode 100644 index 000000000..248c8cf53 --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/data-sources/github/create.mdx @@ -0,0 +1,4 @@ +--- +title: "Create" +openapi: "POST /api/v2/secret-scanning/data-sources/github" +--- diff --git a/docs/api-reference/endpoints/secret-scanning/data-sources/github/delete.mdx b/docs/api-reference/endpoints/secret-scanning/data-sources/github/delete.mdx new file mode 100644 index 000000000..eb791e6db --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/data-sources/github/delete.mdx @@ -0,0 +1,4 @@ +--- +title: "Delete" +openapi: "DELETE /api/v2/secret-scanning/data-sources/github/{dataSourceId}" +--- diff --git a/docs/api-reference/endpoints/secret-scanning/data-sources/github/get-by-id.mdx b/docs/api-reference/endpoints/secret-scanning/data-sources/github/get-by-id.mdx new file mode 100644 index 000000000..0103ad447 --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/data-sources/github/get-by-id.mdx @@ -0,0 +1,4 @@ +--- +title: "Get by ID" +openapi: "GET /api/v2/secret-scanning/data-sources/github/{dataSourceId}" +--- diff --git a/docs/api-reference/endpoints/secret-scanning/data-sources/github/get-by-name.mdx b/docs/api-reference/endpoints/secret-scanning/data-sources/github/get-by-name.mdx new file mode 100644 index 000000000..c86dcc948 --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/data-sources/github/get-by-name.mdx @@ -0,0 +1,4 @@ +--- +title: "Get by Name" +openapi: "GET /api/v2/secret-scanning/data-sources/github/data-source-name/{dataSourceName}" +--- diff --git a/docs/api-reference/endpoints/secret-scanning/data-sources/github/list-resources.mdx b/docs/api-reference/endpoints/secret-scanning/data-sources/github/list-resources.mdx new file mode 100644 index 000000000..f2627827f --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/data-sources/github/list-resources.mdx @@ -0,0 +1,4 @@ +--- +title: "List Resources" +openapi: "GET /api/v2/secret-scanning/data-sources/github/{dataSourceId}/resources" +--- diff --git a/docs/api-reference/endpoints/secret-scanning/data-sources/github/list-scans.mdx b/docs/api-reference/endpoints/secret-scanning/data-sources/github/list-scans.mdx new file mode 100644 index 000000000..839b66355 --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/data-sources/github/list-scans.mdx @@ -0,0 +1,4 @@ +--- +title: "List Scans" +openapi: "GET /api/v2/secret-scanning/data-sources/github/{dataSourceId}/scans" +--- diff --git a/docs/api-reference/endpoints/secret-scanning/data-sources/github/list.mdx b/docs/api-reference/endpoints/secret-scanning/data-sources/github/list.mdx new file mode 100644 index 000000000..951f827a8 --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/data-sources/github/list.mdx @@ -0,0 +1,4 @@ +--- +title: "List" +openapi: "GET /api/v2/secret-scanning/data-sources/github" +--- diff --git a/docs/api-reference/endpoints/secret-scanning/data-sources/github/scan-resource.mdx b/docs/api-reference/endpoints/secret-scanning/data-sources/github/scan-resource.mdx new file mode 100644 index 000000000..5025126fd --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/data-sources/github/scan-resource.mdx @@ -0,0 +1,4 @@ +--- +title: "Scan Resource" +openapi: "POST /api/v2/secret-scanning/data-sources/github/{dataSourceId}/resources/{resourceId}/scan" +--- diff --git a/docs/api-reference/endpoints/secret-scanning/data-sources/github/scan.mdx b/docs/api-reference/endpoints/secret-scanning/data-sources/github/scan.mdx new file mode 100644 index 000000000..f6b21b485 --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/data-sources/github/scan.mdx @@ -0,0 +1,4 @@ +--- +title: "Scan" +openapi: "POST /api/v2/secret-scanning/data-sources/github/{dataSourceId}/scan" +--- diff --git a/docs/api-reference/endpoints/secret-scanning/data-sources/github/update.mdx b/docs/api-reference/endpoints/secret-scanning/data-sources/github/update.mdx new file mode 100644 index 000000000..2d800d9d5 --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/data-sources/github/update.mdx @@ -0,0 +1,4 @@ +--- +title: "Update" +openapi: "PATCH /api/v2/secret-scanning/data-sources/github/{dataSourceId}" +--- diff --git a/docs/api-reference/endpoints/secret-scanning/data-sources/list.mdx b/docs/api-reference/endpoints/secret-scanning/data-sources/list.mdx new file mode 100644 index 000000000..0958dc382 --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/data-sources/list.mdx @@ -0,0 +1,4 @@ +--- +title: "List" +openapi: "GET /api/v2/secret-scanning/data-sources" +--- diff --git a/docs/api-reference/endpoints/secret-scanning/data-sources/options.mdx b/docs/api-reference/endpoints/secret-scanning/data-sources/options.mdx new file mode 100644 index 000000000..3affb5270 --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/data-sources/options.mdx @@ -0,0 +1,4 @@ +--- +title: "Options" +openapi: "GET /api/v2/secret-scanning/data-sources/options" +--- diff --git a/docs/api-reference/endpoints/secret-scanning/findings/list.mdx b/docs/api-reference/endpoints/secret-scanning/findings/list.mdx new file mode 100644 index 000000000..6a97a3a20 --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/findings/list.mdx @@ -0,0 +1,4 @@ +--- +title: "List" +openapi: "GET /api/v2/secret-scanning/findings" +--- diff --git a/docs/api-reference/endpoints/secret-scanning/findings/update.mdx b/docs/api-reference/endpoints/secret-scanning/findings/update.mdx new file mode 100644 index 000000000..1c9614f8b --- /dev/null +++ b/docs/api-reference/endpoints/secret-scanning/findings/update.mdx @@ -0,0 +1,4 @@ +--- +title: "Update" +openapi: "PATCH /api/v2/secret-scanning/findings/{findingId}" +--- \ No newline at end of file diff --git a/docs/documentation/platform/secret-scanning.mdx b/docs/documentation/platform/secret-scanning.mdx deleted file mode 100644 index da28bfa55..000000000 --- a/docs/documentation/platform/secret-scanning.mdx +++ /dev/null @@ -1,175 +0,0 @@ ---- -title: 'Secret Scanning' -description: "Scan and prevent secret leaks in your code repositories" ---- - -The Infisical Secret Scanner allows you to keep an overview and stay alert of exposed secrets across your entire GitHub organization and repositories. - -To further enhance security, we recommend you also use our [CLI Secret Scanner](/cli/scanning-overview#automatically-scan-changes-before-you-commit) to scan for exposed secrets prior to pushing your changes. - - - - - To setup secret scanning on your own instance of Infisical, you can follow the steps below. - - - - Create a new GitHub app in your GitHub organization or personal [Developer Settings](https://github.com/settings/apps). - - ![Create GitHub App](/images/platform/secret-scanning/github-create-app.png) - - ### Configure the GitHub App - To configure the GitHub app to work with Infisical, you'll need to modify the following settings: - - **Homepage URL**: Required to be set. Set it to the URL of your Infisical instance. (e.g. `https://app.infisical.com`) - - **Setup URL**: Set this to `https:///organization/secret-scanning` - - **Webhook URL**: Set this to `https:///api/v1/secret-scanning/webhook` - - **Webhook Secret**: Set this to a random string. This is used to verify the webhook request from Infisical. Use `openssl rand -base64 32` in your terminal to generate a random secret. - - - Remember to save the webhook secret as you will need it in the next step. - - - ![GitHub App Settings](/images/platform/secret-scanning/github-configure-app.png) - - ### Configure the GitHub App Permissions - The GitHub app needs the following permissions: - - Repository permissions: - - `Checks`: Read and Write - - `Contents`: Read-only - - `Issues`: Read and Write - - `Pull Requests`: Read and Write - - `Metadata`: Read-only (enabled by default) - - ![Github App Repository Permissions](/images/platform/secret-scanning/github-repo-permissions.png) - - Subscribed events: - - `Check run` - - `Pull request` - - `Push` - - ![Github App Subscribed Events](/images/platform/secret-scanning/github-subscribed-events.png) - - - ### Create the GitHub App - Now you can create the GitHub app by clicking on the "Create GitHub App" button. - - - If you want other Github users to be able to install the app, you need to tick the "Any account" option under "Where can this GitHub App be installed?" - - - ![Create GitHub App](/images/platform/secret-scanning/github-create-app-button.png) - - - - After clicking the "Create GitHub App" button, you will be redirected to the GitHub settings page. Here you can copy the "App ID" and save it for later when you need to configure your environment variables for your Infisical instance. - - ![Github App ID](/images/platform/secret-scanning/github-app-copy-app-id.png) - - - - The GitHub App slug is the name of the app you created in a slug friendly format. You can find the slug in the URL of the app you created. - - ![Github App Slug](/images/platform/secret-scanning/github-app-copy-slug.png) - - - - Create a new app private key by clicking on the "Generate a private key" button under the "Private keys" section. - - Once you click the "Generate a private key" button, the private key will be downloaded to your computer. Save this file for later as you will need the private key when configuring Infisical. - - ![Github App Private Key](/images/platform/secret-scanning/github-app-create-private-key.png) - - - Remember to save the private key as you will need it in the next step. - - - - - - - Now you can configure your Infisical instance by setting the following environment variables: - - - `SECRET_SCANNING_GIT_APP_ID`: The App ID of your GitHub App. - - `SECRET_SCANNING_GIT_APP_SLUG`: The slug of your GitHub App. - - `SECRET_SCANNING_PRIVATE_KEY`: The private key of your GitHub App that you created in a previous step. - - `SECRET_SCANNING_WEBHOOK_SECRET`: The webhook secret of your GitHub App that you created in a previous step. - - - - After restarting your Infisical instance, you should be able to use the secret scanning feature within your organization. Follow the steps below to add the GitHub App to your Infisical organization. - - -## Install the Infisical Radar GitHub App - -To install the GitHub App, press the "Integrate With GitHub" button in the top right corner of your Infisical Secret Scanning dashboard. - -![Integrate With GitHub](/images/platform/secret-scanning/infisical-connect-secret-scanner.png) - -Next, you'll be prompted to select which organization you'd like to install the app into. Select the organization you'd like to install the app into by clicking the organization in the menu. - -![Select Organization](/images/platform/secret-scanning/github-select-org-2.png) - -Select the repositories you'd like to scan for secrets and press the "Install" button. - -![Select Repositories](/images/platform/secret-scanning/github-select-repos.png) - -## Code Scanning - -![Scanning Overview](/images/platform/secret-scanning/overview.png) - -Secret scans are built on event-driven architecture. This means that every time a push is made to one of your selected repositories, Infisical will scan the modified files for any exposed secrets. - -If one or more exposed secrets are detected, it will be displayed in your Infisical dashboard. An exposed secret is known as a **"Risk"**. Each risk has the following data associated with it: -- **Date**: When the risk was first detected. -- **Secret Type**: Which type of secret was detected. -- **Info**: Information about the secret, such as the repository, file name, and the committer who made the change. - -Once an exposed secret is detected, all organization admins will be sent an e-mail notification containing details about the exposed secret. - - - Each risk also contains a "View Exposed Secret" button, which will take you directly to the GitHub commit and to the line where the secret was exposed. - - - - -![Exposed Secret](/images/platform/secret-scanning/exposed-secret.png) - - -## Responding to Exposed Secrets - -After an exposed secret is detected, it will be marked as `Needs Attention`. When there are risks marked as needs attention, it's important to address them as soon as possible. - -You can mark the risk as `Resolved` by changing the status to one of the following states: -- **This Is a False Positive**: The secret was not exposed, but was detected by the scanner. -- **I Have Rotated The Secret**: The secret was exposed, but it has now been removed. -- **No Rotation Needed**: You are choosing to ignore this risk. You may choose to do this if the risk is non-sensitive or otherwise not a security risk. - -![Needs Attention](/images/platform/secret-scanning/needs-attention.png) - - - - -## Ignoring Known Secrets -If you're intentionally committing a test secret that the secret scanner might flag, you can instruct Infisical to overlook that secret with the methods listed below. - -### infisical-scan:ignore - -To ignore a secret contained in line of code, simply add `infisical-scan:ignore ` at the end of the line as comment in the given programming. - -```js example.js -function helloWorld() { - console.log("8dyfuiRyq=vVc3RRr_edRk-fK__JItpZ"); // infisical-scan:ignore -} -``` - -### .infisicalignore -An alternative method to exclude specific findings involves creating a .infisicalignore file at your repository's root. -You can then add the fingerprints of the findings you wish to exclude. The [Infisical scan](/cli/scanning-overview) report provides a unique Fingerprint for each secret found. -By incorporating these Fingerprints into the .infisicalignore file, Infisical will skip the corresponding secret findings in subsequent scans. - -```.ignore .infisicalignore -bea0ff6e05a4de73a5db625d4ae181a015b50855:frontend/components/utilities/attemptLogin.js:stripe-access-token:147 -bea0ff6e05a4de73a5db625d4ae181a015b50855:backend/src/json/integrations.json:generic-api-key:5 -1961b92340e5d2613acae528b886c842427ce5d0:frontend/components/utilities/attemptLogin.js:stripe-access-token:148 -``` diff --git a/docs/documentation/platform/secret-scanning/overview.mdx b/docs/documentation/platform/secret-scanning/overview.mdx new file mode 100644 index 000000000..81d1b6171 --- /dev/null +++ b/docs/documentation/platform/secret-scanning/overview.mdx @@ -0,0 +1,117 @@ +--- +title: "Secret Scanning" +sidebarTitle: "Overview" +description: "Scan and prevent secret leaks in your code repositories" +--- + +## Introduction + +Monitor and detect exposed secrets across your data sources, including code repositories, with Infisical Secret Scanning. + +For additional security, we recommend using our [CLI Secret Scanner](/cli/scanning-overview#automatically-scan-changes-before-you-commit) to check for exposed secrets before pushing your code changes. + +## How Secret Scanning Works + +Secret Scanning consists of several components that enable you to quickly respond to secret leaks: + +- **Scanner Engine**: The core component that analyzes your code and detects potential secrets using pattern matching and entropy analysis +- **Real-time Monitoring**: Provides continuous surveillance of your repositories for immediate detection of exposed secrets +- **Alert System**: Notifies organization admins via email when secrets are detected +- **Risk Management**: Allows tracking and managing detected secrets with different status options +- **Data Sources**: Integrates with various data sources and version control systems +- **Customizable Rules**: Supports ignore patterns and custom configurations to reduce false positives + +These components work together to provide comprehensive secret detection and incident response capabilities. + +### Data Sources + +Data sources are configured integrations with external platforms, such as a GitHub organization or a GitLab group, that establish secure connections for scanning purposes using [App Connections](/integrations/app-connections/overview). + +A data source acts as a secure intermediary between the external system and the scanner engine. It manages a collection of scannable resources (such as repositories) and handles the authentication and communication required for scanning operations. + +[data source page image] + +### Resources + +Resources are the atomic, scannable units, such as a repository, that can be monitored for secret exposure. Resources are added automatically when a data source is scanned and updated when scanning events are triggered, such as when a user pushes changes to GitHub. + +Each resource maintains its own scanning history and status, allowing for granular monitoring and management of secret scanning across your organization. + +[resource table image] + +### Scans + +Scans can be initiated in two ways: + +1. **Full Scan** - Manually triggered scan that comprehensively checks either: + - All resources associated with a data source + - A single selected resource + +2. **Diff Scan** - Automatically executed when **Auto-Scan** is enabled on a data source. This scan type specifically focuses on updates to existing resources. + +All scan activities can be monitored in real-time through the Infisical UI, which displays: +- Current scan status +- Timestamp of the scan +- Resource(s) being scanned +- Detection results (whether any secrets were found) + +[scan table image] + +## [In Progress - old below] + +## Code Scanning + +![Scanning Overview](/images/platform/secret-scanning/overview.png) + +Secret scans are built on event-driven architecture. This means that every time a push is made to one of your selected repositories, Infisical will scan the modified files for any exposed secrets. + +If one or more exposed secrets are detected, it will be displayed in your Infisical dashboard. An exposed secret is known as a **"Risk"**. Each risk has the following data associated with it: +- **Date**: When the risk was first detected. +- **Secret Type**: Which type of secret was detected. +- **Info**: Information about the secret, such as the repository, file name, and the committer who made the change. + +Once an exposed secret is detected, all organization admins will be sent an e-mail notification containing details about the exposed secret. + + + Each risk also contains a "View Exposed Secret" button, which will take you directly to the GitHub commit and to the line where the secret was exposed. + + + + +![Exposed Secret](/images/platform/secret-scanning/exposed-secret.png) + + +## Responding to Exposed Secrets + +After an exposed secret is detected, it will be marked as `Needs Attention`. When there are risks marked as needs attention, it's important to address them as soon as possible. + +You can mark the risk as `Resolved` by changing the status to one of the following states: +- **This Is a False Positive**: The secret was not exposed, but was detected by the scanner. +- **I Have Rotated The Secret**: The secret was exposed, but it has now been removed. +- **No Rotation Needed**: You are choosing to ignore this risk. You may choose to do this if the risk is non-sensitive or otherwise not a security risk. + +![Needs Attention](/images/platform/secret-scanning/needs-attention.png) + +## Ignoring Known Secrets +If you're intentionally committing a test secret that the secret scanner might flag, you can instruct Infisical to overlook that secret with the methods listed below. + +### infisical-scan:ignore + +To ignore a secret contained in line of code, simply add `infisical-scan:ignore ` at the end of the line as comment in the given programming. + +```js example.js +function helloWorld() { + console.log("8dyfuiRyq=vVc3RRr_edRk-fK__JItpZ"); // infisical-scan:ignore +} +``` + +### .infisicalignore +An alternative method to exclude specific findings involves creating a .infisicalignore file at your repository's root. +You can then add the fingerprints of the findings you wish to exclude. The [Infisical scan](/cli/scanning-overview) report provides a unique Fingerprint for each secret found. +By incorporating these Fingerprints into the .infisicalignore file, Infisical will skip the corresponding secret findings in subsequent scans. + +```.ignore .infisicalignore +bea0ff6e05a4de73a5db625d4ae181a015b50855:frontend/components/utilities/attemptLogin.js:stripe-access-token:147 +bea0ff6e05a4de73a5db625d4ae181a015b50855:backend/src/json/integrations.json:generic-api-key:5 +1961b92340e5d2613acae528b886c842427ce5d0:frontend/components/utilities/attemptLogin.js:stripe-access-token:148 +``` diff --git a/docs/images/app-connections/github-radar/create-github-radar-app-method.png b/docs/images/app-connections/github-radar/create-github-radar-app-method.png new file mode 100644 index 000000000..2aa403dbc Binary files /dev/null and b/docs/images/app-connections/github-radar/create-github-radar-app-method.png differ diff --git a/docs/images/app-connections/github-radar/github-radar-app-created.png b/docs/images/app-connections/github-radar/github-radar-app-created.png new file mode 100644 index 000000000..d4b9a9374 Binary files /dev/null and b/docs/images/app-connections/github-radar/github-radar-app-created.png differ diff --git a/docs/images/app-connections/github-radar/github-radar-authorize.png b/docs/images/app-connections/github-radar/github-radar-authorize.png new file mode 100644 index 000000000..a113b761c Binary files /dev/null and b/docs/images/app-connections/github-radar/github-radar-authorize.png differ diff --git a/docs/images/app-connections/github-radar/select-github-radar-connection.png b/docs/images/app-connections/github-radar/select-github-radar-connection.png new file mode 100644 index 000000000..35a9e574d Binary files /dev/null and b/docs/images/app-connections/github-radar/select-github-radar-connection.png differ diff --git a/docs/images/app-connections/github-radar/self-hosted-github-radar-step-1.png b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-1.png new file mode 100644 index 000000000..4c55482e2 Binary files /dev/null and b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-1.png differ diff --git a/docs/images/app-connections/github-radar/self-hosted-github-radar-step-10.png b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-10.png new file mode 100644 index 000000000..50959a5c2 Binary files /dev/null and b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-10.png differ diff --git a/docs/images/app-connections/github-radar/self-hosted-github-radar-step-2.png b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-2.png new file mode 100644 index 000000000..5b9aa6ebc Binary files /dev/null and b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-2.png differ diff --git a/docs/images/app-connections/github-radar/self-hosted-github-radar-step-3.png b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-3.png new file mode 100644 index 000000000..b9fae23ed Binary files /dev/null and b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-3.png differ diff --git a/docs/images/app-connections/github-radar/self-hosted-github-radar-step-4.png b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-4.png new file mode 100644 index 000000000..6d4bdfc3b Binary files /dev/null and b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-4.png differ diff --git a/docs/images/app-connections/github-radar/self-hosted-github-radar-step-5.png b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-5.png new file mode 100644 index 000000000..2e37745c4 Binary files /dev/null and b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-5.png differ diff --git a/docs/images/app-connections/github-radar/self-hosted-github-radar-step-6.png b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-6.png new file mode 100644 index 000000000..4e4717cfe Binary files /dev/null and b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-6.png differ diff --git a/docs/images/app-connections/github-radar/self-hosted-github-radar-step-7.png b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-7.png new file mode 100644 index 000000000..414b0231b Binary files /dev/null and b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-7.png differ diff --git a/docs/images/app-connections/github-radar/self-hosted-github-radar-step-8.png b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-8.png new file mode 100644 index 000000000..012e09796 Binary files /dev/null and b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-8.png differ diff --git a/docs/images/app-connections/github-radar/self-hosted-github-radar-step-9.png b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-9.png new file mode 100644 index 000000000..d53849e6b Binary files /dev/null and b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-9.png differ diff --git a/docs/images/app-connections/gitlab/create-gitlab-access-token-connection.png b/docs/images/app-connections/gitlab/create-gitlab-access-token-connection.png new file mode 100644 index 000000000..379d6e0e1 Binary files /dev/null and b/docs/images/app-connections/gitlab/create-gitlab-access-token-connection.png differ diff --git a/docs/images/app-connections/gitlab/gitlab-access-token-connection-created.png b/docs/images/app-connections/gitlab/gitlab-access-token-connection-created.png new file mode 100644 index 000000000..85da8a929 Binary files /dev/null and b/docs/images/app-connections/gitlab/gitlab-access-token-connection-created.png differ diff --git a/docs/images/app-connections/gitlab/gitlab-add-access-token.png b/docs/images/app-connections/gitlab/gitlab-add-access-token.png new file mode 100644 index 000000000..b1307f774 Binary files /dev/null and b/docs/images/app-connections/gitlab/gitlab-add-access-token.png differ diff --git a/docs/images/app-connections/gitlab/gitlab-copy-token.png b/docs/images/app-connections/gitlab/gitlab-copy-token.png new file mode 100644 index 000000000..e425ac3ef Binary files /dev/null and b/docs/images/app-connections/gitlab/gitlab-copy-token.png differ diff --git a/docs/images/app-connections/gitlab/gitlab-secret-scanning-token.png b/docs/images/app-connections/gitlab/gitlab-secret-scanning-token.png new file mode 100644 index 000000000..69575afe7 Binary files /dev/null and b/docs/images/app-connections/gitlab/gitlab-secret-scanning-token.png differ diff --git a/docs/images/app-connections/gitlab/select-gitlab-connection.png b/docs/images/app-connections/gitlab/select-gitlab-connection.png new file mode 100644 index 000000000..0f559477d Binary files /dev/null and b/docs/images/app-connections/gitlab/select-gitlab-connection.png differ diff --git a/docs/integrations/app-connections/github-radar.mdx b/docs/integrations/app-connections/github-radar.mdx new file mode 100644 index 000000000..04c357499 --- /dev/null +++ b/docs/integrations/app-connections/github-radar.mdx @@ -0,0 +1,120 @@ +--- +title: "GitHub Radar Connection" +description: "Learn how to configure a GitHub Radar Connection for Infisical." +--- + +Infisical supports GitHub App installation for creating a GitHub Radar Connection. + + + Using a GitHub Radar Connection with app authentication on a self-hosted instance of Infisical requires configuring an application on GitHub + and registering your instance with it. + + + + Navigate to the GitHub App Settings [here](https://github.com/settings/apps). Click **New GitHub App**. + + + If you have a GitHub organization, you can create an application under it + in your organization Settings > Developer settings > GitHub Apps > New GitHub App. + + + ![create github radar app](/images/app-connections/github-radar/self-hosted-github-radar-step-1.png) + + Configure the following fields: + + 1. **Name** - give your app a name + 2. **Homepage URL** - your self-hosted domain (i.e. `https://your-domain.com`) + 3. **Callback URL** - the callback URL for your domain (i.e. `https://your-domain.com/organization/app-connections/github-radar/oauth/callback`) + 4. **User Authorization** - enable request user authorization on app installation + + ![github radar app details](/images/app-connections/github-radar/self-hosted-github-radar-step-2.png) + + Enable and configure the Webhook fields: + + - **Webhook URL** - the webhook URL for your domain (i.e. `https://your-domain.com/secret-scanning/webhooks/github`) + - **Webhook Secret** - a strong, generated secret to verify webhook payloads + - **SSL Verification** - enable SSL verification + + ![github radar app webhook](/images/app-connections/github-radar/self-hosted-github-radar-step-3.png) + + Set the following repository permissions: + 1. **Checks**: `Read and Write` + 2. **Content**: `Read-only` + 3. **Issues**: `Read and Write` + 4. **Metadata**: `Read-only` + 5. **Pull Requests**: `Read and Write` + + ![github radar app permissions 1](/images/app-connections/github-radar/self-hosted-github-radar-step-4.png) + ![github radar app permissions 2](/images/app-connections/github-radar/self-hosted-github-radar-step-5.png) + + Subscribe to the following events: + 1. **Check run** + 2. **Pull request** + 3. **Push** + + ![github radar app events](/images/app-connections/github-radar/self-hosted-github-radar-step-6.png) + + Create the Github application. + ![github radar app complete](/images/app-connections/github-radar/self-hosted-github-radar-step-7.png) + + + Generate a new **Client Secret** for your GitHub application. + ![github radar app client secret](/images/app-connections/github-radar/self-hosted-github-radar-step-8.png) + + Generate a new **Private Key** for your Github application. + + You will need to copy the contents of the .pem file downloaded + + ![github radar app private key](/images/app-connections/github-radar/self-hosted-github-radar-step-9.png) + + Obtain the following credentials: + + 1. **Slug** - the slug of your application found in the URL + 2. **App ID** - the ID of your application + 3. **Client ID** - the client ID of your application + 4. **Client Secret** - the client secret generated above + 5. **Private Key** - the contents of the private key .pem file generated above + 6. **Webhook Secret** - the secret generated in the previous step when configuring the webhook + + ![github radar app credentials](/images/app-connections/github-radar/self-hosted-github-radar-step-10.png) + + Back in your Infisical instance, add the six new environment variables for the credentials of your GitHub Radar application: + + - `INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_ID`: The **Client ID** of your GitHub application. + - `INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_SECRET`: The **Client Secret** of your GitHub application. + - `INF_APP_CONNECTION_GITHUB_RADAR_APP_SLUG`: The **Slug** of your GitHub application. This is the one found in the URL. + - `INF_APP_CONNECTION_GITHUB_RADAR_APP_ID`: The **App ID** of your GitHub application. + - `INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY`: The **Private Key** of your GitHub application. + - `INF_APP_CONNECTION_GITHUB_RADAR_APP_WEBHOOK_SECRET`: The **Webhook Secret** of your GitHub application. + + Once added, restart your Infisical instance and use the GitHub integration via app authentication. + + + + +## Setup GitHub Connection in Infisical + + + + Navigate to the **App Connections** tab on the **Organization Settings** page. + ![App Connections Tab](/images/app-connections/general/add-connection.png) + + + Select the **GitHub Connection** option from the connection options modal. + ![Select GitHub Radar Connection](/images/app-connections/github-radar/select-github-radar-connection.png) + + + Select the **GitHub App** method and click **Connect to GitHub**. + ![Connect via GitHub App](/images/app-connections/github-radar/create-github-radar-app-method.png) + + + You will then be redirected to the GitHub App installation page. + + Install and authorize the GitHub application. This will redirect you back to Infisical's App Connections page. + ![Install GitHub App](/images/app-connections/github-radar/github-radar-authorize.png) + + + Your **GitHub Radar Connection** is now available for use. + ![GitHub Radar Connection](/images/app-connections/github-radar/github-radar-app-created.png) + + \ No newline at end of file diff --git a/docs/integrations/app-connections/teamcity.mdx b/docs/integrations/app-connections/teamcity.mdx index 1ffafe637..889355954 100644 --- a/docs/integrations/app-connections/teamcity.mdx +++ b/docs/integrations/app-connections/teamcity.mdx @@ -3,7 +3,7 @@ title: "TeamCity Connection" description: "Learn how to configure a TeamCity Connection for Infisical." --- -Infisical supports connecting to TeamCity using an Access Token to securely sync your secrets to TeamCity. +Infisical supports connecting to TeamCity using Access Tokens. ## Setup TeamCity Connection in Infisical diff --git a/docs/integrations/app-connections/vercel.mdx b/docs/integrations/app-connections/vercel.mdx index 8ef4a5647..7ab7bea1b 100644 --- a/docs/integrations/app-connections/vercel.mdx +++ b/docs/integrations/app-connections/vercel.mdx @@ -3,7 +3,7 @@ title: "Vercel Connection" description: "Learn how to configure a Vercel Connection for Infisical." --- -Infisical supports connecting to Vercel using an API Token to securely sync your secrets to Vercel. +Infisical supports connecting to Vercel using API Tokens. ## Setup Vercel Connection in Infisical diff --git a/docs/integrations/app-connections/windmill.mdx b/docs/integrations/app-connections/windmill.mdx index ca4aa7da4..5cab9fa38 100644 --- a/docs/integrations/app-connections/windmill.mdx +++ b/docs/integrations/app-connections/windmill.mdx @@ -3,7 +3,7 @@ title: "Windmill Connection" description: "Learn how to configure a Windmill Connection for Infisical." --- -Infisical supports connecting to Windmill using an **Access Token** to securely sync your secrets to Windmill. +Infisical supports connecting to Windmill using Access Tokens. ## Get a Windmill Access Token diff --git a/docs/mint.json b/docs/mint.json index 5335c7423..8f488f0ec 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -242,7 +242,12 @@ ] }, "documentation/platform/secret-sharing", - "documentation/platform/secret-scanning" + { + "group": "Secret Scanning", + "pages": [ + "documentation/platform/secret-scanning/overview" + ] + } ] }, { @@ -478,6 +483,7 @@ "integrations/app-connections/databricks", "integrations/app-connections/gcp", "integrations/app-connections/github", + "integrations/app-connections/github-radar", "integrations/app-connections/hashicorp-vault", "integrations/app-connections/humanitec", "integrations/app-connections/ldap", @@ -1008,6 +1014,40 @@ } ] }, + { + "group": "Secret Scanning", + "pages": [ + { + "group": "Data Sources", + "pages": [ + "api-reference/endpoints/secret-scanning/data-sources/list", + "api-reference/endpoints/secret-scanning/data-sources/options", + { + "group": "GitHub", + "pages": [ + "api-reference/endpoints/secret-scanning/data-sources/github/list", + "api-reference/endpoints/secret-scanning/data-sources/github/get-by-id", + "api-reference/endpoints/secret-scanning/data-sources/github/get-by-name", + "api-reference/endpoints/secret-scanning/data-sources/github/list-resources", + "api-reference/endpoints/secret-scanning/data-sources/github/list-scans", + "api-reference/endpoints/secret-scanning/data-sources/github/create", + "api-reference/endpoints/secret-scanning/data-sources/github/update", + "api-reference/endpoints/secret-scanning/data-sources/github/delete", + "api-reference/endpoints/secret-scanning/data-sources/github/scan", + "api-reference/endpoints/secret-scanning/data-sources/github/scan-resource" + ] + } + ] + }, + { + "group": "Findings", + "pages": [ + "api-reference/endpoints/secret-scanning/findings/list", + "api-reference/endpoints/secret-scanning/findings/update" + ] + } + ] + }, { "group": "Identity Specific Privilege", "pages": [ @@ -1148,6 +1188,18 @@ "api-reference/endpoints/app-connections/github/delete" ] }, + { + "group": "GitHub Radar", + "pages": [ + "api-reference/endpoints/app-connections/github-radar/list", + "api-reference/endpoints/app-connections/github-radar/available", + "api-reference/endpoints/app-connections/github-radar/get-by-id", + "api-reference/endpoints/app-connections/github-radar/get-by-name", + "api-reference/endpoints/app-connections/github-radar/create", + "api-reference/endpoints/app-connections/github-radar/update", + "api-reference/endpoints/app-connections/github-radar/delete" + ] + }, { "group": "Hashicorp Vault", "pages": [ diff --git a/docs/self-hosting/configuration/envars.mdx b/docs/self-hosting/configuration/envars.mdx index b63c58d3a..e08f8239d 100644 --- a/docs/self-hosting/configuration/envars.mdx +++ b/docs/self-hosting/configuration/envars.mdx @@ -510,6 +510,32 @@ You can configure third-party app connections for re-use across Infisical Projec + + + The ID of the GitHub Radar App + + + + The slug of the GitHub Radar App + + + + The client ID for the GitHub Radar App + + + + The client secret for the GitHub Radar App + + + + The private key for the GitHub Radar App + + + + The webhook secret configured for payload verification in the GitHub Radar App + + + The OAuth2 client ID for GitHub OAuth Connection diff --git a/frontend/public/lotties/search.json b/frontend/public/lotties/search.json new file mode 100644 index 000000000..a650cc5a0 --- /dev/null +++ b/frontend/public/lotties/search.json @@ -0,0 +1 @@ +{"v":"5.12.1","fr":60,"ip":0,"op":60,"w":500,"h":500,"nm":"system-regular-42-search","ddd":0,"assets":[{"id":"comp_1","nm":"hover-search","fr":60,"layers":[{"ddd":0,"ind":1,"ty":4,"nm":".primary.design","cl":"primary design","parent":2,"sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[150.995,147.901,0],"ix":2,"l":2},"a":{"a":0,"k":[0,0,0],"ix":1,"l":2},"s":{"a":0,"k":[100,100,100],"ix":6,"l":2}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-41.707,-41.707],[41.707,41.707]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.91,0.91,0.914,1],"ix":3,"x":"var $bm_rt;\n$bm_rt = comp('system-regular-42-search').layer('control').effect('primary')('Color');"},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":41.73,"ix":5},"lc":2,"lj":2,"bm":0,"nm":".primary","mn":"ADBE Vector Graphic - Stroke","hd":false,"cl":"primary"},{"ty":"fl","c":{"a":0,"k":[0.91,0.91,0.914,1],"ix":4,"x":"var $bm_rt;\n$bm_rt = comp('system-regular-42-search').layer('control').effect('primary')('Color');"},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":".primary","mn":"ADBE Vector Graphic - Fill","hd":false,"cl":"primary"},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":1,"op":60,"st":0,"ct":1,"bm":0},{"ddd":0,"ind":2,"ty":4,"nm":".primary.design","cl":"primary design","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.218],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":1,"s":[0]},{"i":{"x":[0.35],"y":[1]},"o":{"x":[0.522],"y":[0]},"t":29,"s":[29]},{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":48,"s":[-4]},{"t":59,"s":[0]}],"ix":10},"p":{"a":1,"k":[{"i":{"x":0.218,"y":1},"o":{"x":0.333,"y":0},"t":1,"s":[223.962,223.958,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.35,"y":1},"o":{"x":0.522,"y":0},"t":29,"s":[310.962,188.958,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":48,"s":[203.962,233.958,0],"to":[0,0,0],"ti":[0,0,0]},{"t":59,"s":[223.962,223.958,0]}],"ix":2,"l":2},"a":{"a":0,"k":[0,0,0],"ix":1,"l":2},"s":{"a":1,"k":[{"i":{"x":[0.218,0.218,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":1,"s":[100,100,100]},{"i":{"x":[0.35,0.35,0.667],"y":[1,1,1]},"o":{"x":[0.522,0.522,0.333],"y":[0,0,0]},"t":29,"s":[100,100,100]},{"i":{"x":[0.667,0.667,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":48,"s":[100,100,100]},{"t":59,"s":[100,100,100]}],"ix":6,"l":2}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,-80.542],[80.542,0],[0,80.542],[-80.542,0]],"o":[[0,80.542],[-80.542,0],[0,-80.542],[80.542,0]],"v":[[145.834,0],[0,145.834],[-145.834,0],[0,-145.834]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.91,0.91,0.914,1],"ix":3,"x":"var $bm_rt;\n$bm_rt = comp('system-regular-42-search').layer('control').effect('primary')('Color');"},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":31.3,"ix":5},"lc":2,"lj":2,"bm":0,"nm":".primary","mn":"ADBE Vector Graphic - Stroke","hd":false,"cl":"primary"},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 1","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":1,"op":60,"st":0,"ct":1,"bm":0},{"ddd":0,"ind":3,"ty":4,"nm":".primary.design","cl":"primary design","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[250,250,0],"ix":2,"l":2},"a":{"a":0,"k":[0,0,0],"ix":1,"l":2},"s":{"a":0,"k":[2083,2083,100],"ix":6,"l":2}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,3.446],[-3.446,0],[0,-3.446],[3.446,0]],"o":[[0,-3.446],[3.446,0],[0,3.446],[-3.446,0]],"v":[[-7.5,-1.25],[-1.25,-7.5],[5,-1.25],[-1.25,5]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ind":1,"ty":"sh","ix":2,"ks":{"a":0,"k":{"i":[[0.391,0.391],[0,0],[0,1.739],[4.273,0],[0,-4.273],[-4.273,0],[-1.322,1.049],[0,0],[-0.256,0],[-0.195,0.195]],"o":[[0,0],[0.971,-1.294],[0,-4.273],[-4.273,0],[0,4.273],[1.815,0],[0,0],[0.195,0.195],[0.256,0],[0.391,-0.391]],"v":[[8.707,7.144],[4.947,3.385],[6.5,-1.25],[-1.25,-9],[-9,-1.25],[-1.25,6.5],[3.554,4.82],[7.293,8.558],[8,8.851],[8.707,8.558]],"c":true},"ix":2},"nm":"Path 2","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[0.91,0.91,0.914,1],"ix":4,"x":"var $bm_rt;\n$bm_rt = comp('system-regular-42-search').layer('control').effect('primary')('Color');"},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":".primary","mn":"ADBE Vector Graphic - Fill","hd":false,"cl":"primary"},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":60,"op":384,"st":60,"ct":1,"bm":0},{"ddd":0,"ind":4,"ty":4,"nm":".primary.design","cl":"primary design","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[250,250,0],"ix":2,"l":2},"a":{"a":0,"k":[0,0,0],"ix":1,"l":2},"s":{"a":0,"k":[2083,2083,100],"ix":6,"l":2}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,3.446],[-3.446,0],[0,-3.446],[3.446,0]],"o":[[0,-3.446],[3.446,0],[0,3.446],[-3.446,0]],"v":[[-7.5,-1.25],[-1.25,-7.5],[5,-1.25],[-1.25,5]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ind":1,"ty":"sh","ix":2,"ks":{"a":0,"k":{"i":[[0.391,0.391],[0,0],[0,1.739],[4.273,0],[0,-4.273],[-4.273,0],[-1.322,1.049],[0,0],[-0.256,0],[-0.195,0.195]],"o":[[0,0],[0.971,-1.294],[0,-4.273],[-4.273,0],[0,4.273],[1.815,0],[0,0],[0.195,0.195],[0.256,0],[0.391,-0.391]],"v":[[8.707,7.144],[4.947,3.385],[6.5,-1.25],[-1.25,-9],[-9,-1.25],[-1.25,6.5],[3.554,4.82],[7.293,8.558],[8,8.851],[8.707,8.558]],"c":true},"ix":2},"nm":"Path 2","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[0.91,0.91,0.914,1],"ix":4,"x":"var $bm_rt;\n$bm_rt = comp('system-regular-42-search').layer('control').effect('primary')('Color');"},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":".primary","mn":"ADBE Vector Graphic - Fill","hd":false,"cl":"primary"},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":1,"st":0,"ct":1,"bm":0}]}],"layers":[{"ddd":0,"ind":1,"ty":3,"nm":"control","sr":1,"ks":{"o":{"a":0,"k":0,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[0,0],"ix":2,"l":2},"a":{"a":0,"k":[0,0,0],"ix":1,"l":2},"s":{"a":0,"k":[100,100,100],"ix":6,"l":2}},"ao":0,"ef":[{"ty":5,"nm":"primary","np":3,"mn":"ADBE Color Control","ix":1,"en":1,"ef":[{"ty":2,"nm":"Color","mn":"ADBE Color Control-0001","ix":1,"v":{"a":0,"k":[0.91,0.91,0.914],"ix":1}}]}],"ip":0,"op":131,"st":0,"bm":0},{"ddd":0,"ind":3,"ty":0,"nm":"hover-search","refId":"comp_1","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[250,250,0],"ix":2,"l":2},"a":{"a":0,"k":[250,250,0],"ix":1,"l":2},"s":{"a":0,"k":[100,100,100],"ix":6,"l":2}},"ao":0,"w":500,"h":500,"ip":0,"op":70,"st":0,"bm":0}],"markers":[{"tm":0,"cm":"default:hover-search","dr":60}],"props":{}} \ No newline at end of file diff --git a/frontend/src/components/secret-scanning/CreateSecretScanningDataSourceModal.tsx b/frontend/src/components/secret-scanning/CreateSecretScanningDataSourceModal.tsx new file mode 100644 index 000000000..a3943cf35 --- /dev/null +++ b/frontend/src/components/secret-scanning/CreateSecretScanningDataSourceModal.tsx @@ -0,0 +1,97 @@ +import { useState } from "react"; +import { faArrowUpRightFromSquare, faBookOpen } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +import { Modal, ModalContent } from "@app/components/v2"; +import { + SecretScanningDataSource, + TSecretScanningDataSource +} from "@app/hooks/api/secretScanningV2"; + +import { SecretScanningDataSourceForm } from "./forms"; +import { SecretScanningDataSourceModalHeader } from "./SecretScanningDataSourceModalHeader"; +import { SecretScanningDataSourceSelect } from "./SecretScanningDataSourceSelect"; + +type Props = { + isOpen: boolean; + onOpenChange: (isOpen: boolean) => void; +}; + +type ContentProps = { + onComplete: (dataSource: TSecretScanningDataSource) => void; + selectedDataSource: SecretScanningDataSource | null; + setSelectedDataSource: (selectedDataSource: SecretScanningDataSource | null) => void; +}; + +const Content = ({ setSelectedDataSource, selectedDataSource, ...props }: ContentProps) => { + if (selectedDataSource) { + return ( + setSelectedDataSource(null)} + type={selectedDataSource} + {...props} + /> + ); + } + + return ; +}; + +export const CreateSecretScanningDataSourceModal = ({ onOpenChange, isOpen, ...props }: Props) => { + const [selectedDataSource, setSelectedDataSource] = useState( + null + ); + + return ( + { + if (!open) setSelectedDataSource(null); + onOpenChange(open); + }} + > + + ) : ( +

+ Add Data Source + +
+ + Docs + +
+ +
+ ) + } + onPointerDownOutside={(e) => e.preventDefault()} + className={selectedDataSource ? "max-w-2xl" : "max-w-3xl"} + subTitle={ + selectedDataSource ? undefined : "Select a data source to configure secret scanning for." + } + bodyClassName="overflow-visible" + > + { + setSelectedDataSource(null); + onOpenChange(false); + }} + selectedDataSource={selectedDataSource} + setSelectedDataSource={setSelectedDataSource} + {...props} + /> + + + ); +}; diff --git a/frontend/src/components/secret-scanning/DeleteSecretScanningDataSourceModal.tsx b/frontend/src/components/secret-scanning/DeleteSecretScanningDataSourceModal.tsx new file mode 100644 index 000000000..08b661b02 --- /dev/null +++ b/frontend/src/components/secret-scanning/DeleteSecretScanningDataSourceModal.tsx @@ -0,0 +1,66 @@ +import { createNotification } from "@app/components/notifications"; +import { DeleteActionModal } from "@app/components/v2"; +import { SECRET_SCANNING_DATA_SOURCE_MAP } from "@app/helpers/secretScanningV2"; +import { + TSecretScanningDataSource, + useDeleteSecretScanningDataSource +} from "@app/hooks/api/secretScanningV2"; + +type Props = { + dataSource?: TSecretScanningDataSource; + isOpen: boolean; + onOpenChange: (isOpen: boolean) => void; + onComplete?: () => void; +}; + +export const DeleteSecretScanningDataSourceModal = ({ + isOpen, + onOpenChange, + dataSource, + onComplete +}: Props) => { + const deleteSecretRotation = useDeleteSecretScanningDataSource(); + + if (!dataSource) return null; + + const { id: dataSourceId, name, type, projectId } = dataSource; + + const handleDeleteDataSource = async () => { + const dataSourceType = SECRET_SCANNING_DATA_SOURCE_MAP[type].name; + + try { + await deleteSecretRotation.mutateAsync({ + dataSourceId, + type, + projectId + }); + + createNotification({ + text: `Successfully deleted ${dataSourceType} Data Source`, + type: "success" + }); + + if (onComplete) onComplete(); + onOpenChange(false); + } catch { + createNotification({ + text: `Failed to delete ${dataSourceType} Data Source`, + type: "error" + }); + } + }; + + return ( + +

+ Findings associated with this data source will be preserved. +

+ + ); +}; diff --git a/frontend/src/components/secret-scanning/EditSecretScanningDataSourceModal.tsx b/frontend/src/components/secret-scanning/EditSecretScanningDataSourceModal.tsx new file mode 100644 index 000000000..318a96bed --- /dev/null +++ b/frontend/src/components/secret-scanning/EditSecretScanningDataSourceModal.tsx @@ -0,0 +1,36 @@ +import { Modal, ModalContent } from "@app/components/v2"; +import { TSecretScanningDataSource } from "@app/hooks/api/secretScanningV2"; + +import { SecretScanningDataSourceForm } from "./forms"; +import { SecretScanningDataSourceModalHeader } from "./SecretScanningDataSourceModalHeader"; + +type Props = { + isOpen: boolean; + onOpenChange: (isOpen: boolean) => void; + dataSource?: TSecretScanningDataSource; +}; + +export const EditSecretScanningDataSourceModal = ({ + dataSource, + onOpenChange, + ...props +}: Props) => { + if (!dataSource) return null; + + return ( + + } + className="max-w-2xl" + bodyClassName="overflow-visible" + > + onOpenChange(false)} + onCancel={() => onOpenChange(false)} + dataSource={dataSource} + type={dataSource.type} + /> + + + ); +}; diff --git a/frontend/src/components/secret-scanning/SecretScanningDataSourceModalHeader.tsx b/frontend/src/components/secret-scanning/SecretScanningDataSourceModalHeader.tsx new file mode 100644 index 000000000..fb1f4236b --- /dev/null +++ b/frontend/src/components/secret-scanning/SecretScanningDataSourceModalHeader.tsx @@ -0,0 +1,47 @@ +import { faArrowUpRightFromSquare, faBookOpen } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +import { SECRET_SCANNING_DATA_SOURCE_MAP } from "@app/helpers/secretScanningV2"; +import { SecretScanningDataSource } from "@app/hooks/api/secretScanningV2"; + +type Props = { + type: SecretScanningDataSource; + isConfigured: boolean; +}; + +export const SecretScanningDataSourceModalHeader = ({ type, isConfigured }: Props) => { + const dataSourceDetails = SECRET_SCANNING_DATA_SOURCE_MAP[type]; + + return ( +
+ {`${dataSourceDetails.name} +
+
+ {dataSourceDetails.name} Data Source + +
+ + Docs + +
+ +
+

+ {isConfigured ? "Edit" : "Connect a"} {dataSourceDetails.name} Data Source +

+
+
+ ); +}; diff --git a/frontend/src/components/secret-scanning/SecretScanningDataSourceSelect.tsx b/frontend/src/components/secret-scanning/SecretScanningDataSourceSelect.tsx new file mode 100644 index 000000000..c507a6ef3 --- /dev/null +++ b/frontend/src/components/secret-scanning/SecretScanningDataSourceSelect.tsx @@ -0,0 +1,91 @@ +import { faWrench } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +import { Spinner, Tooltip } from "@app/components/v2"; +import { SECRET_SCANNING_DATA_SOURCE_MAP } from "@app/helpers/secretScanningV2"; +import { + SecretScanningDataSource, + useSecretScanningDataSourceOptions +} from "@app/hooks/api/secretScanningV2"; + +type Props = { + onSelect: (type: SecretScanningDataSource) => void; +}; + +export const SecretScanningDataSourceSelect = ({ onSelect }: Props) => { + const { isPending, data: dataSourceOptions } = useSecretScanningDataSourceOptions(); + + if (isPending) { + return ( +
+ +

Loading options...

+
+ ); + } + + return ( +
+ {dataSourceOptions?.map(({ type }) => { + const { image, name, size } = SECRET_SCANNING_DATA_SOURCE_MAP[type]; + + return ( + + ); + })} + +

Infisical is constantly adding support for more services.

+

+ {`If you don't see the third-party + service you're looking for,`}{" "} + + let us know on Slack + {" "} + or{" "} + + make a request on GitHub + + . +

+ + } + > +
+ +
+ Coming Soon +
+
+ +
+ ); +}; diff --git a/frontend/src/components/secret-scanning/SecretScanningScanStatus.tsx b/frontend/src/components/secret-scanning/SecretScanningScanStatus.tsx new file mode 100644 index 000000000..c74ee477f --- /dev/null +++ b/frontend/src/components/secret-scanning/SecretScanningScanStatus.tsx @@ -0,0 +1,90 @@ +import { faArrowRotateForward, faCheck, faXmark } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { formatDistance } from "date-fns"; +import { twMerge } from "tailwind-merge"; + +import { Badge, Tooltip } from "@app/components/v2"; +import { SecretScanningScanStatus } from "@app/hooks/api/secretScanningV2"; + +type Props = { + status: SecretScanningScanStatus; + statusMessage?: string | null; + className?: string; + scannedAt?: string | null; +}; + +export const SecretScanningScanStatusBadge = ({ + status, + statusMessage, + className, + scannedAt +}: Props) => { + if (status === SecretScanningScanStatus.Failed) { + let errorMessage = statusMessage; + if (statusMessage) { + try { + errorMessage = JSON.stringify(JSON.parse(statusMessage), null, 2); + } catch { + errorMessage = statusMessage; + } + } + + return ( + +
+
+ +
Failure Reason
+
+
{errorMessage}
+ {scannedAt && ( +
+ Attempted {formatDistance(new Date(scannedAt), new Date(), { addSuffix: true })} +
+ )} +
+ + } + > +
+ + + Scan Error + +
+ + ); + } + + if (status === SecretScanningScanStatus.Queued || status === SecretScanningScanStatus.Scanning) { + return ( + + + Scanning + + ); + } + + return ( + + + Complete + + ); +}; diff --git a/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceConfigFields/GitHubDataSourceConfigFields.tsx b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceConfigFields/GitHubDataSourceConfigFields.tsx new file mode 100644 index 000000000..a208738d2 --- /dev/null +++ b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceConfigFields/GitHubDataSourceConfigFields.tsx @@ -0,0 +1,126 @@ +import { useEffect } from "react"; +import { Controller, useFormContext, useWatch } from "react-hook-form"; +import { MultiValue } from "react-select"; +import { faCircleInfo } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +import { FilterableSelect, FormControl, Select, SelectItem, Tooltip } from "@app/components/v2"; +import { + TGitHubRadarConnectionRepository, + useGitHubRadarConnectionListRepositories +} from "@app/hooks/api/appConnections/github-radar"; +import { SecretScanningDataSource } from "@app/hooks/api/secretScanningV2"; + +import { TSecretScanningDataSourceForm } from "../schemas"; +import { SecretScanningDataSourceConnectionField } from "../SecretScanningDataSourceConnectionField"; + +enum ScanMethod { + AllRepositories = "all-repositories", + SelectRepositories = "select-repositories" +} + +export const GitHubDataSourceConfigFields = () => { + const { control, watch, setValue } = useFormContext< + TSecretScanningDataSourceForm & { + type: SecretScanningDataSource.GitHub; + } + >(); + + const connectionId = useWatch({ control, name: "connection.id" }); + const isUpdate = Boolean(watch("id")); + + const { data: repositories, isPending: areRepositoriesLoading } = + useGitHubRadarConnectionListRepositories(connectionId, { enabled: Boolean(connectionId) }); + + const includeRepos = watch("config.includeRepos"); + + const scanMethod = + !includeRepos || includeRepos[0] === "*" + ? ScanMethod.AllRepositories + : ScanMethod.SelectRepositories; + + useEffect(() => { + if (!includeRepos) { + setValue("config.includeRepos", ["*"]); + } + }, [includeRepos]); + + return ( + <> + { + if (scanMethod === ScanMethod.SelectRepositories) { + setValue("config.includeRepos", []); + } + }} + /> + + + + {scanMethod === ScanMethod.SelectRepositories && ( + ( + Ensure that your connection has the correct permissions.} + > +
+ Don't see the repository you're looking for?{" "} + +
+ + } + > + value.includes(project.name))} + onChange={(newValue) => { + onChange( + newValue + ? (newValue as MultiValue).map( + (p) => p.name + ) + : null + ); + }} + options={repositories} + placeholder="Select repositories..." + getOptionLabel={(option) => option.name} + getOptionValue={(option) => option.name} + /> + + )} + /> + )} + + ); +}; diff --git a/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceConfigFields/SecretScanningDataSourceConfigFields.tsx b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceConfigFields/SecretScanningDataSourceConfigFields.tsx new file mode 100644 index 000000000..bebcf28fc --- /dev/null +++ b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceConfigFields/SecretScanningDataSourceConfigFields.tsx @@ -0,0 +1,55 @@ +import { Controller, useFormContext } from "react-hook-form"; + +import { FormControl, Switch } from "@app/components/v2"; +import { RESOURCE_DESCRIPTION_HELPER } from "@app/helpers/secretScanningV2"; +import { SecretScanningDataSource } from "@app/hooks/api/secretScanningV2"; + +import { TSecretScanningDataSourceForm } from "../schemas"; +import { GitHubDataSourceConfigFields } from "./GitHubDataSourceConfigFields"; + +const COMPONENT_MAP: Record = { + [SecretScanningDataSource.GitHub]: GitHubDataSourceConfigFields +}; + +export const SecretScanningDataSourceConfigFields = () => { + const { watch, control } = useFormContext(); + + const type = watch("type"); + + const Component = COMPONENT_MAP[type]; + const autoScanDescription = RESOURCE_DESCRIPTION_HELPER[type]; + + return ( + <> +

Connect and configure your Data Source.

+ + { + return ( + + +

Auto-Scan {value ? "Enabled" : "Disabled"}

+ + + ); + }} + /> + + ); +}; diff --git a/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceConfigFields/index.ts b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceConfigFields/index.ts new file mode 100644 index 000000000..31d24284f --- /dev/null +++ b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceConfigFields/index.ts @@ -0,0 +1 @@ +export * from "./SecretScanningDataSourceConfigFields"; diff --git a/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceConnectionField.tsx b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceConnectionField.tsx new file mode 100644 index 000000000..4484cda61 --- /dev/null +++ b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceConnectionField.tsx @@ -0,0 +1,105 @@ +import { Controller, useFormContext } from "react-hook-form"; +import { faInfoCircle } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { Link } from "@tanstack/react-router"; + +import { FilterableSelect, FormControl } from "@app/components/v2"; +import { OrgPermissionSubjects, useOrgPermission } from "@app/context"; +import { OrgPermissionAppConnectionActions } from "@app/context/OrgPermissionContext/types"; +import { APP_CONNECTION_MAP } from "@app/helpers/appConnections"; +import { SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP } from "@app/helpers/secretScanningV2"; +import { useListAvailableAppConnections } from "@app/hooks/api/appConnections"; + +import { TSecretScanningDataSourceForm } from "./schemas"; + +type Props = { + onChange?: VoidFunction; + isUpdate?: boolean; +}; + +export const SecretScanningDataSourceConnectionField = ({ + onChange: callback, + isUpdate +}: Props) => { + const { permission } = useOrgPermission(); + const { control, watch } = useFormContext(); + + const dataSourceType = watch("type"); + const app = SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[dataSourceType]; + + const { data: availableConnections, isPending } = useListAvailableAppConnections(app); + + const connectionName = APP_CONNECTION_MAP[app].name; + + const canCreateConnection = permission.can( + OrgPermissionAppConnectionActions.Create, + OrgPermissionSubjects.AppConnections + ); + + const appName = APP_CONNECTION_MAP[app].name; + + return ( + <> + ( + + Check out{" "} + + our docs + {" "} + to ensure your connection has the required permissions for secret scanning. +

+ ) + } + > + { + onChange(newValue); + if (callback) callback(); + }} + isLoading={isPending} + options={availableConnections} + isDisabled={isUpdate} + placeholder="Select connection..." + getOptionLabel={(option) => option.name} + getOptionValue={(option) => option.id} + /> + + )} + control={control} + name="connection" + /> + {!isUpdate && availableConnections?.length === 0 && ( +

+ + {canCreateConnection ? ( + <> + You do not have access to any {appName} Connections. Create one from the{" "} + + App Connections + {" "} + page. + + ) : ( + `You do not have access to any ${appName} Connections. Contact an admin to create one.` + )} +

+ )} + + ); +}; diff --git a/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceDetailsFields.tsx b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceDetailsFields.tsx new file mode 100644 index 000000000..7c5565421 --- /dev/null +++ b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceDetailsFields.tsx @@ -0,0 +1,51 @@ +import { Controller, useFormContext } from "react-hook-form"; + +import { FormControl, Input, TextArea } from "@app/components/v2"; + +import { TSecretScanningDataSourceForm } from "./schemas"; + +export const SecretScanningDataSourceDetailsFields = () => { + const { control } = useFormContext(); + + return ( + <> +

+ Provide a name and description for this Data Source. +

+ ( + + + + )} + control={control} + name="name" + /> + ( + +