mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-10-16 15:31:19 +00:00
Merge remote-tracking branch 'origin/main' into feat/dynamicSecretIdentityName
This commit is contained in:
@@ -10,6 +10,18 @@ import {
|
||||
TSecretRotationV2Raw,
|
||||
TUpdateSecretRotationV2DTO
|
||||
} from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-types";
|
||||
import {
|
||||
SecretScanningDataSource,
|
||||
SecretScanningScanStatus,
|
||||
SecretScanningScanType
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import {
|
||||
TCreateSecretScanningDataSourceDTO,
|
||||
TDeleteSecretScanningDataSourceDTO,
|
||||
TTriggerSecretScanningDataSourceDTO,
|
||||
TUpdateSecretScanningDataSourceDTO,
|
||||
TUpdateSecretScanningFindingDTO
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
|
||||
import { SshCaStatus, SshCertType } from "@app/ee/services/ssh/ssh-certificate-authority-types";
|
||||
import { SshCertKeyAlgorithm } from "@app/ee/services/ssh-certificate/ssh-certificate-types";
|
||||
import { SshCertTemplateStatus } from "@app/ee/services/ssh-certificate-template/ssh-certificate-template-types";
|
||||
@@ -381,6 +393,20 @@ export enum EventType {
|
||||
PROJECT_ASSUME_PRIVILEGE_SESSION_START = "project-assume-privileges-session-start",
|
||||
PROJECT_ASSUME_PRIVILEGE_SESSION_END = "project-assume-privileges-session-end",
|
||||
|
||||
SECRET_SCANNING_DATA_SOURCE_LIST = "secret-scanning-data-source-list",
|
||||
SECRET_SCANNING_DATA_SOURCE_CREATE = "secret-scanning-data-source-create",
|
||||
SECRET_SCANNING_DATA_SOURCE_UPDATE = "secret-scanning-data-source-update",
|
||||
SECRET_SCANNING_DATA_SOURCE_DELETE = "secret-scanning-data-source-delete",
|
||||
SECRET_SCANNING_DATA_SOURCE_GET = "secret-scanning-data-source-get",
|
||||
SECRET_SCANNING_DATA_SOURCE_TRIGGER_SCAN = "secret-scanning-data-source-trigger-scan",
|
||||
SECRET_SCANNING_DATA_SOURCE_SCAN = "secret-scanning-data-source-scan",
|
||||
SECRET_SCANNING_RESOURCE_LIST = "secret-scanning-resource-list",
|
||||
SECRET_SCANNING_SCAN_LIST = "secret-scanning-scan-list",
|
||||
SECRET_SCANNING_FINDING_LIST = "secret-scanning-finding-list",
|
||||
SECRET_SCANNING_FINDING_UPDATE = "secret-scanning-finding-update",
|
||||
SECRET_SCANNING_CONFIG_GET = "secret-scanning-config-get",
|
||||
SECRET_SCANNING_CONFIG_UPDATE = "secret-scanning-config-update",
|
||||
|
||||
UPDATE_ORG = "update-org",
|
||||
|
||||
CREATE_PROJECT = "create-project",
|
||||
@@ -2953,6 +2979,101 @@ interface MicrosoftTeamsWorkflowIntegrationUpdateEvent {
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningDataSourceListEvent {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST;
|
||||
metadata: {
|
||||
type?: SecretScanningDataSource;
|
||||
count: number;
|
||||
dataSourceIds: string[];
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningDataSourceGetEvent {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_GET;
|
||||
metadata: {
|
||||
type: SecretScanningDataSource;
|
||||
dataSourceId: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningDataSourceCreateEvent {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_CREATE;
|
||||
metadata: Omit<TCreateSecretScanningDataSourceDTO, "projectId"> & { dataSourceId: string };
|
||||
}
|
||||
|
||||
interface SecretScanningDataSourceUpdateEvent {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_UPDATE;
|
||||
metadata: TUpdateSecretScanningDataSourceDTO;
|
||||
}
|
||||
|
||||
interface SecretScanningDataSourceDeleteEvent {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_DELETE;
|
||||
metadata: TDeleteSecretScanningDataSourceDTO;
|
||||
}
|
||||
|
||||
interface SecretScanningDataSourceTriggerScanEvent {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_TRIGGER_SCAN;
|
||||
metadata: TTriggerSecretScanningDataSourceDTO;
|
||||
}
|
||||
|
||||
interface SecretScanningDataSourceScanEvent {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN;
|
||||
metadata: {
|
||||
scanId: string;
|
||||
resourceId: string;
|
||||
resourceType: string;
|
||||
dataSourceId: string;
|
||||
dataSourceType: string;
|
||||
scanStatus: SecretScanningScanStatus;
|
||||
scanType: SecretScanningScanType;
|
||||
numberOfSecretsDetected?: number;
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningResourceListEvent {
|
||||
type: EventType.SECRET_SCANNING_RESOURCE_LIST;
|
||||
metadata: {
|
||||
type: SecretScanningDataSource;
|
||||
dataSourceId: string;
|
||||
resourceIds: string[];
|
||||
count: number;
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningScanListEvent {
|
||||
type: EventType.SECRET_SCANNING_SCAN_LIST;
|
||||
metadata: {
|
||||
type: SecretScanningDataSource;
|
||||
dataSourceId: string;
|
||||
count: number;
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningFindingListEvent {
|
||||
type: EventType.SECRET_SCANNING_FINDING_LIST;
|
||||
metadata: {
|
||||
findingIds: string[];
|
||||
count: number;
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningFindingUpdateEvent {
|
||||
type: EventType.SECRET_SCANNING_FINDING_UPDATE;
|
||||
metadata: TUpdateSecretScanningFindingDTO;
|
||||
}
|
||||
|
||||
interface SecretScanningConfigUpdateEvent {
|
||||
type: EventType.SECRET_SCANNING_CONFIG_UPDATE;
|
||||
metadata: {
|
||||
content: string | null;
|
||||
};
|
||||
}
|
||||
|
||||
interface SecretScanningConfigReadEvent {
|
||||
type: EventType.SECRET_SCANNING_CONFIG_GET;
|
||||
metadata?: Record<string, never>; // not needed, based off projectId
|
||||
}
|
||||
|
||||
interface OrgUpdateEvent {
|
||||
type: EventType.UPDATE_ORG;
|
||||
metadata: {
|
||||
@@ -3276,6 +3397,19 @@ export type Event =
|
||||
| MicrosoftTeamsWorkflowIntegrationGetEvent
|
||||
| MicrosoftTeamsWorkflowIntegrationListEvent
|
||||
| MicrosoftTeamsWorkflowIntegrationUpdateEvent
|
||||
| SecretScanningDataSourceListEvent
|
||||
| SecretScanningDataSourceGetEvent
|
||||
| SecretScanningDataSourceCreateEvent
|
||||
| SecretScanningDataSourceUpdateEvent
|
||||
| SecretScanningDataSourceDeleteEvent
|
||||
| SecretScanningDataSourceTriggerScanEvent
|
||||
| SecretScanningDataSourceScanEvent
|
||||
| SecretScanningResourceListEvent
|
||||
| SecretScanningScanListEvent
|
||||
| SecretScanningFindingListEvent
|
||||
| SecretScanningFindingUpdateEvent
|
||||
| SecretScanningConfigUpdateEvent
|
||||
| SecretScanningConfigReadEvent
|
||||
| OrgUpdateEvent
|
||||
| ProjectCreateEvent
|
||||
| ProjectUpdateEvent
|
||||
|
||||
@@ -99,7 +99,9 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
|
||||
secretManagerDecryptor({ cipherTextBlob: dynamicSecretCfg.encryptedInput }).toString()
|
||||
) as object;
|
||||
|
||||
await selectedProvider.revoke(decryptedStoredInput, dynamicSecretLease.externalEntityId);
|
||||
await selectedProvider.revoke(decryptedStoredInput, dynamicSecretLease.externalEntityId, {
|
||||
projectId: folder.projectId
|
||||
});
|
||||
await dynamicSecretLeaseDAL.deleteById(dynamicSecretLease.id);
|
||||
return;
|
||||
}
|
||||
@@ -133,7 +135,9 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
|
||||
await Promise.all(dynamicSecretLeases.map(({ id }) => unsetLeaseRevocation(id)));
|
||||
await Promise.all(
|
||||
dynamicSecretLeases.map(({ externalEntityId }) =>
|
||||
selectedProvider.revoke(decryptedStoredInput, externalEntityId)
|
||||
selectedProvider.revoke(decryptedStoredInput, externalEntityId, {
|
||||
projectId: folder.projectId
|
||||
})
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -162,7 +162,8 @@ export const dynamicSecretLeaseServiceFactory = ({
|
||||
inputs: decryptedStoredInput,
|
||||
expireAt: expireAt.getTime(),
|
||||
usernameTemplate: dynamicSecretCfg.usernameTemplate,
|
||||
identity
|
||||
identity,
|
||||
metadata: { projectId }
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
if (error && typeof error === "object" && error !== null && "sqlMessage" in error) {
|
||||
@@ -264,7 +265,8 @@ export const dynamicSecretLeaseServiceFactory = ({
|
||||
const { entityId } = await selectedProvider.renew(
|
||||
decryptedStoredInput,
|
||||
dynamicSecretLease.externalEntityId,
|
||||
expireAt.getTime()
|
||||
expireAt.getTime(),
|
||||
{ projectId }
|
||||
);
|
||||
|
||||
await dynamicSecretQueueService.unsetLeaseRevocation(dynamicSecretLease.id);
|
||||
@@ -340,7 +342,7 @@ export const dynamicSecretLeaseServiceFactory = ({
|
||||
) as object;
|
||||
|
||||
const revokeResponse = await selectedProvider
|
||||
.revoke(decryptedStoredInput, dynamicSecretLease.externalEntityId)
|
||||
.revoke(decryptedStoredInput, dynamicSecretLease.externalEntityId, { projectId })
|
||||
.catch(async (err) => {
|
||||
// only propogate this error if forced is false
|
||||
if (!isForced) return { error: err as Error };
|
||||
|
||||
@@ -116,7 +116,7 @@ export const dynamicSecretServiceFactory = ({
|
||||
throw new BadRequestError({ message: "Provided dynamic secret already exist under the folder" });
|
||||
|
||||
const selectedProvider = dynamicSecretProviders[provider.type];
|
||||
const inputs = await selectedProvider.validateProviderInputs(provider.inputs);
|
||||
const inputs = await selectedProvider.validateProviderInputs(provider.inputs, { projectId });
|
||||
|
||||
let selectedGatewayId: string | null = null;
|
||||
if (inputs && typeof inputs === "object" && "gatewayId" in inputs && inputs.gatewayId) {
|
||||
@@ -146,7 +146,7 @@ export const dynamicSecretServiceFactory = ({
|
||||
selectedGatewayId = gateway.id;
|
||||
}
|
||||
|
||||
const isConnected = await selectedProvider.validateConnection(provider.inputs);
|
||||
const isConnected = await selectedProvider.validateConnection(provider.inputs, { projectId });
|
||||
if (!isConnected) throw new BadRequestError({ message: "Provider connection failed" });
|
||||
|
||||
const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({
|
||||
@@ -272,7 +272,7 @@ export const dynamicSecretServiceFactory = ({
|
||||
secretManagerDecryptor({ cipherTextBlob: dynamicSecretCfg.encryptedInput }).toString()
|
||||
) as object;
|
||||
const newInput = { ...decryptedStoredInput, ...(inputs || {}) };
|
||||
const updatedInput = await selectedProvider.validateProviderInputs(newInput);
|
||||
const updatedInput = await selectedProvider.validateProviderInputs(newInput, { projectId });
|
||||
|
||||
let selectedGatewayId: string | null = null;
|
||||
if (updatedInput && typeof updatedInput === "object" && "gatewayId" in updatedInput && updatedInput?.gatewayId) {
|
||||
@@ -301,7 +301,7 @@ export const dynamicSecretServiceFactory = ({
|
||||
selectedGatewayId = gateway.id;
|
||||
}
|
||||
|
||||
const isConnected = await selectedProvider.validateConnection(newInput);
|
||||
const isConnected = await selectedProvider.validateConnection(newInput, { projectId });
|
||||
if (!isConnected) throw new BadRequestError({ message: "Provider connection failed" });
|
||||
|
||||
const updatedDynamicCfg = await dynamicSecretDAL.transaction(async (tx) => {
|
||||
@@ -472,7 +472,9 @@ export const dynamicSecretServiceFactory = ({
|
||||
secretManagerDecryptor({ cipherTextBlob: dynamicSecretCfg.encryptedInput }).toString()
|
||||
) as object;
|
||||
const selectedProvider = dynamicSecretProviders[dynamicSecretCfg.type as DynamicSecretProviders];
|
||||
const providerInputs = (await selectedProvider.validateProviderInputs(decryptedStoredInput)) as object;
|
||||
const providerInputs = (await selectedProvider.validateProviderInputs(decryptedStoredInput, {
|
||||
projectId
|
||||
})) as object;
|
||||
|
||||
return { ...dynamicSecretCfg, inputs: providerInputs };
|
||||
};
|
||||
|
||||
@@ -16,13 +16,18 @@ import {
|
||||
PutUserPolicyCommand,
|
||||
RemoveUserFromGroupCommand
|
||||
} from "@aws-sdk/client-iam";
|
||||
import { AssumeRoleCommand, STSClient } from "@aws-sdk/client-sts";
|
||||
import { randomUUID } from "crypto";
|
||||
import handlebars from "handlebars";
|
||||
import { z } from "zod";
|
||||
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { BadRequestError } from "@app/lib/errors";
|
||||
import { alphaNumericNanoId } from "@app/lib/nanoid";
|
||||
|
||||
import { DynamicSecretAwsIamSchema, TDynamicProviderFns } from "./models";
|
||||
import { compileUsernameTemplate } from "./templateUtils";
|
||||
import { AwsIamAuthType, DynamicSecretAwsIamSchema, TDynamicProviderFns } from "./models";
|
||||
|
||||
const generateUsername = (usernameTemplate?: string | null, identity?: { name: string }) => {
|
||||
const randomUsername = alphaNumericNanoId(32);
|
||||
@@ -41,7 +46,43 @@ export const AwsIamProvider = (): TDynamicProviderFns => {
|
||||
return providerInputs;
|
||||
};
|
||||
|
||||
const $getClient = async (providerInputs: z.infer<typeof DynamicSecretAwsIamSchema>) => {
|
||||
const $getClient = async (providerInputs: z.infer<typeof DynamicSecretAwsIamSchema>, projectId: string) => {
|
||||
const appCfg = getConfig();
|
||||
if (providerInputs.method === AwsIamAuthType.AssumeRole) {
|
||||
const stsClient = new STSClient({
|
||||
region: providerInputs.region,
|
||||
credentials:
|
||||
appCfg.DYNAMIC_SECRET_AWS_ACCESS_KEY_ID && appCfg.DYNAMIC_SECRET_AWS_SECRET_ACCESS_KEY
|
||||
? {
|
||||
accessKeyId: appCfg.DYNAMIC_SECRET_AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: appCfg.DYNAMIC_SECRET_AWS_SECRET_ACCESS_KEY
|
||||
}
|
||||
: undefined // if hosting on AWS
|
||||
});
|
||||
|
||||
const command = new AssumeRoleCommand({
|
||||
RoleArn: providerInputs.roleArn,
|
||||
RoleSessionName: `infisical-dynamic-secret-${randomUUID()}`,
|
||||
DurationSeconds: 900, // 15 mins
|
||||
ExternalId: projectId
|
||||
});
|
||||
|
||||
const assumeRes = await stsClient.send(command);
|
||||
|
||||
if (!assumeRes.Credentials?.AccessKeyId || !assumeRes.Credentials?.SecretAccessKey) {
|
||||
throw new BadRequestError({ message: "Failed to assume role - verify credentials and role configuration" });
|
||||
}
|
||||
const client = new IAMClient({
|
||||
region: providerInputs.region,
|
||||
credentials: {
|
||||
accessKeyId: assumeRes.Credentials?.AccessKeyId,
|
||||
secretAccessKey: assumeRes.Credentials?.SecretAccessKey,
|
||||
sessionToken: assumeRes.Credentials?.SessionToken
|
||||
}
|
||||
});
|
||||
return client;
|
||||
}
|
||||
|
||||
const client = new IAMClient({
|
||||
region: providerInputs.region,
|
||||
credentials: {
|
||||
@@ -53,11 +94,23 @@ export const AwsIamProvider = (): TDynamicProviderFns => {
|
||||
return client;
|
||||
};
|
||||
|
||||
const validateConnection = async (inputs: unknown) => {
|
||||
const validateConnection = async (inputs: unknown, { projectId }: { projectId: string }) => {
|
||||
const providerInputs = await validateProviderInputs(inputs);
|
||||
const client = await $getClient(providerInputs);
|
||||
|
||||
const isConnected = await client.send(new GetUserCommand({})).then(() => true);
|
||||
const client = await $getClient(providerInputs, projectId);
|
||||
const isConnected = await client
|
||||
.send(new GetUserCommand({}))
|
||||
.then(() => true)
|
||||
.catch((err) => {
|
||||
const message = (err as Error)?.message;
|
||||
if (
|
||||
providerInputs.method === AwsIamAuthType.AssumeRole &&
|
||||
// assume role will throw an error asking to provider username, but if so this has access in aws correctly
|
||||
message.includes("Must specify userName when calling with non-User credentials")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
return isConnected;
|
||||
};
|
||||
|
||||
@@ -68,11 +121,12 @@ export const AwsIamProvider = (): TDynamicProviderFns => {
|
||||
identity?: {
|
||||
name: string;
|
||||
};
|
||||
metadata: { projectId: string };
|
||||
}) => {
|
||||
const { inputs, usernameTemplate, identity } = data;
|
||||
const { inputs, usernameTemplate, metadata, identity } = data;
|
||||
|
||||
const providerInputs = await validateProviderInputs(inputs);
|
||||
const client = await $getClient(providerInputs);
|
||||
const client = await $getClient(providerInputs, metadata.projectId);
|
||||
|
||||
const username = generateUsername(usernameTemplate, identity);
|
||||
const { policyArns, userGroups, policyDocument, awsPath, permissionBoundaryPolicyArn } = providerInputs;
|
||||
@@ -84,6 +138,7 @@ export const AwsIamProvider = (): TDynamicProviderFns => {
|
||||
UserName: username
|
||||
})
|
||||
);
|
||||
|
||||
if (!createUserRes.User) throw new BadRequestError({ message: "Failed to create AWS IAM User" });
|
||||
if (userGroups) {
|
||||
await Promise.all(
|
||||
@@ -133,9 +188,9 @@ export const AwsIamProvider = (): TDynamicProviderFns => {
|
||||
};
|
||||
};
|
||||
|
||||
const revoke = async (inputs: unknown, entityId: string) => {
|
||||
const revoke = async (inputs: unknown, entityId: string, metadata: { projectId: string }) => {
|
||||
const providerInputs = await validateProviderInputs(inputs);
|
||||
const client = await $getClient(providerInputs);
|
||||
const client = await $getClient(providerInputs, metadata.projectId);
|
||||
|
||||
const username = entityId;
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ import { SapAseProvider } from "./sap-ase";
|
||||
import { SapHanaProvider } from "./sap-hana";
|
||||
import { SqlDatabaseProvider } from "./sql-database";
|
||||
import { TotpProvider } from "./totp";
|
||||
import { VerticaProvider } from "./vertica";
|
||||
|
||||
type TBuildDynamicSecretProviderDTO = {
|
||||
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
|
||||
@@ -40,5 +41,6 @@ export const buildDynamicSecretProviders = ({
|
||||
[DynamicSecretProviders.Snowflake]: SnowflakeProvider(),
|
||||
[DynamicSecretProviders.Totp]: TotpProvider(),
|
||||
[DynamicSecretProviders.SapAse]: SapAseProvider(),
|
||||
[DynamicSecretProviders.Kubernetes]: KubernetesProvider({ gatewayService })
|
||||
[DynamicSecretProviders.Kubernetes]: KubernetesProvider({ gatewayService }),
|
||||
[DynamicSecretProviders.Vertica]: VerticaProvider({ gatewayService })
|
||||
});
|
||||
|
||||
@@ -2,7 +2,7 @@ import axios from "axios";
|
||||
import https from "https";
|
||||
|
||||
import { InternalServerError } from "@app/lib/errors";
|
||||
import { withGatewayProxy } from "@app/lib/gateway";
|
||||
import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
|
||||
import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
|
||||
import { TKubernetesTokenRequest } from "@app/services/identity-kubernetes-auth/identity-kubernetes-auth-types";
|
||||
|
||||
@@ -43,6 +43,7 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO):
|
||||
return res;
|
||||
},
|
||||
{
|
||||
protocol: GatewayProxyProtocol.Tcp,
|
||||
targetHost: inputs.targetHost,
|
||||
targetPort: inputs.targetPort,
|
||||
relayHost,
|
||||
|
||||
@@ -16,7 +16,13 @@ export enum SqlProviders {
|
||||
MySQL = "mysql2",
|
||||
Oracle = "oracledb",
|
||||
MsSQL = "mssql",
|
||||
SapAse = "sap-ase"
|
||||
SapAse = "sap-ase",
|
||||
Vertica = "vertica"
|
||||
}
|
||||
|
||||
export enum AwsIamAuthType {
|
||||
AssumeRole = "assume-role",
|
||||
AccessKey = "access-key"
|
||||
}
|
||||
|
||||
export enum ElasticSearchAuthTypes {
|
||||
@@ -167,16 +173,38 @@ export const DynamicSecretSapAseSchema = z.object({
|
||||
revocationStatement: z.string().trim()
|
||||
});
|
||||
|
||||
export const DynamicSecretAwsIamSchema = z.object({
|
||||
accessKey: z.string().trim().min(1),
|
||||
secretAccessKey: z.string().trim().min(1),
|
||||
region: z.string().trim().min(1),
|
||||
awsPath: z.string().trim().optional(),
|
||||
permissionBoundaryPolicyArn: z.string().trim().optional(),
|
||||
policyDocument: z.string().trim().optional(),
|
||||
userGroups: z.string().trim().optional(),
|
||||
policyArns: z.string().trim().optional()
|
||||
});
|
||||
export const DynamicSecretAwsIamSchema = z.preprocess(
|
||||
(val) => {
|
||||
if (typeof val === "object" && val !== null && !Object.hasOwn(val, "method")) {
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
(val as { method: string }).method = AwsIamAuthType.AccessKey;
|
||||
}
|
||||
return val;
|
||||
},
|
||||
z.discriminatedUnion("method", [
|
||||
z.object({
|
||||
method: z.literal(AwsIamAuthType.AccessKey),
|
||||
accessKey: z.string().trim().min(1),
|
||||
secretAccessKey: z.string().trim().min(1),
|
||||
region: z.string().trim().min(1),
|
||||
awsPath: z.string().trim().optional(),
|
||||
permissionBoundaryPolicyArn: z.string().trim().optional(),
|
||||
policyDocument: z.string().trim().optional(),
|
||||
userGroups: z.string().trim().optional(),
|
||||
policyArns: z.string().trim().optional()
|
||||
}),
|
||||
z.object({
|
||||
method: z.literal(AwsIamAuthType.AssumeRole),
|
||||
roleArn: z.string().trim().min(1, "Role ARN required"),
|
||||
region: z.string().trim().min(1),
|
||||
awsPath: z.string().trim().optional(),
|
||||
permissionBoundaryPolicyArn: z.string().trim().optional(),
|
||||
policyDocument: z.string().trim().optional(),
|
||||
userGroups: z.string().trim().optional(),
|
||||
policyArns: z.string().trim().optional()
|
||||
})
|
||||
])
|
||||
);
|
||||
|
||||
export const DynamicSecretMongoAtlasSchema = z.object({
|
||||
adminPublicKey: z.string().trim().min(1).describe("Admin user public api key"),
|
||||
@@ -293,6 +321,39 @@ export const DynamicSecretKubernetesSchema = z.object({
|
||||
audiences: z.array(z.string().trim().min(1))
|
||||
});
|
||||
|
||||
export const DynamicSecretVerticaSchema = z.object({
|
||||
host: z.string().trim().toLowerCase(),
|
||||
port: z.number(),
|
||||
username: z.string().trim(),
|
||||
password: z.string().trim(),
|
||||
database: z.string().trim(),
|
||||
gatewayId: z.string().nullable().optional(),
|
||||
creationStatement: z.string().trim(),
|
||||
revocationStatement: z.string().trim(),
|
||||
passwordRequirements: z
|
||||
.object({
|
||||
length: z.number().min(1).max(250),
|
||||
required: z
|
||||
.object({
|
||||
lowercase: z.number().min(0),
|
||||
uppercase: z.number().min(0),
|
||||
digits: z.number().min(0),
|
||||
symbols: z.number().min(0)
|
||||
})
|
||||
.refine((data) => {
|
||||
const total = Object.values(data).reduce((sum, count) => sum + count, 0);
|
||||
return total <= 250;
|
||||
}, "Sum of required characters cannot exceed 250"),
|
||||
allowedSymbols: z.string().optional()
|
||||
})
|
||||
.refine((data) => {
|
||||
const total = Object.values(data.required).reduce((sum, count) => sum + count, 0);
|
||||
return total <= data.length;
|
||||
}, "Sum of required characters cannot exceed the total length")
|
||||
.optional()
|
||||
.describe("Password generation requirements")
|
||||
});
|
||||
|
||||
export const DynamicSecretTotpSchema = z.discriminatedUnion("configType", [
|
||||
z.object({
|
||||
configType: z.literal(TotpConfigType.URL),
|
||||
@@ -337,7 +398,8 @@ export enum DynamicSecretProviders {
|
||||
Snowflake = "snowflake",
|
||||
Totp = "totp",
|
||||
SapAse = "sap-ase",
|
||||
Kubernetes = "kubernetes"
|
||||
Kubernetes = "kubernetes",
|
||||
Vertica = "vertica"
|
||||
}
|
||||
|
||||
export const DynamicSecretProviderSchema = z.discriminatedUnion("type", [
|
||||
@@ -356,7 +418,8 @@ export const DynamicSecretProviderSchema = z.discriminatedUnion("type", [
|
||||
z.object({ type: z.literal(DynamicSecretProviders.Ldap), inputs: LdapSchema }),
|
||||
z.object({ type: z.literal(DynamicSecretProviders.Snowflake), inputs: DynamicSecretSnowflakeSchema }),
|
||||
z.object({ type: z.literal(DynamicSecretProviders.Totp), inputs: DynamicSecretTotpSchema }),
|
||||
z.object({ type: z.literal(DynamicSecretProviders.Kubernetes), inputs: DynamicSecretKubernetesSchema })
|
||||
z.object({ type: z.literal(DynamicSecretProviders.Kubernetes), inputs: DynamicSecretKubernetesSchema }),
|
||||
z.object({ type: z.literal(DynamicSecretProviders.Vertica), inputs: DynamicSecretVerticaSchema })
|
||||
]);
|
||||
|
||||
export type TDynamicProviderFns = {
|
||||
@@ -367,9 +430,15 @@ export type TDynamicProviderFns = {
|
||||
identity?: {
|
||||
name: string;
|
||||
};
|
||||
metadata: { projectId: string };
|
||||
}) => Promise<{ entityId: string; data: unknown }>;
|
||||
validateConnection: (inputs: unknown) => Promise<boolean>;
|
||||
validateProviderInputs: (inputs: object) => Promise<unknown>;
|
||||
revoke: (inputs: unknown, entityId: string) => Promise<{ entityId: string }>;
|
||||
renew: (inputs: unknown, entityId: string, expireAt: number) => Promise<{ entityId: string }>;
|
||||
validateConnection: (inputs: unknown, metadata: { projectId: string }) => Promise<boolean>;
|
||||
validateProviderInputs: (inputs: object, metadata: { projectId: string }) => Promise<unknown>;
|
||||
revoke: (inputs: unknown, entityId: string, metadata: { projectId: string }) => Promise<{ entityId: string }>;
|
||||
renew: (
|
||||
inputs: unknown,
|
||||
entityId: string,
|
||||
expireAt: number,
|
||||
metadata: { projectId: string }
|
||||
) => Promise<{ entityId: string }>;
|
||||
};
|
||||
|
||||
@@ -3,7 +3,7 @@ import handlebars from "handlebars";
|
||||
import knex from "knex";
|
||||
import { z } from "zod";
|
||||
|
||||
import { withGatewayProxy } from "@app/lib/gateway";
|
||||
import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
|
||||
import { alphaNumericNanoId } from "@app/lib/nanoid";
|
||||
import { validateHandlebarTemplate } from "@app/lib/template/validate-handlebars";
|
||||
|
||||
@@ -188,6 +188,7 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO)
|
||||
await gatewayCallback("localhost", port);
|
||||
},
|
||||
{
|
||||
protocol: GatewayProxyProtocol.Tcp,
|
||||
targetHost: providerInputs.host,
|
||||
targetPort: providerInputs.port,
|
||||
relayHost,
|
||||
|
||||
@@ -0,0 +1,368 @@
|
||||
import { randomInt } from "crypto";
|
||||
import handlebars from "handlebars";
|
||||
import knex, { Knex } from "knex";
|
||||
import { z } from "zod";
|
||||
|
||||
import { BadRequestError } from "@app/lib/errors";
|
||||
import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
|
||||
import { logger } from "@app/lib/logger";
|
||||
import { alphaNumericNanoId } from "@app/lib/nanoid";
|
||||
import { validateHandlebarTemplate } from "@app/lib/template/validate-handlebars";
|
||||
|
||||
import { TGatewayServiceFactory } from "../../gateway/gateway-service";
|
||||
import { verifyHostInputValidity } from "../dynamic-secret-fns";
|
||||
import { DynamicSecretVerticaSchema, PasswordRequirements, TDynamicProviderFns } from "./models";
|
||||
|
||||
const EXTERNAL_REQUEST_TIMEOUT = 10 * 1000;
|
||||
|
||||
interface VersionResult {
|
||||
version: string;
|
||||
}
|
||||
|
||||
interface SessionResult {
|
||||
session_id?: string;
|
||||
}
|
||||
|
||||
interface DatabaseQueryResult {
|
||||
rows?: Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
// Extended Knex client interface to handle Vertica-specific overrides
|
||||
interface VerticaKnexClient extends Knex {
|
||||
client: {
|
||||
parseVersion?: () => string;
|
||||
};
|
||||
}
|
||||
|
||||
const DEFAULT_PASSWORD_REQUIREMENTS = {
|
||||
length: 48,
|
||||
required: {
|
||||
lowercase: 1,
|
||||
uppercase: 1,
|
||||
digits: 1,
|
||||
symbols: 0
|
||||
},
|
||||
allowedSymbols: "-_.~!*"
|
||||
};
|
||||
|
||||
const generatePassword = (requirements?: PasswordRequirements) => {
|
||||
const finalReqs = requirements || DEFAULT_PASSWORD_REQUIREMENTS;
|
||||
|
||||
try {
|
||||
const { length, required, allowedSymbols } = finalReqs;
|
||||
|
||||
const chars = {
|
||||
lowercase: "abcdefghijklmnopqrstuvwxyz",
|
||||
uppercase: "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
|
||||
digits: "0123456789",
|
||||
symbols: allowedSymbols || "-_.~!*"
|
||||
};
|
||||
|
||||
const parts: string[] = [];
|
||||
|
||||
if (required.lowercase > 0) {
|
||||
parts.push(
|
||||
...Array(required.lowercase)
|
||||
.fill(0)
|
||||
.map(() => chars.lowercase[randomInt(chars.lowercase.length)])
|
||||
);
|
||||
}
|
||||
|
||||
if (required.uppercase > 0) {
|
||||
parts.push(
|
||||
...Array(required.uppercase)
|
||||
.fill(0)
|
||||
.map(() => chars.uppercase[randomInt(chars.uppercase.length)])
|
||||
);
|
||||
}
|
||||
|
||||
if (required.digits > 0) {
|
||||
parts.push(
|
||||
...Array(required.digits)
|
||||
.fill(0)
|
||||
.map(() => chars.digits[randomInt(chars.digits.length)])
|
||||
);
|
||||
}
|
||||
|
||||
if (required.symbols > 0) {
|
||||
parts.push(
|
||||
...Array(required.symbols)
|
||||
.fill(0)
|
||||
.map(() => chars.symbols[randomInt(chars.symbols.length)])
|
||||
);
|
||||
}
|
||||
|
||||
const requiredTotal = Object.values(required).reduce<number>((a, b) => a + b, 0);
|
||||
const remainingLength = Math.max(length - requiredTotal, 0);
|
||||
|
||||
const allowedChars = Object.entries(chars)
|
||||
.filter(([key]) => required[key as keyof typeof required] > 0)
|
||||
.map(([, value]) => value)
|
||||
.join("");
|
||||
|
||||
parts.push(
|
||||
...Array(remainingLength)
|
||||
.fill(0)
|
||||
.map(() => allowedChars[randomInt(allowedChars.length)])
|
||||
);
|
||||
|
||||
// shuffle the array to mix up the characters
|
||||
for (let i = parts.length - 1; i > 0; i -= 1) {
|
||||
const j = randomInt(i + 1);
|
||||
[parts[i], parts[j]] = [parts[j], parts[i]];
|
||||
}
|
||||
|
||||
return parts.join("");
|
||||
} catch (error: unknown) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
throw new Error(`Failed to generate password: ${message}`);
|
||||
}
|
||||
};
|
||||
|
||||
const generateUsername = (usernameTemplate?: string | null) => {
|
||||
const randomUsername = `inf_${alphaNumericNanoId(25)}`; // Username must start with an ascii letter, so we prepend the username with "inf-"
|
||||
if (!usernameTemplate) return randomUsername;
|
||||
|
||||
return handlebars.compile(usernameTemplate)({
|
||||
randomUsername,
|
||||
unixTimestamp: Math.floor(Date.now() / 100)
|
||||
});
|
||||
};
|
||||
|
||||
type TVerticaProviderDTO = {
|
||||
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
|
||||
};
|
||||
|
||||
export const VerticaProvider = ({ gatewayService }: TVerticaProviderDTO): TDynamicProviderFns => {
|
||||
const validateProviderInputs = async (inputs: unknown) => {
|
||||
const providerInputs = await DynamicSecretVerticaSchema.parseAsync(inputs);
|
||||
|
||||
const [hostIp] = await verifyHostInputValidity(providerInputs.host, Boolean(providerInputs.gatewayId));
|
||||
validateHandlebarTemplate("Vertica creation", providerInputs.creationStatement, {
|
||||
allowedExpressions: (val) => ["username", "password"].includes(val)
|
||||
});
|
||||
if (providerInputs.revocationStatement) {
|
||||
validateHandlebarTemplate("Vertica revoke", providerInputs.revocationStatement, {
|
||||
allowedExpressions: (val) => ["username"].includes(val)
|
||||
});
|
||||
}
|
||||
return { ...providerInputs, hostIp };
|
||||
};
|
||||
|
||||
const $getClient = async (providerInputs: z.infer<typeof DynamicSecretVerticaSchema> & { hostIp: string }) => {
|
||||
const config = {
|
||||
client: "pg",
|
||||
connection: {
|
||||
host: providerInputs.hostIp,
|
||||
port: providerInputs.port,
|
||||
database: providerInputs.database,
|
||||
user: providerInputs.username,
|
||||
password: providerInputs.password,
|
||||
ssl: false
|
||||
},
|
||||
acquireConnectionTimeout: EXTERNAL_REQUEST_TIMEOUT,
|
||||
pool: {
|
||||
min: 0,
|
||||
max: 1,
|
||||
acquireTimeoutMillis: 30000,
|
||||
createTimeoutMillis: 30000,
|
||||
destroyTimeoutMillis: 5000,
|
||||
idleTimeoutMillis: 30000,
|
||||
reapIntervalMillis: 1000,
|
||||
createRetryIntervalMillis: 100
|
||||
},
|
||||
// Disable version checking for Vertica compatibility
|
||||
version: "9.6.0" // Fake a compatible PostgreSQL version
|
||||
};
|
||||
|
||||
const client = knex(config) as VerticaKnexClient;
|
||||
|
||||
// Override the version parsing to prevent errors with Vertica
|
||||
if (client.client && typeof client.client.parseVersion !== "undefined") {
|
||||
client.client.parseVersion = () => "9.6.0";
|
||||
}
|
||||
|
||||
return client;
|
||||
};
|
||||
|
||||
const gatewayProxyWrapper = async (
|
||||
providerInputs: z.infer<typeof DynamicSecretVerticaSchema>,
|
||||
gatewayCallback: (host: string, port: number) => Promise<void>
|
||||
) => {
|
||||
const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(providerInputs.gatewayId as string);
|
||||
const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
|
||||
await withGatewayProxy(
|
||||
async (port) => {
|
||||
await gatewayCallback("localhost", port);
|
||||
},
|
||||
{
|
||||
protocol: GatewayProxyProtocol.Tcp,
|
||||
targetHost: providerInputs.host,
|
||||
targetPort: providerInputs.port,
|
||||
relayHost,
|
||||
relayPort: Number(relayPort),
|
||||
identityId: relayDetails.identityId,
|
||||
orgId: relayDetails.orgId,
|
||||
tlsOptions: {
|
||||
ca: relayDetails.certChain,
|
||||
cert: relayDetails.certificate,
|
||||
key: relayDetails.privateKey.toString()
|
||||
}
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
const validateConnection = async (inputs: unknown) => {
|
||||
const providerInputs = await validateProviderInputs(inputs);
|
||||
let isConnected = false;
|
||||
|
||||
const gatewayCallback = async (host = providerInputs.hostIp, port = providerInputs.port) => {
|
||||
let client: VerticaKnexClient | null = null;
|
||||
|
||||
try {
|
||||
client = await $getClient({ ...providerInputs, hostIp: host, port });
|
||||
|
||||
const clientResult: DatabaseQueryResult = await client.raw("SELECT version() AS version");
|
||||
|
||||
const resultFromSelectedDatabase = clientResult.rows?.[0] as VersionResult | undefined;
|
||||
|
||||
if (!resultFromSelectedDatabase?.version) {
|
||||
throw new BadRequestError({
|
||||
message: "Failed to validate Vertica connection, version query failed"
|
||||
});
|
||||
}
|
||||
|
||||
isConnected = true;
|
||||
} finally {
|
||||
if (client) await client.destroy();
|
||||
}
|
||||
};
|
||||
|
||||
if (providerInputs.gatewayId) {
|
||||
await gatewayProxyWrapper(providerInputs, gatewayCallback);
|
||||
} else {
|
||||
await gatewayCallback();
|
||||
}
|
||||
|
||||
return isConnected;
|
||||
};
|
||||
|
||||
const create = async (data: { inputs: unknown; usernameTemplate?: string | null }) => {
|
||||
const { inputs, usernameTemplate } = data;
|
||||
const providerInputs = await validateProviderInputs(inputs);
|
||||
|
||||
const username = generateUsername(usernameTemplate);
|
||||
const password = generatePassword(providerInputs.passwordRequirements);
|
||||
|
||||
const gatewayCallback = async (host = providerInputs.host, port = providerInputs.port) => {
|
||||
let client: VerticaKnexClient | null = null;
|
||||
|
||||
try {
|
||||
client = await $getClient({ ...providerInputs, hostIp: host, port });
|
||||
|
||||
const creationStatement = handlebars.compile(providerInputs.creationStatement, { noEscape: true })({
|
||||
username,
|
||||
password
|
||||
});
|
||||
|
||||
const queries = creationStatement.trim().replaceAll("\n", "").split(";").filter(Boolean);
|
||||
|
||||
// Execute queries sequentially to maintain transaction integrity
|
||||
for (const query of queries) {
|
||||
const trimmedQuery = query.trim();
|
||||
if (trimmedQuery) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await client.raw(trimmedQuery);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (client) await client.destroy();
|
||||
}
|
||||
};
|
||||
|
||||
if (providerInputs.gatewayId) {
|
||||
await gatewayProxyWrapper(providerInputs, gatewayCallback);
|
||||
} else {
|
||||
await gatewayCallback();
|
||||
}
|
||||
|
||||
return { entityId: username, data: { DB_USERNAME: username, DB_PASSWORD: password } };
|
||||
};
|
||||
|
||||
const revoke = async (inputs: unknown, username: string) => {
|
||||
const providerInputs = await validateProviderInputs(inputs);
|
||||
|
||||
const gatewayCallback = async (host = providerInputs.host, port = providerInputs.port) => {
|
||||
let client: VerticaKnexClient | null = null;
|
||||
|
||||
try {
|
||||
client = await $getClient({ ...providerInputs, hostIp: host, port });
|
||||
|
||||
const revokeStatement = handlebars.compile(providerInputs.revocationStatement, { noEscape: true })({
|
||||
username
|
||||
});
|
||||
|
||||
const queries = revokeStatement.trim().replaceAll("\n", "").split(";").filter(Boolean);
|
||||
|
||||
// Check for active sessions and close them
|
||||
try {
|
||||
const sessionResult: DatabaseQueryResult = await client.raw(
|
||||
"SELECT session_id FROM sessions WHERE user_name = ?",
|
||||
[username]
|
||||
);
|
||||
|
||||
const activeSessions = (sessionResult.rows || []) as SessionResult[];
|
||||
|
||||
// Close all sessions in parallel since they're independent operations
|
||||
if (activeSessions.length > 0) {
|
||||
const sessionClosePromises = activeSessions.map(async (session) => {
|
||||
try {
|
||||
await client!.raw("SELECT close_session(?)", [session.session_id]);
|
||||
} catch (error) {
|
||||
// Continue if session is already closed
|
||||
logger.error(error, `Failed to close session ${session.session_id}`);
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.allSettled(sessionClosePromises);
|
||||
}
|
||||
} catch (error) {
|
||||
// Continue if we can't query sessions (permissions, etc.)
|
||||
logger.error(error, "Could not query/close active sessions");
|
||||
}
|
||||
|
||||
// Execute revocation queries sequentially to maintain transaction integrity
|
||||
for (const query of queries) {
|
||||
const trimmedQuery = query.trim();
|
||||
if (trimmedQuery) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await client.raw(trimmedQuery);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (client) await client.destroy();
|
||||
}
|
||||
};
|
||||
|
||||
if (providerInputs.gatewayId) {
|
||||
await gatewayProxyWrapper(providerInputs, gatewayCallback);
|
||||
} else {
|
||||
await gatewayCallback();
|
||||
}
|
||||
|
||||
return { entityId: username };
|
||||
};
|
||||
|
||||
const renew = async (_: unknown, username: string) => {
|
||||
// No need for renewal
|
||||
return { entityId: username };
|
||||
};
|
||||
|
||||
return {
|
||||
validateProviderInputs,
|
||||
validateConnection,
|
||||
create,
|
||||
revoke,
|
||||
renew
|
||||
};
|
||||
};
|
||||
@@ -56,6 +56,7 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
|
||||
kmip: false,
|
||||
gateway: false,
|
||||
sshHostGroups: false,
|
||||
secretScanning: false,
|
||||
enterpriseSecretSyncs: false,
|
||||
enterpriseAppConnections: false
|
||||
});
|
||||
|
||||
@@ -709,6 +709,10 @@ export const licenseServiceFactory = ({
|
||||
return licenses;
|
||||
};
|
||||
|
||||
const invalidateGetPlan = async (orgId: string) => {
|
||||
await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId));
|
||||
};
|
||||
|
||||
return {
|
||||
generateOrgCustomerId,
|
||||
removeOrgCustomer,
|
||||
@@ -723,6 +727,7 @@ export const licenseServiceFactory = ({
|
||||
return onPremFeatures;
|
||||
},
|
||||
getPlan,
|
||||
invalidateGetPlan,
|
||||
updateSubscriptionOrgMemberCount,
|
||||
refreshPlan,
|
||||
getOrgPlan,
|
||||
|
||||
@@ -72,6 +72,7 @@ export type TFeatureSet = {
|
||||
kmip: false;
|
||||
gateway: false;
|
||||
sshHostGroups: false;
|
||||
secretScanning: false;
|
||||
enterpriseSecretSyncs: false;
|
||||
enterpriseAppConnections: false;
|
||||
};
|
||||
|
||||
@@ -13,6 +13,9 @@ import {
|
||||
ProjectPermissionPkiTemplateActions,
|
||||
ProjectPermissionSecretActions,
|
||||
ProjectPermissionSecretRotationActions,
|
||||
ProjectPermissionSecretScanningConfigActions,
|
||||
ProjectPermissionSecretScanningDataSourceActions,
|
||||
ProjectPermissionSecretScanningFindingActions,
|
||||
ProjectPermissionSecretSyncActions,
|
||||
ProjectPermissionSet,
|
||||
ProjectPermissionSshHostActions,
|
||||
@@ -220,6 +223,29 @@ const buildAdminPermissionRules = () => {
|
||||
ProjectPermissionSub.SecretRotation
|
||||
);
|
||||
|
||||
can(
|
||||
[
|
||||
ProjectPermissionSecretScanningDataSourceActions.Create,
|
||||
ProjectPermissionSecretScanningDataSourceActions.Edit,
|
||||
ProjectPermissionSecretScanningDataSourceActions.Delete,
|
||||
ProjectPermissionSecretScanningDataSourceActions.Read,
|
||||
ProjectPermissionSecretScanningDataSourceActions.TriggerScans,
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadResources
|
||||
],
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
can(
|
||||
[ProjectPermissionSecretScanningFindingActions.Read, ProjectPermissionSecretScanningFindingActions.Update],
|
||||
ProjectPermissionSub.SecretScanningFindings
|
||||
);
|
||||
|
||||
can(
|
||||
[ProjectPermissionSecretScanningConfigActions.Read, ProjectPermissionSecretScanningConfigActions.Update],
|
||||
ProjectPermissionSub.SecretScanningConfigs
|
||||
);
|
||||
|
||||
return rules;
|
||||
};
|
||||
|
||||
@@ -401,6 +427,23 @@ const buildMemberPermissionRules = () => {
|
||||
ProjectPermissionSub.SecretSyncs
|
||||
);
|
||||
|
||||
can(
|
||||
[
|
||||
ProjectPermissionSecretScanningDataSourceActions.Read,
|
||||
ProjectPermissionSecretScanningDataSourceActions.TriggerScans,
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadResources
|
||||
],
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
can(
|
||||
[ProjectPermissionSecretScanningFindingActions.Read, ProjectPermissionSecretScanningFindingActions.Update],
|
||||
ProjectPermissionSub.SecretScanningFindings
|
||||
);
|
||||
|
||||
can([ProjectPermissionSecretScanningConfigActions.Read], ProjectPermissionSub.SecretScanningConfigs);
|
||||
|
||||
return rules;
|
||||
};
|
||||
|
||||
@@ -437,6 +480,19 @@ const buildViewerPermissionRules = () => {
|
||||
can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateTemplates);
|
||||
can(ProjectPermissionSecretSyncActions.Read, ProjectPermissionSub.SecretSyncs);
|
||||
|
||||
can(
|
||||
[
|
||||
ProjectPermissionSecretScanningDataSourceActions.Read,
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadResources
|
||||
],
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
can([ProjectPermissionSecretScanningFindingActions.Read], ProjectPermissionSub.SecretScanningFindings);
|
||||
|
||||
can([ProjectPermissionSecretScanningConfigActions.Read], ProjectPermissionSub.SecretScanningConfigs);
|
||||
|
||||
return rules;
|
||||
};
|
||||
|
||||
|
||||
@@ -132,6 +132,26 @@ export enum ProjectPermissionKmipActions {
|
||||
GenerateClientCertificates = "generate-client-certificates"
|
||||
}
|
||||
|
||||
export enum ProjectPermissionSecretScanningDataSourceActions {
|
||||
Read = "read-data-sources",
|
||||
Create = "create-data-sources",
|
||||
Edit = "edit-data-sources",
|
||||
Delete = "delete-data-sources",
|
||||
TriggerScans = "trigger-data-source-scans",
|
||||
ReadScans = "read-data-source-scans",
|
||||
ReadResources = "read-data-source-resources"
|
||||
}
|
||||
|
||||
export enum ProjectPermissionSecretScanningFindingActions {
|
||||
Read = "read-findings",
|
||||
Update = "update-findings"
|
||||
}
|
||||
|
||||
export enum ProjectPermissionSecretScanningConfigActions {
|
||||
Read = "read-configs",
|
||||
Update = "update-configs"
|
||||
}
|
||||
|
||||
export enum ProjectPermissionSub {
|
||||
Role = "role",
|
||||
Member = "member",
|
||||
@@ -167,7 +187,10 @@ export enum ProjectPermissionSub {
|
||||
Kms = "kms",
|
||||
Cmek = "cmek",
|
||||
SecretSyncs = "secret-syncs",
|
||||
Kmip = "kmip"
|
||||
Kmip = "kmip",
|
||||
SecretScanningDataSources = "secret-scanning-data-sources",
|
||||
SecretScanningFindings = "secret-scanning-findings",
|
||||
SecretScanningConfigs = "secret-scanning-configs"
|
||||
}
|
||||
|
||||
export type SecretSubjectFields = {
|
||||
@@ -301,7 +324,10 @@ export type ProjectPermissionSet =
|
||||
| [ProjectPermissionActions.Edit, ProjectPermissionSub.Project]
|
||||
| [ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback]
|
||||
| [ProjectPermissionActions.Create, ProjectPermissionSub.SecretRollback]
|
||||
| [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms];
|
||||
| [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms]
|
||||
| [ProjectPermissionSecretScanningDataSourceActions, ProjectPermissionSub.SecretScanningDataSources]
|
||||
| [ProjectPermissionSecretScanningFindingActions, ProjectPermissionSub.SecretScanningFindings]
|
||||
| [ProjectPermissionSecretScanningConfigActions, ProjectPermissionSub.SecretScanningConfigs];
|
||||
|
||||
const SECRET_PATH_MISSING_SLASH_ERR_MSG = "Invalid Secret Path; it must start with a '/'";
|
||||
const SECRET_PATH_PERMISSION_OPERATOR_SCHEMA = z.union([
|
||||
@@ -350,7 +376,8 @@ const DynamicSecretConditionV2Schema = z
|
||||
.object({
|
||||
[PermissionConditionOperators.$EQ]: PermissionConditionSchema[PermissionConditionOperators.$EQ],
|
||||
[PermissionConditionOperators.$NEQ]: PermissionConditionSchema[PermissionConditionOperators.$NEQ],
|
||||
[PermissionConditionOperators.$IN]: PermissionConditionSchema[PermissionConditionOperators.$IN]
|
||||
[PermissionConditionOperators.$IN]: PermissionConditionSchema[PermissionConditionOperators.$IN],
|
||||
[PermissionConditionOperators.$GLOB]: PermissionConditionSchema[PermissionConditionOperators.$GLOB]
|
||||
})
|
||||
.partial()
|
||||
]),
|
||||
@@ -378,6 +405,23 @@ const DynamicSecretConditionV2Schema = z
|
||||
})
|
||||
.partial();
|
||||
|
||||
const SecretImportConditionSchema = z
|
||||
.object({
|
||||
environment: z.union([
|
||||
z.string(),
|
||||
z
|
||||
.object({
|
||||
[PermissionConditionOperators.$EQ]: PermissionConditionSchema[PermissionConditionOperators.$EQ],
|
||||
[PermissionConditionOperators.$NEQ]: PermissionConditionSchema[PermissionConditionOperators.$NEQ],
|
||||
[PermissionConditionOperators.$IN]: PermissionConditionSchema[PermissionConditionOperators.$IN],
|
||||
[PermissionConditionOperators.$GLOB]: PermissionConditionSchema[PermissionConditionOperators.$GLOB]
|
||||
})
|
||||
.partial()
|
||||
]),
|
||||
secretPath: SECRET_PATH_PERMISSION_OPERATOR_SCHEMA
|
||||
})
|
||||
.partial();
|
||||
|
||||
const SecretConditionV2Schema = z
|
||||
.object({
|
||||
environment: z.union([
|
||||
@@ -631,6 +675,26 @@ const GeneralPermissionSchema = [
|
||||
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionKmipActions).describe(
|
||||
"Describe what action an entity can take."
|
||||
)
|
||||
}),
|
||||
z.object({
|
||||
subject: z
|
||||
.literal(ProjectPermissionSub.SecretScanningDataSources)
|
||||
.describe("The entity this permission pertains to."),
|
||||
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningDataSourceActions).describe(
|
||||
"Describe what action an entity can take."
|
||||
)
|
||||
}),
|
||||
z.object({
|
||||
subject: z.literal(ProjectPermissionSub.SecretScanningFindings).describe("The entity this permission pertains to."),
|
||||
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningFindingActions).describe(
|
||||
"Describe what action an entity can take."
|
||||
)
|
||||
}),
|
||||
z.object({
|
||||
subject: z.literal(ProjectPermissionSub.SecretScanningConfigs).describe("The entity this permission pertains to."),
|
||||
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretScanningConfigActions).describe(
|
||||
"Describe what action an entity can take."
|
||||
)
|
||||
})
|
||||
];
|
||||
|
||||
@@ -695,7 +759,7 @@ export const ProjectPermissionV2Schema = z.discriminatedUnion("subject", [
|
||||
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionActions).describe(
|
||||
"Describe what action an entity can take."
|
||||
),
|
||||
conditions: SecretConditionV1Schema.describe(
|
||||
conditions: SecretImportConditionSchema.describe(
|
||||
"When specified, only matching conditions will be allowed to access given resource."
|
||||
).optional()
|
||||
}),
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { TSecretScanningDataSourceListItem } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
|
||||
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
|
||||
|
||||
export const GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION: TSecretScanningDataSourceListItem = {
|
||||
name: "GitHub",
|
||||
type: SecretScanningDataSource.GitHub,
|
||||
connection: AppConnection.GitHubRadar
|
||||
};
|
||||
@@ -0,0 +1,230 @@
|
||||
import { join } from "path";
|
||||
import { ProbotOctokit } from "probot";
|
||||
|
||||
import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
|
||||
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
|
||||
import {
|
||||
SecretScanningDataSource,
|
||||
SecretScanningFindingSeverity,
|
||||
SecretScanningResource
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import {
|
||||
cloneRepository,
|
||||
convertPatchLineToFileLineNumber,
|
||||
replaceNonChangesWithNewlines
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
|
||||
import {
|
||||
TSecretScanningFactoryGetDiffScanFindingsPayload,
|
||||
TSecretScanningFactoryGetDiffScanResourcePayload,
|
||||
TSecretScanningFactoryGetFullScanPath,
|
||||
TSecretScanningFactoryInitialize,
|
||||
TSecretScanningFactoryListRawResources,
|
||||
TSecretScanningFactoryPostInitialization
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { BadRequestError } from "@app/lib/errors";
|
||||
import { titleCaseToCamelCase } from "@app/lib/fn";
|
||||
import { GitHubRepositoryRegex } from "@app/lib/regex";
|
||||
import { listGitHubRadarRepositories, TGitHubRadarConnection } from "@app/services/app-connection/github-radar";
|
||||
|
||||
import { TGitHubDataSourceWithConnection, TQueueGitHubResourceDiffScan } from "./github-secret-scanning-types";
|
||||
|
||||
export const GitHubSecretScanningFactory = () => {
|
||||
const initialize: TSecretScanningFactoryInitialize<TGitHubRadarConnection> = async (
|
||||
{ connection, secretScanningV2DAL },
|
||||
callback
|
||||
) => {
|
||||
const externalId = connection.credentials.installationId;
|
||||
|
||||
const existingDataSource = await secretScanningV2DAL.dataSources.findOne({
|
||||
externalId,
|
||||
type: SecretScanningDataSource.GitHub
|
||||
});
|
||||
|
||||
if (existingDataSource)
|
||||
throw new BadRequestError({
|
||||
message: `A Data Source already exists for this GitHub Radar Connection in the Project with ID "${existingDataSource.projectId}"`
|
||||
});
|
||||
|
||||
return callback({
|
||||
externalId
|
||||
});
|
||||
};
|
||||
|
||||
const postInitialization: TSecretScanningFactoryPostInitialization<TGitHubRadarConnection> = async () => {
|
||||
// no post-initialization required
|
||||
};
|
||||
|
||||
const listRawResources: TSecretScanningFactoryListRawResources<TGitHubDataSourceWithConnection> = async (
|
||||
dataSource
|
||||
) => {
|
||||
const {
|
||||
connection,
|
||||
config: { includeRepos }
|
||||
} = dataSource;
|
||||
|
||||
const repos = await listGitHubRadarRepositories(connection);
|
||||
|
||||
const filteredRepos: typeof repos = [];
|
||||
if (includeRepos.includes("*")) {
|
||||
filteredRepos.push(...repos);
|
||||
} else {
|
||||
filteredRepos.push(...repos.filter((repo) => includeRepos.includes(repo.full_name)));
|
||||
}
|
||||
|
||||
return filteredRepos.map(({ id, full_name }) => ({
|
||||
name: full_name,
|
||||
externalId: id.toString(),
|
||||
type: SecretScanningResource.Repository
|
||||
}));
|
||||
};
|
||||
|
||||
const getFullScanPath: TSecretScanningFactoryGetFullScanPath<TGitHubDataSourceWithConnection> = async ({
|
||||
dataSource,
|
||||
resourceName,
|
||||
tempFolder
|
||||
}) => {
|
||||
const appCfg = getConfig();
|
||||
const {
|
||||
connection: {
|
||||
credentials: { installationId }
|
||||
}
|
||||
} = dataSource;
|
||||
|
||||
const octokit = new ProbotOctokit({
|
||||
auth: {
|
||||
appId: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID,
|
||||
privateKey: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY,
|
||||
installationId
|
||||
}
|
||||
});
|
||||
|
||||
const {
|
||||
data: { token }
|
||||
} = await octokit.apps.createInstallationAccessToken({
|
||||
installation_id: Number(installationId)
|
||||
});
|
||||
|
||||
const repoPath = join(tempFolder, "repo.git");
|
||||
|
||||
if (!GitHubRepositoryRegex.test(resourceName)) {
|
||||
throw new Error("Invalid GitHub repository name");
|
||||
}
|
||||
|
||||
await cloneRepository({
|
||||
cloneUrl: `https://x-access-token:${token}@github.com/${resourceName}.git`,
|
||||
repoPath
|
||||
});
|
||||
|
||||
return repoPath;
|
||||
};
|
||||
|
||||
const getDiffScanResourcePayload: TSecretScanningFactoryGetDiffScanResourcePayload<
|
||||
TQueueGitHubResourceDiffScan["payload"]
|
||||
> = ({ repository }) => {
|
||||
return {
|
||||
name: repository.full_name,
|
||||
externalId: repository.id.toString(),
|
||||
type: SecretScanningResource.Repository
|
||||
};
|
||||
};
|
||||
|
||||
const getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<
|
||||
TGitHubDataSourceWithConnection,
|
||||
TQueueGitHubResourceDiffScan["payload"]
|
||||
> = async ({ dataSource, payload, resourceName, configPath }) => {
|
||||
const appCfg = getConfig();
|
||||
const {
|
||||
connection: {
|
||||
credentials: { installationId }
|
||||
}
|
||||
} = dataSource;
|
||||
|
||||
const octokit = new ProbotOctokit({
|
||||
auth: {
|
||||
appId: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_ID,
|
||||
privateKey: appCfg.INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY,
|
||||
installationId
|
||||
}
|
||||
});
|
||||
|
||||
const { commits, repository } = payload;
|
||||
|
||||
const [owner, repo] = repository.full_name.split("/");
|
||||
|
||||
const allFindings: SecretMatch[] = [];
|
||||
|
||||
for (const commit of commits) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const commitData = await octokit.repos.getCommit({
|
||||
owner,
|
||||
repo,
|
||||
ref: commit.id
|
||||
});
|
||||
|
||||
// eslint-disable-next-line no-continue
|
||||
if (!commitData.data.files) continue;
|
||||
|
||||
for (const file of commitData.data.files) {
|
||||
if ((file.status === "added" || file.status === "modified") && file.patch) {
|
||||
// eslint-disable-next-line
|
||||
const findings = await scanContentAndGetFindings(
|
||||
replaceNonChangesWithNewlines(`\n${file.patch}`),
|
||||
configPath
|
||||
);
|
||||
|
||||
const adjustedFindings = findings.map((finding) => {
|
||||
const startLine = convertPatchLineToFileLineNumber(file.patch!, finding.StartLine);
|
||||
const endLine =
|
||||
finding.StartLine === finding.EndLine
|
||||
? startLine
|
||||
: convertPatchLineToFileLineNumber(file.patch!, finding.EndLine);
|
||||
const startColumn = finding.StartColumn - 1; // subtract 1 for +
|
||||
const endColumn = finding.EndColumn - 1; // subtract 1 for +
|
||||
|
||||
return {
|
||||
...finding,
|
||||
StartLine: startLine,
|
||||
EndLine: endLine,
|
||||
StartColumn: startColumn,
|
||||
EndColumn: endColumn,
|
||||
File: file.filename,
|
||||
Commit: commit.id,
|
||||
Author: commit.author.name,
|
||||
Email: commit.author.email ?? "",
|
||||
Message: commit.message,
|
||||
Fingerprint: `${commit.id}:${file.filename}:${finding.RuleID}:${startLine}:${startColumn}`,
|
||||
Date: commit.timestamp,
|
||||
Link: `https://github.com/${resourceName}/blob/${commit.id}/${file.filename}#L${startLine}`
|
||||
};
|
||||
});
|
||||
|
||||
allFindings.push(...adjustedFindings);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return allFindings.map(
|
||||
({
|
||||
// discard match and secret as we don't want to store
|
||||
Match,
|
||||
Secret,
|
||||
...finding
|
||||
}) => ({
|
||||
details: titleCaseToCamelCase(finding),
|
||||
fingerprint: finding.Fingerprint,
|
||||
severity: SecretScanningFindingSeverity.High,
|
||||
rule: finding.RuleID
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
return {
|
||||
initialize,
|
||||
postInitialization,
|
||||
listRawResources,
|
||||
getFullScanPath,
|
||||
getDiffScanResourcePayload,
|
||||
getDiffScanFindingsPayload
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,85 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
SecretScanningDataSource,
|
||||
SecretScanningResource
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import {
|
||||
BaseCreateSecretScanningDataSourceSchema,
|
||||
BaseSecretScanningDataSourceSchema,
|
||||
BaseSecretScanningFindingSchema,
|
||||
BaseUpdateSecretScanningDataSourceSchema,
|
||||
GitRepositoryScanFindingDetailsSchema
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-schemas";
|
||||
import { SecretScanningDataSources } from "@app/lib/api-docs";
|
||||
import { GitHubRepositoryRegex } from "@app/lib/regex";
|
||||
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
|
||||
|
||||
export const GitHubDataSourceConfigSchema = z.object({
|
||||
includeRepos: z
|
||||
.array(
|
||||
z
|
||||
.string()
|
||||
.min(1)
|
||||
.max(256)
|
||||
.refine((value) => value === "*" || GitHubRepositoryRegex.test(value), "Invalid repository name format")
|
||||
)
|
||||
.nonempty("One or more repositories required")
|
||||
.max(100, "Cannot configure more than 100 repositories")
|
||||
.default(["*"])
|
||||
.describe(SecretScanningDataSources.CONFIG.GITHUB.includeRepos)
|
||||
});
|
||||
|
||||
export const GitHubDataSourceSchema = BaseSecretScanningDataSourceSchema({
|
||||
type: SecretScanningDataSource.GitHub,
|
||||
isConnectionRequired: true
|
||||
})
|
||||
.extend({
|
||||
config: GitHubDataSourceConfigSchema
|
||||
})
|
||||
.describe(
|
||||
JSON.stringify({
|
||||
title: "GitHub"
|
||||
})
|
||||
);
|
||||
|
||||
export const CreateGitHubDataSourceSchema = BaseCreateSecretScanningDataSourceSchema({
|
||||
type: SecretScanningDataSource.GitHub,
|
||||
isConnectionRequired: true
|
||||
})
|
||||
.extend({
|
||||
config: GitHubDataSourceConfigSchema
|
||||
})
|
||||
.describe(
|
||||
JSON.stringify({
|
||||
title: "GitHub"
|
||||
})
|
||||
);
|
||||
|
||||
export const UpdateGitHubDataSourceSchema = BaseUpdateSecretScanningDataSourceSchema(SecretScanningDataSource.GitHub)
|
||||
.extend({
|
||||
config: GitHubDataSourceConfigSchema.optional()
|
||||
})
|
||||
.describe(
|
||||
JSON.stringify({
|
||||
title: "GitHub"
|
||||
})
|
||||
);
|
||||
|
||||
export const GitHubDataSourceListItemSchema = z
|
||||
.object({
|
||||
name: z.literal("GitHub"),
|
||||
connection: z.literal(AppConnection.GitHubRadar),
|
||||
type: z.literal(SecretScanningDataSource.GitHub)
|
||||
})
|
||||
.describe(
|
||||
JSON.stringify({
|
||||
title: "GitHub"
|
||||
})
|
||||
);
|
||||
|
||||
export const GitHubFindingSchema = BaseSecretScanningFindingSchema.extend({
|
||||
resourceType: z.literal(SecretScanningResource.Repository),
|
||||
dataSourceType: z.literal(SecretScanningDataSource.GitHub),
|
||||
details: GitRepositoryScanFindingDetailsSchema
|
||||
});
|
||||
@@ -0,0 +1,87 @@
|
||||
import { PushEvent } from "@octokit/webhooks-types";
|
||||
|
||||
import { TSecretScanningV2DALFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-dal";
|
||||
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { TSecretScanningV2QueueServiceFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-queue";
|
||||
import { logger } from "@app/lib/logger";
|
||||
|
||||
import { TGitHubDataSource } from "./github-secret-scanning-types";
|
||||
|
||||
export const githubSecretScanningService = (
|
||||
secretScanningV2DAL: TSecretScanningV2DALFactory,
|
||||
secretScanningV2Queue: Pick<TSecretScanningV2QueueServiceFactory, "queueResourceDiffScan">
|
||||
) => {
|
||||
const handleInstallationDeletedEvent = async (installationId: number) => {
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findOne({
|
||||
externalId: String(installationId),
|
||||
type: SecretScanningDataSource.GitHub
|
||||
});
|
||||
|
||||
if (!dataSource) {
|
||||
logger.error(
|
||||
`secretScanningV2RemoveEvent: GitHub - Could not find data source [installationId=${installationId}]`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`secretScanningV2RemoveEvent: GitHub - installation deleted [installationId=${installationId}] [dataSourceId=${dataSource.id}]`
|
||||
);
|
||||
|
||||
await secretScanningV2DAL.dataSources.updateById(dataSource.id, {
|
||||
isDisconnected: true
|
||||
});
|
||||
};
|
||||
|
||||
const handlePushEvent = async (payload: PushEvent) => {
|
||||
const { commits, repository, installation } = payload;
|
||||
|
||||
if (!commits || !repository || !installation) {
|
||||
logger.warn(
|
||||
`secretScanningV2PushEvent: GitHub - Insufficient data [commits=${commits?.length ?? 0}] [repository=${repository.name}] [installationId=${installation?.id}]`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const dataSource = (await secretScanningV2DAL.dataSources.findOne({
|
||||
externalId: String(installation.id),
|
||||
type: SecretScanningDataSource.GitHub
|
||||
})) as TGitHubDataSource | undefined;
|
||||
|
||||
if (!dataSource) {
|
||||
logger.error(
|
||||
`secretScanningV2PushEvent: GitHub - Could not find data source [installationId=${installation.id}]`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const {
|
||||
isAutoScanEnabled,
|
||||
config: { includeRepos }
|
||||
} = dataSource;
|
||||
|
||||
if (!isAutoScanEnabled) {
|
||||
logger.info(
|
||||
`secretScanningV2PushEvent: GitHub - ignoring due to auto scan disabled [dataSourceId=${dataSource.id}] [installationId=${installation.id}]`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (includeRepos.includes("*") || includeRepos.includes(repository.full_name)) {
|
||||
await secretScanningV2Queue.queueResourceDiffScan({
|
||||
dataSourceType: SecretScanningDataSource.GitHub,
|
||||
payload,
|
||||
dataSourceId: dataSource.id
|
||||
});
|
||||
} else {
|
||||
logger.info(
|
||||
`secretScanningV2PushEvent: GitHub - ignoring due to repository not being present in config [installationId=${installation.id}] [dataSourceId=${dataSource.id}]`
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
handlePushEvent,
|
||||
handleInstallationDeletedEvent
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,32 @@
|
||||
import { PushEvent } from "@octokit/webhooks-types";
|
||||
import { z } from "zod";
|
||||
|
||||
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { TGitHubRadarConnection } from "@app/services/app-connection/github-radar";
|
||||
|
||||
import {
|
||||
CreateGitHubDataSourceSchema,
|
||||
GitHubDataSourceListItemSchema,
|
||||
GitHubDataSourceSchema,
|
||||
GitHubFindingSchema
|
||||
} from "./github-secret-scanning-schemas";
|
||||
|
||||
export type TGitHubDataSource = z.infer<typeof GitHubDataSourceSchema>;
|
||||
|
||||
export type TGitHubDataSourceInput = z.infer<typeof CreateGitHubDataSourceSchema>;
|
||||
|
||||
export type TGitHubDataSourceListItem = z.infer<typeof GitHubDataSourceListItemSchema>;
|
||||
|
||||
export type TGitHubFinding = z.infer<typeof GitHubFindingSchema>;
|
||||
|
||||
export type TGitHubDataSourceWithConnection = TGitHubDataSource & {
|
||||
connection: TGitHubRadarConnection;
|
||||
};
|
||||
|
||||
export type TQueueGitHubResourceDiffScan = {
|
||||
dataSourceType: SecretScanningDataSource.GitHub;
|
||||
payload: PushEvent;
|
||||
dataSourceId: string;
|
||||
resourceId: string;
|
||||
scanId: string;
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from "./github-secret-scanning-constants";
|
||||
export * from "./github-secret-scanning-schemas";
|
||||
export * from "./github-secret-scanning-types";
|
||||
@@ -0,0 +1,460 @@
|
||||
import { Knex } from "knex";
|
||||
|
||||
import { TDbClient } from "@app/db";
|
||||
import {
|
||||
SecretScanningResourcesSchema,
|
||||
SecretScanningScansSchema,
|
||||
TableName,
|
||||
TSecretScanningDataSources
|
||||
} from "@app/db/schemas";
|
||||
import { SecretScanningFindingStatus } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { DatabaseError } from "@app/lib/errors";
|
||||
import {
|
||||
buildFindFilter,
|
||||
ormify,
|
||||
prependTableNameToFindFilter,
|
||||
selectAllTableCols,
|
||||
sqlNestRelationships,
|
||||
TFindOpt
|
||||
} from "@app/lib/knex";
|
||||
|
||||
export type TSecretScanningV2DALFactory = ReturnType<typeof secretScanningV2DALFactory>;
|
||||
|
||||
type TSecretScanningDataSourceFindFilter = Parameters<typeof buildFindFilter<TSecretScanningDataSources>>[0];
|
||||
type TSecretScanningDataSourceFindOptions = TFindOpt<TSecretScanningDataSources, true, "name">;
|
||||
|
||||
const baseSecretScanningDataSourceQuery = ({
|
||||
filter = {},
|
||||
db,
|
||||
tx
|
||||
}: {
|
||||
db: TDbClient;
|
||||
filter?: TSecretScanningDataSourceFindFilter;
|
||||
options?: TSecretScanningDataSourceFindOptions;
|
||||
tx?: Knex;
|
||||
}) => {
|
||||
const query = (tx || db.replicaNode())(TableName.SecretScanningDataSource)
|
||||
.join(
|
||||
TableName.AppConnection,
|
||||
`${TableName.SecretScanningDataSource}.connectionId`,
|
||||
`${TableName.AppConnection}.id`
|
||||
)
|
||||
.select(selectAllTableCols(TableName.SecretScanningDataSource))
|
||||
.select(
|
||||
// entire connection
|
||||
db.ref("name").withSchema(TableName.AppConnection).as("connectionName"),
|
||||
db.ref("method").withSchema(TableName.AppConnection).as("connectionMethod"),
|
||||
db.ref("app").withSchema(TableName.AppConnection).as("connectionApp"),
|
||||
db.ref("orgId").withSchema(TableName.AppConnection).as("connectionOrgId"),
|
||||
db.ref("encryptedCredentials").withSchema(TableName.AppConnection).as("connectionEncryptedCredentials"),
|
||||
db.ref("description").withSchema(TableName.AppConnection).as("connectionDescription"),
|
||||
db.ref("version").withSchema(TableName.AppConnection).as("connectionVersion"),
|
||||
db.ref("createdAt").withSchema(TableName.AppConnection).as("connectionCreatedAt"),
|
||||
db.ref("updatedAt").withSchema(TableName.AppConnection).as("connectionUpdatedAt"),
|
||||
db
|
||||
.ref("isPlatformManagedCredentials")
|
||||
.withSchema(TableName.AppConnection)
|
||||
.as("connectionIsPlatformManagedCredentials")
|
||||
);
|
||||
|
||||
if (filter) {
|
||||
/* eslint-disable @typescript-eslint/no-misused-promises */
|
||||
void query.where(buildFindFilter(prependTableNameToFindFilter(TableName.SecretScanningDataSource, filter)));
|
||||
}
|
||||
|
||||
return query;
|
||||
};
|
||||
|
||||
const expandSecretScanningDataSource = <
|
||||
T extends Awaited<ReturnType<typeof baseSecretScanningDataSourceQuery>>[number]
|
||||
>(
|
||||
dataSource: T
|
||||
) => {
|
||||
const {
|
||||
connectionApp,
|
||||
connectionName,
|
||||
connectionId,
|
||||
connectionOrgId,
|
||||
connectionEncryptedCredentials,
|
||||
connectionMethod,
|
||||
connectionDescription,
|
||||
connectionCreatedAt,
|
||||
connectionUpdatedAt,
|
||||
connectionVersion,
|
||||
connectionIsPlatformManagedCredentials,
|
||||
...el
|
||||
} = dataSource;
|
||||
|
||||
return {
|
||||
...el,
|
||||
connectionId,
|
||||
connection: connectionId
|
||||
? {
|
||||
app: connectionApp,
|
||||
id: connectionId,
|
||||
name: connectionName,
|
||||
orgId: connectionOrgId,
|
||||
encryptedCredentials: connectionEncryptedCredentials,
|
||||
method: connectionMethod,
|
||||
description: connectionDescription,
|
||||
createdAt: connectionCreatedAt,
|
||||
updatedAt: connectionUpdatedAt,
|
||||
version: connectionVersion,
|
||||
isPlatformManagedCredentials: connectionIsPlatformManagedCredentials
|
||||
}
|
||||
: undefined
|
||||
};
|
||||
};
|
||||
|
||||
export const secretScanningV2DALFactory = (db: TDbClient) => {
|
||||
const dataSourceOrm = ormify(db, TableName.SecretScanningDataSource);
|
||||
const resourceOrm = ormify(db, TableName.SecretScanningResource);
|
||||
const scanOrm = ormify(db, TableName.SecretScanningScan);
|
||||
const findingOrm = ormify(db, TableName.SecretScanningFinding);
|
||||
const configOrm = ormify(db, TableName.SecretScanningConfig);
|
||||
|
||||
const findDataSource = async (filter: Parameters<(typeof dataSourceOrm)["find"]>[0], tx?: Knex) => {
|
||||
try {
|
||||
const dataSources = await baseSecretScanningDataSourceQuery({ filter, db, tx });
|
||||
|
||||
if (!dataSources.length) return [];
|
||||
|
||||
return dataSources.map(expandSecretScanningDataSource);
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find - Secret Scanning Data Source" });
|
||||
}
|
||||
};
|
||||
|
||||
const findDataSourceById = async (id: string, tx?: Knex) => {
|
||||
try {
|
||||
const dataSource = await baseSecretScanningDataSourceQuery({ filter: { id }, db, tx }).first();
|
||||
|
||||
if (dataSource) return expandSecretScanningDataSource(dataSource);
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find By ID - Secret Scanning Data Source" });
|
||||
}
|
||||
};
|
||||
|
||||
const createDataSource = async (data: Parameters<(typeof dataSourceOrm)["create"]>[0], tx?: Knex) => {
|
||||
const source = await dataSourceOrm.create(data, tx);
|
||||
|
||||
const dataSource = (await baseSecretScanningDataSourceQuery({
|
||||
filter: { id: source.id },
|
||||
db,
|
||||
tx
|
||||
}).first())!;
|
||||
|
||||
return expandSecretScanningDataSource(dataSource);
|
||||
};
|
||||
|
||||
const updateDataSourceById = async (
|
||||
dataSourceId: string,
|
||||
data: Parameters<(typeof dataSourceOrm)["updateById"]>[1],
|
||||
tx?: Knex
|
||||
) => {
|
||||
const source = await dataSourceOrm.updateById(dataSourceId, data, tx);
|
||||
|
||||
const dataSource = (await baseSecretScanningDataSourceQuery({
|
||||
filter: { id: source.id },
|
||||
db,
|
||||
tx
|
||||
}).first())!;
|
||||
|
||||
return expandSecretScanningDataSource(dataSource);
|
||||
};
|
||||
|
||||
const deleteDataSourceById = async (dataSourceId: string, tx?: Knex) => {
|
||||
const dataSource = (await baseSecretScanningDataSourceQuery({
|
||||
filter: { id: dataSourceId },
|
||||
db,
|
||||
tx
|
||||
}).first())!;
|
||||
|
||||
await dataSourceOrm.deleteById(dataSourceId, tx);
|
||||
|
||||
return expandSecretScanningDataSource(dataSource);
|
||||
};
|
||||
|
||||
const findOneDataSource = async (filter: Parameters<(typeof dataSourceOrm)["findOne"]>[0], tx?: Knex) => {
|
||||
try {
|
||||
const dataSource = await baseSecretScanningDataSourceQuery({ filter, db, tx }).first();
|
||||
|
||||
if (dataSource) {
|
||||
return expandSecretScanningDataSource(dataSource);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find One - Secret Scanning Data Source" });
|
||||
}
|
||||
};
|
||||
|
||||
const findDataSourceWithDetails = async (filter: Parameters<(typeof dataSourceOrm)["find"]>[0], tx?: Knex) => {
|
||||
try {
|
||||
// TODO (scott): this query will probably need to be optimized
|
||||
|
||||
const dataSources = await baseSecretScanningDataSourceQuery({ filter, db, tx })
|
||||
.leftJoin(
|
||||
TableName.SecretScanningResource,
|
||||
`${TableName.SecretScanningResource}.dataSourceId`,
|
||||
`${TableName.SecretScanningDataSource}.id`
|
||||
)
|
||||
.leftJoin(
|
||||
TableName.SecretScanningScan,
|
||||
`${TableName.SecretScanningScan}.resourceId`,
|
||||
`${TableName.SecretScanningResource}.id`
|
||||
)
|
||||
.leftJoin(
|
||||
TableName.SecretScanningFinding,
|
||||
`${TableName.SecretScanningFinding}.scanId`,
|
||||
`${TableName.SecretScanningScan}.id`
|
||||
)
|
||||
.where((qb) => {
|
||||
void qb
|
||||
.where(`${TableName.SecretScanningFinding}.status`, SecretScanningFindingStatus.Unresolved)
|
||||
.orWhereNull(`${TableName.SecretScanningFinding}.status`);
|
||||
})
|
||||
.select(
|
||||
db.ref("id").withSchema(TableName.SecretScanningScan).as("scanId"),
|
||||
db.ref("status").withSchema(TableName.SecretScanningScan).as("scanStatus"),
|
||||
db.ref("statusMessage").withSchema(TableName.SecretScanningScan).as("scanStatusMessage"),
|
||||
db.ref("createdAt").withSchema(TableName.SecretScanningScan).as("scanCreatedAt"),
|
||||
db.ref("status").withSchema(TableName.SecretScanningFinding).as("findingStatus"),
|
||||
db.ref("id").withSchema(TableName.SecretScanningFinding).as("findingId")
|
||||
);
|
||||
|
||||
if (!dataSources.length) return [];
|
||||
|
||||
const results = sqlNestRelationships({
|
||||
data: dataSources,
|
||||
key: "id",
|
||||
parentMapper: (dataSource) => expandSecretScanningDataSource(dataSource),
|
||||
childrenMapper: [
|
||||
{
|
||||
key: "scanId",
|
||||
label: "scans" as const,
|
||||
mapper: ({ scanId, scanCreatedAt, scanStatus, scanStatusMessage }) => ({
|
||||
id: scanId,
|
||||
createdAt: scanCreatedAt,
|
||||
status: scanStatus,
|
||||
statusMessage: scanStatusMessage
|
||||
})
|
||||
},
|
||||
{
|
||||
key: "findingId",
|
||||
label: "findings" as const,
|
||||
mapper: ({ findingId }) => ({
|
||||
id: findingId
|
||||
})
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
return results.map(({ scans, findings, ...dataSource }) => {
|
||||
const lastScan =
|
||||
scans && scans.length
|
||||
? scans.reduce((latest, current) => {
|
||||
return new Date(current.createdAt) > new Date(latest.createdAt) ? current : latest;
|
||||
})
|
||||
: null;
|
||||
|
||||
return {
|
||||
...dataSource,
|
||||
lastScanStatus: lastScan?.status ?? null,
|
||||
lastScanStatusMessage: lastScan?.statusMessage ?? null,
|
||||
lastScannedAt: lastScan?.createdAt ?? null,
|
||||
unresolvedFindings: scans.length ? findings.length : null
|
||||
};
|
||||
});
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find with Details - Secret Scanning Data Source" });
|
||||
}
|
||||
};
|
||||
|
||||
const findResourcesWithDetails = async (filter: Parameters<(typeof resourceOrm)["find"]>[0], tx?: Knex) => {
|
||||
try {
|
||||
// TODO (scott): this query will probably need to be optimized
|
||||
|
||||
const resources = await (tx || db.replicaNode())(TableName.SecretScanningResource)
|
||||
.where((qb) => {
|
||||
if (filter)
|
||||
void qb.where(buildFindFilter(prependTableNameToFindFilter(TableName.SecretScanningResource, filter)));
|
||||
})
|
||||
.leftJoin(
|
||||
TableName.SecretScanningScan,
|
||||
`${TableName.SecretScanningScan}.resourceId`,
|
||||
`${TableName.SecretScanningResource}.id`
|
||||
)
|
||||
.leftJoin(
|
||||
TableName.SecretScanningFinding,
|
||||
`${TableName.SecretScanningFinding}.scanId`,
|
||||
`${TableName.SecretScanningScan}.id`
|
||||
)
|
||||
.where((qb) => {
|
||||
void qb
|
||||
.where(`${TableName.SecretScanningFinding}.status`, SecretScanningFindingStatus.Unresolved)
|
||||
.orWhereNull(`${TableName.SecretScanningFinding}.status`);
|
||||
})
|
||||
.select(selectAllTableCols(TableName.SecretScanningResource))
|
||||
.select(
|
||||
db.ref("id").withSchema(TableName.SecretScanningScan).as("scanId"),
|
||||
db.ref("status").withSchema(TableName.SecretScanningScan).as("scanStatus"),
|
||||
db.ref("type").withSchema(TableName.SecretScanningScan).as("scanType"),
|
||||
db.ref("statusMessage").withSchema(TableName.SecretScanningScan).as("scanStatusMessage"),
|
||||
db.ref("createdAt").withSchema(TableName.SecretScanningScan).as("scanCreatedAt"),
|
||||
db.ref("status").withSchema(TableName.SecretScanningFinding).as("findingStatus"),
|
||||
db.ref("id").withSchema(TableName.SecretScanningFinding).as("findingId")
|
||||
);
|
||||
|
||||
if (!resources.length) return [];
|
||||
|
||||
const results = sqlNestRelationships({
|
||||
data: resources,
|
||||
key: "id",
|
||||
parentMapper: (resource) => SecretScanningResourcesSchema.parse(resource),
|
||||
childrenMapper: [
|
||||
{
|
||||
key: "scanId",
|
||||
label: "scans" as const,
|
||||
mapper: ({ scanId, scanCreatedAt, scanStatus, scanStatusMessage, scanType }) => ({
|
||||
id: scanId,
|
||||
type: scanType,
|
||||
createdAt: scanCreatedAt,
|
||||
status: scanStatus,
|
||||
statusMessage: scanStatusMessage
|
||||
})
|
||||
},
|
||||
{
|
||||
key: "findingId",
|
||||
label: "findings" as const,
|
||||
mapper: ({ findingId }) => ({
|
||||
id: findingId
|
||||
})
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
return results.map(({ scans, findings, ...resource }) => {
|
||||
const lastScan =
|
||||
scans && scans.length
|
||||
? scans.reduce((latest, current) => {
|
||||
return new Date(current.createdAt) > new Date(latest.createdAt) ? current : latest;
|
||||
})
|
||||
: null;
|
||||
|
||||
return {
|
||||
...resource,
|
||||
lastScanStatus: lastScan?.status ?? null,
|
||||
lastScanStatusMessage: lastScan?.statusMessage ?? null,
|
||||
lastScannedAt: lastScan?.createdAt ?? null,
|
||||
unresolvedFindings: findings?.length ?? 0
|
||||
};
|
||||
});
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find with Details - Secret Scanning Resource" });
|
||||
}
|
||||
};
|
||||
|
||||
const findScansWithDetailsByDataSourceId = async (dataSourceId: string, tx?: Knex) => {
|
||||
try {
|
||||
// TODO (scott): this query will probably need to be optimized
|
||||
|
||||
const scans = await (tx || db.replicaNode())(TableName.SecretScanningScan)
|
||||
.leftJoin(
|
||||
TableName.SecretScanningResource,
|
||||
`${TableName.SecretScanningResource}.id`,
|
||||
`${TableName.SecretScanningScan}.resourceId`
|
||||
)
|
||||
.where(`${TableName.SecretScanningResource}.dataSourceId`, dataSourceId)
|
||||
.leftJoin(
|
||||
TableName.SecretScanningFinding,
|
||||
`${TableName.SecretScanningFinding}.scanId`,
|
||||
`${TableName.SecretScanningScan}.id`
|
||||
)
|
||||
.select(selectAllTableCols(TableName.SecretScanningScan))
|
||||
.select(
|
||||
db.ref("status").withSchema(TableName.SecretScanningFinding).as("findingStatus"),
|
||||
db.ref("id").withSchema(TableName.SecretScanningFinding).as("findingId"),
|
||||
db.ref("name").withSchema(TableName.SecretScanningResource).as("resourceName")
|
||||
);
|
||||
|
||||
if (!scans.length) return [];
|
||||
|
||||
const results = sqlNestRelationships({
|
||||
data: scans,
|
||||
key: "id",
|
||||
parentMapper: (scan) => SecretScanningScansSchema.parse(scan),
|
||||
childrenMapper: [
|
||||
{
|
||||
key: "findingId",
|
||||
label: "findings" as const,
|
||||
mapper: ({ findingId, findingStatus }) => ({
|
||||
id: findingId,
|
||||
status: findingStatus
|
||||
})
|
||||
},
|
||||
{
|
||||
key: "resourceId",
|
||||
label: "resources" as const,
|
||||
mapper: ({ resourceName }) => ({
|
||||
name: resourceName
|
||||
})
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
return results.map(({ findings, resources, ...scan }) => {
|
||||
return {
|
||||
...scan,
|
||||
unresolvedFindings:
|
||||
findings?.filter((finding) => finding.status === SecretScanningFindingStatus.Unresolved).length ?? 0,
|
||||
resolvedFindings:
|
||||
findings?.filter((finding) => finding.status !== SecretScanningFindingStatus.Unresolved).length ?? 0,
|
||||
resourceName: resources[0].name
|
||||
};
|
||||
});
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find with Details By Data Source ID - Secret Scanning Scan" });
|
||||
}
|
||||
};
|
||||
|
||||
const findScansByDataSourceId = async (dataSourceId: string, tx?: Knex) => {
|
||||
try {
|
||||
const scans = await (tx || db.replicaNode())(TableName.SecretScanningScan)
|
||||
.leftJoin(
|
||||
TableName.SecretScanningResource,
|
||||
`${TableName.SecretScanningResource}.id`,
|
||||
`${TableName.SecretScanningScan}.resourceId`
|
||||
)
|
||||
.where(`${TableName.SecretScanningResource}.dataSourceId`, dataSourceId)
|
||||
|
||||
.select(selectAllTableCols(TableName.SecretScanningScan));
|
||||
|
||||
return scans;
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find By Data Source ID - Secret Scanning Scan" });
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
dataSources: {
|
||||
...dataSourceOrm,
|
||||
find: findDataSource,
|
||||
findById: findDataSourceById,
|
||||
findOne: findOneDataSource,
|
||||
create: createDataSource,
|
||||
updateById: updateDataSourceById,
|
||||
deleteById: deleteDataSourceById,
|
||||
findWithDetails: findDataSourceWithDetails
|
||||
},
|
||||
resources: {
|
||||
...resourceOrm,
|
||||
findWithDetails: findResourcesWithDetails
|
||||
},
|
||||
scans: {
|
||||
...scanOrm,
|
||||
findWithDetailsByDataSourceId: findScansWithDetailsByDataSourceId,
|
||||
findByDataSourceId: findScansByDataSourceId
|
||||
},
|
||||
findings: findingOrm,
|
||||
configs: configOrm
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,33 @@
|
||||
export enum SecretScanningDataSource {
|
||||
GitHub = "github"
|
||||
}
|
||||
|
||||
export enum SecretScanningScanStatus {
|
||||
Completed = "completed",
|
||||
Failed = "failed",
|
||||
Queued = "queued",
|
||||
Scanning = "scanning"
|
||||
}
|
||||
|
||||
export enum SecretScanningScanType {
|
||||
FullScan = "full-scan",
|
||||
DiffScan = "diff-scan"
|
||||
}
|
||||
|
||||
export enum SecretScanningFindingStatus {
|
||||
Resolved = "resolved",
|
||||
Unresolved = "unresolved",
|
||||
FalsePositive = "false-positive",
|
||||
Ignore = "ignore"
|
||||
}
|
||||
|
||||
export enum SecretScanningResource {
|
||||
Repository = "repository",
|
||||
Project = "project"
|
||||
}
|
||||
|
||||
export enum SecretScanningFindingSeverity {
|
||||
High = "high",
|
||||
Medium = "medium",
|
||||
Low = "low"
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { GitHubSecretScanningFactory } from "@app/ee/services/secret-scanning-v2/github/github-secret-scanning-factory";
|
||||
|
||||
import { SecretScanningDataSource } from "./secret-scanning-v2-enums";
|
||||
import {
|
||||
TQueueSecretScanningResourceDiffScan,
|
||||
TSecretScanningDataSourceCredentials,
|
||||
TSecretScanningDataSourceWithConnection,
|
||||
TSecretScanningFactory
|
||||
} from "./secret-scanning-v2-types";
|
||||
|
||||
type TSecretScanningFactoryImplementation = TSecretScanningFactory<
|
||||
TSecretScanningDataSourceWithConnection,
|
||||
TSecretScanningDataSourceCredentials,
|
||||
TQueueSecretScanningResourceDiffScan["payload"]
|
||||
>;
|
||||
|
||||
export const SECRET_SCANNING_FACTORY_MAP: Record<SecretScanningDataSource, TSecretScanningFactoryImplementation> = {
|
||||
[SecretScanningDataSource.GitHub]: GitHubSecretScanningFactory as TSecretScanningFactoryImplementation
|
||||
};
|
||||
@@ -0,0 +1,140 @@
|
||||
import { AxiosError } from "axios";
|
||||
import { exec } from "child_process";
|
||||
import RE2 from "re2";
|
||||
|
||||
import { readFindingsFile } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
|
||||
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
|
||||
import { GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/github";
|
||||
import { titleCaseToCamelCase } from "@app/lib/fn";
|
||||
|
||||
import { SecretScanningDataSource, SecretScanningFindingSeverity } from "./secret-scanning-v2-enums";
|
||||
import { TCloneRepository, TGetFindingsPayload, TSecretScanningDataSourceListItem } from "./secret-scanning-v2-types";
|
||||
|
||||
const SECRET_SCANNING_SOURCE_LIST_OPTIONS: Record<SecretScanningDataSource, TSecretScanningDataSourceListItem> = {
|
||||
[SecretScanningDataSource.GitHub]: GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION
|
||||
};
|
||||
|
||||
export const listSecretScanningDataSourceOptions = () => {
|
||||
return Object.values(SECRET_SCANNING_SOURCE_LIST_OPTIONS).sort((a, b) => a.name.localeCompare(b.name));
|
||||
};
|
||||
|
||||
export const cloneRepository = async ({ cloneUrl, repoPath }: TCloneRepository): Promise<void> => {
|
||||
const command = `git clone ${cloneUrl} ${repoPath} --bare`;
|
||||
return new Promise((resolve, reject) => {
|
||||
exec(command, (error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
export function scanDirectory(inputPath: string, outputPath: string, configPath?: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const command = `cd ${inputPath} && infisical scan --exit-code=77 -r "${outputPath}" ${configPath ? `-c ${configPath}` : ""}`;
|
||||
exec(command, (error) => {
|
||||
if (error && error.code !== 77) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export const scanGitRepositoryAndGetFindings = async (
|
||||
scanPath: string,
|
||||
findingsPath: string,
|
||||
configPath?: string
|
||||
): TGetFindingsPayload => {
|
||||
await scanDirectory(scanPath, findingsPath, configPath);
|
||||
|
||||
const findingsData = JSON.parse(await readFindingsFile(findingsPath)) as SecretMatch[];
|
||||
|
||||
return findingsData.map(
|
||||
({
|
||||
// discard match and secret as we don't want to store
|
||||
Match,
|
||||
Secret,
|
||||
...finding
|
||||
}) => ({
|
||||
details: titleCaseToCamelCase(finding),
|
||||
fingerprint: `${finding.Fingerprint}:${finding.StartColumn}`,
|
||||
severity: SecretScanningFindingSeverity.High,
|
||||
rule: finding.RuleID
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
export const replaceNonChangesWithNewlines = (patch: string) => {
|
||||
return patch
|
||||
.split("\n")
|
||||
.map((line) => {
|
||||
// Keep added lines (remove the + prefix)
|
||||
if (line.startsWith("+") && !line.startsWith("+++")) {
|
||||
return line.substring(1);
|
||||
}
|
||||
|
||||
// Replace everything else with newlines to maintain line positioning
|
||||
|
||||
return "";
|
||||
})
|
||||
.join("\n");
|
||||
};
|
||||
|
||||
const HunkHeaderRegex = new RE2(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/);
|
||||
|
||||
export const convertPatchLineToFileLineNumber = (patch: string, patchLineNumber: number) => {
|
||||
const lines = patch.split("\n");
|
||||
let currentPatchLine = 0;
|
||||
let currentNewLine = 0;
|
||||
|
||||
for (const line of lines) {
|
||||
currentPatchLine += 1;
|
||||
|
||||
// Hunk header: @@ -a,b +c,d @@
|
||||
const hunkHeaderMatch = HunkHeaderRegex.match(line);
|
||||
if (hunkHeaderMatch) {
|
||||
const startLine = parseInt(hunkHeaderMatch[1], 10);
|
||||
currentNewLine = startLine;
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
if (currentPatchLine === patchLineNumber) {
|
||||
return currentNewLine;
|
||||
}
|
||||
|
||||
if (line.startsWith("+++")) {
|
||||
// eslint-disable-next-line no-continue
|
||||
continue; // skip file metadata lines
|
||||
}
|
||||
|
||||
// Advance only if the line exists in the new file
|
||||
if (line.startsWith("+") || line.startsWith(" ")) {
|
||||
currentNewLine += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return currentNewLine;
|
||||
};
|
||||
|
||||
const MAX_MESSAGE_LENGTH = 1024;
|
||||
|
||||
export const parseScanErrorMessage = (err: unknown): string => {
|
||||
let errorMessage: string;
|
||||
|
||||
if (err instanceof AxiosError) {
|
||||
errorMessage = err?.response?.data
|
||||
? JSON.stringify(err?.response?.data)
|
||||
: (err?.message ?? "An unknown error occurred.");
|
||||
} else {
|
||||
errorMessage = (err as Error)?.message || "An unknown error occurred.";
|
||||
}
|
||||
|
||||
return errorMessage.length <= MAX_MESSAGE_LENGTH
|
||||
? errorMessage
|
||||
: `${errorMessage.substring(0, MAX_MESSAGE_LENGTH - 3)}...`;
|
||||
};
|
||||
@@ -0,0 +1,14 @@
|
||||
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
|
||||
|
||||
export const SECRET_SCANNING_DATA_SOURCE_NAME_MAP: Record<SecretScanningDataSource, string> = {
|
||||
[SecretScanningDataSource.GitHub]: "GitHub"
|
||||
};
|
||||
|
||||
export const SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP: Record<SecretScanningDataSource, AppConnection> = {
|
||||
[SecretScanningDataSource.GitHub]: AppConnection.GitHubRadar
|
||||
};
|
||||
|
||||
export const AUTO_SYNC_DESCRIPTION_HELPER: Record<SecretScanningDataSource, { verb: string; noun: string }> = {
|
||||
[SecretScanningDataSource.GitHub]: { verb: "push", noun: "repositories" }
|
||||
};
|
||||
@@ -0,0 +1,626 @@
|
||||
import { join } from "path";
|
||||
|
||||
import { ProjectMembershipRole, TSecretScanningFindings } from "@app/db/schemas";
|
||||
import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service";
|
||||
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
|
||||
import {
|
||||
createTempFolder,
|
||||
deleteTempFolder,
|
||||
writeTextToFile
|
||||
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
|
||||
import {
|
||||
parseScanErrorMessage,
|
||||
scanGitRepositoryAndGetFindings
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
|
||||
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { BadRequestError, InternalServerError } from "@app/lib/errors";
|
||||
import { logger } from "@app/lib/logger";
|
||||
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
|
||||
import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns";
|
||||
import { TAppConnection } from "@app/services/app-connection/app-connection-types";
|
||||
import { ActorType } from "@app/services/auth/auth-type";
|
||||
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
|
||||
import { TProjectDALFactory } from "@app/services/project/project-dal";
|
||||
import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal";
|
||||
import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
|
||||
|
||||
import { TSecretScanningV2DALFactory } from "./secret-scanning-v2-dal";
|
||||
import {
|
||||
SecretScanningDataSource,
|
||||
SecretScanningResource,
|
||||
SecretScanningScanStatus,
|
||||
SecretScanningScanType
|
||||
} from "./secret-scanning-v2-enums";
|
||||
import { SECRET_SCANNING_FACTORY_MAP } from "./secret-scanning-v2-factory";
|
||||
import {
|
||||
TFindingsPayload,
|
||||
TQueueSecretScanningDataSourceFullScan,
|
||||
TQueueSecretScanningResourceDiffScan,
|
||||
TQueueSecretScanningSendNotification,
|
||||
TSecretScanningDataSourceWithConnection
|
||||
} from "./secret-scanning-v2-types";
|
||||
|
||||
type TSecretRotationV2QueueServiceFactoryDep = {
|
||||
queueService: TQueueServiceFactory;
|
||||
secretScanningV2DAL: TSecretScanningV2DALFactory;
|
||||
smtpService: Pick<TSmtpService, "sendMail">;
|
||||
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
|
||||
projectDAL: Pick<TProjectDALFactory, "findById">;
|
||||
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
|
||||
auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
|
||||
keyStore: Pick<TKeyStoreFactory, "acquireLock" | "getItem">;
|
||||
};
|
||||
|
||||
export type TSecretScanningV2QueueServiceFactory = Awaited<ReturnType<typeof secretScanningV2QueueServiceFactory>>;
|
||||
|
||||
export const secretScanningV2QueueServiceFactory = async ({
|
||||
queueService,
|
||||
secretScanningV2DAL,
|
||||
projectMembershipDAL,
|
||||
projectDAL,
|
||||
smtpService,
|
||||
kmsService,
|
||||
auditLogService,
|
||||
keyStore
|
||||
}: TSecretRotationV2QueueServiceFactoryDep) => {
|
||||
const queueDataSourceFullScan = async (
|
||||
dataSource: TSecretScanningDataSourceWithConnection,
|
||||
resourceExternalId?: string
|
||||
) => {
|
||||
try {
|
||||
const { type } = dataSource;
|
||||
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[type]();
|
||||
|
||||
const rawResources = await factory.listRawResources(dataSource);
|
||||
|
||||
let filteredRawResources = rawResources;
|
||||
|
||||
// TODO: should add individual resource fetch to factory
|
||||
if (resourceExternalId) {
|
||||
filteredRawResources = rawResources.filter((resource) => resource.externalId === resourceExternalId);
|
||||
}
|
||||
|
||||
if (!filteredRawResources.length) {
|
||||
throw new BadRequestError({
|
||||
message: `${resourceExternalId ? `Resource with "ID" ${resourceExternalId} could not be found.` : "Data source has no resources to scan"}. Ensure your data source config is correct and not filtering out scanning resources.`
|
||||
});
|
||||
}
|
||||
|
||||
for (const resource of filteredRawResources) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
if (await keyStore.getItem(KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId))) {
|
||||
throw new BadRequestError({ message: `A scan is already in progress for resource "${resource.name}"` });
|
||||
}
|
||||
}
|
||||
|
||||
await secretScanningV2DAL.resources.transaction(async (tx) => {
|
||||
const resources = await secretScanningV2DAL.resources.upsert(
|
||||
filteredRawResources.map((rawResource) => ({
|
||||
...rawResource,
|
||||
dataSourceId: dataSource.id
|
||||
})),
|
||||
["externalId", "dataSourceId"],
|
||||
tx
|
||||
);
|
||||
|
||||
const scans = await secretScanningV2DAL.scans.insertMany(
|
||||
resources.map((resource) => ({
|
||||
resourceId: resource.id,
|
||||
type: SecretScanningScanType.FullScan
|
||||
})),
|
||||
tx
|
||||
);
|
||||
|
||||
for (const scan of scans) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2FullScan, {
|
||||
scanId: scan.id,
|
||||
resourceId: scan.resourceId,
|
||||
dataSourceId: dataSource.id
|
||||
});
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(error, `Failed to queue full-scan for data source with ID "${dataSource.id}"`);
|
||||
|
||||
if (error instanceof BadRequestError) throw error;
|
||||
|
||||
throw new InternalServerError({ message: `Failed to queue scan: ${(error as Error).message}` });
|
||||
}
|
||||
};
|
||||
|
||||
await queueService.startPg<QueueName.SecretScanningV2>(
|
||||
QueueJobs.SecretScanningV2FullScan,
|
||||
async ([job]) => {
|
||||
const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan;
|
||||
const { retryCount, retryLimit } = job;
|
||||
|
||||
const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
|
||||
|
||||
const tempFolder = await createTempFolder();
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
|
||||
|
||||
const resource = await secretScanningV2DAL.resources.findById(resourceId);
|
||||
|
||||
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
|
||||
|
||||
let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
|
||||
|
||||
try {
|
||||
try {
|
||||
lock = await keyStore.acquireLock(
|
||||
[KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
|
||||
60 * 1000 * 5
|
||||
);
|
||||
} catch (e) {
|
||||
throw new Error("Failed to acquire scanning lock.");
|
||||
}
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Scanning
|
||||
}
|
||||
);
|
||||
|
||||
let connection: TAppConnection | null = null;
|
||||
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
|
||||
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]();
|
||||
|
||||
const findingsPath = join(tempFolder, "findings.json");
|
||||
|
||||
const scanPath = await factory.getFullScanPath({
|
||||
dataSource: {
|
||||
...dataSource,
|
||||
connection
|
||||
} as TSecretScanningDataSourceWithConnection,
|
||||
resourceName: resource.name,
|
||||
tempFolder
|
||||
});
|
||||
|
||||
const config = await secretScanningV2DAL.configs.findOne({
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
let configPath: string | undefined;
|
||||
|
||||
if (config && config.content) {
|
||||
configPath = join(tempFolder, "infisical-scan.toml");
|
||||
await writeTextToFile(configPath, config.content);
|
||||
}
|
||||
|
||||
let findingsPayload: TFindingsPayload;
|
||||
switch (resource.type) {
|
||||
case SecretScanningResource.Repository:
|
||||
case SecretScanningResource.Project:
|
||||
findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
|
||||
break;
|
||||
default:
|
||||
throw new Error("Unhandled resource type");
|
||||
}
|
||||
|
||||
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
|
||||
let findings: TSecretScanningFindings[] = [];
|
||||
if (findingsPayload.length) {
|
||||
findings = await secretScanningV2DAL.findings.upsert(
|
||||
findingsPayload.map((finding) => ({
|
||||
...finding,
|
||||
projectId: dataSource.projectId,
|
||||
dataSourceName: dataSource.name,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceName: resource.name,
|
||||
resourceType: resource.type,
|
||||
scanId
|
||||
})),
|
||||
["projectId", "fingerprint"],
|
||||
tx,
|
||||
["resourceName", "dataSourceName"]
|
||||
);
|
||||
}
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Completed,
|
||||
statusMessage: null
|
||||
}
|
||||
);
|
||||
|
||||
return findings;
|
||||
});
|
||||
|
||||
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
|
||||
|
||||
if (newFindings.length) {
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Completed,
|
||||
resourceName: resource.name,
|
||||
isDiffScan: false,
|
||||
dataSource,
|
||||
numberOfSecrets: newFindings.length,
|
||||
scanId
|
||||
});
|
||||
}
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: dataSource.projectId,
|
||||
actor: {
|
||||
type: ActorType.PLATFORM,
|
||||
metadata: {}
|
||||
},
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
|
||||
metadata: {
|
||||
dataSourceId: dataSource.id,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceId: resource.id,
|
||||
resourceType: resource.type,
|
||||
scanId,
|
||||
scanStatus: SecretScanningScanStatus.Completed,
|
||||
scanType: SecretScanningScanType.FullScan,
|
||||
numberOfSecretsDetected: findingsPayload.length
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`);
|
||||
} catch (error) {
|
||||
if (retryCount === retryLimit) {
|
||||
const errorMessage = parseScanErrorMessage(error);
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
statusMessage: errorMessage
|
||||
}
|
||||
);
|
||||
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
resourceName: resource.name,
|
||||
dataSource,
|
||||
errorMessage
|
||||
});
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: dataSource.projectId,
|
||||
actor: {
|
||||
type: ActorType.PLATFORM,
|
||||
metadata: {}
|
||||
},
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
|
||||
metadata: {
|
||||
dataSourceId: dataSource.id,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceId: resource.id,
|
||||
resourceType: resource.type,
|
||||
scanId,
|
||||
scanStatus: SecretScanningScanStatus.Failed,
|
||||
scanType: SecretScanningScanType.FullScan
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
|
||||
throw error;
|
||||
} finally {
|
||||
await deleteTempFolder(tempFolder);
|
||||
await lock?.release();
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 20,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
|
||||
const queueResourceDiffScan = async ({
|
||||
payload,
|
||||
dataSourceId,
|
||||
dataSourceType
|
||||
}: Pick<TQueueSecretScanningResourceDiffScan, "payload" | "dataSourceId" | "dataSourceType">) => {
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[dataSourceType as SecretScanningDataSource]();
|
||||
|
||||
const resourcePayload = factory.getDiffScanResourcePayload(payload);
|
||||
|
||||
try {
|
||||
const { resourceId, scanId } = await secretScanningV2DAL.resources.transaction(async (tx) => {
|
||||
const [resource] = await secretScanningV2DAL.resources.upsert(
|
||||
[
|
||||
{
|
||||
...resourcePayload,
|
||||
dataSourceId
|
||||
}
|
||||
],
|
||||
["externalId", "dataSourceId"],
|
||||
tx
|
||||
);
|
||||
|
||||
const scan = await secretScanningV2DAL.scans.create(
|
||||
{
|
||||
resourceId: resource.id,
|
||||
type: SecretScanningScanType.DiffScan
|
||||
},
|
||||
tx
|
||||
);
|
||||
|
||||
return {
|
||||
resourceId: resource.id,
|
||||
scanId: scan.id
|
||||
};
|
||||
});
|
||||
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2DiffScan, {
|
||||
payload,
|
||||
dataSourceId,
|
||||
dataSourceType,
|
||||
scanId,
|
||||
resourceId
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
error,
|
||||
`secretScanningV2Queue: Failed to queue diff scan [dataSourceId=${dataSourceId}] [resourceExternalId=${resourcePayload.externalId}]`
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
await queueService.startPg<QueueName.SecretScanningV2>(
|
||||
QueueJobs.SecretScanningV2DiffScan,
|
||||
async ([job]) => {
|
||||
const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan;
|
||||
const { retryCount, retryLimit } = job;
|
||||
|
||||
const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
|
||||
|
||||
const resource = await secretScanningV2DAL.resources.findById(resourceId);
|
||||
|
||||
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
|
||||
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]();
|
||||
|
||||
const tempFolder = await createTempFolder();
|
||||
|
||||
try {
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Scanning
|
||||
}
|
||||
);
|
||||
|
||||
let connection: TAppConnection | null = null;
|
||||
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
|
||||
|
||||
const config = await secretScanningV2DAL.configs.findOne({
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
let configPath: string | undefined;
|
||||
|
||||
if (config && config.content) {
|
||||
configPath = join(tempFolder, "infisical-scan.toml");
|
||||
await writeTextToFile(configPath, config.content);
|
||||
}
|
||||
|
||||
const findingsPayload = await factory.getDiffScanFindingsPayload({
|
||||
dataSource: {
|
||||
...dataSource,
|
||||
connection
|
||||
} as TSecretScanningDataSourceWithConnection,
|
||||
resourceName: resource.name,
|
||||
payload,
|
||||
configPath
|
||||
});
|
||||
|
||||
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
|
||||
let findings: TSecretScanningFindings[] = [];
|
||||
|
||||
if (findingsPayload.length) {
|
||||
findings = await secretScanningV2DAL.findings.upsert(
|
||||
findingsPayload.map((finding) => ({
|
||||
...finding,
|
||||
projectId: dataSource.projectId,
|
||||
dataSourceName: dataSource.name,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceName: resource.name,
|
||||
resourceType: resource.type,
|
||||
scanId
|
||||
})),
|
||||
["projectId", "fingerprint"],
|
||||
tx,
|
||||
["resourceName", "dataSourceName"]
|
||||
);
|
||||
}
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Completed
|
||||
}
|
||||
);
|
||||
|
||||
return findings;
|
||||
});
|
||||
|
||||
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
|
||||
|
||||
if (newFindings.length) {
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Completed,
|
||||
resourceName: resource.name,
|
||||
isDiffScan: true,
|
||||
dataSource,
|
||||
numberOfSecrets: newFindings.length,
|
||||
scanId
|
||||
});
|
||||
}
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: dataSource.projectId,
|
||||
actor: {
|
||||
type: ActorType.PLATFORM,
|
||||
metadata: {}
|
||||
},
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
|
||||
metadata: {
|
||||
dataSourceId: dataSource.id,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceId,
|
||||
resourceType: resource.type,
|
||||
scanId,
|
||||
scanStatus: SecretScanningScanStatus.Completed,
|
||||
scanType: SecretScanningScanType.DiffScan,
|
||||
numberOfSecretsDetected: findingsPayload.length
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`);
|
||||
} catch (error) {
|
||||
if (retryCount === retryLimit) {
|
||||
const errorMessage = parseScanErrorMessage(error);
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
statusMessage: errorMessage
|
||||
}
|
||||
);
|
||||
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
resourceName: resource.name,
|
||||
dataSource,
|
||||
errorMessage
|
||||
});
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: dataSource.projectId,
|
||||
actor: {
|
||||
type: ActorType.PLATFORM,
|
||||
metadata: {}
|
||||
},
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
|
||||
metadata: {
|
||||
dataSourceId: dataSource.id,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceId: resource.id,
|
||||
resourceType: resource.type,
|
||||
scanId,
|
||||
scanStatus: SecretScanningScanStatus.Failed,
|
||||
scanType: SecretScanningScanType.DiffScan
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
|
||||
throw error;
|
||||
} finally {
|
||||
await deleteTempFolder(tempFolder);
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 20,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
|
||||
await queueService.startPg<QueueName.SecretScanningV2>(
|
||||
QueueJobs.SecretScanningV2SendNotification,
|
||||
async ([job]) => {
|
||||
const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification;
|
||||
|
||||
const appCfg = getConfig();
|
||||
|
||||
if (!appCfg.isSmtpConfigured) return;
|
||||
|
||||
try {
|
||||
const { projectId } = dataSource;
|
||||
|
||||
logger.info(
|
||||
`secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
|
||||
);
|
||||
|
||||
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
|
||||
const project = await projectDAL.findById(projectId);
|
||||
|
||||
const projectAdmins = projectMembers.filter((member) =>
|
||||
member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
|
||||
);
|
||||
|
||||
const timestamp = new Date().toISOString();
|
||||
|
||||
await smtpService.sendMail({
|
||||
recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
|
||||
template:
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? SmtpTemplates.SecretScanningV2SecretsDetected
|
||||
: SmtpTemplates.SecretScanningV2ScanFailed,
|
||||
subjectLine:
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? "Incident Alert: Secret(s) Leaked"
|
||||
: `Secret Scanning Failed`,
|
||||
substitutions:
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? {
|
||||
authorName: "Jim",
|
||||
authorEmail: "[email protected]",
|
||||
resourceName,
|
||||
numberOfSecrets: payload.numberOfSecrets,
|
||||
isDiffScan: payload.isDiffScan,
|
||||
url: encodeURI(
|
||||
`${appCfg.SITE_URL}/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
|
||||
),
|
||||
timestamp
|
||||
}
|
||||
: {
|
||||
dataSourceName: dataSource.name,
|
||||
resourceName,
|
||||
projectName: project.name,
|
||||
timestamp,
|
||||
errorMessage: payload.errorMessage,
|
||||
url: encodeURI(
|
||||
`${appCfg.SITE_URL}/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
|
||||
)
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
error,
|
||||
`secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 5,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
queueDataSourceFullScan,
|
||||
queueResourceDiffScan
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,99 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { SecretScanningDataSourcesSchema, SecretScanningFindingsSchema } from "@app/db/schemas";
|
||||
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps";
|
||||
import { SecretScanningDataSources } from "@app/lib/api-docs";
|
||||
import { slugSchema } from "@app/server/lib/schemas";
|
||||
|
||||
type SecretScanningDataSourceSchemaOpts = {
|
||||
type: SecretScanningDataSource;
|
||||
isConnectionRequired: boolean;
|
||||
};
|
||||
|
||||
export const BaseSecretScanningDataSourceSchema = ({
|
||||
type,
|
||||
isConnectionRequired
|
||||
}: SecretScanningDataSourceSchemaOpts) =>
|
||||
SecretScanningDataSourcesSchema.omit({
|
||||
// unique to provider
|
||||
type: true,
|
||||
connectionId: true,
|
||||
config: true,
|
||||
encryptedCredentials: true
|
||||
}).extend({
|
||||
type: z.literal(type),
|
||||
connectionId: isConnectionRequired ? z.string().uuid() : z.null(),
|
||||
connection: isConnectionRequired
|
||||
? z.object({
|
||||
app: z.literal(SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[type]),
|
||||
name: z.string(),
|
||||
id: z.string().uuid()
|
||||
})
|
||||
: z.null()
|
||||
});
|
||||
|
||||
export const BaseCreateSecretScanningDataSourceSchema = ({
|
||||
type,
|
||||
isConnectionRequired
|
||||
}: SecretScanningDataSourceSchemaOpts) =>
|
||||
z.object({
|
||||
name: slugSchema({ field: "name" }).describe(SecretScanningDataSources.CREATE(type).name),
|
||||
projectId: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1, "Project ID required")
|
||||
.describe(SecretScanningDataSources.CREATE(type).projectId),
|
||||
description: z
|
||||
.string()
|
||||
.trim()
|
||||
.max(256, "Description cannot exceed 256 characters")
|
||||
.nullish()
|
||||
.describe(SecretScanningDataSources.CREATE(type).description),
|
||||
connectionId: isConnectionRequired
|
||||
? z.string().uuid().describe(SecretScanningDataSources.CREATE(type).connectionId)
|
||||
: z.undefined(),
|
||||
isAutoScanEnabled: z
|
||||
.boolean()
|
||||
.optional()
|
||||
.default(true)
|
||||
.describe(SecretScanningDataSources.CREATE(type).isAutoScanEnabled)
|
||||
});
|
||||
|
||||
export const BaseUpdateSecretScanningDataSourceSchema = (type: SecretScanningDataSource) =>
|
||||
z.object({
|
||||
name: slugSchema({ field: "name" }).describe(SecretScanningDataSources.UPDATE(type).name).optional(),
|
||||
description: z
|
||||
.string()
|
||||
.trim()
|
||||
.max(256, "Description cannot exceed 256 characters")
|
||||
.nullish()
|
||||
.describe(SecretScanningDataSources.UPDATE(type).description),
|
||||
isAutoScanEnabled: z.boolean().optional().describe(SecretScanningDataSources.UPDATE(type).isAutoScanEnabled)
|
||||
});
|
||||
|
||||
export const GitRepositoryScanFindingDetailsSchema = z.object({
|
||||
description: z.string(),
|
||||
startLine: z.number(),
|
||||
endLine: z.number(),
|
||||
startColumn: z.number(),
|
||||
endColumn: z.number(),
|
||||
file: z.string(),
|
||||
link: z.string(),
|
||||
symlinkFile: z.string(),
|
||||
commit: z.string(),
|
||||
entropy: z.number(),
|
||||
author: z.string(),
|
||||
email: z.string(),
|
||||
date: z.string(),
|
||||
message: z.string(),
|
||||
tags: z.string().array(),
|
||||
ruleID: z.string(),
|
||||
fingerprint: z.string()
|
||||
});
|
||||
|
||||
export const BaseSecretScanningFindingSchema = SecretScanningFindingsSchema.omit({
|
||||
dataSourceType: true,
|
||||
resourceType: true,
|
||||
details: true
|
||||
});
|
||||
@@ -0,0 +1,875 @@
|
||||
import { ForbiddenError } from "@casl/ability";
|
||||
import { join } from "path";
|
||||
|
||||
import { ActionProjectType } from "@app/db/schemas";
|
||||
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
|
||||
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
|
||||
import {
|
||||
ProjectPermissionSecretScanningConfigActions,
|
||||
ProjectPermissionSecretScanningDataSourceActions,
|
||||
ProjectPermissionSecretScanningFindingActions,
|
||||
ProjectPermissionSub
|
||||
} from "@app/ee/services/permission/project-permission";
|
||||
import {
|
||||
createTempFolder,
|
||||
deleteTempFolder,
|
||||
scanContentAndGetFindings,
|
||||
writeTextToFile
|
||||
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
|
||||
import { githubSecretScanningService } from "@app/ee/services/secret-scanning-v2/github/github-secret-scanning-service";
|
||||
import { SecretScanningFindingStatus } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
import { SECRET_SCANNING_FACTORY_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-factory";
|
||||
import { listSecretScanningDataSourceOptions } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
|
||||
import {
|
||||
SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP,
|
||||
SECRET_SCANNING_DATA_SOURCE_NAME_MAP
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps";
|
||||
import {
|
||||
TCreateSecretScanningDataSourceDTO,
|
||||
TDeleteSecretScanningDataSourceDTO,
|
||||
TFindSecretScanningDataSourceByIdDTO,
|
||||
TFindSecretScanningDataSourceByNameDTO,
|
||||
TListSecretScanningDataSourcesByProjectId,
|
||||
TSecretScanningDataSource,
|
||||
TSecretScanningDataSourceWithConnection,
|
||||
TSecretScanningDataSourceWithDetails,
|
||||
TSecretScanningFinding,
|
||||
TSecretScanningResourceWithDetails,
|
||||
TSecretScanningScanWithDetails,
|
||||
TTriggerSecretScanningDataSourceDTO,
|
||||
TUpdateSecretScanningDataSourceDTO,
|
||||
TUpdateSecretScanningFindingDTO,
|
||||
TUpsertSecretScanningConfigDTO
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
|
||||
import { DatabaseErrorCode } from "@app/lib/error-codes";
|
||||
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
|
||||
import { OrgServiceActor } from "@app/lib/types";
|
||||
import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns";
|
||||
import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service";
|
||||
import { TAppConnection } from "@app/services/app-connection/app-connection-types";
|
||||
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
|
||||
import { KmsDataKey } from "@app/services/kms/kms-types";
|
||||
|
||||
import { TSecretScanningV2DALFactory } from "./secret-scanning-v2-dal";
|
||||
import { TSecretScanningV2QueueServiceFactory } from "./secret-scanning-v2-queue";
|
||||
|
||||
export type TSecretScanningV2ServiceFactoryDep = {
|
||||
secretScanningV2DAL: TSecretScanningV2DALFactory;
|
||||
appConnectionService: Pick<TAppConnectionServiceFactory, "connectAppConnectionById">;
|
||||
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission" | "getOrgPermission">;
|
||||
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
|
||||
secretScanningV2Queue: Pick<
|
||||
TSecretScanningV2QueueServiceFactory,
|
||||
"queueDataSourceFullScan" | "queueResourceDiffScan"
|
||||
>;
|
||||
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
|
||||
};
|
||||
|
||||
export type TSecretScanningV2ServiceFactory = ReturnType<typeof secretScanningV2ServiceFactory>;
|
||||
|
||||
export const secretScanningV2ServiceFactory = ({
|
||||
secretScanningV2DAL,
|
||||
permissionService,
|
||||
appConnectionService,
|
||||
licenseService,
|
||||
secretScanningV2Queue,
|
||||
kmsService
|
||||
}: TSecretScanningV2ServiceFactoryDep) => {
|
||||
const $checkListSecretScanningDataSourcesByProjectIdPermissions = async (
|
||||
projectId: string,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Data Sources due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.Read,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
};
|
||||
|
||||
const listSecretScanningDataSourcesByProjectId = async (
|
||||
{ projectId, type }: TListSecretScanningDataSourcesByProjectId,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
await $checkListSecretScanningDataSourcesByProjectIdPermissions(projectId, actor);
|
||||
|
||||
const dataSources = await secretScanningV2DAL.dataSources.find({
|
||||
...(type && { type }),
|
||||
projectId
|
||||
});
|
||||
|
||||
return dataSources as TSecretScanningDataSource[];
|
||||
};
|
||||
|
||||
const listSecretScanningDataSourcesWithDetailsByProjectId = async (
|
||||
{ projectId, type }: TListSecretScanningDataSourcesByProjectId,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
await $checkListSecretScanningDataSourcesByProjectIdPermissions(projectId, actor);
|
||||
|
||||
const dataSources = await secretScanningV2DAL.dataSources.findWithDetails({
|
||||
...(type && { type }),
|
||||
projectId
|
||||
});
|
||||
|
||||
return dataSources as TSecretScanningDataSourceWithDetails[];
|
||||
};
|
||||
|
||||
const findSecretScanningDataSourceById = async (
|
||||
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.Read,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
return dataSource as TSecretScanningDataSource;
|
||||
};
|
||||
|
||||
const findSecretScanningDataSourceByName = async (
|
||||
{ type, sourceName, projectId }: TFindSecretScanningDataSourceByNameDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
// we prevent conflicting names within a folder
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findOne({
|
||||
name: sourceName,
|
||||
projectId
|
||||
});
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with name "${sourceName}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.Read,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSource.id}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
return dataSource as TSecretScanningDataSource;
|
||||
};
|
||||
|
||||
const createSecretScanningDataSource = async (
|
||||
payload: TCreateSecretScanningDataSourceDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to create Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: payload.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.Create,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
let connection: TAppConnection | null = null;
|
||||
if (payload.connectionId) {
|
||||
// validates permission to connect and app is valid for data source
|
||||
connection = await appConnectionService.connectAppConnectionById(
|
||||
SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[payload.type],
|
||||
payload.connectionId,
|
||||
actor
|
||||
);
|
||||
}
|
||||
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[payload.type]();
|
||||
|
||||
try {
|
||||
const createdDataSource = await factory.initialize(
|
||||
{
|
||||
payload,
|
||||
connection: connection as TSecretScanningDataSourceWithConnection["connection"],
|
||||
secretScanningV2DAL
|
||||
},
|
||||
async ({ credentials, externalId }) => {
|
||||
let encryptedCredentials: Buffer | null = null;
|
||||
|
||||
if (credentials) {
|
||||
const { encryptor } = await kmsService.createCipherPairWithDataKey({
|
||||
type: KmsDataKey.SecretManager,
|
||||
projectId: payload.projectId
|
||||
});
|
||||
|
||||
const { cipherTextBlob } = encryptor({
|
||||
plainText: Buffer.from(JSON.stringify(credentials))
|
||||
});
|
||||
|
||||
encryptedCredentials = cipherTextBlob;
|
||||
}
|
||||
|
||||
return secretScanningV2DAL.dataSources.transaction(async (tx) => {
|
||||
const dataSource = await secretScanningV2DAL.dataSources.create(
|
||||
{
|
||||
encryptedCredentials,
|
||||
externalId,
|
||||
...payload
|
||||
},
|
||||
tx
|
||||
);
|
||||
|
||||
await factory.postInitialization({
|
||||
payload,
|
||||
connection: connection as TSecretScanningDataSourceWithConnection["connection"],
|
||||
dataSourceId: dataSource.id,
|
||||
credentials
|
||||
});
|
||||
|
||||
return dataSource;
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
if (payload.isAutoScanEnabled) {
|
||||
try {
|
||||
await secretScanningV2Queue.queueDataSourceFullScan({
|
||||
...createdDataSource,
|
||||
connection
|
||||
} as TSecretScanningDataSourceWithConnection);
|
||||
} catch {
|
||||
// silently fail, don't want to block creation, they'll try scanning when they don't see anything and get the error
|
||||
}
|
||||
}
|
||||
|
||||
return createdDataSource as TSecretScanningDataSource;
|
||||
} catch (err) {
|
||||
if (err instanceof DatabaseError && (err.error as { code: string })?.code === DatabaseErrorCode.UniqueViolation) {
|
||||
throw new BadRequestError({
|
||||
message: `A Secret Scanning Data Source with the name "${payload.name}" already exists for the project with ID "${payload.projectId}"`
|
||||
});
|
||||
}
|
||||
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
const updateSecretScanningDataSource = async (
|
||||
{ type, dataSourceId, ...payload }: TUpdateSecretScanningDataSourceDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to update Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.Edit,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
try {
|
||||
const updatedDataSource = await secretScanningV2DAL.dataSources.updateById(dataSourceId, payload);
|
||||
|
||||
return updatedDataSource as TSecretScanningDataSource;
|
||||
} catch (err) {
|
||||
if (err instanceof DatabaseError && (err.error as { code: string })?.code === DatabaseErrorCode.UniqueViolation) {
|
||||
throw new BadRequestError({
|
||||
message: `A Secret Scanning Data Source with the name "${payload.name}" already exists for the project with ID "${dataSource.projectId}"`
|
||||
});
|
||||
}
|
||||
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
const deleteSecretScanningDataSource = async (
|
||||
{ type, dataSourceId }: TDeleteSecretScanningDataSourceDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to delete Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.Delete,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
// TODO: clean up webhooks
|
||||
|
||||
await secretScanningV2DAL.dataSources.deleteById(dataSourceId);
|
||||
|
||||
return dataSource as TSecretScanningDataSource;
|
||||
};
|
||||
|
||||
const triggerSecretScanningDataSourceScan = async (
|
||||
{ type, dataSourceId, resourceId }: TTriggerSecretScanningDataSourceDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to trigger scan for Secret Scanning Data Source due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.TriggerScans,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
let connection: TAppConnection | null = null;
|
||||
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
|
||||
|
||||
let resourceExternalId: string | undefined;
|
||||
|
||||
if (resourceId) {
|
||||
const resource = await secretScanningV2DAL.resources.findOne({ id: resourceId, dataSourceId });
|
||||
if (!resource) {
|
||||
throw new NotFoundError({
|
||||
message: `Could not find Secret Scanning Resource with ID "${resourceId}" for Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
}
|
||||
resourceExternalId = resource.externalId;
|
||||
}
|
||||
|
||||
await secretScanningV2Queue.queueDataSourceFullScan(
|
||||
{
|
||||
...dataSource,
|
||||
connection
|
||||
} as TSecretScanningDataSourceWithConnection,
|
||||
resourceExternalId
|
||||
);
|
||||
|
||||
return dataSource as TSecretScanningDataSource;
|
||||
};
|
||||
|
||||
const listSecretScanningResourcesByDataSourceId = async (
|
||||
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Resources due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadResources,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
const resources = await secretScanningV2DAL.resources.find({
|
||||
dataSourceId
|
||||
});
|
||||
|
||||
return { resources, projectId: dataSource.projectId };
|
||||
};
|
||||
|
||||
const listSecretScanningScansByDataSourceId = async (
|
||||
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Resources due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
const scans = await secretScanningV2DAL.scans.findByDataSourceId(dataSourceId);
|
||||
|
||||
return { scans, projectId: dataSource.projectId };
|
||||
};
|
||||
|
||||
const listSecretScanningResourcesWithDetailsByDataSourceId = async (
|
||||
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Resources due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadResources,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
const resources = await secretScanningV2DAL.resources.findWithDetails({ dataSourceId });
|
||||
|
||||
return { resources: resources as TSecretScanningResourceWithDetails[], projectId: dataSource.projectId };
|
||||
};
|
||||
|
||||
const listSecretScanningScansWithDetailsByDataSourceId = async (
|
||||
{ type, dataSourceId }: TFindSecretScanningDataSourceByIdDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Scans due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source with ID "${dataSourceId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningDataSourceActions.ReadScans,
|
||||
ProjectPermissionSub.SecretScanningDataSources
|
||||
);
|
||||
|
||||
if (type !== dataSource.type)
|
||||
throw new BadRequestError({
|
||||
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
|
||||
});
|
||||
|
||||
const scans = await secretScanningV2DAL.scans.findWithDetailsByDataSourceId(dataSourceId);
|
||||
|
||||
return { scans: scans as TSecretScanningScanWithDetails[], projectId: dataSource.projectId };
|
||||
};
|
||||
|
||||
const getSecretScanningUnresolvedFindingsCountByProjectId = async (projectId: string, actor: OrgServiceActor) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningFindingActions.Read,
|
||||
ProjectPermissionSub.SecretScanningFindings
|
||||
);
|
||||
|
||||
const [finding] = await secretScanningV2DAL.findings.find(
|
||||
{
|
||||
projectId,
|
||||
status: SecretScanningFindingStatus.Unresolved
|
||||
},
|
||||
{ count: true }
|
||||
);
|
||||
|
||||
return Number(finding?.count ?? 0);
|
||||
};
|
||||
|
||||
const listSecretScanningFindingsByProjectId = async (projectId: string, actor: OrgServiceActor) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningFindingActions.Read,
|
||||
ProjectPermissionSub.SecretScanningFindings
|
||||
);
|
||||
|
||||
const findings = await secretScanningV2DAL.findings.find({
|
||||
projectId
|
||||
});
|
||||
|
||||
return findings as TSecretScanningFinding[];
|
||||
};
|
||||
|
||||
const updateSecretScanningFindingById = async (
|
||||
{ findingId, remarks, status }: TUpdateSecretScanningFindingDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const finding = await secretScanningV2DAL.findings.findById(findingId);
|
||||
|
||||
if (!finding)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find Secret Scanning Finding with ID "${findingId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId: finding.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningFindingActions.Update,
|
||||
ProjectPermissionSub.SecretScanningFindings
|
||||
);
|
||||
|
||||
const updatedFinding = await secretScanningV2DAL.findings.updateById(findingId, {
|
||||
remarks,
|
||||
status
|
||||
});
|
||||
|
||||
return { finding: updatedFinding as TSecretScanningFinding, projectId: finding.projectId };
|
||||
};
|
||||
|
||||
const findSecretScanningConfigByProjectId = async (projectId: string, actor: OrgServiceActor) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Configuration due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningConfigActions.Read,
|
||||
ProjectPermissionSub.SecretScanningConfigs
|
||||
);
|
||||
|
||||
const config = await secretScanningV2DAL.configs.findOne({
|
||||
projectId
|
||||
});
|
||||
|
||||
return (
|
||||
config ?? { content: null, projectId, updatedAt: null } // using default config
|
||||
);
|
||||
};
|
||||
|
||||
const upsertSecretScanningConfig = async (
|
||||
{ projectId, content }: TUpsertSecretScanningConfigDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const plan = await licenseService.getPlan(actor.orgId);
|
||||
|
||||
if (!plan.secretScanning)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to access Secret Scanning Configuration due to plan restriction. Upgrade plan to enable Secret Scanning."
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretScanning,
|
||||
projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionSecretScanningConfigActions.Update,
|
||||
ProjectPermissionSub.SecretScanningConfigs
|
||||
);
|
||||
|
||||
if (content) {
|
||||
const tempFolder = await createTempFolder();
|
||||
try {
|
||||
const configPath = join(tempFolder, "infisical-scan.toml");
|
||||
await writeTextToFile(configPath, content);
|
||||
|
||||
// just checking if config parses
|
||||
await scanContentAndGetFindings("", configPath);
|
||||
} catch (e) {
|
||||
throw new BadRequestError({
|
||||
message: "Unable to parse configuration: Check syntax and formatting."
|
||||
});
|
||||
} finally {
|
||||
await deleteTempFolder(tempFolder);
|
||||
}
|
||||
}
|
||||
|
||||
const [config] = await secretScanningV2DAL.configs.upsert(
|
||||
[
|
||||
{
|
||||
projectId,
|
||||
content
|
||||
}
|
||||
],
|
||||
"projectId"
|
||||
);
|
||||
|
||||
return config;
|
||||
};
|
||||
|
||||
return {
|
||||
listSecretScanningDataSourceOptions,
|
||||
listSecretScanningDataSourcesByProjectId,
|
||||
listSecretScanningDataSourcesWithDetailsByProjectId,
|
||||
findSecretScanningDataSourceById,
|
||||
findSecretScanningDataSourceByName,
|
||||
createSecretScanningDataSource,
|
||||
updateSecretScanningDataSource,
|
||||
deleteSecretScanningDataSource,
|
||||
triggerSecretScanningDataSourceScan,
|
||||
listSecretScanningResourcesByDataSourceId,
|
||||
listSecretScanningScansByDataSourceId,
|
||||
listSecretScanningResourcesWithDetailsByDataSourceId,
|
||||
listSecretScanningScansWithDetailsByDataSourceId,
|
||||
getSecretScanningUnresolvedFindingsCountByProjectId,
|
||||
listSecretScanningFindingsByProjectId,
|
||||
updateSecretScanningFindingById,
|
||||
findSecretScanningConfigByProjectId,
|
||||
upsertSecretScanningConfig,
|
||||
github: githubSecretScanningService(secretScanningV2DAL, secretScanningV2Queue)
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,189 @@
|
||||
import {
|
||||
TSecretScanningDataSources,
|
||||
TSecretScanningFindingsInsert,
|
||||
TSecretScanningResources,
|
||||
TSecretScanningScans
|
||||
} from "@app/db/schemas";
|
||||
import {
|
||||
TGitHubDataSource,
|
||||
TGitHubDataSourceInput,
|
||||
TGitHubDataSourceListItem,
|
||||
TGitHubDataSourceWithConnection,
|
||||
TGitHubFinding,
|
||||
TQueueGitHubResourceDiffScan
|
||||
} from "@app/ee/services/secret-scanning-v2/github";
|
||||
import { TSecretScanningV2DALFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-dal";
|
||||
import {
|
||||
SecretScanningDataSource,
|
||||
SecretScanningFindingStatus,
|
||||
SecretScanningScanStatus
|
||||
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
|
||||
|
||||
export type TSecretScanningDataSource = TGitHubDataSource;
|
||||
|
||||
export type TSecretScanningDataSourceWithDetails = TSecretScanningDataSource & {
|
||||
lastScannedAt?: Date | null;
|
||||
lastScanStatus?: SecretScanningScanStatus | null;
|
||||
lastScanStatusMessage?: string | null;
|
||||
unresolvedFindings: number;
|
||||
};
|
||||
|
||||
export type TSecretScanningResourceWithDetails = TSecretScanningResources & {
|
||||
lastScannedAt?: Date | null;
|
||||
lastScanStatus?: SecretScanningScanStatus | null;
|
||||
lastScanStatusMessage?: string | null;
|
||||
unresolvedFindings: number;
|
||||
};
|
||||
|
||||
export type TSecretScanningScanWithDetails = TSecretScanningScans & {
|
||||
unresolvedFindings: number;
|
||||
resolvedFindings: number;
|
||||
resourceName: string;
|
||||
};
|
||||
|
||||
export type TSecretScanningDataSourceWithConnection = TGitHubDataSourceWithConnection;
|
||||
|
||||
export type TSecretScanningDataSourceInput = TGitHubDataSourceInput;
|
||||
|
||||
export type TSecretScanningDataSourceListItem = TGitHubDataSourceListItem;
|
||||
|
||||
export type TSecretScanningFinding = TGitHubFinding;
|
||||
|
||||
export type TListSecretScanningDataSourcesByProjectId = {
|
||||
projectId: string;
|
||||
type?: SecretScanningDataSource;
|
||||
};
|
||||
|
||||
export type TFindSecretScanningDataSourceByIdDTO = {
|
||||
dataSourceId: string;
|
||||
type: SecretScanningDataSource;
|
||||
};
|
||||
|
||||
export type TFindSecretScanningDataSourceByNameDTO = {
|
||||
sourceName: string;
|
||||
projectId: string;
|
||||
type: SecretScanningDataSource;
|
||||
};
|
||||
|
||||
export type TCreateSecretScanningDataSourceDTO = Pick<
|
||||
TSecretScanningDataSource,
|
||||
"description" | "name" | "projectId"
|
||||
> & {
|
||||
connectionId?: string;
|
||||
type: SecretScanningDataSource;
|
||||
isAutoScanEnabled?: boolean;
|
||||
config: Partial<TSecretScanningDataSourceInput["config"]>;
|
||||
};
|
||||
|
||||
export type TUpdateSecretScanningDataSourceDTO = Partial<
|
||||
Omit<TCreateSecretScanningDataSourceDTO, "projectId" | "connectionId">
|
||||
> & {
|
||||
dataSourceId: string;
|
||||
type: SecretScanningDataSource;
|
||||
};
|
||||
|
||||
export type TDeleteSecretScanningDataSourceDTO = {
|
||||
type: SecretScanningDataSource;
|
||||
dataSourceId: string;
|
||||
};
|
||||
|
||||
export type TTriggerSecretScanningDataSourceDTO = {
|
||||
type: SecretScanningDataSource;
|
||||
dataSourceId: string;
|
||||
resourceId?: string;
|
||||
};
|
||||
|
||||
export type TQueueSecretScanningDataSourceFullScan = {
|
||||
dataSourceId: string;
|
||||
resourceId: string;
|
||||
scanId: string;
|
||||
};
|
||||
|
||||
export type TQueueSecretScanningResourceDiffScan = TQueueGitHubResourceDiffScan;
|
||||
|
||||
export type TQueueSecretScanningSendNotification = {
|
||||
dataSource: TSecretScanningDataSources;
|
||||
resourceName: string;
|
||||
} & (
|
||||
| { status: SecretScanningScanStatus.Failed; errorMessage: string }
|
||||
| { status: SecretScanningScanStatus.Completed; numberOfSecrets: number; scanId: string; isDiffScan: boolean }
|
||||
);
|
||||
|
||||
export type TCloneRepository = {
|
||||
cloneUrl: string;
|
||||
repoPath: string;
|
||||
};
|
||||
|
||||
export type TSecretScanningFactoryListRawResources<T extends TSecretScanningDataSourceWithConnection> = (
|
||||
dataSource: T
|
||||
) => Promise<Pick<TSecretScanningResources, "externalId" | "name" | "type">[]>;
|
||||
|
||||
export type TSecretScanningFactoryGetDiffScanResourcePayload<
|
||||
P extends TQueueSecretScanningResourceDiffScan["payload"]
|
||||
> = (payload: P) => Pick<TSecretScanningResources, "externalId" | "name" | "type">;
|
||||
|
||||
export type TSecretScanningFactoryGetFullScanPath<T extends TSecretScanningDataSourceWithConnection> = (parameters: {
|
||||
dataSource: T;
|
||||
resourceName: string;
|
||||
tempFolder: string;
|
||||
}) => Promise<string>;
|
||||
|
||||
export type TSecretScanningFactoryGetDiffScanFindingsPayload<
|
||||
T extends TSecretScanningDataSourceWithConnection,
|
||||
P extends TQueueSecretScanningResourceDiffScan["payload"]
|
||||
> = (parameters: { dataSource: T; resourceName: string; payload: P; configPath?: string }) => Promise<TFindingsPayload>;
|
||||
|
||||
export type TSecretScanningDataSourceRaw = NonNullable<
|
||||
Awaited<ReturnType<TSecretScanningV2DALFactory["dataSources"]["findById"]>>
|
||||
>;
|
||||
|
||||
export type TSecretScanningFactoryInitialize<
|
||||
T extends TSecretScanningDataSourceWithConnection["connection"] | undefined = undefined,
|
||||
C extends TSecretScanningDataSourceCredentials = undefined
|
||||
> = (
|
||||
params: {
|
||||
payload: TCreateSecretScanningDataSourceDTO;
|
||||
connection: T;
|
||||
secretScanningV2DAL: TSecretScanningV2DALFactory;
|
||||
},
|
||||
callback: (parameters: { credentials?: C; externalId?: string }) => Promise<TSecretScanningDataSourceRaw>
|
||||
) => Promise<TSecretScanningDataSourceRaw>;
|
||||
|
||||
export type TSecretScanningFactoryPostInitialization<
|
||||
T extends TSecretScanningDataSourceWithConnection["connection"] | undefined = undefined,
|
||||
C extends TSecretScanningDataSourceCredentials = undefined
|
||||
> = (params: {
|
||||
payload: TCreateSecretScanningDataSourceDTO;
|
||||
connection: T;
|
||||
credentials: C;
|
||||
dataSourceId: string;
|
||||
}) => Promise<void>;
|
||||
|
||||
export type TSecretScanningFactory<
|
||||
T extends TSecretScanningDataSourceWithConnection,
|
||||
C extends TSecretScanningDataSourceCredentials,
|
||||
P extends TQueueSecretScanningResourceDiffScan["payload"]
|
||||
> = () => {
|
||||
listRawResources: TSecretScanningFactoryListRawResources<T>;
|
||||
getFullScanPath: TSecretScanningFactoryGetFullScanPath<T>;
|
||||
initialize: TSecretScanningFactoryInitialize<T["connection"] | undefined, C>;
|
||||
postInitialization: TSecretScanningFactoryPostInitialization<T["connection"] | undefined, C>;
|
||||
getDiffScanResourcePayload: TSecretScanningFactoryGetDiffScanResourcePayload<P>;
|
||||
getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<T, P>;
|
||||
};
|
||||
|
||||
export type TFindingsPayload = Pick<TSecretScanningFindingsInsert, "details" | "fingerprint" | "severity" | "rule">[];
|
||||
export type TGetFindingsPayload = Promise<TFindingsPayload>;
|
||||
|
||||
export type TUpdateSecretScanningFindingDTO = {
|
||||
status?: SecretScanningFindingStatus;
|
||||
remarks?: string | null;
|
||||
findingId: string;
|
||||
};
|
||||
|
||||
export type TUpsertSecretScanningConfigDTO = {
|
||||
projectId: string;
|
||||
content: string | null;
|
||||
};
|
||||
|
||||
export type TSecretScanningDataSourceCredentials = undefined;
|
||||
@@ -0,0 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { GitHubDataSourceSchema, GitHubFindingSchema } from "@app/ee/services/secret-scanning-v2/github";
|
||||
|
||||
export const SecretScanningDataSourceSchema = z.discriminatedUnion("type", [GitHubDataSourceSchema]);
|
||||
|
||||
export const SecretScanningFindingSchema = z.discriminatedUnion("resourceType", [GitHubFindingSchema]);
|
||||
+4
-4
@@ -65,9 +65,9 @@ export function runInfisicalScanOnRepo(repoPath: string, outputPath: string): Pr
|
||||
});
|
||||
}
|
||||
|
||||
export function runInfisicalScan(inputPath: string, outputPath: string): Promise<void> {
|
||||
export function runInfisicalScan(inputPath: string, outputPath: string, configPath?: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}"`;
|
||||
const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}" ${configPath ? `-c "${configPath}"` : ""}`;
|
||||
exec(command, (error) => {
|
||||
if (error && error.code !== 77) {
|
||||
reject(error);
|
||||
@@ -138,14 +138,14 @@ export async function scanFullRepoContentAndGetFindings(
|
||||
}
|
||||
}
|
||||
|
||||
export async function scanContentAndGetFindings(textContent: string): Promise<SecretMatch[]> {
|
||||
export async function scanContentAndGetFindings(textContent: string, configPath?: string): Promise<SecretMatch[]> {
|
||||
const tempFolder = await createTempFolder();
|
||||
const filePath = join(tempFolder, "content.txt");
|
||||
const findingsPath = join(tempFolder, "findings.json");
|
||||
|
||||
try {
|
||||
await writeTextToFile(filePath, textContent);
|
||||
await runInfisicalScan(filePath, findingsPath);
|
||||
await runInfisicalScan(filePath, findingsPath, configPath);
|
||||
const findingsData = await readFindingsFile(findingsPath);
|
||||
return JSON.parse(findingsData) as SecretMatch[];
|
||||
} finally {
|
||||
|
||||
+1
@@ -9,6 +9,7 @@ export type SecretMatch = {
|
||||
Match: string;
|
||||
Secret: string;
|
||||
File: string;
|
||||
Link: string;
|
||||
SymlinkFile: string;
|
||||
Commit: string;
|
||||
Entropy: number;
|
||||
|
||||
@@ -117,6 +117,7 @@ export const OCIVaultSyncFns = {
|
||||
syncSecrets: async (secretSync: TOCIVaultSyncWithCredentials, secretMap: TSecretMap) => {
|
||||
const {
|
||||
connection,
|
||||
environment,
|
||||
destinationConfig: { compartmentOcid, vaultOcid, keyOcid }
|
||||
} = secretSync;
|
||||
|
||||
@@ -213,7 +214,7 @@ export const OCIVaultSyncFns = {
|
||||
// Update and delete secrets
|
||||
for await (const [key, variable] of Object.entries(variables)) {
|
||||
// eslint-disable-next-line no-continue
|
||||
if (!matchesSchema(key, secretSync.syncOptions.keySchema)) continue;
|
||||
if (!matchesSchema(key, environment?.slug || "", secretSync.syncOptions.keySchema)) continue;
|
||||
|
||||
// Only update / delete active secrets
|
||||
if (variable.lifecycleState === vault.models.SecretSummary.LifecycleState.Active) {
|
||||
|
||||
Reference in New Issue
Block a user