Merge branch 'main' into feat/secretsBatchUI

This commit is contained in:
Carlos Monastyrski
2025-07-14 11:54:56 -03:00
259 changed files with 5578 additions and 534 deletions
@@ -2,6 +2,7 @@ import { nanoid } from "nanoid";
import { z } from "zod";
import { ApproverType, BypasserType } from "@app/ee/services/access-approval-policy/access-approval-policy-types";
import { removeTrailingSlash } from "@app/lib/fn";
import { EnforcementLevel } from "@app/lib/types";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
@@ -19,7 +20,7 @@ export const registerAccessApprovalPolicyRouter = async (server: FastifyZodProvi
body: z.object({
projectSlug: z.string().trim(),
name: z.string().optional(),
secretPath: z.string().trim().default("/"),
secretPath: z.string().trim().min(1, { message: "Secret path cannot be empty" }).transform(removeTrailingSlash),
environment: z.string(),
approvers: z
.discriminatedUnion("type", [
@@ -174,8 +175,9 @@ export const registerAccessApprovalPolicyRouter = async (server: FastifyZodProvi
secretPath: z
.string()
.trim()
.min(1, { message: "Secret path cannot be empty" })
.optional()
.transform((val) => (val === "" ? "/" : val)),
.transform((val) => (val ? removeTrailingSlash(val) : val)),
approvers: z
.discriminatedUnion("type", [
z.object({
@@ -23,10 +23,8 @@ export const registerSecretApprovalPolicyRouter = async (server: FastifyZodProvi
environment: z.string(),
secretPath: z
.string()
.optional()
.nullable()
.default("/")
.transform((val) => (val ? removeTrailingSlash(val) : val)),
.min(1, { message: "Secret path cannot be empty" })
.transform((val) => removeTrailingSlash(val)),
approvers: z
.discriminatedUnion("type", [
z.object({ type: z.literal(ApproverType.Group), id: z.string() }),
@@ -100,10 +98,10 @@ export const registerSecretApprovalPolicyRouter = async (server: FastifyZodProvi
approvals: z.number().min(1).default(1),
secretPath: z
.string()
.trim()
.min(1, { message: "Secret path cannot be empty" })
.optional()
.nullable()
.transform((val) => (val ? removeTrailingSlash(val) : val))
.transform((val) => (val === "" ? "/" : val)),
.transform((val) => (val ? removeTrailingSlash(val) : undefined)),
enforcementLevel: z.nativeEnum(EnforcementLevel).optional(),
allowedSelfApprovals: z.boolean().default(true)
}),
@@ -141,14 +141,39 @@ export const registerSecretApprovalRequestRouter = async (server: FastifyZodProv
},
onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => {
const { approval } = await server.services.secretApprovalRequest.mergeSecretApprovalRequest({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
approvalId: req.params.id,
bypassReason: req.body.bypassReason
const { approval, projectId, secretMutationEvents } =
await server.services.secretApprovalRequest.mergeSecretApprovalRequest({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
approvalId: req.params.id,
bypassReason: req.body.bypassReason
});
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
orgId: req.permission.orgId,
projectId,
event: {
type: EventType.SECRET_APPROVAL_MERGED,
metadata: {
mergedBy: req.permission.id,
secretApprovalRequestSlug: approval.slug,
secretApprovalRequestId: approval.id
}
}
});
for await (const event of secretMutationEvents) {
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
orgId: req.permission.orgId,
projectId,
event
});
}
return { approval };
}
});
@@ -0,0 +1,16 @@
import { registerSecretScanningEndpoints } from "@app/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints";
import {
BitbucketDataSourceSchema,
CreateBitbucketDataSourceSchema,
UpdateBitbucketDataSourceSchema
} from "@app/ee/services/secret-scanning-v2/bitbucket";
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
export const registerBitbucketSecretScanningRouter = async (server: FastifyZodProvider) =>
registerSecretScanningEndpoints({
type: SecretScanningDataSource.Bitbucket,
server,
responseSchema: BitbucketDataSourceSchema,
createSchema: CreateBitbucketDataSourceSchema,
updateSchema: UpdateBitbucketDataSourceSchema
});
@@ -1,5 +1,6 @@
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { registerBitbucketSecretScanningRouter } from "./bitbucket-secret-scanning-router";
import { registerGitHubSecretScanningRouter } from "./github-secret-scanning-router";
export * from "./secret-scanning-v2-router";
@@ -8,5 +9,6 @@ export const SECRET_SCANNING_REGISTER_ROUTER_MAP: Record<
SecretScanningDataSource,
(server: FastifyZodProvider) => Promise<void>
> = {
[SecretScanningDataSource.GitHub]: registerGitHubSecretScanningRouter
[SecretScanningDataSource.GitHub]: registerGitHubSecretScanningRouter,
[SecretScanningDataSource.Bitbucket]: registerBitbucketSecretScanningRouter
};
@@ -2,6 +2,7 @@ import { z } from "zod";
import { SecretScanningConfigsSchema } from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { BitbucketDataSourceListItemSchema } from "@app/ee/services/secret-scanning-v2/bitbucket";
import { GitHubDataSourceListItemSchema } from "@app/ee/services/secret-scanning-v2/github";
import {
SecretScanningFindingStatus,
@@ -21,7 +22,10 @@ import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
const SecretScanningDataSourceOptionsSchema = z.discriminatedUnion("type", [GitHubDataSourceListItemSchema]);
const SecretScanningDataSourceOptionsSchema = z.discriminatedUnion("type", [
GitHubDataSourceListItemSchema,
BitbucketDataSourceListItemSchema
]);
export const registerSecretScanningV2Router = async (server: FastifyZodProvider) => {
server.route({
@@ -53,7 +53,7 @@ export interface TAccessApprovalPolicyDALFactory
envId: string;
enforcementLevel: string;
allowedSelfApprovals: boolean;
secretPath?: string | null | undefined;
secretPath: string;
deletedAt?: Date | null | undefined;
environment: {
id: string;
@@ -93,7 +93,7 @@ export interface TAccessApprovalPolicyDALFactory
envId: string;
enforcementLevel: string;
allowedSelfApprovals: boolean;
secretPath?: string | null | undefined;
secretPath: string;
deletedAt?: Date | null | undefined;
environment: {
id: string;
@@ -116,7 +116,7 @@ export interface TAccessApprovalPolicyDALFactory
envId: string;
enforcementLevel: string;
allowedSelfApprovals: boolean;
secretPath?: string | null | undefined;
secretPath: string;
deletedAt?: Date | null | undefined;
}>;
findLastValidPolicy: (
@@ -138,7 +138,7 @@ export interface TAccessApprovalPolicyDALFactory
envId: string;
enforcementLevel: string;
allowedSelfApprovals: boolean;
secretPath?: string | null | undefined;
secretPath: string;
deletedAt?: Date | null | undefined;
}
| undefined
@@ -190,7 +190,7 @@ export interface TAccessApprovalPolicyServiceFactory {
envId: string;
enforcementLevel: string;
allowedSelfApprovals: boolean;
secretPath?: string | null | undefined;
secretPath: string;
deletedAt?: Date | null | undefined;
}>;
deleteAccessApprovalPolicy: ({
@@ -214,7 +214,7 @@ export interface TAccessApprovalPolicyServiceFactory {
envId: string;
enforcementLevel: string;
allowedSelfApprovals: boolean;
secretPath?: string | null | undefined;
secretPath: string;
deletedAt?: Date | null | undefined;
environment: {
id: string;
@@ -252,7 +252,7 @@ export interface TAccessApprovalPolicyServiceFactory {
envId: string;
enforcementLevel: string;
allowedSelfApprovals: boolean;
secretPath?: string | null | undefined;
secretPath: string;
deletedAt?: Date | null | undefined;
}>;
getAccessApprovalPolicyByProjectSlug: ({
@@ -286,7 +286,7 @@ export interface TAccessApprovalPolicyServiceFactory {
envId: string;
enforcementLevel: string;
allowedSelfApprovals: boolean;
secretPath?: string | null | undefined;
secretPath: string;
deletedAt?: Date | null | undefined;
environment: {
id: string;
@@ -337,7 +337,7 @@ export interface TAccessApprovalPolicyServiceFactory {
envId: string;
enforcementLevel: string;
allowedSelfApprovals: boolean;
secretPath?: string | null | undefined;
secretPath: string;
deletedAt?: Date | null | undefined;
environment: {
id: string;
@@ -60,6 +60,26 @@ export const accessApprovalPolicyServiceFactory = ({
accessApprovalRequestReviewerDAL,
orgMembershipDAL
}: TAccessApprovalPolicyServiceFactoryDep): TAccessApprovalPolicyServiceFactory => {
const $policyExists = async ({
envId,
secretPath,
policyId
}: {
envId: string;
secretPath: string;
policyId?: string;
}) => {
const policy = await accessApprovalPolicyDAL
.findOne({
envId,
secretPath,
deletedAt: null
})
.catch(() => null);
return policyId ? policy && policy.id !== policyId : Boolean(policy);
};
const createAccessApprovalPolicy: TAccessApprovalPolicyServiceFactory["createAccessApprovalPolicy"] = async ({
name,
actor,
@@ -106,6 +126,12 @@ export const accessApprovalPolicyServiceFactory = ({
const env = await projectEnvDAL.findOne({ slug: environment, projectId: project.id });
if (!env) throw new NotFoundError({ message: `Environment with slug '${environment}' not found` });
if (await $policyExists({ envId: env.id, secretPath })) {
throw new BadRequestError({
message: `A policy for secret path '${secretPath}' already exists in environment '${environment}'`
});
}
let approverUserIds = userApprovers;
if (userApproverNames.length) {
const approverUsersInDB = await userDAL.find({
@@ -279,7 +305,11 @@ export const accessApprovalPolicyServiceFactory = ({
) as { username: string; sequence?: number }[];
const accessApprovalPolicy = await accessApprovalPolicyDAL.findById(policyId);
if (!accessApprovalPolicy) throw new BadRequestError({ message: "Approval policy not found" });
if (!accessApprovalPolicy) {
throw new NotFoundError({
message: `Access approval policy with ID '${policyId}' not found`
});
}
const currentApprovals = approvals || accessApprovalPolicy.approvals;
if (
@@ -290,9 +320,18 @@ export const accessApprovalPolicyServiceFactory = ({
throw new BadRequestError({ message: "Approvals cannot be greater than approvers" });
}
if (!accessApprovalPolicy) {
throw new NotFoundError({ message: `Secret approval policy with ID '${policyId}' not found` });
if (
await $policyExists({
envId: accessApprovalPolicy.envId,
secretPath: secretPath || accessApprovalPolicy.secretPath,
policyId: accessApprovalPolicy.id
})
) {
throw new BadRequestError({
message: `A policy for secret path '${secretPath}' already exists in environment '${accessApprovalPolicy.environment.slug}'`
});
}
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
@@ -122,7 +122,7 @@ export interface TAccessApprovalPolicyServiceFactory {
envId: string;
enforcementLevel: string;
allowedSelfApprovals: boolean;
secretPath?: string | null | undefined;
secretPath: string;
deletedAt?: Date | null | undefined;
}>;
deleteAccessApprovalPolicy: ({
@@ -146,7 +146,7 @@ export interface TAccessApprovalPolicyServiceFactory {
envId: string;
enforcementLevel: string;
allowedSelfApprovals: boolean;
secretPath?: string | null | undefined;
secretPath: string;
deletedAt?: Date | null | undefined;
environment: {
id: string;
@@ -218,7 +218,7 @@ export interface TAccessApprovalPolicyServiceFactory {
envId: string;
enforcementLevel: string;
allowedSelfApprovals: boolean;
secretPath?: string | null | undefined;
secretPath: string;
deletedAt?: Date | null | undefined;
environment: {
id: string;
@@ -269,7 +269,7 @@ export interface TAccessApprovalPolicyServiceFactory {
envId: string;
enforcementLevel: string;
allowedSelfApprovals: boolean;
secretPath?: string | null | undefined;
secretPath: string;
deletedAt?: Date | null | undefined;
environment: {
id: string;
@@ -116,6 +116,15 @@ interface BaseAuthData {
userAgentType?: UserAgentType;
}
export enum SecretApprovalEvent {
Create = "create",
Update = "update",
Delete = "delete",
CreateMany = "create-many",
UpdateMany = "update-many",
DeleteMany = "delete-many"
}
export enum UserAgentType {
WEB = "web",
CLI = "cli",
@@ -1705,6 +1714,17 @@ interface SecretApprovalRequest {
committedBy: string;
secretApprovalRequestSlug: string;
secretApprovalRequestId: string;
eventType: SecretApprovalEvent;
secretKey?: string;
secretId?: string;
secrets?: {
secretKey?: string;
secretId?: string;
environment?: string;
secretPath?: string;
}[];
environment: string;
secretPath: string;
};
}
@@ -21,7 +21,7 @@ import { randomUUID } from "crypto";
import { z } from "zod";
import { getConfig } from "@app/lib/config/env";
import { BadRequestError } from "@app/lib/errors";
import { BadRequestError, UnauthorizedError } from "@app/lib/errors";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { AwsIamAuthType, DynamicSecretAwsIamSchema, TDynamicProviderFns } from "./models";
@@ -81,6 +81,21 @@ export const AwsIamProvider = (): TDynamicProviderFns => {
return client;
}
if (providerInputs.method === AwsIamAuthType.IRSA) {
// Allow instances to disable automatic service account token fetching (e.g. for shared cloud)
if (!appCfg.KUBERNETES_AUTO_FETCH_SERVICE_ACCOUNT_TOKEN) {
throw new UnauthorizedError({
message: "Failed to get AWS credentials via IRSA: KUBERNETES_AUTO_FETCH_SERVICE_ACCOUNT_TOKEN is not enabled."
});
}
// The SDK will automatically pick up credentials from the environment
const client = new IAMClient({
region: providerInputs.region
});
return client;
}
const client = new IAMClient({
region: providerInputs.region,
credentials: {
@@ -101,7 +116,7 @@ export const AwsIamProvider = (): TDynamicProviderFns => {
.catch((err) => {
const message = (err as Error)?.message;
if (
providerInputs.method === AwsIamAuthType.AssumeRole &&
(providerInputs.method === AwsIamAuthType.AssumeRole || providerInputs.method === AwsIamAuthType.IRSA) &&
// assume role will throw an error asking to provider username, but if so this has access in aws correctly
message.includes("Must specify userName when calling with non-User credentials")
) {
@@ -28,7 +28,8 @@ export enum SqlProviders {
export enum AwsIamAuthType {
AssumeRole = "assume-role",
AccessKey = "access-key"
AccessKey = "access-key",
IRSA = "irsa"
}
export enum ElasticSearchAuthTypes {
@@ -221,6 +222,16 @@ export const DynamicSecretAwsIamSchema = z.preprocess(
userGroups: z.string().trim().optional(),
policyArns: z.string().trim().optional(),
tags: ResourceMetadataSchema.optional()
}),
z.object({
method: z.literal(AwsIamAuthType.IRSA),
region: z.string().trim().min(1),
awsPath: z.string().trim().optional(),
permissionBoundaryPolicyArn: z.string().trim().optional(),
policyDocument: z.string().trim().optional(),
userGroups: z.string().trim().optional(),
policyArns: z.string().trim().optional(),
tags: ResourceMetadataSchema.optional()
})
])
);
@@ -361,13 +361,6 @@ export const ldapConfigServiceFactory = ({
});
} else {
const plan = await licenseService.getPlan(orgId);
if (plan?.slug !== "enterprise" && plan?.memberLimit && plan.membersUsed >= plan.memberLimit) {
// limit imposed on number of members allowed / number of members used exceeds the number of members allowed
throw new BadRequestError({
message: "Failed to create new member via LDAP due to member limit reached. Upgrade plan to add more members."
});
}
if (plan?.slug !== "enterprise" && plan?.identityLimit && plan.identitiesUsed >= plan.identityLimit) {
// limit imposed on number of identities allowed / number of identities used exceeds the number of identities allowed
throw new BadRequestError({
@@ -1,5 +1,4 @@
export const BillingPlanRows = {
MemberLimit: { name: "Organization member limit", field: "memberLimit" },
IdentityLimit: { name: "Organization identity limit", field: "identityLimit" },
WorkspaceLimit: { name: "Project limit", field: "workspaceLimit" },
EnvironmentLimit: { name: "Environment limit", field: "environmentLimit" },
@@ -442,9 +442,7 @@ export const licenseServiceFactory = ({
rows: data.rows.map((el) => {
let used = "-";
if (el.name === BillingPlanRows.MemberLimit.name) {
used = orgMembersUsed.toString();
} else if (el.name === BillingPlanRows.WorkspaceLimit.name) {
if (el.name === BillingPlanRows.WorkspaceLimit.name) {
used = projectCount.toString();
} else if (el.name === BillingPlanRows.IdentityLimit.name) {
used = (identityUsed + orgMembersUsed).toString();
@@ -464,12 +462,10 @@ export const licenseServiceFactory = ({
const allowed = onPremFeatures[field as keyof TFeatureSet];
let used = "-";
if (field === BillingPlanRows.MemberLimit.field) {
used = orgMembersUsed.toString();
} else if (field === BillingPlanRows.WorkspaceLimit.field) {
if (field === BillingPlanRows.WorkspaceLimit.field) {
used = projectCount.toString();
} else if (field === BillingPlanRows.IdentityLimit.field) {
used = identityUsed.toString();
used = (identityUsed + orgMembersUsed).toString();
}
return {
@@ -311,13 +311,6 @@ export const samlConfigServiceFactory = ({
});
} else {
const plan = await licenseService.getPlan(orgId);
if (plan?.slug !== "enterprise" && plan?.memberLimit && plan.membersUsed >= plan.memberLimit) {
// limit imposed on number of members allowed / number of members used exceeds the number of members allowed
throw new BadRequestError({
message: "Failed to create new member via SAML due to member limit reached. Upgrade plan to add more members."
});
}
if (plan?.slug !== "enterprise" && plan?.identityLimit && plan.identitiesUsed >= plan.identityLimit) {
// limit imposed on number of identities allowed / number of identities used exceeds the number of identities allowed
throw new BadRequestError({
@@ -55,6 +55,26 @@ export const secretApprovalPolicyServiceFactory = ({
licenseService,
secretApprovalRequestDAL
}: TSecretApprovalPolicyServiceFactoryDep) => {
const $policyExists = async ({
envId,
secretPath,
policyId
}: {
envId: string;
secretPath: string;
policyId?: string;
}) => {
const policy = await secretApprovalPolicyDAL
.findOne({
envId,
secretPath,
deletedAt: null
})
.catch(() => null);
return policyId ? policy && policy.id !== policyId : Boolean(policy);
};
const createSecretApprovalPolicy = async ({
name,
actor,
@@ -106,10 +126,17 @@ export const secretApprovalPolicyServiceFactory = ({
}
const env = await projectEnvDAL.findOne({ slug: environment, projectId });
if (!env)
if (!env) {
throw new NotFoundError({
message: `Environment with slug '${environment}' not found in project with ID ${projectId}`
});
}
if (await $policyExists({ envId: env.id, secretPath })) {
throw new BadRequestError({
message: `A policy for secret path '${secretPath}' already exists in environment '${environment}'`
});
}
let groupBypassers: string[] = [];
let bypasserUserIds: string[] = [];
@@ -260,6 +287,18 @@ export const secretApprovalPolicyServiceFactory = ({
});
}
if (
await $policyExists({
envId: secretApprovalPolicy.envId,
secretPath: secretPath || secretApprovalPolicy.secretPath,
policyId: secretApprovalPolicy.id
})
) {
throw new BadRequestError({
message: `A policy for secret path '${secretPath}' already exists in environment '${secretApprovalPolicy.environment.slug}'`
});
}
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
@@ -4,7 +4,7 @@ import { ApproverType, BypasserType } from "../access-approval-policy/access-app
export type TCreateSapDTO = {
approvals: number;
secretPath?: string | null;
secretPath: string;
environment: string;
approvers: ({ type: ApproverType.Group; id: string } | { type: ApproverType.User; id?: string; username?: string })[];
bypassers?: (
@@ -20,7 +20,7 @@ export type TCreateSapDTO = {
export type TUpdateSapDTO = {
secretPolicyId: string;
approvals?: number;
secretPath?: string | null;
secretPath?: string;
approvers: ({ type: ApproverType.Group; id: string } | { type: ApproverType.User; id?: string; username?: string })[];
bypassers?: (
| { type: BypasserType.Group; id: string }
@@ -11,6 +11,7 @@ import {
TSecretApprovalRequestsSecretsInsert,
TSecretApprovalRequestsSecretsV2Insert
} from "@app/db/schemas";
import { Event, EventType } from "@app/ee/services/audit-log/audit-log-types";
import { getConfig } from "@app/lib/config/env";
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors";
@@ -524,7 +525,7 @@ export const secretApprovalRequestServiceFactory = ({
});
}
const { policy, folderId, projectId, bypassers } = secretApprovalRequest;
const { policy, folderId, projectId, bypassers, environment } = secretApprovalRequest;
if (policy.deletedAt) {
throw new BadRequestError({
message: "The policy associated with this secret approval request has been deleted."
@@ -958,7 +959,112 @@ export const secretApprovalRequestServiceFactory = ({
});
}
return mergeStatus;
const { created, updated, deleted } = mergeStatus.secrets;
const secretMutationEvents: Event[] = [];
if (created.length) {
if (created.length > 1) {
secretMutationEvents.push({
type: EventType.CREATE_SECRETS,
metadata: {
environment,
secretPath: folder.path,
secrets: created.map((secret) => ({
secretId: secret.id,
secretVersion: 1,
// @ts-expect-error not present on v1 secrets
secretKey: secret.key as string,
// @ts-expect-error not present on v1 secrets
secretMetadata: secret.secretMetadata as ResourceMetadataDTO
}))
}
});
} else {
const [secret] = created;
secretMutationEvents.push({
type: EventType.CREATE_SECRET,
metadata: {
environment,
secretPath: folder.path,
secretId: secret.id,
secretVersion: 1,
// @ts-expect-error not present on v1 secrets
secretKey: secret.key as string,
// @ts-expect-error not present on v1 secrets
secretMetadata: secret.secretMetadata as ResourceMetadataDTO
}
});
}
}
if (updated.length) {
if (updated.length > 1) {
secretMutationEvents.push({
type: EventType.UPDATE_SECRETS,
metadata: {
environment,
secretPath: folder.path,
secrets: updated.map((secret) => ({
secretId: secret.id,
secretVersion: secret.version,
// @ts-expect-error not present on v1 secrets
secretKey: secret.key as string,
// @ts-expect-error not present on v1 secrets
secretMetadata: secret.secretMetadata as ResourceMetadataDTO
}))
}
});
} else {
const [secret] = updated;
secretMutationEvents.push({
type: EventType.UPDATE_SECRET,
metadata: {
environment,
secretPath: folder.path,
secretId: secret.id,
secretVersion: secret.version,
// @ts-expect-error not present on v1 secrets
secretKey: secret.key as string,
// @ts-expect-error not present on v1 secrets
secretMetadata: secret.secretMetadata as ResourceMetadataDTO
}
});
}
}
if (deleted.length) {
if (deleted.length > 1) {
secretMutationEvents.push({
type: EventType.DELETE_SECRETS,
metadata: {
environment,
secretPath: folder.path,
secrets: deleted.map((secret) => ({
secretId: secret.id,
secretVersion: secret.version,
// @ts-expect-error not present on v1 secrets
secretKey: secret.key as string
}))
}
});
} else {
const [secret] = deleted;
secretMutationEvents.push({
type: EventType.DELETE_SECRET,
metadata: {
environment,
secretPath: folder.path,
secretId: secret.id,
secretVersion: secret.version,
// @ts-expect-error not present on v1 secrets
secretKey: secret.key as string
}
});
}
}
return { ...mergeStatus, projectId, secretMutationEvents };
};
// function to save secret change to secret approval
@@ -0,0 +1,9 @@
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { TSecretScanningDataSourceListItem } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
export const BITBUCKET_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION: TSecretScanningDataSourceListItem = {
name: "Bitbucket",
type: SecretScanningDataSource.Bitbucket,
connection: AppConnection.Bitbucket
};
@@ -0,0 +1,314 @@
import { join } from "path";
import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import {
SecretScanningFindingSeverity,
SecretScanningResource
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
cloneRepository,
convertPatchLineToFileLineNumber,
replaceNonChangesWithNewlines
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
import {
TSecretScanningFactoryGetDiffScanFindingsPayload,
TSecretScanningFactoryGetDiffScanResourcePayload,
TSecretScanningFactoryGetFullScanPath,
TSecretScanningFactoryInitialize,
TSecretScanningFactoryListRawResources,
TSecretScanningFactoryPostInitialization,
TSecretScanningFactoryTeardown
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { getConfig } from "@app/lib/config/env";
import { request } from "@app/lib/config/request";
import { titleCaseToCamelCase } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { BasicRepositoryRegex } from "@app/lib/regex";
import {
getBitbucketUser,
listBitbucketRepositories,
TBitbucketConnection
} from "@app/services/app-connection/bitbucket";
import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
import {
TBitbucketDataSourceCredentials,
TBitbucketDataSourceInput,
TBitbucketDataSourceWithConnection,
TQueueBitbucketResourceDiffScan
} from "./bitbucket-secret-scanning-types";
export const BitbucketSecretScanningFactory = () => {
const initialize: TSecretScanningFactoryInitialize<
TBitbucketDataSourceInput,
TBitbucketConnection,
TBitbucketDataSourceCredentials
> = async ({ connection, payload }, callback) => {
const cfg = getConfig();
const { email, apiToken } = connection.credentials;
const authHeader = `Basic ${Buffer.from(`${email}:${apiToken}`).toString("base64")}`;
const { data } = await request.post<{ uuid: string }>(
`${IntegrationUrls.BITBUCKET_API_URL}/2.0/workspaces/${encodeURIComponent(payload.config.workspaceSlug)}/hooks`,
{
description: "Infisical webhook for push events",
url: `${cfg.SITE_URL}/secret-scanning/webhooks/bitbucket`,
active: false,
events: ["repo:push"]
},
{
headers: {
Authorization: authHeader,
Accept: "application/json"
}
}
);
return callback({
credentials: { webhookId: data.uuid, webhookSecret: alphaNumericNanoId(64) }
});
};
const postInitialization: TSecretScanningFactoryPostInitialization<
TBitbucketDataSourceInput,
TBitbucketConnection,
TBitbucketDataSourceCredentials
> = async ({ dataSourceId, credentials, connection, payload }) => {
const { email, apiToken } = connection.credentials;
const { webhookId, webhookSecret } = credentials;
const authHeader = `Basic ${Buffer.from(`${email}:${apiToken}`).toString("base64")}`;
const cfg = getConfig();
const newWebhookUrl = `${cfg.SITE_URL}/secret-scanning/webhooks/bitbucket?dataSourceId=${dataSourceId}`;
await request.put(
`${IntegrationUrls.BITBUCKET_API_URL}/2.0/workspaces/${encodeURIComponent(payload.config.workspaceSlug)}/hooks/${webhookId}`,
{
description: "Infisical webhook for push events",
url: newWebhookUrl,
active: true,
events: ["repo:push"],
secret: webhookSecret
},
{
headers: {
Authorization: authHeader,
Accept: "application/json"
}
}
);
};
const teardown: TSecretScanningFactoryTeardown<
TBitbucketDataSourceWithConnection,
TBitbucketDataSourceCredentials
> = async ({ credentials, dataSource }) => {
const {
connection: {
credentials: { email, apiToken }
},
config
} = dataSource;
const { webhookId } = credentials;
const authHeader = `Basic ${Buffer.from(`${email}:${apiToken}`).toString("base64")}`;
try {
await request.delete(
`${IntegrationUrls.BITBUCKET_API_URL}/2.0/workspaces/${config.workspaceSlug}/hooks/${webhookId}`,
{
headers: {
Authorization: authHeader,
Accept: "application/json"
}
}
);
} catch (err) {
logger.error(`teardown: Bitbucket - Failed to call delete on webhook [webhookId=${webhookId}]`);
}
};
const listRawResources: TSecretScanningFactoryListRawResources<TBitbucketDataSourceWithConnection> = async (
dataSource
) => {
const {
connection,
config: { includeRepos, workspaceSlug }
} = dataSource;
const repos = await listBitbucketRepositories(connection, workspaceSlug);
const filteredRepos: typeof repos = [];
if (includeRepos.includes("*")) {
filteredRepos.push(...repos);
} else {
filteredRepos.push(...repos.filter((repo) => includeRepos.includes(repo.full_name)));
}
return filteredRepos.map(({ full_name, uuid }) => ({
name: full_name,
externalId: uuid,
type: SecretScanningResource.Repository
}));
};
const getFullScanPath: TSecretScanningFactoryGetFullScanPath<TBitbucketDataSourceWithConnection> = async ({
dataSource,
resourceName,
tempFolder
}) => {
const {
connection: {
credentials: { apiToken, email }
}
} = dataSource;
const repoPath = join(tempFolder, "repo.git");
if (!BasicRepositoryRegex.test(resourceName)) {
throw new Error("Invalid Bitbucket repository name");
}
const { username } = await getBitbucketUser({ email, apiToken });
await cloneRepository({
cloneUrl: `https://${encodeURIComponent(username)}:${apiToken}@bitbucket.org/${resourceName}.git`,
repoPath
});
return repoPath;
};
const getDiffScanResourcePayload: TSecretScanningFactoryGetDiffScanResourcePayload<
TQueueBitbucketResourceDiffScan["payload"]
> = ({ repository }) => {
return {
name: repository.full_name,
externalId: repository.uuid,
type: SecretScanningResource.Repository
};
};
const getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<
TBitbucketDataSourceWithConnection,
TQueueBitbucketResourceDiffScan["payload"]
> = async ({ dataSource, payload, resourceName, configPath }) => {
const {
connection: {
credentials: { apiToken, email }
}
} = dataSource;
const { push, repository } = payload;
const allFindings: SecretMatch[] = [];
const authHeader = `Basic ${Buffer.from(`${email}:${apiToken}`).toString("base64")}`;
for (const change of push.changes) {
for (const commit of change.commits) {
// eslint-disable-next-line no-await-in-loop
const { data: diffstat } = await request.get<{
values: {
status: "added" | "modified" | "removed" | "renamed";
new?: { path: string };
old?: { path: string };
}[];
}>(`${IntegrationUrls.BITBUCKET_API_URL}/2.0/repositories/${repository.full_name}/diffstat/${commit.hash}`, {
headers: {
Authorization: authHeader,
Accept: "application/json"
}
});
// eslint-disable-next-line no-continue
if (!diffstat.values) continue;
for (const file of diffstat.values) {
if ((file.status === "added" || file.status === "modified") && file.new?.path) {
const filePath = file.new.path;
// eslint-disable-next-line no-await-in-loop
const { data: patch } = await request.get<string>(
`https://api.bitbucket.org/2.0/repositories/${repository.full_name}/diff/${commit.hash}`,
{
params: {
path: filePath
},
headers: {
Authorization: authHeader
},
responseType: "text"
}
);
// eslint-disable-next-line no-continue
if (!patch) continue;
// eslint-disable-next-line no-await-in-loop
const findings = await scanContentAndGetFindings(replaceNonChangesWithNewlines(`\n${patch}`), configPath);
const adjustedFindings = findings.map((finding) => {
const startLine = convertPatchLineToFileLineNumber(patch, finding.StartLine);
const endLine =
finding.StartLine === finding.EndLine
? startLine
: convertPatchLineToFileLineNumber(patch, finding.EndLine);
const startColumn = finding.StartColumn - 1; // subtract 1 for +
const endColumn = finding.EndColumn - 1; // subtract 1 for +
const authorName = commit.author.user?.display_name || commit.author.raw.split(" <")[0];
const emailMatch = commit.author.raw.match(/<(.*)>/);
const authorEmail = emailMatch?.[1] ?? "";
return {
...finding,
StartLine: startLine,
EndLine: endLine,
StartColumn: startColumn,
EndColumn: endColumn,
File: filePath,
Commit: commit.hash,
Author: authorName,
Email: authorEmail,
Message: commit.message,
Fingerprint: `${commit.hash}:${filePath}:${finding.RuleID}:${startLine}:${startColumn}`,
Date: commit.date,
Link: `https://bitbucket.org/${resourceName}/src/${commit.hash}/${filePath}#lines-${startLine}`
};
});
allFindings.push(...adjustedFindings);
}
}
}
}
return allFindings.map(
({
// discard match and secret as we don't want to store
Match,
Secret,
...finding
}) => ({
details: titleCaseToCamelCase(finding),
fingerprint: finding.Fingerprint,
severity: SecretScanningFindingSeverity.High,
rule: finding.RuleID
})
);
};
return {
initialize,
postInitialization,
listRawResources,
getFullScanPath,
getDiffScanResourcePayload,
getDiffScanFindingsPayload,
teardown
};
};
@@ -0,0 +1,97 @@
import { z } from "zod";
import {
SecretScanningDataSource,
SecretScanningResource
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
BaseCreateSecretScanningDataSourceSchema,
BaseSecretScanningDataSourceSchema,
BaseSecretScanningFindingSchema,
BaseUpdateSecretScanningDataSourceSchema,
GitRepositoryScanFindingDetailsSchema
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-schemas";
import { SecretScanningDataSources } from "@app/lib/api-docs";
import { BasicRepositoryRegex } from "@app/lib/regex";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
export const BitbucketDataSourceConfigSchema = z.object({
workspaceSlug: z
.string()
.min(1, "Workspace slug required")
.max(128)
.describe(SecretScanningDataSources.CONFIG.BITBUCKET.workspaceSlug),
includeRepos: z
.array(
z
.string()
.min(1)
.max(256)
.refine((value) => value === "*" || BasicRepositoryRegex.test(value), "Invalid repository name format")
)
.nonempty("One or more repositories required")
.max(100, "Cannot configure more than 100 repositories")
.default(["*"])
.describe(SecretScanningDataSources.CONFIG.BITBUCKET.includeRepos)
});
export const BitbucketDataSourceSchema = BaseSecretScanningDataSourceSchema({
type: SecretScanningDataSource.Bitbucket,
isConnectionRequired: true
})
.extend({
config: BitbucketDataSourceConfigSchema
})
.describe(
JSON.stringify({
title: "Bitbucket"
})
);
export const CreateBitbucketDataSourceSchema = BaseCreateSecretScanningDataSourceSchema({
type: SecretScanningDataSource.Bitbucket,
isConnectionRequired: true
})
.extend({
config: BitbucketDataSourceConfigSchema
})
.describe(
JSON.stringify({
title: "Bitbucket"
})
);
export const UpdateBitbucketDataSourceSchema = BaseUpdateSecretScanningDataSourceSchema(
SecretScanningDataSource.Bitbucket
)
.extend({
config: BitbucketDataSourceConfigSchema.optional()
})
.describe(
JSON.stringify({
title: "Bitbucket"
})
);
export const BitbucketDataSourceListItemSchema = z
.object({
name: z.literal("Bitbucket"),
connection: z.literal(AppConnection.Bitbucket),
type: z.literal(SecretScanningDataSource.Bitbucket)
})
.describe(
JSON.stringify({
title: "Bitbucket"
})
);
export const BitbucketFindingSchema = BaseSecretScanningFindingSchema.extend({
resourceType: z.literal(SecretScanningResource.Repository),
dataSourceType: z.literal(SecretScanningDataSource.Bitbucket),
details: GitRepositoryScanFindingDetailsSchema
});
export const BitbucketDataSourceCredentialsSchema = z.object({
webhookId: z.string(),
webhookSecret: z.string()
});
@@ -0,0 +1,104 @@
import crypto from "crypto";
import { TSecretScanningV2DALFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-dal";
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { TSecretScanningV2QueueServiceFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-queue";
import { logger } from "@app/lib/logger";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
import {
TBitbucketDataSource,
TBitbucketDataSourceCredentials,
TBitbucketPushEvent
} from "./bitbucket-secret-scanning-types";
export const bitbucketSecretScanningService = (
secretScanningV2DAL: TSecretScanningV2DALFactory,
secretScanningV2Queue: Pick<TSecretScanningV2QueueServiceFactory, "queueResourceDiffScan">,
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">
) => {
const handlePushEvent = async (
payload: TBitbucketPushEvent & { dataSourceId: string; receivedSignature: string; bodyString: string }
) => {
const { push, repository, bodyString, receivedSignature } = payload;
if (!push?.changes?.length || !repository?.workspace?.uuid) {
logger.warn(
`secretScanningV2PushEvent: Bitbucket - Insufficient data [changes=${
push?.changes?.length ?? 0
}] [repository=${repository?.name}] [workspaceUuid=${repository?.workspace?.uuid}]`
);
return;
}
const dataSource = (await secretScanningV2DAL.dataSources.findOne({
id: payload.dataSourceId,
type: SecretScanningDataSource.Bitbucket
})) as TBitbucketDataSource | undefined;
if (!dataSource) {
logger.error(
`secretScanningV2PushEvent: Bitbucket - Could not find data source [workspaceUuid=${repository.workspace.uuid}]`
);
return;
}
const {
isAutoScanEnabled,
config: { includeRepos },
encryptedCredentials,
projectId
} = dataSource;
if (!encryptedCredentials) {
logger.info(
`secretScanningV2PushEvent: Bitbucket - Could not find encrypted credentials [dataSourceId=${dataSource.id}] [workspaceUuid=${repository.workspace.uuid}]`
);
return;
}
const { decryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
});
const decryptedCredentials = decryptor({ cipherTextBlob: encryptedCredentials });
const credentials = JSON.parse(decryptedCredentials.toString()) as TBitbucketDataSourceCredentials;
const hmac = crypto.createHmac("sha256", credentials.webhookSecret);
hmac.update(bodyString);
const calculatedSignature = hmac.digest("hex");
if (calculatedSignature !== receivedSignature) {
logger.error(
`secretScanningV2PushEvent: Bitbucket - Invalid signature for webhook [dataSourceId=${dataSource.id}] [workspaceUuid=${repository.workspace.uuid}]`
);
return;
}
if (!isAutoScanEnabled) {
logger.info(
`secretScanningV2PushEvent: Bitbucket - ignoring due to auto scan disabled [dataSourceId=${dataSource.id}] [workspaceUuid=${repository.workspace.uuid}]`
);
return;
}
if (includeRepos.includes("*") || includeRepos.includes(repository.full_name)) {
await secretScanningV2Queue.queueResourceDiffScan({
dataSourceType: SecretScanningDataSource.Bitbucket,
payload,
dataSourceId: dataSource.id
});
} else {
logger.info(
`secretScanningV2PushEvent: Bitbucket - ignoring due to repository not being present in config [workspaceUuid=${repository.workspace.uuid}] [dataSourceId=${dataSource.id}]`
);
}
};
return {
handlePushEvent
};
};
@@ -0,0 +1,85 @@
import { z } from "zod";
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { TBitbucketConnection } from "@app/services/app-connection/bitbucket";
import {
BitbucketDataSourceCredentialsSchema,
BitbucketDataSourceListItemSchema,
BitbucketDataSourceSchema,
BitbucketFindingSchema,
CreateBitbucketDataSourceSchema
} from "./bitbucket-secret-scanning-schemas";
export type TBitbucketDataSource = z.infer<typeof BitbucketDataSourceSchema>;
export type TBitbucketDataSourceInput = z.infer<typeof CreateBitbucketDataSourceSchema>;
export type TBitbucketDataSourceListItem = z.infer<typeof BitbucketDataSourceListItemSchema>;
export type TBitbucketDataSourceCredentials = z.infer<typeof BitbucketDataSourceCredentialsSchema>;
export type TBitbucketFinding = z.infer<typeof BitbucketFindingSchema>;
export type TBitbucketDataSourceWithConnection = TBitbucketDataSource & {
connection: TBitbucketConnection;
};
export type TBitbucketPushEventRepository = {
full_name: string;
name: string;
workspace: {
slug: string;
uuid: string;
};
uuid: string;
};
export type TBitbucketPushEventCommit = {
hash: string;
message: string;
author: {
raw: string;
user?: {
display_name: string;
uuid: string;
nickname: string;
};
};
date: string;
};
export type TBitbucketPushEventChange = {
new?: {
name: string;
type: string;
};
old?: {
name: string;
type: string;
};
created: boolean;
closed: boolean;
forced: boolean;
commits: TBitbucketPushEventCommit[];
};
export type TBitbucketPushEvent = {
push: {
changes: TBitbucketPushEventChange[];
};
repository: TBitbucketPushEventRepository;
actor: {
display_name: string;
uuid: string;
nickname: string;
};
};
export type TQueueBitbucketResourceDiffScan = {
dataSourceType: SecretScanningDataSource.Bitbucket;
payload: TBitbucketPushEvent & { dataSourceId: string };
dataSourceId: string;
resourceId: string;
scanId: string;
};
@@ -0,0 +1,3 @@
export * from "./bitbucket-secret-scanning-constants";
export * from "./bitbucket-secret-scanning-schemas";
export * from "./bitbucket-secret-scanning-types";
@@ -19,18 +19,23 @@ import {
TSecretScanningFactoryGetFullScanPath,
TSecretScanningFactoryInitialize,
TSecretScanningFactoryListRawResources,
TSecretScanningFactoryPostInitialization
TSecretScanningFactoryPostInitialization,
TSecretScanningFactoryTeardown
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { getConfig } from "@app/lib/config/env";
import { BadRequestError } from "@app/lib/errors";
import { titleCaseToCamelCase } from "@app/lib/fn";
import { GitHubRepositoryRegex } from "@app/lib/regex";
import { BasicRepositoryRegex } from "@app/lib/regex";
import { listGitHubRadarRepositories, TGitHubRadarConnection } from "@app/services/app-connection/github-radar";
import { TGitHubDataSourceWithConnection, TQueueGitHubResourceDiffScan } from "./github-secret-scanning-types";
import {
TGitHubDataSourceInput,
TGitHubDataSourceWithConnection,
TQueueGitHubResourceDiffScan
} from "./github-secret-scanning-types";
export const GitHubSecretScanningFactory = () => {
const initialize: TSecretScanningFactoryInitialize<TGitHubRadarConnection> = async (
const initialize: TSecretScanningFactoryInitialize<TGitHubDataSourceInput, TGitHubRadarConnection> = async (
{ connection, secretScanningV2DAL },
callback
) => {
@@ -51,10 +56,17 @@ export const GitHubSecretScanningFactory = () => {
});
};
const postInitialization: TSecretScanningFactoryPostInitialization<TGitHubRadarConnection> = async () => {
const postInitialization: TSecretScanningFactoryPostInitialization<
TGitHubDataSourceInput,
TGitHubRadarConnection
> = async () => {
// no post-initialization required
};
const teardown: TSecretScanningFactoryTeardown<TGitHubDataSourceWithConnection> = async () => {
// no termination required
};
const listRawResources: TSecretScanningFactoryListRawResources<TGitHubDataSourceWithConnection> = async (
dataSource
) => {
@@ -107,7 +119,7 @@ export const GitHubSecretScanningFactory = () => {
const repoPath = join(tempFolder, "repo.git");
if (!GitHubRepositoryRegex.test(resourceName)) {
if (!BasicRepositoryRegex.test(resourceName)) {
throw new Error("Invalid GitHub repository name");
}
@@ -225,6 +237,7 @@ export const GitHubSecretScanningFactory = () => {
listRawResources,
getFullScanPath,
getDiffScanResourcePayload,
getDiffScanFindingsPayload
getDiffScanFindingsPayload,
teardown
};
};
@@ -12,7 +12,7 @@ import {
GitRepositoryScanFindingDetailsSchema
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-schemas";
import { SecretScanningDataSources } from "@app/lib/api-docs";
import { GitHubRepositoryRegex } from "@app/lib/regex";
import { BasicRepositoryRegex } from "@app/lib/regex";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
export const GitHubDataSourceConfigSchema = z.object({
@@ -22,7 +22,7 @@ export const GitHubDataSourceConfigSchema = z.object({
.string()
.min(1)
.max(256)
.refine((value) => value === "*" || GitHubRepositoryRegex.test(value), "Invalid repository name format")
.refine((value) => value === "*" || BasicRepositoryRegex.test(value), "Invalid repository name format")
)
.nonempty("One or more repositories required")
.max(100, "Cannot configure more than 100 repositories")
@@ -1,5 +1,6 @@
export enum SecretScanningDataSource {
GitHub = "github"
GitHub = "github",
Bitbucket = "bitbucket"
}
export enum SecretScanningScanStatus {
@@ -1,19 +1,23 @@
import { BitbucketSecretScanningFactory } from "@app/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-factory";
import { GitHubSecretScanningFactory } from "@app/ee/services/secret-scanning-v2/github/github-secret-scanning-factory";
import { SecretScanningDataSource } from "./secret-scanning-v2-enums";
import {
TQueueSecretScanningResourceDiffScan,
TSecretScanningDataSourceCredentials,
TSecretScanningDataSourceInput,
TSecretScanningDataSourceWithConnection,
TSecretScanningFactory
} from "./secret-scanning-v2-types";
type TSecretScanningFactoryImplementation = TSecretScanningFactory<
TSecretScanningDataSourceWithConnection,
TSecretScanningDataSourceCredentials,
TQueueSecretScanningResourceDiffScan["payload"]
TQueueSecretScanningResourceDiffScan["payload"],
TSecretScanningDataSourceInput,
TSecretScanningDataSourceCredentials
>;
export const SECRET_SCANNING_FACTORY_MAP: Record<SecretScanningDataSource, TSecretScanningFactoryImplementation> = {
[SecretScanningDataSource.GitHub]: GitHubSecretScanningFactory as TSecretScanningFactoryImplementation
[SecretScanningDataSource.GitHub]: GitHubSecretScanningFactory as TSecretScanningFactoryImplementation,
[SecretScanningDataSource.Bitbucket]: BitbucketSecretScanningFactory as TSecretScanningFactoryImplementation
};
@@ -4,6 +4,7 @@ import RE2 from "re2";
import { readFindingsFile } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import { BITBUCKET_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/bitbucket";
import { GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/github";
import { titleCaseToCamelCase } from "@app/lib/fn";
@@ -11,7 +12,8 @@ import { SecretScanningDataSource, SecretScanningFindingSeverity } from "./secre
import { TCloneRepository, TGetFindingsPayload, TSecretScanningDataSourceListItem } from "./secret-scanning-v2-types";
const SECRET_SCANNING_SOURCE_LIST_OPTIONS: Record<SecretScanningDataSource, TSecretScanningDataSourceListItem> = {
[SecretScanningDataSource.GitHub]: GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION
[SecretScanningDataSource.GitHub]: GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION,
[SecretScanningDataSource.Bitbucket]: BITBUCKET_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION
};
export const listSecretScanningDataSourceOptions = () => {
@@ -2,13 +2,16 @@ import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/se
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
export const SECRET_SCANNING_DATA_SOURCE_NAME_MAP: Record<SecretScanningDataSource, string> = {
[SecretScanningDataSource.GitHub]: "GitHub"
[SecretScanningDataSource.GitHub]: "GitHub",
[SecretScanningDataSource.Bitbucket]: "Bitbucket"
};
export const SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP: Record<SecretScanningDataSource, AppConnection> = {
[SecretScanningDataSource.GitHub]: AppConnection.GitHubRadar
[SecretScanningDataSource.GitHub]: AppConnection.GitHubRadar,
[SecretScanningDataSource.Bitbucket]: AppConnection.Bitbucket
};
export const AUTO_SYNC_DESCRIPTION_HELPER: Record<SecretScanningDataSource, { verb: string; noun: string }> = {
[SecretScanningDataSource.GitHub]: { verb: "push", noun: "repositories" }
[SecretScanningDataSource.GitHub]: { verb: "push", noun: "repositories" },
[SecretScanningDataSource.Bitbucket]: { verb: "push", noun: "repositories" }
};
@@ -37,7 +37,8 @@ import {
TQueueSecretScanningDataSourceFullScan,
TQueueSecretScanningResourceDiffScan,
TQueueSecretScanningSendNotification,
TSecretScanningDataSourceWithConnection
TSecretScanningDataSourceWithConnection,
TSecretScanningFinding
} from "./secret-scanning-v2-types";
type TSecretRotationV2QueueServiceFactoryDep = {
@@ -459,13 +460,16 @@ export const secretScanningV2QueueServiceFactory = async ({
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
if (newFindings.length) {
const finding = newFindings[0] as TSecretScanningFinding;
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Completed,
resourceName: resource.name,
isDiffScan: true,
dataSource,
numberOfSecrets: newFindings.length,
scanId
scanId,
authorName: finding?.details?.author,
authorEmail: finding?.details?.email
});
}
@@ -582,8 +586,8 @@ export const secretScanningV2QueueServiceFactory = async ({
substitutions:
payload.status === SecretScanningScanStatus.Completed
? {
authorName: "Jim",
authorEmail: "[email protected]",
authorName: payload.authorName,
authorEmail: payload.authorEmail,
resourceName,
numberOfSecrets: payload.numberOfSecrets,
isDiffScan: payload.isDiffScan,
@@ -19,8 +19,7 @@ export const BaseSecretScanningDataSourceSchema = ({
// unique to provider
type: true,
connectionId: true,
config: true,
encryptedCredentials: true
config: true
}).extend({
type: z.literal(type),
connectionId: isConnectionRequired ? z.string().uuid() : z.null(),
@@ -30,6 +30,8 @@ import {
TFindSecretScanningDataSourceByNameDTO,
TListSecretScanningDataSourcesByProjectId,
TSecretScanningDataSource,
TSecretScanningDataSourceCredentials,
TSecretScanningDataSourceInput,
TSecretScanningDataSourceWithConnection,
TSecretScanningDataSourceWithDetails,
TSecretScanningFinding,
@@ -49,6 +51,7 @@ import { TAppConnection } from "@app/services/app-connection/app-connection-type
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
import { bitbucketSecretScanningService } from "./bitbucket/bitbucket-secret-scanning-service";
import { TSecretScanningV2DALFactory } from "./secret-scanning-v2-dal";
import { TSecretScanningV2QueueServiceFactory } from "./secret-scanning-v2-queue";
@@ -256,7 +259,7 @@ export const secretScanningV2ServiceFactory = ({
try {
const createdDataSource = await factory.initialize(
{
payload,
payload: payload as TSecretScanningDataSourceInput,
connection: connection as TSecretScanningDataSourceWithConnection["connection"],
secretScanningV2DAL
},
@@ -287,7 +290,7 @@ export const secretScanningV2ServiceFactory = ({
);
await factory.postInitialization({
payload,
payload: payload as TSecretScanningDataSourceInput,
connection: connection as TSecretScanningDataSourceWithConnection["connection"],
dataSourceId: dataSource.id,
credentials
@@ -398,7 +401,6 @@ export const secretScanningV2ServiceFactory = ({
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
projectId: dataSource.projectId
});
@@ -412,7 +414,36 @@ export const secretScanningV2ServiceFactory = ({
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
});
// TODO: clean up webhooks
const factory = SECRET_SCANNING_FACTORY_MAP[type]();
let connection: TAppConnection | null = null;
if (dataSource.connection) {
connection = await decryptAppConnection(dataSource.connection, kmsService);
}
let credentials: TSecretScanningDataSourceCredentials | undefined;
if (dataSource.encryptedCredentials) {
const { decryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId: dataSource.projectId
});
credentials = JSON.parse(
decryptor({
cipherTextBlob: dataSource.encryptedCredentials
}).toString()
) as TSecretScanningDataSourceCredentials;
}
await factory.teardown({
dataSource: {
...dataSource,
// @ts-expect-error currently we don't have a null connection data source
connection
},
credentials
});
await secretScanningV2DAL.dataSources.deleteById(dataSourceId);
@@ -869,6 +900,7 @@ export const secretScanningV2ServiceFactory = ({
updateSecretScanningFindingById,
findSecretScanningConfigByProjectId,
upsertSecretScanningConfig,
github: githubSecretScanningService(secretScanningV2DAL, secretScanningV2Queue)
github: githubSecretScanningService(secretScanningV2DAL, secretScanningV2Queue),
bitbucket: bitbucketSecretScanningService(secretScanningV2DAL, secretScanningV2Queue, kmsService)
};
};
@@ -4,6 +4,15 @@ import {
TSecretScanningResources,
TSecretScanningScans
} from "@app/db/schemas";
import {
TBitbucketDataSource,
TBitbucketDataSourceCredentials,
TBitbucketDataSourceInput,
TBitbucketDataSourceListItem,
TBitbucketDataSourceWithConnection,
TBitbucketFinding,
TQueueBitbucketResourceDiffScan
} from "@app/ee/services/secret-scanning-v2/bitbucket";
import {
TGitHubDataSource,
TGitHubDataSourceInput,
@@ -19,7 +28,7 @@ import {
SecretScanningScanStatus
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
export type TSecretScanningDataSource = TGitHubDataSource;
export type TSecretScanningDataSource = TGitHubDataSource | TBitbucketDataSource;
export type TSecretScanningDataSourceWithDetails = TSecretScanningDataSource & {
lastScannedAt?: Date | null;
@@ -41,13 +50,17 @@ export type TSecretScanningScanWithDetails = TSecretScanningScans & {
resourceName: string;
};
export type TSecretScanningDataSourceWithConnection = TGitHubDataSourceWithConnection;
export type TSecretScanningDataSourceWithConnection =
| TGitHubDataSourceWithConnection
| TBitbucketDataSourceWithConnection;
export type TSecretScanningDataSourceInput = TGitHubDataSourceInput;
export type TSecretScanningDataSourceInput = TGitHubDataSourceInput | TBitbucketDataSourceInput;
export type TSecretScanningDataSourceListItem = TGitHubDataSourceListItem;
export type TSecretScanningDataSourceListItem = TGitHubDataSourceListItem | TBitbucketDataSourceListItem;
export type TSecretScanningFinding = TGitHubFinding;
export type TSecretScanningDataSourceCredentials = TBitbucketDataSourceCredentials | undefined;
export type TSecretScanningFinding = TGitHubFinding | TBitbucketFinding;
export type TListSecretScanningDataSourcesByProjectId = {
projectId: string;
@@ -99,14 +112,21 @@ export type TQueueSecretScanningDataSourceFullScan = {
scanId: string;
};
export type TQueueSecretScanningResourceDiffScan = TQueueGitHubResourceDiffScan;
export type TQueueSecretScanningResourceDiffScan = TQueueGitHubResourceDiffScan | TQueueBitbucketResourceDiffScan;
export type TQueueSecretScanningSendNotification = {
dataSource: TSecretScanningDataSources;
resourceName: string;
} & (
| { status: SecretScanningScanStatus.Failed; errorMessage: string }
| { status: SecretScanningScanStatus.Completed; numberOfSecrets: number; scanId: string; isDiffScan: boolean }
| {
status: SecretScanningScanStatus.Completed;
numberOfSecrets: number;
scanId: string;
isDiffScan: boolean;
authorName?: string;
authorEmail?: string;
}
);
export type TCloneRepository = {
@@ -138,11 +158,12 @@ export type TSecretScanningDataSourceRaw = NonNullable<
>;
export type TSecretScanningFactoryInitialize<
P extends TSecretScanningDataSourceInput,
T extends TSecretScanningDataSourceWithConnection["connection"] | undefined = undefined,
C extends TSecretScanningDataSourceCredentials = undefined
> = (
params: {
payload: TCreateSecretScanningDataSourceDTO;
payload: P;
connection: T;
secretScanningV2DAL: TSecretScanningV2DALFactory;
},
@@ -150,24 +171,27 @@ export type TSecretScanningFactoryInitialize<
) => Promise<TSecretScanningDataSourceRaw>;
export type TSecretScanningFactoryPostInitialization<
P extends TSecretScanningDataSourceInput,
T extends TSecretScanningDataSourceWithConnection["connection"] | undefined = undefined,
C extends TSecretScanningDataSourceCredentials = undefined
> = (params: {
payload: TCreateSecretScanningDataSourceDTO;
connection: T;
credentials: C;
dataSourceId: string;
}) => Promise<void>;
> = (params: { payload: P; connection: T; credentials: C; dataSourceId: string }) => Promise<void>;
export type TSecretScanningFactoryTeardown<
T extends TSecretScanningDataSourceWithConnection,
C extends TSecretScanningDataSourceCredentials = undefined
> = (params: { dataSource: T; credentials: C }) => Promise<void>;
export type TSecretScanningFactory<
T extends TSecretScanningDataSourceWithConnection,
C extends TSecretScanningDataSourceCredentials,
P extends TQueueSecretScanningResourceDiffScan["payload"]
P extends TQueueSecretScanningResourceDiffScan["payload"],
I extends TSecretScanningDataSourceInput,
C extends TSecretScanningDataSourceCredentials | undefined = undefined
> = () => {
listRawResources: TSecretScanningFactoryListRawResources<T>;
getFullScanPath: TSecretScanningFactoryGetFullScanPath<T>;
initialize: TSecretScanningFactoryInitialize<T["connection"] | undefined, C>;
postInitialization: TSecretScanningFactoryPostInitialization<T["connection"] | undefined, C>;
initialize: TSecretScanningFactoryInitialize<I, T["connection"] | undefined, C>;
postInitialization: TSecretScanningFactoryPostInitialization<I, T["connection"] | undefined, C>;
teardown: TSecretScanningFactoryTeardown<T, C>;
getDiffScanResourcePayload: TSecretScanningFactoryGetDiffScanResourcePayload<P>;
getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<T, P>;
};
@@ -185,5 +209,3 @@ export type TUpsertSecretScanningConfigDTO = {
projectId: string;
content: string | null;
};
export type TSecretScanningDataSourceCredentials = undefined;
@@ -1,7 +1,22 @@
import { z } from "zod";
import { BitbucketDataSourceSchema, BitbucketFindingSchema } from "@app/ee/services/secret-scanning-v2/bitbucket";
import { GitHubDataSourceSchema, GitHubFindingSchema } from "@app/ee/services/secret-scanning-v2/github";
export const SecretScanningDataSourceSchema = z.discriminatedUnion("type", [GitHubDataSourceSchema]);
export const SecretScanningDataSourceSchema = z.discriminatedUnion("type", [
GitHubDataSourceSchema,
BitbucketDataSourceSchema
]);
export const SecretScanningFindingSchema = z.discriminatedUnion("resourceType", [GitHubFindingSchema]);
export const SecretScanningFindingSchema = z.discriminatedUnion("dataSourceType", [
GitHubFindingSchema.describe(
JSON.stringify({
title: "GitHub"
})
),
BitbucketFindingSchema.describe(
JSON.stringify({
title: "Bitbucket"
})
)
]);