Merge branch 'heads/main' into daniel/srp-removal-round-2

This commit is contained in:
Daniel Hougaard
2025-08-05 22:17:01 +04:00
446 changed files with 13524 additions and 3367 deletions
@@ -1,8 +1,10 @@
// weird commonjs-related error in the CI requires us to do the import like this
import knex from "knex";
import { v4 as uuidv4 } from "uuid";
import { TDbClient } from "@app/db";
import { TableName, TAuditLogs } from "@app/db/schemas";
import { getConfig } from "@app/lib/config/env";
import { DatabaseError, GatewayTimeoutError } from "@app/lib/errors";
import { ormify, selectAllTableCols, TOrmify } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
@@ -150,43 +152,70 @@ export const auditLogDALFactory = (db: TDbClient) => {
// delete all audit log that have expired
const pruneAuditLog: TAuditLogDALFactory["pruneAuditLog"] = async (tx) => {
const AUDIT_LOG_PRUNE_BATCH_SIZE = 10000;
const MAX_RETRY_ON_FAILURE = 3;
const runPrune = async (dbClient: knex.Knex) => {
const AUDIT_LOG_PRUNE_BATCH_SIZE = 10000;
const MAX_RETRY_ON_FAILURE = 3;
const today = new Date();
let deletedAuditLogIds: { id: string }[] = [];
let numberOfRetryOnFailure = 0;
let isRetrying = false;
const today = new Date();
let deletedAuditLogIds: { id: string }[] = [];
let numberOfRetryOnFailure = 0;
let isRetrying = false;
logger.info(`${QueueName.DailyResourceCleanUp}: audit log started`);
do {
try {
const findExpiredLogSubQuery = (tx || db)(TableName.AuditLog)
.where("expiresAt", "<", today)
.where("createdAt", "<", today) // to use audit log partition
.orderBy(`${TableName.AuditLog}.createdAt`, "desc")
.select("id")
.limit(AUDIT_LOG_PRUNE_BATCH_SIZE);
logger.info(`${QueueName.DailyResourceCleanUp}: audit log started`);
do {
try {
const findExpiredLogSubQuery = dbClient(TableName.AuditLog)
.where("expiresAt", "<", today)
.where("createdAt", "<", today) // to use audit log partition
.orderBy(`${TableName.AuditLog}.createdAt`, "desc")
.select("id")
.limit(AUDIT_LOG_PRUNE_BATCH_SIZE);
// eslint-disable-next-line no-await-in-loop
deletedAuditLogIds = await (tx || db)(TableName.AuditLog)
.whereIn("id", findExpiredLogSubQuery)
.del()
.returning("id");
numberOfRetryOnFailure = 0; // reset
} catch (error) {
numberOfRetryOnFailure += 1;
logger.error(error, "Failed to delete audit log on pruning");
} finally {
// eslint-disable-next-line no-await-in-loop
await new Promise((resolve) => {
setTimeout(resolve, 10); // time to breathe for db
});
}
isRetrying = numberOfRetryOnFailure > 0;
} while (deletedAuditLogIds.length > 0 || (isRetrying && numberOfRetryOnFailure < MAX_RETRY_ON_FAILURE));
logger.info(`${QueueName.DailyResourceCleanUp}: audit log completed`);
// eslint-disable-next-line no-await-in-loop
deletedAuditLogIds = await dbClient(TableName.AuditLog)
.whereIn("id", findExpiredLogSubQuery)
.del()
.returning("id");
numberOfRetryOnFailure = 0; // reset
} catch (error) {
numberOfRetryOnFailure += 1;
logger.error(error, "Failed to delete audit log on pruning");
} finally {
// eslint-disable-next-line no-await-in-loop
await new Promise((resolve) => {
setTimeout(resolve, 10); // time to breathe for db
});
}
isRetrying = numberOfRetryOnFailure > 0;
} while (deletedAuditLogIds.length > 0 || (isRetrying && numberOfRetryOnFailure < MAX_RETRY_ON_FAILURE));
logger.info(`${QueueName.DailyResourceCleanUp}: audit log completed`);
};
if (tx) {
await runPrune(tx);
} else {
const QUERY_TIMEOUT_MS = 10 * 60 * 1000; // 10 minutes
await db.transaction(async (trx) => {
await trx.raw(`SET statement_timeout = ${QUERY_TIMEOUT_MS}`);
await runPrune(trx);
});
}
};
return { ...auditLogOrm, pruneAuditLog, find };
const create: TAuditLogDALFactory["create"] = async (tx) => {
const config = getConfig();
if (config.DISABLE_AUDIT_LOG_STORAGE) {
return {
...tx,
id: uuidv4(),
createdAt: new Date(),
updatedAt: new Date()
};
}
return auditLogOrm.create(tx);
};
return { ...auditLogOrm, create, pruneAuditLog, find };
};
@@ -1,7 +1,8 @@
import { AxiosError, RawAxiosRequestHeaders } from "axios";
import { SecretKeyEncoding } from "@app/db/schemas";
import { getConfig } from "@app/lib/config/env";
import { ProjectType, SecretKeyEncoding } from "@app/db/schemas";
import { TEventBusService } from "@app/ee/services/event/event-bus-service";
import { TopicName, toPublishableEvent } from "@app/ee/services/event/types";
import { request } from "@app/lib/config/request";
import { crypto } from "@app/lib/crypto/cryptography";
import { logger } from "@app/lib/logger";
@@ -21,6 +22,7 @@ type TAuditLogQueueServiceFactoryDep = {
queueService: TQueueServiceFactory;
projectDAL: Pick<TProjectDALFactory, "findById">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
eventBusService: TEventBusService;
};
export type TAuditLogQueueServiceFactory = {
@@ -36,133 +38,17 @@ export const auditLogQueueServiceFactory = async ({
queueService,
projectDAL,
licenseService,
auditLogStreamDAL
auditLogStreamDAL,
eventBusService
}: TAuditLogQueueServiceFactoryDep): Promise<TAuditLogQueueServiceFactory> => {
const appCfg = getConfig();
const pushToLog = async (data: TCreateAuditLogDTO) => {
if (appCfg.USE_PG_QUEUE && appCfg.SHOULD_INIT_PG_QUEUE) {
await queueService.queuePg<QueueName.AuditLog>(QueueJobs.AuditLog, data, {
retryLimit: 10,
retryBackoff: true
});
} else {
await queueService.queue<QueueName.AuditLog>(QueueName.AuditLog, QueueJobs.AuditLog, data, {
removeOnFail: {
count: 3
},
removeOnComplete: true
});
}
};
if (appCfg.SHOULD_INIT_PG_QUEUE) {
await queueService.startPg<QueueName.AuditLog>(
QueueJobs.AuditLog,
async ([job]) => {
const { actor, event, ipAddress, projectId, userAgent, userAgentType } = job.data;
let { orgId } = job.data;
const MS_IN_DAY = 24 * 60 * 60 * 1000;
let project;
if (!orgId) {
// it will never be undefined for both org and project id
// TODO(akhilmhdh): use caching here in dal to avoid db calls
project = await projectDAL.findById(projectId as string);
orgId = project.orgId;
}
const plan = await licenseService.getPlan(orgId);
if (plan.auditLogsRetentionDays === 0) {
// skip inserting if audit log retention is 0 meaning its not supported
return;
}
// For project actions, set TTL to project-level audit log retention config
// This condition ensures that the plan's audit log retention days cannot be bypassed
const ttlInDays =
project?.auditLogsRetentionDays && project.auditLogsRetentionDays < plan.auditLogsRetentionDays
? project.auditLogsRetentionDays
: plan.auditLogsRetentionDays;
const ttl = ttlInDays * MS_IN_DAY;
const auditLog = await auditLogDAL.create({
actor: actor.type,
actorMetadata: actor.metadata,
userAgent,
projectId,
projectName: project?.name,
ipAddress,
orgId,
eventType: event.type,
expiresAt: new Date(Date.now() + ttl),
eventMetadata: event.metadata,
userAgentType
});
const logStreams = orgId ? await auditLogStreamDAL.find({ orgId }) : [];
await Promise.allSettled(
logStreams.map(
async ({
url,
encryptedHeadersTag,
encryptedHeadersIV,
encryptedHeadersKeyEncoding,
encryptedHeadersCiphertext
}) => {
const streamHeaders =
encryptedHeadersIV && encryptedHeadersCiphertext && encryptedHeadersTag
? (JSON.parse(
crypto
.encryption()
.symmetric()
.decryptWithRootEncryptionKey({
keyEncoding: encryptedHeadersKeyEncoding as SecretKeyEncoding,
iv: encryptedHeadersIV,
tag: encryptedHeadersTag,
ciphertext: encryptedHeadersCiphertext
})
) as LogStreamHeaders[])
: [];
const headers: RawAxiosRequestHeaders = { "Content-Type": "application/json" };
if (streamHeaders.length)
streamHeaders.forEach(({ key, value }) => {
headers[key] = value;
});
try {
const response = await request.post(
url,
{ ...providerSpecificPayload(url), ...auditLog },
{
headers,
// request timeout
timeout: AUDIT_LOG_STREAM_TIMEOUT,
// connection timeout
signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
}
);
return response;
} catch (error) {
logger.error(
`Failed to stream audit log [url=${url}] for org [orgId=${orgId}] [error=${(error as AxiosError).message}]`
);
return error;
}
}
)
);
await queueService.queue<QueueName.AuditLog>(QueueName.AuditLog, QueueJobs.AuditLog, data, {
removeOnFail: {
count: 3
},
{
batchSize: 1,
workerCount: 30,
pollingIntervalSeconds: 0.5
}
);
}
removeOnComplete: true
});
};
queueService.start(QueueName.AuditLog, async (job) => {
const { actor, event, ipAddress, projectId, userAgent, userAgentType } = job.data;
@@ -178,88 +64,97 @@ export const auditLogQueueServiceFactory = async ({
}
const plan = await licenseService.getPlan(orgId);
if (plan.auditLogsRetentionDays === 0) {
// skip inserting if audit log retention is 0 meaning its not supported
return;
// skip inserting if audit log retention is 0 meaning its not supported
if (plan.auditLogsRetentionDays !== 0) {
// For project actions, set TTL to project-level audit log retention config
// This condition ensures that the plan's audit log retention days cannot be bypassed
const ttlInDays =
project?.auditLogsRetentionDays && project.auditLogsRetentionDays < plan.auditLogsRetentionDays
? project.auditLogsRetentionDays
: plan.auditLogsRetentionDays;
const ttl = ttlInDays * MS_IN_DAY;
const auditLog = await auditLogDAL.create({
actor: actor.type,
actorMetadata: actor.metadata,
userAgent,
projectId,
projectName: project?.name,
ipAddress,
orgId,
eventType: event.type,
expiresAt: new Date(Date.now() + ttl),
eventMetadata: event.metadata,
userAgentType
});
const logStreams = orgId ? await auditLogStreamDAL.find({ orgId }) : [];
await Promise.allSettled(
logStreams.map(
async ({
url,
encryptedHeadersTag,
encryptedHeadersIV,
encryptedHeadersKeyEncoding,
encryptedHeadersCiphertext
}) => {
const streamHeaders =
encryptedHeadersIV && encryptedHeadersCiphertext && encryptedHeadersTag
? (JSON.parse(
crypto
.encryption()
.symmetric()
.decryptWithRootEncryptionKey({
keyEncoding: encryptedHeadersKeyEncoding as SecretKeyEncoding,
iv: encryptedHeadersIV,
tag: encryptedHeadersTag,
ciphertext: encryptedHeadersCiphertext
})
) as LogStreamHeaders[])
: [];
const headers: RawAxiosRequestHeaders = { "Content-Type": "application/json" };
if (streamHeaders.length)
streamHeaders.forEach(({ key, value }) => {
headers[key] = value;
});
try {
const response = await request.post(
url,
{ ...providerSpecificPayload(url), ...auditLog },
{
headers,
// request timeout
timeout: AUDIT_LOG_STREAM_TIMEOUT,
// connection timeout
signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
}
);
return response;
} catch (error) {
logger.error(
`Failed to stream audit log [url=${url}] for org [orgId=${orgId}] [error=${(error as AxiosError).message}]`
);
return error;
}
}
)
);
}
// For project actions, set TTL to project-level audit log retention config
// This condition ensures that the plan's audit log retention days cannot be bypassed
const ttlInDays =
project?.auditLogsRetentionDays && project.auditLogsRetentionDays < plan.auditLogsRetentionDays
? project.auditLogsRetentionDays
: plan.auditLogsRetentionDays;
const publishable = toPublishableEvent(event);
const ttl = ttlInDays * MS_IN_DAY;
const auditLog = await auditLogDAL.create({
actor: actor.type,
actorMetadata: actor.metadata,
userAgent,
projectId,
projectName: project?.name,
ipAddress,
orgId,
eventType: event.type,
expiresAt: new Date(Date.now() + ttl),
eventMetadata: event.metadata,
userAgentType
});
const logStreams = orgId ? await auditLogStreamDAL.find({ orgId }) : [];
await Promise.allSettled(
logStreams.map(
async ({
url,
encryptedHeadersTag,
encryptedHeadersIV,
encryptedHeadersKeyEncoding,
encryptedHeadersCiphertext
}) => {
const streamHeaders =
encryptedHeadersIV && encryptedHeadersCiphertext && encryptedHeadersTag
? (JSON.parse(
crypto
.encryption()
.symmetric()
.decryptWithRootEncryptionKey({
keyEncoding: encryptedHeadersKeyEncoding as SecretKeyEncoding,
iv: encryptedHeadersIV,
tag: encryptedHeadersTag,
ciphertext: encryptedHeadersCiphertext
})
) as LogStreamHeaders[])
: [];
const headers: RawAxiosRequestHeaders = { "Content-Type": "application/json" };
if (streamHeaders.length)
streamHeaders.forEach(({ key, value }) => {
headers[key] = value;
});
try {
const response = await request.post(
url,
{ ...providerSpecificPayload(url), ...auditLog },
{
headers,
// request timeout
timeout: AUDIT_LOG_STREAM_TIMEOUT,
// connection timeout
signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
}
);
return response;
} catch (error) {
logger.error(
`Failed to stream audit log [url=${url}] for org [orgId=${orgId}] [error=${(error as AxiosError).message}]`
);
return error;
}
}
)
);
if (publishable) {
await eventBusService.publish(TopicName.CoreServers, {
type: ProjectType.SecretManager,
source: "infiscal",
data: publishable.data
});
}
});
return {
@@ -161,6 +161,9 @@ export enum EventType {
CREATE_IDENTITY = "create-identity",
UPDATE_IDENTITY = "update-identity",
DELETE_IDENTITY = "delete-identity",
MACHINE_IDENTITY_AUTH_TEMPLATE_CREATE = "machine-identity-auth-template-create",
MACHINE_IDENTITY_AUTH_TEMPLATE_UPDATE = "machine-identity-auth-template-update",
MACHINE_IDENTITY_AUTH_TEMPLATE_DELETE = "machine-identity-auth-template-delete",
LOGIN_IDENTITY_UNIVERSAL_AUTH = "login-identity-universal-auth",
ADD_IDENTITY_UNIVERSAL_AUTH = "add-identity-universal-auth",
UPDATE_IDENTITY_UNIVERSAL_AUTH = "update-identity-universal-auth",
@@ -830,6 +833,30 @@ interface LoginIdentityUniversalAuthEvent {
};
}
interface MachineIdentityAuthTemplateCreateEvent {
type: EventType.MACHINE_IDENTITY_AUTH_TEMPLATE_CREATE;
metadata: {
templateId: string;
name: string;
};
}
interface MachineIdentityAuthTemplateUpdateEvent {
type: EventType.MACHINE_IDENTITY_AUTH_TEMPLATE_UPDATE;
metadata: {
templateId: string;
name: string;
};
}
interface MachineIdentityAuthTemplateDeleteEvent {
type: EventType.MACHINE_IDENTITY_AUTH_TEMPLATE_DELETE;
metadata: {
templateId: string;
name: string;
};
}
interface AddIdentityUniversalAuthEvent {
type: EventType.ADD_IDENTITY_UNIVERSAL_AUTH;
metadata: {
@@ -1325,6 +1352,7 @@ interface AddIdentityLdapAuthEvent {
accessTokenTrustedIps?: Array<TIdentityTrustedIp>;
allowedFields?: TAllowedFields[];
url: string;
templateId?: string | null;
};
}
@@ -1338,6 +1366,7 @@ interface UpdateIdentityLdapAuthEvent {
accessTokenTrustedIps?: Array<TIdentityTrustedIp>;
allowedFields?: TAllowedFields[];
url?: string;
templateId?: string | null;
};
}
@@ -3439,6 +3468,9 @@ export type Event =
| UpdateIdentityEvent
| DeleteIdentityEvent
| LoginIdentityUniversalAuthEvent
| MachineIdentityAuthTemplateCreateEvent
| MachineIdentityAuthTemplateUpdateEvent
| MachineIdentityAuthTemplateDeleteEvent
| AddIdentityUniversalAuthEvent
| UpdateIdentityUniversalAuthEvent
| DeleteIdentityUniversalAuthEvent
@@ -0,0 +1,83 @@
import Redis from "ioredis";
import { z } from "zod";
import { logger } from "@app/lib/logger";
import { EventSchema, TopicName } from "./types";
export const eventBusFactory = (redis: Redis) => {
const publisher = redis.duplicate();
// Duplicate the publisher to create a subscriber.
// This is necessary because Redis does not allow a single connection to both publish and subscribe.
const subscriber = publisher.duplicate();
const init = async (topics: TopicName[] = Object.values(TopicName)) => {
subscriber.on("error", (e) => {
logger.error(e, "Event Bus subscriber error");
});
publisher.on("error", (e) => {
logger.error(e, "Event Bus publisher error");
});
await subscriber.subscribe(...topics);
};
/**
* Publishes an event to the specified topic.
* @param topic - The topic to publish the event to.
* @param event - The event data to publish.
*/
const publish = async <T extends z.input<typeof EventSchema>>(topic: TopicName, event: T) => {
const json = JSON.stringify(event);
return publisher.publish(topic, json, (err) => {
if (err) {
return logger.error(err, `Error publishing to channel ${topic}`);
}
});
};
/**
* @param fn - The function to call when a message is received.
* It should accept the parsed event data as an argument.
* @template T - The type of the event data, which should match the schema defined in EventSchema.
* @returns A function that can be called to unsubscribe from the event bus.
*/
const subscribe = <T extends z.infer<typeof EventSchema>>(fn: (data: T) => Promise<void> | void) => {
// Not using async await cause redis client's `on` method does not expect async listeners.
const listener = (channel: string, message: string) => {
try {
const parsed = JSON.parse(message) as T;
const thenable = fn(parsed);
// If the function returns a Promise, catch any errors that occur during processing.
if (thenable instanceof Promise) {
thenable.catch((error) => {
logger.error(error, `Error processing message from channel ${channel}`);
});
}
} catch (error) {
logger.error(error, `Error parsing message data from channel ${channel}`);
}
};
subscriber.on("message", listener);
return () => {
subscriber.off("message", listener);
};
};
const close = async () => {
try {
await publisher.quit();
await subscriber.quit();
} catch (error) {
logger.error(error, "Error closing event bus connections");
}
};
return { init, publish, subscribe, close };
};
export type TEventBusService = ReturnType<typeof eventBusFactory>;
@@ -0,0 +1,164 @@
/* eslint-disable no-continue */
import { subject } from "@casl/ability";
import Redis from "ioredis";
import { KeyStorePrefixes } from "@app/keystore/keystore";
import { logger } from "@app/lib/logger";
import { TEventBusService } from "./event-bus-service";
import { createEventStreamClient, EventStreamClient, IEventStreamClientOpts } from "./event-sse-stream";
import { EventData, RegisteredEvent, toBusEventName } from "./types";
const AUTH_REFRESH_INTERVAL = 60 * 1000;
const HEART_BEAT_INTERVAL = 15 * 1000;
export const sseServiceFactory = (bus: TEventBusService, redis: Redis) => {
let heartbeatInterval: NodeJS.Timeout | null = null;
const clients = new Set<EventStreamClient>();
heartbeatInterval = setInterval(() => {
for (const client of clients) {
if (client.stream.closed) continue;
void client.ping();
}
}, HEART_BEAT_INTERVAL);
const refreshInterval = setInterval(() => {
for (const client of clients) {
if (client.stream.closed) continue;
void client.refresh();
}
}, AUTH_REFRESH_INTERVAL);
const removeActiveConnection = async (projectId: string, identityId: string, connectionId: string) => {
const set = KeyStorePrefixes.ActiveSSEConnectionsSet(projectId, identityId);
const key = KeyStorePrefixes.ActiveSSEConnections(projectId, identityId, connectionId);
await Promise.all([redis.lrem(set, 0, connectionId), redis.del(key)]);
};
const getActiveConnectionsCount = async (projectId: string, identityId: string) => {
const set = KeyStorePrefixes.ActiveSSEConnectionsSet(projectId, identityId);
const connections = await redis.lrange(set, 0, -1);
if (connections.length === 0) {
return 0; // No active connections
}
const keys = connections.map((c) => KeyStorePrefixes.ActiveSSEConnections(projectId, identityId, c));
const values = await redis.mget(...keys);
// eslint-disable-next-line no-plusplus
for (let i = 0; i < values.length; i++) {
if (values[i] === null) {
// eslint-disable-next-line no-await-in-loop
await removeActiveConnection(projectId, identityId, connections[i]);
}
}
return redis.llen(set);
};
const onDisconnect = async (client: EventStreamClient) => {
try {
client.close();
clients.delete(client);
await removeActiveConnection(client.auth.projectId, client.auth.actorId, client.id);
} catch (error) {
logger.error(error, "Error during SSE stream disconnection");
}
};
function filterEventsForClient(client: EventStreamClient, event: EventData, registered: RegisteredEvent[]) {
const eventType = toBusEventName(event.data.eventType);
const match = registered.find((r) => r.event === eventType);
if (!match) return;
const item = event.data.payload;
if (Array.isArray(item)) {
if (item.length === 0) return;
const baseSubject = {
eventType,
environment: undefined as string | undefined,
secretPath: undefined as string | undefined
};
const filtered = item.filter((ev) => {
baseSubject.secretPath = ev.secretPath ?? "/";
baseSubject.environment = ev.environment;
return client.matcher.can("subscribe", subject(event.type, baseSubject));
});
if (filtered.length === 0) return;
return client.send({
...event,
data: {
...event.data,
payload: filtered
}
});
}
// For single item
const baseSubject = {
eventType,
secretPath: item.secretPath ?? "/",
environment: item.environment
};
if (client.matcher.can("subscribe", subject(event.type, baseSubject))) {
client.send(event);
}
}
const subscribe = async (
opts: IEventStreamClientOpts & {
onClose?: () => void;
}
) => {
const client = createEventStreamClient(redis, opts);
// Set up event listener on event bus
const unsubscribe = bus.subscribe((event) => {
if (event.type !== opts.type) return;
filterEventsForClient(client, event, opts.registered);
});
client.stream.on("close", () => {
unsubscribe();
void onDisconnect(client); // This will never throw
});
await client.open();
clients.add(client);
return client;
};
const close = () => {
if (heartbeatInterval) {
clearInterval(heartbeatInterval);
}
if (refreshInterval) {
clearInterval(refreshInterval);
}
for (const client of clients) {
client.close();
}
clients.clear();
};
return { subscribe, close, getActiveConnectionsCount };
};
export type TServerSentEventsService = ReturnType<typeof sseServiceFactory>;
@@ -0,0 +1,178 @@
/* eslint-disable no-underscore-dangle */
import { Readable } from "node:stream";
import { MongoAbility, PureAbility } from "@casl/ability";
import { MongoQuery } from "@ucast/mongo2js";
import Redis from "ioredis";
import { nanoid } from "nanoid";
import { ProjectType } from "@app/db/schemas";
import { ProjectPermissionSet } from "@app/ee/services/permission/project-permission";
import { KeyStorePrefixes } from "@app/keystore/keystore";
import { conditionsMatcher } from "@app/lib/casl";
import { logger } from "@app/lib/logger";
import { EventData, RegisteredEvent } from "./types";
export const getServerSentEventsHeaders = () =>
({
"Cache-Control": "no-cache",
"Content-Type": "text/event-stream",
Connection: "keep-alive",
"X-Accel-Buffering": "no"
}) as const;
type TAuthInfo = {
actorId: string;
projectId: string;
permission: MongoAbility<ProjectPermissionSet, MongoQuery>;
};
export interface IEventStreamClientOpts {
type: ProjectType;
registered: RegisteredEvent[];
onAuthRefresh: (info: TAuthInfo) => Promise<void> | void;
getAuthInfo: () => Promise<TAuthInfo> | TAuthInfo;
}
interface EventMessage {
time?: string | number;
type: string;
data?: unknown;
}
function serializeSseEvent(chunk: EventMessage): string {
let payload = "";
if (chunk.time) payload += `id: ${chunk.time}\n`;
if (chunk.type) payload += `event: ${chunk.type}\n`;
if (chunk.data) payload += `data: ${JSON.stringify(chunk)}\n`;
return `${payload}\n`;
}
export type EventStreamClient = {
id: string;
stream: Readable;
open: () => Promise<void>;
send: (data: EventMessage | EventData) => void;
ping: () => Promise<void>;
refresh: () => Promise<void>;
close: () => void;
get auth(): TAuthInfo;
signal: AbortSignal;
abort: () => void;
matcher: PureAbility;
};
export function createEventStreamClient(redis: Redis, options: IEventStreamClientOpts): EventStreamClient {
const rules = options.registered.map((r) => ({
subject: options.type,
action: "subscribe",
conditions: {
eventType: r.event,
secretPath: r.conditions?.secretPath ?? "/",
environment: r.conditions?.environmentSlug
}
}));
const id = `sse-${nanoid()}`;
const control = new AbortController();
const matcher = new PureAbility(rules, { conditionsMatcher });
let auth: TAuthInfo | undefined;
const stream = new Readable({
objectMode: true
});
// We will manually push data to the stream
stream._read = () => {};
const send = (data: EventMessage | EventData) => {
const chunk = serializeSseEvent(data);
if (!stream.push(chunk)) {
logger.debug("Backpressure detected: dropped manual event");
}
};
stream.on("error", (error: Error) => stream.destroy(error));
const open = async () => {
auth = await options.getAuthInfo();
await options.onAuthRefresh(auth);
const { actorId, projectId } = auth;
const set = KeyStorePrefixes.ActiveSSEConnectionsSet(projectId, actorId);
const key = KeyStorePrefixes.ActiveSSEConnections(projectId, actorId, id);
await Promise.all([redis.rpush(set, id), redis.set(key, "1", "EX", 60)]);
};
const ping = async () => {
if (!auth) return; // Avoid race condition if ping is called before open
const { actorId, projectId } = auth;
const key = KeyStorePrefixes.ActiveSSEConnections(projectId, actorId, id);
await redis.set(key, "1", "EX", 60);
stream.push("1");
};
const close = () => {
if (stream.closed) return;
stream.push(null);
stream.destroy();
};
/**
* Refreshes the connection's auth permissions
* Must be called atleast once when connection is opened
*/
const refresh = async () => {
try {
auth = await options.getAuthInfo();
await options.onAuthRefresh(auth);
} catch (error) {
if (error instanceof Error) {
send({
type: "error",
data: {
...error
}
});
return close();
}
stream.emit("error", error);
}
};
const abort = () => {
try {
control.abort();
} catch (error) {
logger.debug(error, "Error aborting SSE stream");
}
};
return {
id,
stream,
open,
send,
ping,
refresh,
close,
signal: control.signal,
abort,
matcher,
get auth() {
if (!auth) {
throw new Error("Auth info not set");
}
return auth;
}
};
}
+125
View File
@@ -0,0 +1,125 @@
import { z } from "zod";
import { ProjectType } from "@app/db/schemas";
import { Event, EventType } from "@app/ee/services/audit-log/audit-log-types";
export enum TopicName {
CoreServers = "infisical::core-servers"
}
export enum BusEventName {
CreateSecret = "secret:create",
UpdateSecret = "secret:update",
DeleteSecret = "secret:delete"
}
type PublisableEventTypes =
| EventType.CREATE_SECRET
| EventType.CREATE_SECRETS
| EventType.DELETE_SECRET
| EventType.DELETE_SECRETS
| EventType.UPDATE_SECRETS
| EventType.UPDATE_SECRET;
export function toBusEventName(input: EventType) {
switch (input) {
case EventType.CREATE_SECRET:
case EventType.CREATE_SECRETS:
return BusEventName.CreateSecret;
case EventType.UPDATE_SECRET:
case EventType.UPDATE_SECRETS:
return BusEventName.UpdateSecret;
case EventType.DELETE_SECRET:
case EventType.DELETE_SECRETS:
return BusEventName.DeleteSecret;
default:
return null;
}
}
const isBulkEvent = (event: Event): event is Extract<Event, { metadata: { secrets: Array<unknown> } }> => {
return event.type.endsWith("-secrets"); // Feels so wrong
};
export const toPublishableEvent = (event: Event) => {
const name = toBusEventName(event.type);
if (!name) return null;
const e = event as Extract<Event, { type: PublisableEventTypes }>;
if (isBulkEvent(e)) {
return {
name,
isBulk: true,
data: {
eventType: e.type,
payload: e.metadata.secrets.map((s) => ({
environment: e.metadata.environment,
secretPath: e.metadata.secretPath,
...s
}))
}
} as const;
}
return {
name,
isBulk: false,
data: {
eventType: e.type,
payload: {
...e.metadata,
environment: e.metadata.environment
}
}
} as const;
};
export const EventName = z.nativeEnum(BusEventName);
const EventSecretPayload = z.object({
secretPath: z.string().optional(),
secretId: z.string(),
secretKey: z.string(),
environment: z.string()
});
export type EventSecret = z.infer<typeof EventSecretPayload>;
export const EventSchema = z.object({
datacontenttype: z.literal("application/json").optional().default("application/json"),
type: z.nativeEnum(ProjectType),
source: z.string(),
time: z
.string()
.optional()
.default(() => new Date().toISOString()),
data: z.discriminatedUnion("eventType", [
z.object({
specversion: z.number().optional().default(1),
eventType: z.enum([EventType.CREATE_SECRET, EventType.UPDATE_SECRET, EventType.DELETE_SECRET]),
payload: EventSecretPayload
}),
z.object({
specversion: z.number().optional().default(1),
eventType: z.enum([EventType.CREATE_SECRETS, EventType.UPDATE_SECRETS, EventType.DELETE_SECRETS]),
payload: EventSecretPayload.array()
})
// Add more event types as needed
])
});
export type EventData = z.infer<typeof EventSchema>;
export const EventRegisterSchema = z.object({
event: EventName,
conditions: z
.object({
secretPath: z.string().optional().default("/"),
environmentSlug: z.string()
})
.optional()
});
export type RegisteredEvent = z.infer<typeof EventRegisterSchema>;
@@ -0,0 +1,83 @@
/* eslint-disable no-case-declarations */
import { Knex } from "knex";
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { buildFindFilter, ormify } from "@app/lib/knex";
import { IdentityAuthTemplateMethod } from "./identity-auth-template-enums";
export type TIdentityAuthTemplateDALFactory = ReturnType<typeof identityAuthTemplateDALFactory>;
export const identityAuthTemplateDALFactory = (db: TDbClient) => {
const identityAuthTemplateOrm = ormify(db, TableName.IdentityAuthTemplate);
const findByOrgId = async (
orgId: string,
{ limit, offset, search, tx }: { limit?: number; offset?: number; search?: string; tx?: Knex } = {}
) => {
let query = (tx || db.replicaNode())(TableName.IdentityAuthTemplate).where({ orgId });
let countQuery = (tx || db.replicaNode())(TableName.IdentityAuthTemplate).where({ orgId });
if (search) {
const searchFilter = `%${search.toLowerCase()}%`;
query = query.whereRaw("LOWER(name) LIKE ?", [searchFilter]);
countQuery = countQuery.whereRaw("LOWER(name) LIKE ?", [searchFilter]);
}
query = query.orderBy("createdAt", "desc");
if (limit !== undefined) {
query = query.limit(limit);
}
if (offset !== undefined) {
query = query.offset(offset);
}
const docs = await query;
const [{ count }] = (await countQuery.count("* as count")) as [{ count: string | number }];
return { docs, totalCount: Number(count) };
};
const findByAuthMethod = async (authMethod: string, orgId: string, tx?: Knex) => {
const query = (tx || db.replicaNode())(TableName.IdentityAuthTemplate)
.where({ authMethod, orgId })
.orderBy("createdAt", "desc");
const docs = await query;
return docs;
};
const findTemplateUsages = async (templateId: string, authMethod: string, tx?: Knex) => {
switch (authMethod) {
case IdentityAuthTemplateMethod.LDAP:
const query = (tx || db.replicaNode())(TableName.IdentityLdapAuth)
.join(TableName.Identity, `${TableName.IdentityLdapAuth}.identityId`, `${TableName.Identity}.id`)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter({ templateId }, TableName.IdentityLdapAuth))
.select(
db.ref("identityId").withSchema(TableName.IdentityLdapAuth),
db.ref("name").withSchema(TableName.Identity).as("identityName")
);
const docs = await query;
return docs;
default:
return [];
}
};
const findByIdAndOrgId = async (id: string, orgId: string, tx?: Knex) => {
const query = (tx || db.replicaNode())(TableName.IdentityAuthTemplate).where({ id, orgId });
const doc = await query;
return doc?.[0];
};
return {
...identityAuthTemplateOrm,
findByOrgId,
findByAuthMethod,
findTemplateUsages,
findByIdAndOrgId
};
};
@@ -0,0 +1,22 @@
export enum IdentityAuthTemplateMethod {
LDAP = "ldap"
}
export const TEMPLATE_VALIDATION_MESSAGES = {
TEMPLATE_NAME_REQUIRED: "Template name is required",
TEMPLATE_NAME_MAX_LENGTH: "Template name must be at most 64 characters long",
AUTH_METHOD_REQUIRED: "Auth method is required",
TEMPLATE_ID_REQUIRED: "Template ID is required",
LDAP: {
URL_REQUIRED: "LDAP URL is required",
BIND_DN_REQUIRED: "Bind DN is required",
BIND_PASSWORD_REQUIRED: "Bind password is required",
SEARCH_BASE_REQUIRED: "Search base is required"
}
} as const;
export const TEMPLATE_SUCCESS_MESSAGES = {
CREATED: "Template created successfully",
UPDATED: "Template updated successfully",
DELETED: "Template deleted successfully"
} as const;
@@ -0,0 +1,454 @@
import { ForbiddenError } from "@casl/ability";
import { EventType, TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-types";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import {
OrgPermissionMachineIdentityAuthTemplateActions,
OrgPermissionSubjects
} from "@app/ee/services/permission/org-permission";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { TOrgPermission } from "@app/lib/types";
import { ActorType } from "@app/services/auth/auth-type";
import { TIdentityLdapAuthDALFactory } from "@app/services/identity-ldap-auth/identity-ldap-auth-dal";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
import { TIdentityAuthTemplateDALFactory } from "./identity-auth-template-dal";
import { IdentityAuthTemplateMethod } from "./identity-auth-template-enums";
import {
TDeleteIdentityAuthTemplateDTO,
TFindTemplateUsagesDTO,
TGetIdentityAuthTemplateDTO,
TGetTemplatesByAuthMethodDTO,
TLdapTemplateFields,
TListIdentityAuthTemplatesDTO,
TUnlinkTemplateUsageDTO
} from "./identity-auth-template-types";
type TIdentityAuthTemplateServiceFactoryDep = {
identityAuthTemplateDAL: TIdentityAuthTemplateDALFactory;
identityLdapAuthDAL: TIdentityLdapAuthDALFactory;
permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey" | "encryptWithInputKey" | "decryptWithInputKey">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
};
export type TIdentityAuthTemplateServiceFactory = ReturnType<typeof identityAuthTemplateServiceFactory>;
export const identityAuthTemplateServiceFactory = ({
identityAuthTemplateDAL,
identityLdapAuthDAL,
permissionService,
kmsService,
licenseService,
auditLogService
}: TIdentityAuthTemplateServiceFactoryDep) => {
// Plan check
const $checkPlan = async (orgId: string) => {
const plan = await licenseService.getPlan(orgId);
if (!plan.machineIdentityAuthTemplates)
throw new BadRequestError({
message:
"Failed to use identity auth template due to plan restriction. Upgrade plan to access machine identity auth templates."
});
};
const createTemplate = async ({
name,
authMethod,
templateFields,
actorId,
actorAuthMethod,
actor,
actorOrgId
}: {
name: string;
authMethod: string;
templateFields: Record<string, unknown>;
} & Omit<TOrgPermission, "orgId">) => {
await $checkPlan(actorOrgId);
const { permission } = await permissionService.getOrgPermission(
actor,
actorId,
actorOrgId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionMachineIdentityAuthTemplateActions.CreateTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
const { encryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.Organization,
orgId: actorOrgId
});
const template = await identityAuthTemplateDAL.create({
name,
authMethod,
templateFields: encryptor({ plainText: Buffer.from(JSON.stringify(templateFields)) }).cipherTextBlob,
orgId: actorOrgId
});
return { ...template, templateFields };
};
const updateTemplate = async ({
templateId,
name,
templateFields,
actorId,
actorAuthMethod,
actor,
actorOrgId
}: {
templateId: string;
name?: string;
templateFields?: Record<string, unknown>;
} & Omit<TOrgPermission, "orgId">) => {
await $checkPlan(actorOrgId);
const template = await identityAuthTemplateDAL.findByIdAndOrgId(templateId, actorOrgId);
if (!template) {
throw new NotFoundError({ message: "Template not found" });
}
const { permission } = await permissionService.getOrgPermission(
actor,
actorId,
template.orgId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionMachineIdentityAuthTemplateActions.EditTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
const { encryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.Organization,
orgId: template.orgId
});
let finalTemplateFields: Record<string, unknown> = {};
const updatedTemplate = await identityAuthTemplateDAL.transaction(async (tx) => {
const authTemplate = await identityAuthTemplateDAL.updateById(
templateId,
{
name,
...(templateFields && {
templateFields: encryptor({ plainText: Buffer.from(JSON.stringify(templateFields)) }).cipherTextBlob
})
},
tx
);
if (templateFields && template.authMethod === IdentityAuthTemplateMethod.LDAP) {
const { decryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.Organization,
orgId: template.orgId
});
const currentTemplateFields = JSON.parse(
decryptor({ cipherTextBlob: template.templateFields }).toString()
) as TLdapTemplateFields;
const mergedTemplateFields: TLdapTemplateFields = { ...currentTemplateFields, ...templateFields };
finalTemplateFields = mergedTemplateFields;
const ldapUpdateData: {
url?: string;
searchBase?: string;
encryptedBindDN?: Buffer;
encryptedBindPass?: Buffer;
encryptedLdapCaCertificate?: Buffer;
} = {};
if ("url" in templateFields) {
ldapUpdateData.url = mergedTemplateFields.url;
}
if ("searchBase" in templateFields) {
ldapUpdateData.searchBase = mergedTemplateFields.searchBase;
}
if ("bindDN" in templateFields) {
ldapUpdateData.encryptedBindDN = encryptor({
plainText: Buffer.from(mergedTemplateFields.bindDN)
}).cipherTextBlob;
}
if ("bindPass" in templateFields) {
ldapUpdateData.encryptedBindPass = encryptor({
plainText: Buffer.from(mergedTemplateFields.bindPass)
}).cipherTextBlob;
}
if ("ldapCaCertificate" in templateFields) {
ldapUpdateData.encryptedLdapCaCertificate = encryptor({
plainText: Buffer.from(mergedTemplateFields.ldapCaCertificate || "")
}).cipherTextBlob;
}
if (Object.keys(ldapUpdateData).length > 0) {
const updatedLdapAuths = await identityLdapAuthDAL.update({ templateId }, ldapUpdateData, tx);
await Promise.all(
updatedLdapAuths.map(async (updatedLdapAuth) => {
await auditLogService.createAuditLog({
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
orgId: actorOrgId,
event: {
type: EventType.UPDATE_IDENTITY_LDAP_AUTH,
metadata: {
identityId: updatedLdapAuth.identityId,
templateId: template.id
}
}
});
})
);
}
}
return authTemplate;
});
return { ...updatedTemplate, templateFields: finalTemplateFields };
};
const deleteTemplate = async ({
templateId,
actorId,
actorAuthMethod,
actor,
actorOrgId
}: TDeleteIdentityAuthTemplateDTO) => {
await $checkPlan(actorOrgId);
const template = await identityAuthTemplateDAL.findByIdAndOrgId(templateId, actorOrgId);
if (!template) {
throw new NotFoundError({ message: "Template not found" });
}
const { permission } = await permissionService.getOrgPermission(
actor,
actorId,
template.orgId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionMachineIdentityAuthTemplateActions.DeleteTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
const deletedTemplate = await identityAuthTemplateDAL.transaction(async (tx) => {
// Remove template reference from identityLdapAuth records
const updatedLdapAuths = await identityLdapAuthDAL.update({ templateId }, { templateId: null }, tx);
await Promise.all(
updatedLdapAuths.map(async (updatedLdapAuth) => {
await auditLogService.createAuditLog({
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
orgId: actorOrgId,
event: {
type: EventType.UPDATE_IDENTITY_LDAP_AUTH,
metadata: {
identityId: updatedLdapAuth.identityId,
templateId: template.id
}
}
});
})
);
// Delete the template
const [deletedTpl] = await identityAuthTemplateDAL.delete({ id: templateId }, tx);
return deletedTpl;
});
return deletedTemplate;
};
const getTemplate = async ({
templateId,
actorId,
actorAuthMethod,
actor,
actorOrgId
}: TGetIdentityAuthTemplateDTO) => {
await $checkPlan(actorOrgId);
const template = await identityAuthTemplateDAL.findByIdAndOrgId(templateId, actorOrgId);
if (!template) {
throw new NotFoundError({ message: "Template not found" });
}
const { permission } = await permissionService.getOrgPermission(
actor,
actorId,
template.orgId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionMachineIdentityAuthTemplateActions.ListTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
const { decryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.Organization,
orgId: template.orgId
});
const decryptedTemplateFields = decryptor({ cipherTextBlob: template.templateFields }).toString();
return {
...template,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
templateFields: JSON.parse(decryptedTemplateFields)
};
};
const listTemplates = async ({
limit,
offset,
search,
actorId,
actorAuthMethod,
actor,
actorOrgId
}: TListIdentityAuthTemplatesDTO) => {
await $checkPlan(actorOrgId);
const { permission } = await permissionService.getOrgPermission(
actor,
actorId,
actorOrgId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionMachineIdentityAuthTemplateActions.ListTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
const { docs, totalCount } = await identityAuthTemplateDAL.findByOrgId(actorOrgId, { limit, offset, search });
const { decryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.Organization,
orgId: actorOrgId
});
return {
totalCount,
templates: docs.map((doc) => ({
...doc,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
templateFields: JSON.parse(decryptor({ cipherTextBlob: doc.templateFields }).toString())
}))
};
};
const getTemplatesByAuthMethod = async ({
authMethod,
actorId,
actorAuthMethod,
actor,
actorOrgId
}: TGetTemplatesByAuthMethodDTO) => {
await $checkPlan(actorOrgId);
const { permission } = await permissionService.getOrgPermission(
actor,
actorId,
actorOrgId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionMachineIdentityAuthTemplateActions.AttachTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
const docs = await identityAuthTemplateDAL.findByAuthMethod(authMethod, actorOrgId);
const { decryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.Organization,
orgId: actorOrgId
});
return docs.map((doc) => ({
...doc,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
templateFields: JSON.parse(decryptor({ cipherTextBlob: doc.templateFields }).toString())
}));
};
const findTemplateUsages = async ({
templateId,
actorId,
actorAuthMethod,
actor,
actorOrgId
}: TFindTemplateUsagesDTO) => {
await $checkPlan(actorOrgId);
const { permission } = await permissionService.getOrgPermission(
actor,
actorId,
actorOrgId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionMachineIdentityAuthTemplateActions.ListTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
const template = await identityAuthTemplateDAL.findByIdAndOrgId(templateId, actorOrgId);
if (!template) {
throw new NotFoundError({ message: "Template not found" });
}
const docs = await identityAuthTemplateDAL.findTemplateUsages(templateId, template.authMethod);
return docs;
};
const unlinkTemplateUsage = async ({
templateId,
identityIds,
actorId,
actorAuthMethod,
actor,
actorOrgId
}: TUnlinkTemplateUsageDTO) => {
await $checkPlan(actorOrgId);
const { permission } = await permissionService.getOrgPermission(
actor,
actorId,
actorOrgId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionMachineIdentityAuthTemplateActions.UnlinkTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
const template = await identityAuthTemplateDAL.findByIdAndOrgId(templateId, actorOrgId);
if (!template) {
throw new NotFoundError({ message: "Template not found" });
}
switch (template.authMethod) {
case IdentityAuthTemplateMethod.LDAP:
await identityLdapAuthDAL.update({ $in: { identityId: identityIds }, templateId }, { templateId: null });
break;
default:
break;
}
};
return {
createTemplate,
updateTemplate,
deleteTemplate,
getTemplate,
listTemplates,
getTemplatesByAuthMethod,
findTemplateUsages,
unlinkTemplateUsage
};
};
@@ -0,0 +1,61 @@
import { TProjectPermission } from "@app/lib/types";
import { IdentityAuthTemplateMethod } from "./identity-auth-template-enums";
// Method-specific template field types
export type TLdapTemplateFields = {
url: string;
bindDN: string;
bindPass: string;
searchBase: string;
ldapCaCertificate?: string;
};
// Union type for all template field types
export type TTemplateFieldsByMethod = {
[IdentityAuthTemplateMethod.LDAP]: TLdapTemplateFields;
};
// Generic base types that use conditional types for type safety
export type TCreateIdentityAuthTemplateDTO = {
name: string;
authMethod: IdentityAuthTemplateMethod;
templateFields: TTemplateFieldsByMethod[IdentityAuthTemplateMethod];
} & Omit<TProjectPermission, "projectId">;
export type TUpdateIdentityAuthTemplateDTO = {
templateId: string;
name?: string;
templateFields?: Partial<TTemplateFieldsByMethod[IdentityAuthTemplateMethod]>;
} & Omit<TProjectPermission, "projectId">;
export type TDeleteIdentityAuthTemplateDTO = {
templateId: string;
} & Omit<TProjectPermission, "projectId">;
export type TGetIdentityAuthTemplateDTO = {
templateId: string;
} & Omit<TProjectPermission, "projectId">;
export type TListIdentityAuthTemplatesDTO = {
limit?: number;
offset?: number;
search?: string;
} & Omit<TProjectPermission, "projectId">;
export type TGetTemplatesByAuthMethodDTO = {
authMethod: string;
} & Omit<TProjectPermission, "projectId">;
export type TFindTemplateUsagesDTO = {
templateId: string;
} & Omit<TProjectPermission, "projectId">;
export type TUnlinkTemplateUsageDTO = {
templateId: string;
identityIds: string[];
} & Omit<TProjectPermission, "projectId">;
// Specific LDAP types for convenience
export type TCreateLdapTemplateDTO = TCreateIdentityAuthTemplateDTO;
export type TUpdateLdapTemplateDTO = TUpdateIdentityAuthTemplateDTO;
@@ -0,0 +1,6 @@
export type { TIdentityAuthTemplateDALFactory } from "./identity-auth-template-dal";
export { identityAuthTemplateDALFactory } from "./identity-auth-template-dal";
export * from "./identity-auth-template-enums";
export type { TIdentityAuthTemplateServiceFactory } from "./identity-auth-template-service";
export { identityAuthTemplateServiceFactory } from "./identity-auth-template-service";
export type * from "./identity-auth-template-types";
@@ -31,7 +31,8 @@ export const getDefaultOnPremFeatures = () => {
caCrl: false,
sshHostGroups: false,
enterpriseSecretSyncs: false,
enterpriseAppConnections: false
enterpriseAppConnections: false,
machineIdentityAuthTemplates: false
};
};
@@ -59,7 +59,9 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
secretScanning: false,
enterpriseSecretSyncs: false,
enterpriseAppConnections: false,
fips: false
fips: false,
eventSubscriptions: false,
machineIdentityAuthTemplates: false
});
export const setupLicenseRequestWithStore = (
@@ -5,13 +5,14 @@
// TODO(akhilmhdh): With tony find out the api structure and fill it here
import { ForbiddenError } from "@casl/ability";
import { AxiosError } from "axios";
import { CronJob } from "cron";
import { Knex } from "knex";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { verifyOfflineLicense } from "@app/lib/crypto";
import { NotFoundError } from "@app/lib/errors";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { TIdentityOrgDALFactory } from "@app/services/identity/identity-org-dal";
import { TOrgDALFactory } from "@app/services/org/org-dal";
@@ -603,10 +604,22 @@ export const licenseServiceFactory = ({
});
}
const { data } = await licenseServerCloudApi.request.delete(
`/api/license-server/v1/customers/${organization.customerId}/billing-details/payment-methods/${pmtMethodId}`
);
return data;
try {
const { data } = await licenseServerCloudApi.request.delete(
`/api/license-server/v1/customers/${organization.customerId}/billing-details/payment-methods/${pmtMethodId}`
);
return data;
} catch (error) {
if (error instanceof AxiosError) {
throw new BadRequestError({
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
message: `Failed to remove payment method: ${error.response?.data?.message}`
});
}
throw new BadRequestError({
message: "Unable to remove payment method"
});
}
};
const getOrgTaxIds = async ({ orgId, actor, actorId, actorAuthMethod, actorOrgId }: TGetOrgTaxIdDTO) => {
@@ -75,7 +75,9 @@ export type TFeatureSet = {
secretScanning: false;
enterpriseSecretSyncs: false;
enterpriseAppConnections: false;
machineIdentityAuthTemplates: false;
fips: false;
eventSubscriptions: false;
};
export type TOrgPlansTableDTO = {
@@ -161,7 +161,8 @@ const buildAdminPermissionRules = () => {
ProjectPermissionSecretActions.ReadValue,
ProjectPermissionSecretActions.Create,
ProjectPermissionSecretActions.Edit,
ProjectPermissionSecretActions.Delete
ProjectPermissionSecretActions.Delete,
ProjectPermissionSecretActions.Subscribe
],
ProjectPermissionSub.Secrets
);
@@ -265,7 +266,8 @@ const buildMemberPermissionRules = () => {
ProjectPermissionSecretActions.ReadValue,
ProjectPermissionSecretActions.Edit,
ProjectPermissionSecretActions.Create,
ProjectPermissionSecretActions.Delete
ProjectPermissionSecretActions.Delete,
ProjectPermissionSecretActions.Subscribe
],
ProjectPermissionSub.Secrets
);
@@ -28,6 +28,15 @@ export enum OrgPermissionKmipActions {
Setup = "setup"
}
export enum OrgPermissionMachineIdentityAuthTemplateActions {
ListTemplates = "list-templates",
EditTemplates = "edit-templates",
CreateTemplates = "create-templates",
DeleteTemplates = "delete-templates",
UnlinkTemplates = "unlink-templates",
AttachTemplates = "attach-templates"
}
export enum OrgPermissionAdminConsoleAction {
AccessAllProjects = "access-all-projects"
}
@@ -88,6 +97,7 @@ export enum OrgPermissionSubjects {
Identity = "identity",
Kms = "kms",
AdminConsole = "organization-admin-console",
MachineIdentityAuthTemplate = "machine-identity-auth-template",
AuditLogs = "audit-logs",
ProjectTemplates = "project-templates",
AppConnections = "app-connections",
@@ -126,6 +136,7 @@ export type OrgPermissionSet =
)
]
| [OrgPermissionAdminConsoleAction, OrgPermissionSubjects.AdminConsole]
| [OrgPermissionMachineIdentityAuthTemplateActions, OrgPermissionSubjects.MachineIdentityAuthTemplate]
| [OrgPermissionKmipActions, OrgPermissionSubjects.Kmip]
| [OrgPermissionSecretShareAction, OrgPermissionSubjects.SecretShare];
@@ -237,6 +248,14 @@ export const OrgPermissionSchema = z.discriminatedUnion("subject", [
"Describe what action an entity can take."
)
}),
z.object({
subject: z
.literal(OrgPermissionSubjects.MachineIdentityAuthTemplate)
.describe("The entity this permission pertains to."),
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionMachineIdentityAuthTemplateActions).describe(
"Describe what action an entity can take."
)
}),
z.object({
subject: z.literal(OrgPermissionSubjects.Gateway).describe("The entity this permission pertains to."),
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionGatewayActions).describe(
@@ -350,6 +369,25 @@ const buildAdminPermission = () => {
// the proxy assignment is temporary in order to prevent "more privilege" error during role assignment to MI
can(OrgPermissionKmipActions.Proxy, OrgPermissionSubjects.Kmip);
can(OrgPermissionMachineIdentityAuthTemplateActions.ListTemplates, OrgPermissionSubjects.MachineIdentityAuthTemplate);
can(OrgPermissionMachineIdentityAuthTemplateActions.EditTemplates, OrgPermissionSubjects.MachineIdentityAuthTemplate);
can(
OrgPermissionMachineIdentityAuthTemplateActions.CreateTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
can(
OrgPermissionMachineIdentityAuthTemplateActions.DeleteTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
can(
OrgPermissionMachineIdentityAuthTemplateActions.UnlinkTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
can(
OrgPermissionMachineIdentityAuthTemplateActions.AttachTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
can(OrgPermissionSecretShareAction.ManageSettings, OrgPermissionSubjects.SecretShare);
return rules;
@@ -385,6 +423,16 @@ const buildMemberPermission = () => {
can(OrgPermissionGatewayActions.CreateGateways, OrgPermissionSubjects.Gateway);
can(OrgPermissionGatewayActions.AttachGateways, OrgPermissionSubjects.Gateway);
can(OrgPermissionMachineIdentityAuthTemplateActions.ListTemplates, OrgPermissionSubjects.MachineIdentityAuthTemplate);
can(
OrgPermissionMachineIdentityAuthTemplateActions.UnlinkTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
can(
OrgPermissionMachineIdentityAuthTemplateActions.AttachTemplates,
OrgPermissionSubjects.MachineIdentityAuthTemplate
);
return rules;
};
@@ -36,7 +36,8 @@ export enum ProjectPermissionSecretActions {
ReadValue = "readValue",
Create = "create",
Edit = "edit",
Delete = "delete"
Delete = "delete",
Subscribe = "subscribe"
}
export enum ProjectPermissionCmekActions {
@@ -204,6 +205,7 @@ export type SecretSubjectFields = {
secretPath: string;
secretName?: string;
secretTags?: string[];
eventType?: string;
};
export type SecretFolderSubjectFields = {
@@ -483,7 +485,17 @@ const SecretConditionV2Schema = z
.object({
[PermissionConditionOperators.$IN]: PermissionConditionSchema[PermissionConditionOperators.$IN]
})
.partial()
.partial(),
eventType: z.union([
z.string(),
z
.object({
[PermissionConditionOperators.$EQ]: PermissionConditionSchema[PermissionConditionOperators.$EQ],
[PermissionConditionOperators.$NEQ]: PermissionConditionSchema[PermissionConditionOperators.$NEQ],
[PermissionConditionOperators.$IN]: PermissionConditionSchema[PermissionConditionOperators.$IN]
})
.partial()
])
})
.partial();
@@ -65,7 +65,10 @@ import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
import { TUserDALFactory } from "@app/services/user/user-dal";
import { TLicenseServiceFactory } from "../license/license-service";
import { throwIfMissingSecretReadValueOrDescribePermission } from "../permission/permission-fns";
import {
hasSecretReadValueOrDescribePermission,
throwIfMissingSecretReadValueOrDescribePermission
} from "../permission/permission-fns";
import { TPermissionServiceFactory } from "../permission/permission-service-types";
import { ProjectPermissionSecretActions, ProjectPermissionSub } from "../permission/project-permission";
import { TSecretApprovalPolicyDALFactory } from "../secret-approval-policy/secret-approval-policy-dal";
@@ -277,13 +280,19 @@ export const secretApprovalRequestServiceFactory = ({
) {
throw new ForbiddenRequestError({ message: "User has insufficient privileges" });
}
const hasSecretReadAccess = permission.can(
ProjectPermissionSecretActions.DescribeAndReadValue,
ProjectPermissionSub.Secrets
);
const getHasSecretReadAccess = (environment: string, tags: { slug: string }[], secretPath?: string) => {
const canRead = hasSecretReadValueOrDescribePermission(permission, ProjectPermissionSecretActions.ReadValue, {
environment,
secretPath: secretPath || "/",
secretTags: tags.map((i) => i.slug)
});
return canRead;
};
let secrets;
const secretPath = await folderDAL.findSecretPathByFolderIds(secretApprovalRequest.projectId, [
secretApprovalRequest.folderId
]);
if (shouldUseSecretV2Bridge) {
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
@@ -299,8 +308,8 @@ export const secretApprovalRequestServiceFactory = ({
version: el.version,
secretMetadata: el.secretMetadata as ResourceMetadataDTO,
isRotatedSecret: el.secret?.isRotatedSecret ?? false,
secretValueHidden: !hasSecretReadAccess,
secretValue: !hasSecretReadAccess
secretValueHidden: !getHasSecretReadAccess(secretApprovalRequest.environment, el.tags, secretPath?.[0]?.path),
secretValue: !getHasSecretReadAccess(secretApprovalRequest.environment, el.tags, secretPath?.[0]?.path)
? INFISICAL_SECRET_VALUE_HIDDEN_MASK
: el.secret && el.secret.isRotatedSecret
? undefined
@@ -315,8 +324,12 @@ export const secretApprovalRequestServiceFactory = ({
secretKey: el.secret.key,
id: el.secret.id,
version: el.secret.version,
secretValueHidden: !hasSecretReadAccess,
secretValue: !hasSecretReadAccess
secretValueHidden: !getHasSecretReadAccess(
secretApprovalRequest.environment,
el.tags,
secretPath?.[0]?.path
),
secretValue: !getHasSecretReadAccess(secretApprovalRequest.environment, el.tags, secretPath?.[0]?.path)
? INFISICAL_SECRET_VALUE_HIDDEN_MASK
: el.secret.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedValue }).toString()
@@ -331,8 +344,12 @@ export const secretApprovalRequestServiceFactory = ({
secretKey: el.secretVersion.key,
id: el.secretVersion.id,
version: el.secretVersion.version,
secretValueHidden: !hasSecretReadAccess,
secretValue: !hasSecretReadAccess
secretValueHidden: !getHasSecretReadAccess(
secretApprovalRequest.environment,
el.tags,
secretPath?.[0]?.path
),
secretValue: !getHasSecretReadAccess(secretApprovalRequest.environment, el.tags, secretPath?.[0]?.path)
? INFISICAL_SECRET_VALUE_HIDDEN_MASK
: el.secretVersion.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedValue }).toString()
@@ -350,7 +367,7 @@ export const secretApprovalRequestServiceFactory = ({
const encryptedSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
secrets = encryptedSecrets.map((el) => ({
...el,
secretValueHidden: !hasSecretReadAccess,
secretValueHidden: !getHasSecretReadAccess(secretApprovalRequest.environment, el.tags, secretPath?.[0]?.path),
...decryptSecretWithBot(el, botKey),
secret: el.secret
? {
@@ -370,9 +387,6 @@ export const secretApprovalRequestServiceFactory = ({
: undefined
}));
}
const secretPath = await folderDAL.findSecretPathByFolderIds(secretApprovalRequest.projectId, [
secretApprovalRequest.folderId
]);
return { ...secretApprovalRequest, secretPath: secretPath?.[0]?.path || "/", commits: secrets };
};
@@ -21,6 +21,8 @@ const GRAPH_API_BASE = "https://graph.microsoft.com/v1.0";
type AzureErrorResponse = { error: { message: string } };
const EXPIRY_PADDING_IN_DAYS = 3;
const sleep = async () =>
new Promise((resolve) => {
setTimeout(resolve, 1000);
@@ -33,7 +35,8 @@ export const azureClientSecretRotationFactory: TRotationFactory<
const {
connection,
parameters: { objectId, clientId: clientIdParam },
secretsMapping
secretsMapping,
rotationInterval
} = secretRotation;
/**
@@ -50,7 +53,7 @@ export const azureClientSecretRotationFactory: TRotationFactory<
)}-${now.getFullYear()}`;
const endDateTime = new Date();
endDateTime.setFullYear(now.getFullYear() + 5);
endDateTime.setDate(now.getDate() + rotationInterval * 2 + EXPIRY_PADDING_IN_DAYS); // give 72 hour buffer
try {
const { data } = await request.post<AzureAddPasswordResponse>(
@@ -195,6 +198,12 @@ export const azureClientSecretRotationFactory: TRotationFactory<
callback
) => {
const credentials = await $rotateClientSecret();
// 2.5 years as expiry is set to x2 interval for the inactive period of credential
if (rotationInterval > Math.floor(365 * 2.5) - EXPIRY_PADDING_IN_DAYS) {
throw new BadRequestError({ message: "Azure does not support token duration over 5 years" });
}
return callback(credentials);
};
@@ -51,6 +51,7 @@ const baseSecretRotationV2Query = ({
db.ref("encryptedCredentials").withSchema(TableName.AppConnection).as("connectionEncryptedCredentials"),
db.ref("description").withSchema(TableName.AppConnection).as("connectionDescription"),
db.ref("version").withSchema(TableName.AppConnection).as("connectionVersion"),
db.ref("gatewayId").withSchema(TableName.AppConnection).as("connectionGatewayId"),
db.ref("createdAt").withSchema(TableName.AppConnection).as("connectionCreatedAt"),
db.ref("updatedAt").withSchema(TableName.AppConnection).as("connectionUpdatedAt"),
db
@@ -104,6 +105,7 @@ const expandSecretRotation = <T extends Awaited<ReturnType<typeof baseSecretRota
connectionCreatedAt,
connectionUpdatedAt,
connectionVersion,
connectionGatewayId,
connectionIsPlatformManagedCredentials,
...el
} = secretRotation;
@@ -123,6 +125,7 @@ const expandSecretRotation = <T extends Awaited<ReturnType<typeof baseSecretRota
createdAt: connectionCreatedAt,
updatedAt: connectionUpdatedAt,
version: connectionVersion,
gatewayId: connectionGatewayId,
isPlatformManagedCredentials: connectionIsPlatformManagedCredentials
},
folder: {
@@ -18,7 +18,8 @@ import {
TSecretScanningFactoryInitialize,
TSecretScanningFactoryListRawResources,
TSecretScanningFactoryPostInitialization,
TSecretScanningFactoryTeardown
TSecretScanningFactoryTeardown,
TSecretScanningFactoryValidateConfigUpdate
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { getConfig } from "@app/lib/config/env";
import { request } from "@app/lib/config/request";
@@ -302,6 +303,13 @@ export const BitbucketSecretScanningFactory = () => {
);
};
const validateConfigUpdate: TSecretScanningFactoryValidateConfigUpdate<
TBitbucketDataSourceInput["config"],
TBitbucketDataSourceWithConnection
> = async () => {
// no validation required
};
return {
initialize,
postInitialization,
@@ -309,6 +317,7 @@ export const BitbucketSecretScanningFactory = () => {
getFullScanPath,
getDiffScanResourcePayload,
getDiffScanFindingsPayload,
teardown
teardown,
validateConfigUpdate
};
};
@@ -20,7 +20,8 @@ import {
TSecretScanningFactoryInitialize,
TSecretScanningFactoryListRawResources,
TSecretScanningFactoryPostInitialization,
TSecretScanningFactoryTeardown
TSecretScanningFactoryTeardown,
TSecretScanningFactoryValidateConfigUpdate
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { getConfig } from "@app/lib/config/env";
import { BadRequestError } from "@app/lib/errors";
@@ -64,7 +65,14 @@ export const GitHubSecretScanningFactory = () => {
};
const teardown: TSecretScanningFactoryTeardown<TGitHubDataSourceWithConnection> = async () => {
// no termination required
// no teardown required
};
const validateConfigUpdate: TSecretScanningFactoryValidateConfigUpdate<
TGitHubDataSourceInput["config"],
TGitHubDataSourceWithConnection
> = async () => {
// no validation required
};
const listRawResources: TSecretScanningFactoryListRawResources<TGitHubDataSourceWithConnection> = async (
@@ -238,6 +246,7 @@ export const GitHubSecretScanningFactory = () => {
getFullScanPath,
getDiffScanResourcePayload,
getDiffScanFindingsPayload,
teardown
teardown,
validateConfigUpdate
};
};
@@ -0,0 +1,9 @@
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { TSecretScanningDataSourceListItem } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
export const GITLAB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION: TSecretScanningDataSourceListItem = {
name: "GitLab",
type: SecretScanningDataSource.GitLab,
connection: AppConnection.GitLab
};
@@ -0,0 +1,8 @@
export enum GitLabDataSourceScope {
Project = "project",
Group = "group"
}
export enum GitLabWebHookEvent {
Push = "Push Hook"
}
@@ -0,0 +1,409 @@
import { Camelize, GitbeakerRequestError, GroupHookSchema, ProjectHookSchema } from "@gitbeaker/rest";
import { join } from "path";
import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import {
SecretScanningFindingSeverity,
SecretScanningResource
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
cloneRepository,
convertPatchLineToFileLineNumber,
replaceNonChangesWithNewlines
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
import {
TSecretScanningFactoryGetDiffScanFindingsPayload,
TSecretScanningFactoryGetDiffScanResourcePayload,
TSecretScanningFactoryGetFullScanPath,
TSecretScanningFactoryInitialize,
TSecretScanningFactoryListRawResources,
TSecretScanningFactoryParams,
TSecretScanningFactoryPostInitialization,
TSecretScanningFactoryTeardown,
TSecretScanningFactoryValidateConfigUpdate
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { getConfig } from "@app/lib/config/env";
import { BadRequestError } from "@app/lib/errors";
import { titleCaseToCamelCase } from "@app/lib/fn";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { GitLabProjectRegex } from "@app/lib/regex";
import {
getGitLabConnectionClient,
getGitLabInstanceUrl,
TGitLabConnection
} from "@app/services/app-connection/gitlab";
import { GitLabDataSourceScope } from "./gitlab-secret-scanning-enums";
import {
TGitLabDataSourceCredentials,
TGitLabDataSourceInput,
TGitLabDataSourceWithConnection,
TQueueGitLabResourceDiffScan
} from "./gitlab-secret-scanning-types";
const getMainDomain = (instanceUrl: string) => {
const url = new URL(instanceUrl);
const { hostname } = url;
const parts = hostname.split(".");
if (parts.length >= 2) {
return parts.slice(-2).join(".");
}
return hostname;
};
export const GitLabSecretScanningFactory = ({ appConnectionDAL, kmsService }: TSecretScanningFactoryParams) => {
const initialize: TSecretScanningFactoryInitialize<
TGitLabDataSourceInput,
TGitLabConnection,
TGitLabDataSourceCredentials
> = async ({ payload: { config, name }, connection }, callback) => {
const token = alphaNumericNanoId(64);
const client = await getGitLabConnectionClient(connection, appConnectionDAL, kmsService);
const appCfg = getConfig();
if (config.scope === GitLabDataSourceScope.Project) {
const { projectId } = config;
const project = await client.Projects.show(projectId);
if (!project) {
throw new BadRequestError({ message: `Could not find project with ID ${projectId}.` });
}
let hook: Camelize<ProjectHookSchema>;
try {
hook = await client.ProjectHooks.add(projectId, `${appCfg.SITE_URL}/secret-scanning/webhooks/gitlab`, {
token,
pushEvents: true,
enableSslVerification: true,
// @ts-expect-error gitbeaker is outdated, and the types don't support this field yet
name: `Infisical Secret Scanning - ${name}`
});
} catch (error) {
if (error instanceof GitbeakerRequestError) {
throw new BadRequestError({ message: `${error.message}: ${error.cause?.description ?? "Unknown Error"}` });
}
throw error;
}
try {
return await callback({
credentials: {
token,
hookId: hook.id
}
});
} catch (error) {
try {
await client.ProjectHooks.remove(projectId, hook.id);
} catch {
// do nothing, just try to clean up webhook
}
throw error;
}
}
// group scope
const { groupId } = config;
const group = await client.Groups.show(groupId);
if (!group) {
throw new BadRequestError({ message: `Could not find group with ID ${groupId}.` });
}
let hook: Camelize<GroupHookSchema>;
try {
hook = await client.GroupHooks.add(groupId, `${appCfg.SITE_URL}/secret-scanning/webhooks/gitlab`, {
token,
pushEvents: true,
enableSslVerification: true,
// @ts-expect-error gitbeaker is outdated, and the types don't support this field yet
name: `Infisical Secret Scanning - ${name}`
});
} catch (error) {
if (error instanceof GitbeakerRequestError) {
throw new BadRequestError({ message: `${error.message}: ${error.cause?.description ?? "Unknown Error"}` });
}
throw error;
}
try {
return await callback({
credentials: {
token,
hookId: hook.id
}
});
} catch (error) {
try {
await client.GroupHooks.remove(groupId, hook.id);
} catch {
// do nothing, just try to clean up webhook
}
throw error;
}
};
const postInitialization: TSecretScanningFactoryPostInitialization<
TGitLabDataSourceInput,
TGitLabConnection,
TGitLabDataSourceCredentials
> = async ({ connection, dataSourceId, credentials, payload: { config } }) => {
const client = await getGitLabConnectionClient(connection, appConnectionDAL, kmsService);
const appCfg = getConfig();
const hookUrl = `${appCfg.SITE_URL}/secret-scanning/webhooks/gitlab`;
const { hookId } = credentials;
if (config.scope === GitLabDataSourceScope.Project) {
const { projectId } = config;
try {
await client.ProjectHooks.edit(projectId, hookId, hookUrl, {
// @ts-expect-error gitbeaker is outdated, and the types don't support this field yet
name: `Infisical Secret Scanning - ${dataSourceId}`,
custom_headers: [{ key: "x-data-source-id", value: dataSourceId }]
});
} catch (error) {
try {
await client.ProjectHooks.remove(projectId, hookId);
} catch {
// do nothing, just try to clean up webhook
}
throw error;
}
return;
}
// group-scope
const { groupId } = config;
try {
await client.GroupHooks.edit(groupId, hookId, hookUrl, {
// @ts-expect-error gitbeaker is outdated, and the types don't support this field yet
name: `Infisical Secret Scanning - ${dataSourceId}`,
custom_headers: [{ key: "x-data-source-id", value: dataSourceId }]
});
} catch (error) {
try {
await client.GroupHooks.remove(groupId, hookId);
} catch {
// do nothing, just try to clean up webhook
}
throw error;
}
};
const listRawResources: TSecretScanningFactoryListRawResources<TGitLabDataSourceWithConnection> = async (
dataSource
) => {
const { connection, config } = dataSource;
const client = await getGitLabConnectionClient(connection, appConnectionDAL, kmsService);
if (config.scope === GitLabDataSourceScope.Project) {
const { projectId } = config;
const project = await client.Projects.show(projectId);
if (!project) {
throw new BadRequestError({ message: `Could not find project with ID ${projectId}.` });
}
// scott: even though we have this data we want to get potentially updated name
return [
{
name: project.pathWithNamespace,
externalId: project.id.toString(),
type: SecretScanningResource.Project
}
];
}
// group-scope
const { groupId, includeProjects } = config;
const projects = await client.Groups.allProjects(groupId, {
archived: false
});
const filteredProjects: typeof projects = [];
if (!includeProjects || includeProjects.includes("*")) {
filteredProjects.push(...projects);
} else {
filteredProjects.push(...projects.filter((project) => includeProjects.includes(project.pathWithNamespace)));
}
return filteredProjects.map(({ id, pathWithNamespace }) => ({
name: pathWithNamespace,
externalId: id.toString(),
type: SecretScanningResource.Project
}));
};
const getFullScanPath: TSecretScanningFactoryGetFullScanPath<TGitLabDataSourceWithConnection> = async ({
dataSource,
resourceName,
tempFolder
}) => {
const { connection } = dataSource;
const instanceUrl = await getGitLabInstanceUrl(connection.credentials.instanceUrl);
const client = await getGitLabConnectionClient(connection, appConnectionDAL, kmsService);
const user = await client.Users.showCurrentUser();
const repoPath = join(tempFolder, "repo.git");
if (!GitLabProjectRegex.test(resourceName)) {
throw new Error("Invalid GitLab project name");
}
await cloneRepository({
cloneUrl: `https://${user.username}:${connection.credentials.accessToken}@${getMainDomain(instanceUrl)}/${resourceName}.git`,
repoPath
});
return repoPath;
};
const teardown: TSecretScanningFactoryTeardown<
TGitLabDataSourceWithConnection,
TGitLabDataSourceCredentials
> = async ({ dataSource: { connection, config }, credentials: { hookId } }) => {
const client = await getGitLabConnectionClient(connection, appConnectionDAL, kmsService);
if (config.scope === GitLabDataSourceScope.Project) {
const { projectId } = config;
try {
await client.ProjectHooks.remove(projectId, hookId);
} catch (error) {
// do nothing, just try to clean up webhook
}
return;
}
const { groupId } = config;
try {
await client.GroupHooks.remove(groupId, hookId);
} catch (error) {
// do nothing, just try to clean up webhook
}
};
const getDiffScanResourcePayload: TSecretScanningFactoryGetDiffScanResourcePayload<
TQueueGitLabResourceDiffScan["payload"]
> = ({ project }) => {
return {
name: project.path_with_namespace,
externalId: project.id.toString(),
type: SecretScanningResource.Project
};
};
const getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<
TGitLabDataSourceWithConnection,
TQueueGitLabResourceDiffScan["payload"]
> = async ({ dataSource, payload, resourceName, configPath }) => {
const { connection } = dataSource;
const client = await getGitLabConnectionClient(connection, appConnectionDAL, kmsService);
const { commits, project } = payload;
const allFindings: SecretMatch[] = [];
for (const commit of commits) {
// eslint-disable-next-line no-await-in-loop
const commitDiffs = await client.Commits.showDiff(project.id, commit.id);
for (const commitDiff of commitDiffs) {
// eslint-disable-next-line no-continue
if (commitDiff.deletedFile) continue;
// eslint-disable-next-line no-await-in-loop
const findings = await scanContentAndGetFindings(
replaceNonChangesWithNewlines(`\n${commitDiff.diff}`),
configPath
);
const adjustedFindings = findings.map((finding) => {
const startLine = convertPatchLineToFileLineNumber(commitDiff.diff, finding.StartLine);
const endLine =
finding.StartLine === finding.EndLine
? startLine
: convertPatchLineToFileLineNumber(commitDiff.diff, finding.EndLine);
const startColumn = finding.StartColumn - 1; // subtract 1 for +
const endColumn = finding.EndColumn - 1; // subtract 1 for +
const authorName = commit.author.name;
const authorEmail = commit.author.email;
return {
...finding,
StartLine: startLine,
EndLine: endLine,
StartColumn: startColumn,
EndColumn: endColumn,
File: commitDiff.newPath,
Commit: commit.id,
Author: authorName,
Email: authorEmail,
Message: commit.message,
Fingerprint: `${commit.id}:${commitDiff.newPath}:${finding.RuleID}:${startLine}:${startColumn}`,
Date: commit.timestamp,
Link: `https://gitlab.com/${resourceName}/blob/${commit.id}/${commitDiff.newPath}#L${startLine}`
};
});
allFindings.push(...adjustedFindings);
}
}
return allFindings.map(
({
// discard match and secret as we don't want to store
Match,
Secret,
...finding
}) => ({
details: titleCaseToCamelCase(finding),
fingerprint: finding.Fingerprint,
severity: SecretScanningFindingSeverity.High,
rule: finding.RuleID
})
);
};
const validateConfigUpdate: TSecretScanningFactoryValidateConfigUpdate<
TGitLabDataSourceInput["config"],
TGitLabDataSourceWithConnection
> = async ({ config, dataSource }) => {
if (dataSource.config.scope !== config.scope) {
throw new BadRequestError({ message: "Cannot change Data Source scope after creation." });
}
};
return {
listRawResources,
getFullScanPath,
initialize,
postInitialization,
teardown,
getDiffScanResourcePayload,
getDiffScanFindingsPayload,
validateConfigUpdate
};
};
@@ -0,0 +1,101 @@
import { z } from "zod";
import { GitLabDataSourceScope } from "@app/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-enums";
import {
SecretScanningDataSource,
SecretScanningResource
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
BaseCreateSecretScanningDataSourceSchema,
BaseSecretScanningDataSourceSchema,
BaseSecretScanningFindingSchema,
BaseUpdateSecretScanningDataSourceSchema,
GitRepositoryScanFindingDetailsSchema
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-schemas";
import { SecretScanningDataSources } from "@app/lib/api-docs";
import { GitLabProjectRegex } from "@app/lib/regex";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
export const GitLabDataSourceConfigSchema = z.discriminatedUnion("scope", [
z.object({
scope: z.literal(GitLabDataSourceScope.Group).describe(SecretScanningDataSources.CONFIG.GITLAB.scope),
groupId: z.number().describe(SecretScanningDataSources.CONFIG.GITLAB.groupId),
groupName: z.string().trim().max(256).optional().describe(SecretScanningDataSources.CONFIG.GITLAB.groupName),
includeProjects: z
.array(
z
.string()
.min(1)
.max(256)
.refine((value) => value === "*" || GitLabProjectRegex.test(value), "Invalid project name format")
)
.nonempty("One or more projects required")
.max(100, "Cannot configure more than 100 projects")
.default(["*"])
.describe(SecretScanningDataSources.CONFIG.GITLAB.includeProjects)
}),
z.object({
scope: z.literal(GitLabDataSourceScope.Project).describe(SecretScanningDataSources.CONFIG.GITLAB.scope),
projectName: z.string().trim().max(256).optional().describe(SecretScanningDataSources.CONFIG.GITLAB.projectName),
projectId: z.number().describe(SecretScanningDataSources.CONFIG.GITLAB.projectId)
})
]);
export const GitLabDataSourceSchema = BaseSecretScanningDataSourceSchema({
type: SecretScanningDataSource.GitLab,
isConnectionRequired: true
})
.extend({
config: GitLabDataSourceConfigSchema
})
.describe(
JSON.stringify({
title: "GitLab"
})
);
export const CreateGitLabDataSourceSchema = BaseCreateSecretScanningDataSourceSchema({
type: SecretScanningDataSource.GitLab,
isConnectionRequired: true
})
.extend({
config: GitLabDataSourceConfigSchema
})
.describe(
JSON.stringify({
title: "GitLab"
})
);
export const UpdateGitLabDataSourceSchema = BaseUpdateSecretScanningDataSourceSchema(SecretScanningDataSource.GitLab)
.extend({
config: GitLabDataSourceConfigSchema.optional()
})
.describe(
JSON.stringify({
title: "GitLab"
})
);
export const GitLabDataSourceListItemSchema = z
.object({
name: z.literal("GitLab"),
connection: z.literal(AppConnection.GitLab),
type: z.literal(SecretScanningDataSource.GitLab)
})
.describe(
JSON.stringify({
title: "GitLab"
})
);
export const GitLabFindingSchema = BaseSecretScanningFindingSchema.extend({
resourceType: z.literal(SecretScanningResource.Project),
dataSourceType: z.literal(SecretScanningDataSource.GitLab),
details: GitRepositoryScanFindingDetailsSchema
});
export const GitLabDataSourceCredentialsSchema = z.object({
token: z.string(),
hookId: z.number()
});
@@ -0,0 +1,94 @@
import { GitLabDataSourceScope } from "@app/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-enums";
import { TSecretScanningV2DALFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-dal";
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { TSecretScanningV2QueueServiceFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-queue";
import { logger } from "@app/lib/logger";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
import {
TGitLabDataSource,
TGitLabDataSourceCredentials,
THandleGitLabPushEvent
} from "./gitlab-secret-scanning-types";
export const gitlabSecretScanningService = (
secretScanningV2DAL: TSecretScanningV2DALFactory,
secretScanningV2Queue: Pick<TSecretScanningV2QueueServiceFactory, "queueResourceDiffScan">,
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">
) => {
const handlePushEvent = async ({ payload, token, dataSourceId }: THandleGitLabPushEvent) => {
if (!payload.total_commits_count || !payload.project) {
logger.warn(
`secretScanningV2PushEvent: GitLab - Insufficient data [changes=${
payload.total_commits_count ?? 0
}] [projectName=${payload.project?.path_with_namespace ?? "unknown"}] [projectId=${payload.project?.id ?? "unknown"}]`
);
return;
}
const dataSource = (await secretScanningV2DAL.dataSources.findOne({
id: dataSourceId,
type: SecretScanningDataSource.GitLab
})) as TGitLabDataSource | undefined;
if (!dataSource) {
logger.error(
`secretScanningV2PushEvent: GitLab - Could not find data source [dataSourceId=${dataSourceId}] [projectId=${payload.project.id}]`
);
return;
}
const { isAutoScanEnabled, config, encryptedCredentials, projectId } = dataSource;
if (!encryptedCredentials) {
logger.info(
`secretScanningV2PushEvent: GitLab - Could not find encrypted credentials [dataSourceId=${dataSource.id}] [projectId=${payload.project.id}]`
);
return;
}
const { decryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
});
const decryptedCredentials = decryptor({ cipherTextBlob: encryptedCredentials });
const credentials = JSON.parse(decryptedCredentials.toString()) as TGitLabDataSourceCredentials;
if (token !== credentials.token) {
logger.error(
`secretScanningV2PushEvent: GitLab - Invalid webhook token [dataSourceId=${dataSource.id}] [projectId=${payload.project.id}]`
);
return;
}
if (!isAutoScanEnabled) {
logger.info(
`secretScanningV2PushEvent: GitLab - ignoring due to auto scan disabled [dataSourceId=${dataSource.id}] [projectId=${payload.project.id}]`
);
return;
}
if (
config.scope === GitLabDataSourceScope.Project
? config.projectId.toString() === payload.project_id.toString()
: config.includeProjects.includes("*") || config.includeProjects.includes(payload.project.path_with_namespace)
) {
await secretScanningV2Queue.queueResourceDiffScan({
dataSourceType: SecretScanningDataSource.GitLab,
payload,
dataSourceId: dataSource.id
});
} else {
logger.info(
`secretScanningV2PushEvent: GitLab - ignoring due to repository not being present in config [dataSourceId=${dataSource.id}] [projectId=${payload.project.id}]`
);
}
};
return {
handlePushEvent
};
};
@@ -0,0 +1,97 @@
import { z } from "zod";
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { TGitLabConnection } from "@app/services/app-connection/gitlab";
import {
CreateGitLabDataSourceSchema,
GitLabDataSourceCredentialsSchema,
GitLabDataSourceListItemSchema,
GitLabDataSourceSchema,
GitLabFindingSchema
} from "./gitlab-secret-scanning-schemas";
export type TGitLabDataSource = z.infer<typeof GitLabDataSourceSchema>;
export type TGitLabDataSourceInput = z.infer<typeof CreateGitLabDataSourceSchema>;
export type TGitLabDataSourceListItem = z.infer<typeof GitLabDataSourceListItemSchema>;
export type TGitLabFinding = z.infer<typeof GitLabFindingSchema>;
export type TGitLabDataSourceWithConnection = TGitLabDataSource & {
connection: TGitLabConnection;
};
export type TGitLabDataSourceCredentials = z.infer<typeof GitLabDataSourceCredentialsSchema>;
export type TGitLabDataSourcePushEventPayload = {
object_kind: "push";
event_name: "push";
before: string;
after: string;
ref: string;
ref_protected: boolean;
checkout_sha: string;
user_id: number;
user_name: string;
user_username: string;
user_email: string;
user_avatar: string;
project_id: number;
project: {
id: number;
name: string;
description: string;
web_url: string;
avatar_url: string | null;
git_ssh_url: string;
git_http_url: string;
namespace: string;
visibility_level: number;
path_with_namespace: string;
default_branch: string;
homepage: string;
url: string;
ssh_url: string;
http_url: string;
};
repository: {
name: string;
url: string;
description: string;
homepage: string;
git_http_url: string;
git_ssh_url: string;
visibility_level: number;
};
commits: {
id: string;
message: string;
title: string;
timestamp: string;
url: string;
author: {
name: string;
email: string;
};
added: string[];
modified: string[];
removed: string[];
}[];
total_commits_count: number;
};
export type THandleGitLabPushEvent = {
payload: TGitLabDataSourcePushEventPayload;
dataSourceId: string;
token: string;
};
export type TQueueGitLabResourceDiffScan = {
dataSourceType: SecretScanningDataSource.GitLab;
payload: TGitLabDataSourcePushEventPayload;
dataSourceId: string;
resourceId: string;
scanId: string;
};
@@ -0,0 +1,3 @@
export * from "./gitlab-secret-scanning-constants";
export * from "./gitlab-secret-scanning-schemas";
export * from "./gitlab-secret-scanning-types";
@@ -49,6 +49,7 @@ const baseSecretScanningDataSourceQuery = ({
db.ref("encryptedCredentials").withSchema(TableName.AppConnection).as("connectionEncryptedCredentials"),
db.ref("description").withSchema(TableName.AppConnection).as("connectionDescription"),
db.ref("version").withSchema(TableName.AppConnection).as("connectionVersion"),
db.ref("gatewayId").withSchema(TableName.AppConnection).as("connectionGatewayId"),
db.ref("createdAt").withSchema(TableName.AppConnection).as("connectionCreatedAt"),
db.ref("updatedAt").withSchema(TableName.AppConnection).as("connectionUpdatedAt"),
db
@@ -82,6 +83,7 @@ const expandSecretScanningDataSource = <
connectionUpdatedAt,
connectionVersion,
connectionIsPlatformManagedCredentials,
connectionGatewayId,
...el
} = dataSource;
@@ -100,7 +102,8 @@ const expandSecretScanningDataSource = <
createdAt: connectionCreatedAt,
updatedAt: connectionUpdatedAt,
version: connectionVersion,
isPlatformManagedCredentials: connectionIsPlatformManagedCredentials
isPlatformManagedCredentials: connectionIsPlatformManagedCredentials,
gatewayId: connectionGatewayId
}
: undefined
};
@@ -1,6 +1,7 @@
export enum SecretScanningDataSource {
GitHub = "github",
Bitbucket = "bitbucket"
Bitbucket = "bitbucket",
GitLab = "gitlab"
}
export enum SecretScanningScanStatus {
@@ -1,5 +1,6 @@
import { BitbucketSecretScanningFactory } from "@app/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-factory";
import { GitHubSecretScanningFactory } from "@app/ee/services/secret-scanning-v2/github/github-secret-scanning-factory";
import { GitLabSecretScanningFactory } from "@app/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-factory";
import { SecretScanningDataSource } from "./secret-scanning-v2-enums";
import {
@@ -19,5 +20,6 @@ type TSecretScanningFactoryImplementation = TSecretScanningFactory<
export const SECRET_SCANNING_FACTORY_MAP: Record<SecretScanningDataSource, TSecretScanningFactoryImplementation> = {
[SecretScanningDataSource.GitHub]: GitHubSecretScanningFactory as TSecretScanningFactoryImplementation,
[SecretScanningDataSource.Bitbucket]: BitbucketSecretScanningFactory as TSecretScanningFactoryImplementation
[SecretScanningDataSource.Bitbucket]: BitbucketSecretScanningFactory as TSecretScanningFactoryImplementation,
[SecretScanningDataSource.GitLab]: GitLabSecretScanningFactory as TSecretScanningFactoryImplementation
};
@@ -13,6 +13,7 @@ import {
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import { BITBUCKET_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/bitbucket";
import { GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/github";
import { GITLAB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/gitlab";
import { getConfig } from "@app/lib/config/env";
import { crypto } from "@app/lib/crypto";
import { BadRequestError } from "@app/lib/errors";
@@ -23,7 +24,8 @@ import { TCloneRepository, TGetFindingsPayload, TSecretScanningDataSourceListIte
const SECRET_SCANNING_SOURCE_LIST_OPTIONS: Record<SecretScanningDataSource, TSecretScanningDataSourceListItem> = {
[SecretScanningDataSource.GitHub]: GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION,
[SecretScanningDataSource.Bitbucket]: BITBUCKET_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION
[SecretScanningDataSource.Bitbucket]: BITBUCKET_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION,
[SecretScanningDataSource.GitLab]: GITLAB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION
};
export const listSecretScanningDataSourceOptions = () => {
@@ -3,15 +3,18 @@ import { AppConnection } from "@app/services/app-connection/app-connection-enums
export const SECRET_SCANNING_DATA_SOURCE_NAME_MAP: Record<SecretScanningDataSource, string> = {
[SecretScanningDataSource.GitHub]: "GitHub",
[SecretScanningDataSource.Bitbucket]: "Bitbucket"
[SecretScanningDataSource.Bitbucket]: "Bitbucket",
[SecretScanningDataSource.GitLab]: "GitLab"
};
export const SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP: Record<SecretScanningDataSource, AppConnection> = {
[SecretScanningDataSource.GitHub]: AppConnection.GitHubRadar,
[SecretScanningDataSource.Bitbucket]: AppConnection.Bitbucket
[SecretScanningDataSource.Bitbucket]: AppConnection.Bitbucket,
[SecretScanningDataSource.GitLab]: AppConnection.GitLab
};
export const AUTO_SYNC_DESCRIPTION_HELPER: Record<SecretScanningDataSource, { verb: string; noun: string }> = {
[SecretScanningDataSource.GitHub]: { verb: "push", noun: "repositories" },
[SecretScanningDataSource.Bitbucket]: { verb: "push", noun: "repositories" }
[SecretScanningDataSource.Bitbucket]: { verb: "push", noun: "repositories" },
[SecretScanningDataSource.GitLab]: { verb: "push", noun: "projects" }
};
@@ -16,6 +16,7 @@ import { getConfig } from "@app/lib/config/env";
import { BadRequestError, InternalServerError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";
import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns";
import { TAppConnection } from "@app/services/app-connection/app-connection-types";
import { ActorType } from "@app/services/auth/auth-type";
@@ -48,6 +49,7 @@ type TSecretRotationV2QueueServiceFactoryDep = {
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
projectDAL: Pick<TProjectDALFactory, "findById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
appConnectionDAL: Pick<TAppConnectionDALFactory, "updateById">;
auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
keyStore: Pick<TKeyStoreFactory, "acquireLock" | "getItem">;
};
@@ -62,7 +64,8 @@ export const secretScanningV2QueueServiceFactory = async ({
smtpService,
kmsService,
auditLogService,
keyStore
keyStore,
appConnectionDAL
}: TSecretRotationV2QueueServiceFactoryDep) => {
const queueDataSourceFullScan = async (
dataSource: TSecretScanningDataSourceWithConnection,
@@ -71,7 +74,10 @@ export const secretScanningV2QueueServiceFactory = async ({
try {
const { type } = dataSource;
const factory = SECRET_SCANNING_FACTORY_MAP[type]();
const factory = SECRET_SCANNING_FACTORY_MAP[type]({
kmsService,
appConnectionDAL
});
const rawResources = await factory.listRawResources(dataSource);
@@ -171,7 +177,10 @@ export const secretScanningV2QueueServiceFactory = async ({
let connection: TAppConnection | null = null;
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]();
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
kmsService,
appConnectionDAL
});
const findingsPath = join(tempFolder, "findings.json");
@@ -329,7 +338,10 @@ export const secretScanningV2QueueServiceFactory = async ({
dataSourceId,
dataSourceType
}: Pick<TQueueSecretScanningResourceDiffScan, "payload" | "dataSourceId" | "dataSourceType">) => {
const factory = SECRET_SCANNING_FACTORY_MAP[dataSourceType as SecretScanningDataSource]();
const factory = SECRET_SCANNING_FACTORY_MAP[dataSourceType as SecretScanningDataSource]({
kmsService,
appConnectionDAL
});
const resourcePayload = factory.getDiffScanResourcePayload(payload);
@@ -391,7 +403,10 @@ export const secretScanningV2QueueServiceFactory = async ({
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]();
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
kmsService,
appConnectionDAL
});
const tempFolder = await createTempFolder();
@@ -46,6 +46,7 @@ import {
import { DatabaseErrorCode } from "@app/lib/error-codes";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
import { OrgServiceActor } from "@app/lib/types";
import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";
import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns";
import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service";
import { TAppConnection } from "@app/services/app-connection/app-connection-types";
@@ -53,12 +54,14 @@ import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
import { bitbucketSecretScanningService } from "./bitbucket/bitbucket-secret-scanning-service";
import { gitlabSecretScanningService } from "./gitlab/gitlab-secret-scanning-service";
import { TSecretScanningV2DALFactory } from "./secret-scanning-v2-dal";
import { TSecretScanningV2QueueServiceFactory } from "./secret-scanning-v2-queue";
export type TSecretScanningV2ServiceFactoryDep = {
secretScanningV2DAL: TSecretScanningV2DALFactory;
appConnectionService: Pick<TAppConnectionServiceFactory, "connectAppConnectionById">;
appConnectionDAL: Pick<TAppConnectionDALFactory, "updateById">;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission" | "getOrgPermission">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
secretScanningV2Queue: Pick<
@@ -76,6 +79,7 @@ export const secretScanningV2ServiceFactory = ({
appConnectionService,
licenseService,
secretScanningV2Queue,
appConnectionDAL,
kmsService
}: TSecretScanningV2ServiceFactoryDep) => {
const $checkListSecretScanningDataSourcesByProjectIdPermissions = async (
@@ -255,7 +259,10 @@ export const secretScanningV2ServiceFactory = ({
);
}
const factory = SECRET_SCANNING_FACTORY_MAP[payload.type]();
const factory = SECRET_SCANNING_FACTORY_MAP[payload.type]({
appConnectionDAL,
kmsService
});
try {
const createdDataSource = await factory.initialize(
@@ -363,6 +370,31 @@ export const secretScanningV2ServiceFactory = ({
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
});
let connection: TAppConnection | null = null;
if (dataSource.connectionId) {
// validates permission to connect and app is valid for data source
connection = await appConnectionService.connectAppConnectionById(
SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[dataSource.type],
dataSource.connectionId,
actor
);
}
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type]({
appConnectionDAL,
kmsService
});
if (payload.config) {
await factory.validateConfigUpdate({
dataSource: {
...dataSource,
connection
} as TSecretScanningDataSourceWithConnection,
config: payload.config as TSecretScanningDataSourceWithConnection["config"]
});
}
try {
const updatedDataSource = await secretScanningV2DAL.dataSources.updateById(dataSourceId, payload);
@@ -416,7 +448,10 @@ export const secretScanningV2ServiceFactory = ({
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
});
const factory = SECRET_SCANNING_FACTORY_MAP[type]();
const factory = SECRET_SCANNING_FACTORY_MAP[type]({
appConnectionDAL,
kmsService
});
let connection: TAppConnection | null = null;
if (dataSource.connection) {
@@ -903,6 +938,7 @@ export const secretScanningV2ServiceFactory = ({
findSecretScanningConfigByProjectId,
upsertSecretScanningConfig,
github: githubSecretScanningService(secretScanningV2DAL, secretScanningV2Queue),
bitbucket: bitbucketSecretScanningService(secretScanningV2DAL, secretScanningV2Queue, kmsService)
bitbucket: bitbucketSecretScanningService(secretScanningV2DAL, secretScanningV2Queue, kmsService),
gitlab: gitlabSecretScanningService(secretScanningV2DAL, secretScanningV2Queue, kmsService)
};
};
@@ -21,14 +21,25 @@ import {
TGitHubFinding,
TQueueGitHubResourceDiffScan
} from "@app/ee/services/secret-scanning-v2/github";
import {
TGitLabDataSource,
TGitLabDataSourceCredentials,
TGitLabDataSourceInput,
TGitLabDataSourceListItem,
TGitLabDataSourceWithConnection,
TGitLabFinding,
TQueueGitLabResourceDiffScan
} from "@app/ee/services/secret-scanning-v2/gitlab";
import { TSecretScanningV2DALFactory } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-dal";
import {
SecretScanningDataSource,
SecretScanningFindingStatus,
SecretScanningScanStatus
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
export type TSecretScanningDataSource = TGitHubDataSource | TBitbucketDataSource;
export type TSecretScanningDataSource = TGitHubDataSource | TBitbucketDataSource | TGitLabDataSource;
export type TSecretScanningDataSourceWithDetails = TSecretScanningDataSource & {
lastScannedAt?: Date | null;
@@ -52,15 +63,25 @@ export type TSecretScanningScanWithDetails = TSecretScanningScans & {
export type TSecretScanningDataSourceWithConnection =
| TGitHubDataSourceWithConnection
| TBitbucketDataSourceWithConnection;
| TBitbucketDataSourceWithConnection
| TGitLabDataSourceWithConnection;
export type TSecretScanningDataSourceInput = TGitHubDataSourceInput | TBitbucketDataSourceInput;
export type TSecretScanningDataSourceInput =
| TGitHubDataSourceInput
| TBitbucketDataSourceInput
| TGitLabDataSourceInput;
export type TSecretScanningDataSourceListItem = TGitHubDataSourceListItem | TBitbucketDataSourceListItem;
export type TSecretScanningDataSourceListItem =
| TGitHubDataSourceListItem
| TBitbucketDataSourceListItem
| TGitLabDataSourceListItem;
export type TSecretScanningDataSourceCredentials = TBitbucketDataSourceCredentials | undefined;
export type TSecretScanningDataSourceCredentials =
| TBitbucketDataSourceCredentials
| TGitLabDataSourceCredentials
| undefined;
export type TSecretScanningFinding = TGitHubFinding | TBitbucketFinding;
export type TSecretScanningFinding = TGitHubFinding | TBitbucketFinding | TGitLabFinding;
export type TListSecretScanningDataSourcesByProjectId = {
projectId: string;
@@ -112,7 +133,10 @@ export type TQueueSecretScanningDataSourceFullScan = {
scanId: string;
};
export type TQueueSecretScanningResourceDiffScan = TQueueGitHubResourceDiffScan | TQueueBitbucketResourceDiffScan;
export type TQueueSecretScanningResourceDiffScan =
| TQueueGitHubResourceDiffScan
| TQueueBitbucketResourceDiffScan
| TQueueGitLabResourceDiffScan;
export type TQueueSecretScanningSendNotification = {
dataSource: TSecretScanningDataSources;
@@ -170,6 +194,11 @@ export type TSecretScanningFactoryInitialize<
callback: (parameters: { credentials?: C; externalId?: string }) => Promise<TSecretScanningDataSourceRaw>
) => Promise<TSecretScanningDataSourceRaw>;
export type TSecretScanningFactoryValidateConfigUpdate<
C extends TSecretScanningDataSourceInput["config"],
T extends TSecretScanningDataSourceWithConnection
> = (params: { config: C; dataSource: T }) => Promise<void>;
export type TSecretScanningFactoryPostInitialization<
P extends TSecretScanningDataSourceInput,
T extends TSecretScanningDataSourceWithConnection["connection"] | undefined = undefined,
@@ -181,17 +210,23 @@ export type TSecretScanningFactoryTeardown<
C extends TSecretScanningDataSourceCredentials = undefined
> = (params: { dataSource: T; credentials: C }) => Promise<void>;
export type TSecretScanningFactoryParams = {
appConnectionDAL: Pick<TAppConnectionDALFactory, "updateById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
};
export type TSecretScanningFactory<
T extends TSecretScanningDataSourceWithConnection,
P extends TQueueSecretScanningResourceDiffScan["payload"],
I extends TSecretScanningDataSourceInput,
C extends TSecretScanningDataSourceCredentials | undefined = undefined
> = () => {
> = (params: TSecretScanningFactoryParams) => {
listRawResources: TSecretScanningFactoryListRawResources<T>;
getFullScanPath: TSecretScanningFactoryGetFullScanPath<T>;
initialize: TSecretScanningFactoryInitialize<I, T["connection"] | undefined, C>;
postInitialization: TSecretScanningFactoryPostInitialization<I, T["connection"] | undefined, C>;
teardown: TSecretScanningFactoryTeardown<T, C>;
validateConfigUpdate: TSecretScanningFactoryValidateConfigUpdate<I["config"], T>;
getDiffScanResourcePayload: TSecretScanningFactoryGetDiffScanResourcePayload<P>;
getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<T, P>;
};
@@ -2,10 +2,12 @@ import { z } from "zod";
import { BitbucketDataSourceSchema, BitbucketFindingSchema } from "@app/ee/services/secret-scanning-v2/bitbucket";
import { GitHubDataSourceSchema, GitHubFindingSchema } from "@app/ee/services/secret-scanning-v2/github";
import { GitLabDataSourceSchema, GitLabFindingSchema } from "@app/ee/services/secret-scanning-v2/gitlab";
export const SecretScanningDataSourceSchema = z.discriminatedUnion("type", [
GitHubDataSourceSchema,
BitbucketDataSourceSchema
BitbucketDataSourceSchema,
GitLabDataSourceSchema
]);
export const SecretScanningFindingSchema = z.discriminatedUnion("dataSourceType", [
@@ -18,5 +20,10 @@ export const SecretScanningFindingSchema = z.discriminatedUnion("dataSourceType"
JSON.stringify({
title: "Bitbucket"
})
),
GitLabFindingSchema.describe(
JSON.stringify({
title: "GitLab"
})
)
]);