mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-10-15 14:27:23 +00:00
feat: events system implementation (#4246)
* chore: save poc * chore: save wip * fix: undo cors * fix: impl changes * fix: PR changes * fix: mocks * fix: connection tracking and auth changes * fix: PR changes * fix: revert license * feat: frontend change * fix: revert docker compose.dev * fix: duplicate publisher connection * fix: pr changes * chore: move event impl to `ee` * fix: lint errors * fix: check length of events * fix: static permissions matching * fix: secretPath * fix: remove source prefix in bus event name * fix: license check
This commit is contained in:
Sid
@@ -1,7 +1,8 @@
|
||||
import { AxiosError, RawAxiosRequestHeaders } from "axios";
|
||||
|
||||
import { SecretKeyEncoding } from "@app/db/schemas";
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { ProjectType, SecretKeyEncoding } from "@app/db/schemas";
|
||||
import { TEventBusService } from "@app/ee/services/event/event-bus-service";
|
||||
import { TopicName, toPublishableEvent } from "@app/ee/services/event/types";
|
||||
import { request } from "@app/lib/config/request";
|
||||
import { crypto } from "@app/lib/crypto/cryptography";
|
||||
import { logger } from "@app/lib/logger";
|
||||
@@ -21,6 +22,7 @@ type TAuditLogQueueServiceFactoryDep = {
|
||||
queueService: TQueueServiceFactory;
|
||||
projectDAL: Pick<TProjectDALFactory, "findById">;
|
||||
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
|
||||
eventBusService: TEventBusService;
|
||||
};
|
||||
|
||||
export type TAuditLogQueueServiceFactory = {
|
||||
@@ -36,133 +38,17 @@ export const auditLogQueueServiceFactory = async ({
|
||||
queueService,
|
||||
projectDAL,
|
||||
licenseService,
|
||||
auditLogStreamDAL
|
||||
auditLogStreamDAL,
|
||||
eventBusService
|
||||
}: TAuditLogQueueServiceFactoryDep): Promise<TAuditLogQueueServiceFactory> => {
|
||||
const appCfg = getConfig();
|
||||
|
||||
const pushToLog = async (data: TCreateAuditLogDTO) => {
|
||||
if (appCfg.USE_PG_QUEUE && appCfg.SHOULD_INIT_PG_QUEUE) {
|
||||
await queueService.queuePg<QueueName.AuditLog>(QueueJobs.AuditLog, data, {
|
||||
retryLimit: 10,
|
||||
retryBackoff: true
|
||||
});
|
||||
} else {
|
||||
await queueService.queue<QueueName.AuditLog>(QueueName.AuditLog, QueueJobs.AuditLog, data, {
|
||||
removeOnFail: {
|
||||
count: 3
|
||||
},
|
||||
removeOnComplete: true
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
if (appCfg.SHOULD_INIT_PG_QUEUE) {
|
||||
await queueService.startPg<QueueName.AuditLog>(
|
||||
QueueJobs.AuditLog,
|
||||
async ([job]) => {
|
||||
const { actor, event, ipAddress, projectId, userAgent, userAgentType } = job.data;
|
||||
let { orgId } = job.data;
|
||||
const MS_IN_DAY = 24 * 60 * 60 * 1000;
|
||||
let project;
|
||||
|
||||
if (!orgId) {
|
||||
// it will never be undefined for both org and project id
|
||||
// TODO(akhilmhdh): use caching here in dal to avoid db calls
|
||||
project = await projectDAL.findById(projectId as string);
|
||||
orgId = project.orgId;
|
||||
}
|
||||
|
||||
const plan = await licenseService.getPlan(orgId);
|
||||
if (plan.auditLogsRetentionDays === 0) {
|
||||
// skip inserting if audit log retention is 0 meaning its not supported
|
||||
return;
|
||||
}
|
||||
|
||||
// For project actions, set TTL to project-level audit log retention config
|
||||
// This condition ensures that the plan's audit log retention days cannot be bypassed
|
||||
const ttlInDays =
|
||||
project?.auditLogsRetentionDays && project.auditLogsRetentionDays < plan.auditLogsRetentionDays
|
||||
? project.auditLogsRetentionDays
|
||||
: plan.auditLogsRetentionDays;
|
||||
|
||||
const ttl = ttlInDays * MS_IN_DAY;
|
||||
|
||||
const auditLog = await auditLogDAL.create({
|
||||
actor: actor.type,
|
||||
actorMetadata: actor.metadata,
|
||||
userAgent,
|
||||
projectId,
|
||||
projectName: project?.name,
|
||||
ipAddress,
|
||||
orgId,
|
||||
eventType: event.type,
|
||||
expiresAt: new Date(Date.now() + ttl),
|
||||
eventMetadata: event.metadata,
|
||||
userAgentType
|
||||
});
|
||||
|
||||
const logStreams = orgId ? await auditLogStreamDAL.find({ orgId }) : [];
|
||||
await Promise.allSettled(
|
||||
logStreams.map(
|
||||
async ({
|
||||
url,
|
||||
encryptedHeadersTag,
|
||||
encryptedHeadersIV,
|
||||
encryptedHeadersKeyEncoding,
|
||||
encryptedHeadersCiphertext
|
||||
}) => {
|
||||
const streamHeaders =
|
||||
encryptedHeadersIV && encryptedHeadersCiphertext && encryptedHeadersTag
|
||||
? (JSON.parse(
|
||||
crypto
|
||||
.encryption()
|
||||
.symmetric()
|
||||
.decryptWithRootEncryptionKey({
|
||||
keyEncoding: encryptedHeadersKeyEncoding as SecretKeyEncoding,
|
||||
iv: encryptedHeadersIV,
|
||||
tag: encryptedHeadersTag,
|
||||
ciphertext: encryptedHeadersCiphertext
|
||||
})
|
||||
) as LogStreamHeaders[])
|
||||
: [];
|
||||
|
||||
const headers: RawAxiosRequestHeaders = { "Content-Type": "application/json" };
|
||||
|
||||
if (streamHeaders.length)
|
||||
streamHeaders.forEach(({ key, value }) => {
|
||||
headers[key] = value;
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await request.post(
|
||||
url,
|
||||
{ ...providerSpecificPayload(url), ...auditLog },
|
||||
{
|
||||
headers,
|
||||
// request timeout
|
||||
timeout: AUDIT_LOG_STREAM_TIMEOUT,
|
||||
// connection timeout
|
||||
signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
|
||||
}
|
||||
);
|
||||
return response;
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`Failed to stream audit log [url=${url}] for org [orgId=${orgId}] [error=${(error as AxiosError).message}]`
|
||||
);
|
||||
return error;
|
||||
}
|
||||
}
|
||||
)
|
||||
);
|
||||
await queueService.queue<QueueName.AuditLog>(QueueName.AuditLog, QueueJobs.AuditLog, data, {
|
||||
removeOnFail: {
|
||||
count: 3
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 30,
|
||||
pollingIntervalSeconds: 0.5
|
||||
}
|
||||
);
|
||||
}
|
||||
removeOnComplete: true
|
||||
});
|
||||
};
|
||||
|
||||
queueService.start(QueueName.AuditLog, async (job) => {
|
||||
const { actor, event, ipAddress, projectId, userAgent, userAgentType } = job.data;
|
||||
@@ -178,88 +64,97 @@ export const auditLogQueueServiceFactory = async ({
|
||||
}
|
||||
|
||||
const plan = await licenseService.getPlan(orgId);
|
||||
if (plan.auditLogsRetentionDays === 0) {
|
||||
// skip inserting if audit log retention is 0 meaning its not supported
|
||||
return;
|
||||
|
||||
// skip inserting if audit log retention is 0 meaning its not supported
|
||||
if (plan.auditLogsRetentionDays !== 0) {
|
||||
// For project actions, set TTL to project-level audit log retention config
|
||||
// This condition ensures that the plan's audit log retention days cannot be bypassed
|
||||
const ttlInDays =
|
||||
project?.auditLogsRetentionDays && project.auditLogsRetentionDays < plan.auditLogsRetentionDays
|
||||
? project.auditLogsRetentionDays
|
||||
: plan.auditLogsRetentionDays;
|
||||
|
||||
const ttl = ttlInDays * MS_IN_DAY;
|
||||
|
||||
const auditLog = await auditLogDAL.create({
|
||||
actor: actor.type,
|
||||
actorMetadata: actor.metadata,
|
||||
userAgent,
|
||||
projectId,
|
||||
projectName: project?.name,
|
||||
ipAddress,
|
||||
orgId,
|
||||
eventType: event.type,
|
||||
expiresAt: new Date(Date.now() + ttl),
|
||||
eventMetadata: event.metadata,
|
||||
userAgentType
|
||||
});
|
||||
|
||||
const logStreams = orgId ? await auditLogStreamDAL.find({ orgId }) : [];
|
||||
await Promise.allSettled(
|
||||
logStreams.map(
|
||||
async ({
|
||||
url,
|
||||
encryptedHeadersTag,
|
||||
encryptedHeadersIV,
|
||||
encryptedHeadersKeyEncoding,
|
||||
encryptedHeadersCiphertext
|
||||
}) => {
|
||||
const streamHeaders =
|
||||
encryptedHeadersIV && encryptedHeadersCiphertext && encryptedHeadersTag
|
||||
? (JSON.parse(
|
||||
crypto
|
||||
.encryption()
|
||||
.symmetric()
|
||||
.decryptWithRootEncryptionKey({
|
||||
keyEncoding: encryptedHeadersKeyEncoding as SecretKeyEncoding,
|
||||
iv: encryptedHeadersIV,
|
||||
tag: encryptedHeadersTag,
|
||||
ciphertext: encryptedHeadersCiphertext
|
||||
})
|
||||
) as LogStreamHeaders[])
|
||||
: [];
|
||||
|
||||
const headers: RawAxiosRequestHeaders = { "Content-Type": "application/json" };
|
||||
|
||||
if (streamHeaders.length)
|
||||
streamHeaders.forEach(({ key, value }) => {
|
||||
headers[key] = value;
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await request.post(
|
||||
url,
|
||||
{ ...providerSpecificPayload(url), ...auditLog },
|
||||
{
|
||||
headers,
|
||||
// request timeout
|
||||
timeout: AUDIT_LOG_STREAM_TIMEOUT,
|
||||
// connection timeout
|
||||
signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
|
||||
}
|
||||
);
|
||||
return response;
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`Failed to stream audit log [url=${url}] for org [orgId=${orgId}] [error=${(error as AxiosError).message}]`
|
||||
);
|
||||
return error;
|
||||
}
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// For project actions, set TTL to project-level audit log retention config
|
||||
// This condition ensures that the plan's audit log retention days cannot be bypassed
|
||||
const ttlInDays =
|
||||
project?.auditLogsRetentionDays && project.auditLogsRetentionDays < plan.auditLogsRetentionDays
|
||||
? project.auditLogsRetentionDays
|
||||
: plan.auditLogsRetentionDays;
|
||||
const publishable = toPublishableEvent(event);
|
||||
|
||||
const ttl = ttlInDays * MS_IN_DAY;
|
||||
|
||||
const auditLog = await auditLogDAL.create({
|
||||
actor: actor.type,
|
||||
actorMetadata: actor.metadata,
|
||||
userAgent,
|
||||
projectId,
|
||||
projectName: project?.name,
|
||||
ipAddress,
|
||||
orgId,
|
||||
eventType: event.type,
|
||||
expiresAt: new Date(Date.now() + ttl),
|
||||
eventMetadata: event.metadata,
|
||||
userAgentType
|
||||
});
|
||||
|
||||
const logStreams = orgId ? await auditLogStreamDAL.find({ orgId }) : [];
|
||||
await Promise.allSettled(
|
||||
logStreams.map(
|
||||
async ({
|
||||
url,
|
||||
encryptedHeadersTag,
|
||||
encryptedHeadersIV,
|
||||
encryptedHeadersKeyEncoding,
|
||||
encryptedHeadersCiphertext
|
||||
}) => {
|
||||
const streamHeaders =
|
||||
encryptedHeadersIV && encryptedHeadersCiphertext && encryptedHeadersTag
|
||||
? (JSON.parse(
|
||||
crypto
|
||||
.encryption()
|
||||
.symmetric()
|
||||
.decryptWithRootEncryptionKey({
|
||||
keyEncoding: encryptedHeadersKeyEncoding as SecretKeyEncoding,
|
||||
iv: encryptedHeadersIV,
|
||||
tag: encryptedHeadersTag,
|
||||
ciphertext: encryptedHeadersCiphertext
|
||||
})
|
||||
) as LogStreamHeaders[])
|
||||
: [];
|
||||
|
||||
const headers: RawAxiosRequestHeaders = { "Content-Type": "application/json" };
|
||||
|
||||
if (streamHeaders.length)
|
||||
streamHeaders.forEach(({ key, value }) => {
|
||||
headers[key] = value;
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await request.post(
|
||||
url,
|
||||
{ ...providerSpecificPayload(url), ...auditLog },
|
||||
{
|
||||
headers,
|
||||
// request timeout
|
||||
timeout: AUDIT_LOG_STREAM_TIMEOUT,
|
||||
// connection timeout
|
||||
signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
|
||||
}
|
||||
);
|
||||
return response;
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`Failed to stream audit log [url=${url}] for org [orgId=${orgId}] [error=${(error as AxiosError).message}]`
|
||||
);
|
||||
return error;
|
||||
}
|
||||
}
|
||||
)
|
||||
);
|
||||
if (publishable) {
|
||||
await eventBusService.publish(TopicName.CoreServers, {
|
||||
type: ProjectType.SecretManager,
|
||||
source: "infiscal",
|
||||
data: publishable.data
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import Redis from "ioredis";
|
||||
import { z } from "zod";
|
||||
|
||||
import { logger } from "@app/lib/logger";
|
||||
|
||||
import { EventSchema, TopicName } from "./types";
|
||||
|
||||
export const eventBusFactory = (redis: Redis) => {
|
||||
const publisher = redis.duplicate();
|
||||
// Duplicate the publisher to create a subscriber.
|
||||
// This is necessary because Redis does not allow a single connection to both publish and subscribe.
|
||||
const subscriber = publisher.duplicate();
|
||||
|
||||
const init = async (topics: TopicName[] = Object.values(TopicName)) => {
|
||||
subscriber.on("error", (e) => {
|
||||
logger.error(e, "Event Bus subscriber error");
|
||||
});
|
||||
|
||||
publisher.on("error", (e) => {
|
||||
logger.error(e, "Event Bus publisher error");
|
||||
});
|
||||
|
||||
await subscriber.subscribe(...topics);
|
||||
};
|
||||
|
||||
/**
|
||||
* Publishes an event to the specified topic.
|
||||
* @param topic - The topic to publish the event to.
|
||||
* @param event - The event data to publish.
|
||||
*/
|
||||
const publish = async <T extends z.input<typeof EventSchema>>(topic: TopicName, event: T) => {
|
||||
const json = JSON.stringify(event);
|
||||
|
||||
return publisher.publish(topic, json, (err) => {
|
||||
if (err) {
|
||||
return logger.error(err, `Error publishing to channel ${topic}`);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* @param fn - The function to call when a message is received.
|
||||
* It should accept the parsed event data as an argument.
|
||||
* @template T - The type of the event data, which should match the schema defined in EventSchema.
|
||||
* @returns A function that can be called to unsubscribe from the event bus.
|
||||
*/
|
||||
const subscribe = <T extends z.infer<typeof EventSchema>>(fn: (data: T) => Promise<void> | void) => {
|
||||
// Not using async await cause redis client's `on` method does not expect async listeners.
|
||||
const listener = (channel: string, message: string) => {
|
||||
try {
|
||||
const parsed = JSON.parse(message) as T;
|
||||
const thenable = fn(parsed);
|
||||
|
||||
// If the function returns a Promise, catch any errors that occur during processing.
|
||||
if (thenable instanceof Promise) {
|
||||
thenable.catch((error) => {
|
||||
logger.error(error, `Error processing message from channel ${channel}`);
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(error, `Error parsing message data from channel ${channel}`);
|
||||
}
|
||||
};
|
||||
subscriber.on("message", listener);
|
||||
|
||||
return () => {
|
||||
subscriber.off("message", listener);
|
||||
};
|
||||
};
|
||||
|
||||
const close = async () => {
|
||||
try {
|
||||
await publisher.quit();
|
||||
await subscriber.quit();
|
||||
} catch (error) {
|
||||
logger.error(error, "Error closing event bus connections");
|
||||
}
|
||||
};
|
||||
|
||||
return { init, publish, subscribe, close };
|
||||
};
|
||||
|
||||
export type TEventBusService = ReturnType<typeof eventBusFactory>;
|
||||
@@ -0,0 +1,164 @@
|
||||
/* eslint-disable no-continue */
|
||||
import { subject } from "@casl/ability";
|
||||
import Redis from "ioredis";
|
||||
|
||||
import { KeyStorePrefixes } from "@app/keystore/keystore";
|
||||
import { logger } from "@app/lib/logger";
|
||||
|
||||
import { TEventBusService } from "./event-bus-service";
|
||||
import { createEventStreamClient, EventStreamClient, IEventStreamClientOpts } from "./event-sse-stream";
|
||||
import { EventData, RegisteredEvent, toBusEventName } from "./types";
|
||||
|
||||
const AUTH_REFRESH_INTERVAL = 60 * 1000;
|
||||
const HEART_BEAT_INTERVAL = 15 * 1000;
|
||||
|
||||
export const sseServiceFactory = (bus: TEventBusService, redis: Redis) => {
|
||||
let heartbeatInterval: NodeJS.Timeout | null = null;
|
||||
|
||||
const clients = new Set<EventStreamClient>();
|
||||
|
||||
heartbeatInterval = setInterval(() => {
|
||||
for (const client of clients) {
|
||||
if (client.stream.closed) continue;
|
||||
void client.ping();
|
||||
}
|
||||
}, HEART_BEAT_INTERVAL);
|
||||
|
||||
const refreshInterval = setInterval(() => {
|
||||
for (const client of clients) {
|
||||
if (client.stream.closed) continue;
|
||||
void client.refresh();
|
||||
}
|
||||
}, AUTH_REFRESH_INTERVAL);
|
||||
|
||||
const removeActiveConnection = async (projectId: string, identityId: string, connectionId: string) => {
|
||||
const set = KeyStorePrefixes.ActiveSSEConnectionsSet(projectId, identityId);
|
||||
const key = KeyStorePrefixes.ActiveSSEConnections(projectId, identityId, connectionId);
|
||||
|
||||
await Promise.all([redis.lrem(set, 0, connectionId), redis.del(key)]);
|
||||
};
|
||||
|
||||
const getActiveConnectionsCount = async (projectId: string, identityId: string) => {
|
||||
const set = KeyStorePrefixes.ActiveSSEConnectionsSet(projectId, identityId);
|
||||
const connections = await redis.lrange(set, 0, -1);
|
||||
|
||||
if (connections.length === 0) {
|
||||
return 0; // No active connections
|
||||
}
|
||||
|
||||
const keys = connections.map((c) => KeyStorePrefixes.ActiveSSEConnections(projectId, identityId, c));
|
||||
|
||||
const values = await redis.mget(...keys);
|
||||
|
||||
// eslint-disable-next-line no-plusplus
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
if (values[i] === null) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await removeActiveConnection(projectId, identityId, connections[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return redis.llen(set);
|
||||
};
|
||||
|
||||
const onDisconnect = async (client: EventStreamClient) => {
|
||||
try {
|
||||
client.close();
|
||||
clients.delete(client);
|
||||
await removeActiveConnection(client.auth.projectId, client.auth.actorId, client.id);
|
||||
} catch (error) {
|
||||
logger.error(error, "Error during SSE stream disconnection");
|
||||
}
|
||||
};
|
||||
|
||||
function filterEventsForClient(client: EventStreamClient, event: EventData, registered: RegisteredEvent[]) {
|
||||
const eventType = toBusEventName(event.data.eventType);
|
||||
const match = registered.find((r) => r.event === eventType);
|
||||
if (!match) return;
|
||||
|
||||
const item = event.data.payload;
|
||||
|
||||
if (Array.isArray(item)) {
|
||||
if (item.length === 0) return;
|
||||
|
||||
const baseSubject = {
|
||||
eventType,
|
||||
environment: undefined as string | undefined,
|
||||
secretPath: undefined as string | undefined
|
||||
};
|
||||
|
||||
const filtered = item.filter((ev) => {
|
||||
baseSubject.secretPath = ev.secretPath ?? "/";
|
||||
baseSubject.environment = ev.environment;
|
||||
|
||||
return client.matcher.can("subscribe", subject(event.type, baseSubject));
|
||||
});
|
||||
|
||||
if (filtered.length === 0) return;
|
||||
|
||||
return client.send({
|
||||
...event,
|
||||
data: {
|
||||
...event.data,
|
||||
payload: filtered
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// For single item
|
||||
const baseSubject = {
|
||||
eventType,
|
||||
secretPath: item.secretPath ?? "/",
|
||||
environment: item.environment
|
||||
};
|
||||
|
||||
if (client.matcher.can("subscribe", subject(event.type, baseSubject))) {
|
||||
client.send(event);
|
||||
}
|
||||
}
|
||||
|
||||
const subscribe = async (
|
||||
opts: IEventStreamClientOpts & {
|
||||
onClose?: () => void;
|
||||
}
|
||||
) => {
|
||||
const client = createEventStreamClient(redis, opts);
|
||||
|
||||
// Set up event listener on event bus
|
||||
const unsubscribe = bus.subscribe((event) => {
|
||||
if (event.type !== opts.type) return;
|
||||
filterEventsForClient(client, event, opts.registered);
|
||||
});
|
||||
|
||||
client.stream.on("close", () => {
|
||||
unsubscribe();
|
||||
void onDisconnect(client); // This will never throw
|
||||
});
|
||||
|
||||
await client.open();
|
||||
|
||||
clients.add(client);
|
||||
|
||||
return client;
|
||||
};
|
||||
|
||||
const close = () => {
|
||||
if (heartbeatInterval) {
|
||||
clearInterval(heartbeatInterval);
|
||||
}
|
||||
|
||||
if (refreshInterval) {
|
||||
clearInterval(refreshInterval);
|
||||
}
|
||||
|
||||
for (const client of clients) {
|
||||
client.close();
|
||||
}
|
||||
|
||||
clients.clear();
|
||||
};
|
||||
|
||||
return { subscribe, close, getActiveConnectionsCount };
|
||||
};
|
||||
|
||||
export type TServerSentEventsService = ReturnType<typeof sseServiceFactory>;
|
||||
@@ -0,0 +1,178 @@
|
||||
/* eslint-disable no-underscore-dangle */
|
||||
import { Readable } from "node:stream";
|
||||
|
||||
import { MongoAbility, PureAbility } from "@casl/ability";
|
||||
import { MongoQuery } from "@ucast/mongo2js";
|
||||
import Redis from "ioredis";
|
||||
import { nanoid } from "nanoid";
|
||||
|
||||
import { ProjectType } from "@app/db/schemas";
|
||||
import { ProjectPermissionSet } from "@app/ee/services/permission/project-permission";
|
||||
import { KeyStorePrefixes } from "@app/keystore/keystore";
|
||||
import { conditionsMatcher } from "@app/lib/casl";
|
||||
import { logger } from "@app/lib/logger";
|
||||
|
||||
import { EventData, RegisteredEvent } from "./types";
|
||||
|
||||
export const getServerSentEventsHeaders = () =>
|
||||
({
|
||||
"Cache-Control": "no-cache",
|
||||
"Content-Type": "text/event-stream",
|
||||
Connection: "keep-alive",
|
||||
"X-Accel-Buffering": "no"
|
||||
}) as const;
|
||||
|
||||
type TAuthInfo = {
|
||||
actorId: string;
|
||||
projectId: string;
|
||||
permission: MongoAbility<ProjectPermissionSet, MongoQuery>;
|
||||
};
|
||||
|
||||
export interface IEventStreamClientOpts {
|
||||
type: ProjectType;
|
||||
registered: RegisteredEvent[];
|
||||
onAuthRefresh: (info: TAuthInfo) => Promise<void> | void;
|
||||
getAuthInfo: () => Promise<TAuthInfo> | TAuthInfo;
|
||||
}
|
||||
|
||||
interface EventMessage {
|
||||
time?: string | number;
|
||||
type: string;
|
||||
data?: unknown;
|
||||
}
|
||||
|
||||
function serializeSseEvent(chunk: EventMessage): string {
|
||||
let payload = "";
|
||||
|
||||
if (chunk.time) payload += `id: ${chunk.time}\n`;
|
||||
if (chunk.type) payload += `event: ${chunk.type}\n`;
|
||||
if (chunk.data) payload += `data: ${JSON.stringify(chunk)}\n`;
|
||||
|
||||
return `${payload}\n`;
|
||||
}
|
||||
|
||||
export type EventStreamClient = {
|
||||
id: string;
|
||||
stream: Readable;
|
||||
open: () => Promise<void>;
|
||||
send: (data: EventMessage | EventData) => void;
|
||||
ping: () => Promise<void>;
|
||||
refresh: () => Promise<void>;
|
||||
close: () => void;
|
||||
get auth(): TAuthInfo;
|
||||
signal: AbortSignal;
|
||||
abort: () => void;
|
||||
matcher: PureAbility;
|
||||
};
|
||||
|
||||
export function createEventStreamClient(redis: Redis, options: IEventStreamClientOpts): EventStreamClient {
|
||||
const rules = options.registered.map((r) => ({
|
||||
subject: options.type,
|
||||
action: "subscribe",
|
||||
conditions: {
|
||||
eventType: r.event,
|
||||
secretPath: r.conditions?.secretPath ?? "/",
|
||||
environment: r.conditions?.environmentSlug
|
||||
}
|
||||
}));
|
||||
|
||||
const id = `sse-${nanoid()}`;
|
||||
const control = new AbortController();
|
||||
const matcher = new PureAbility(rules, { conditionsMatcher });
|
||||
|
||||
let auth: TAuthInfo | undefined;
|
||||
|
||||
const stream = new Readable({
|
||||
objectMode: true
|
||||
});
|
||||
|
||||
// We will manually push data to the stream
|
||||
stream._read = () => {};
|
||||
|
||||
const send = (data: EventMessage | EventData) => {
|
||||
const chunk = serializeSseEvent(data);
|
||||
if (!stream.push(chunk)) {
|
||||
logger.debug("Backpressure detected: dropped manual event");
|
||||
}
|
||||
};
|
||||
|
||||
stream.on("error", (error: Error) => stream.destroy(error));
|
||||
|
||||
const open = async () => {
|
||||
auth = await options.getAuthInfo();
|
||||
await options.onAuthRefresh(auth);
|
||||
|
||||
const { actorId, projectId } = auth;
|
||||
const set = KeyStorePrefixes.ActiveSSEConnectionsSet(projectId, actorId);
|
||||
const key = KeyStorePrefixes.ActiveSSEConnections(projectId, actorId, id);
|
||||
|
||||
await Promise.all([redis.rpush(set, id), redis.set(key, "1", "EX", 60)]);
|
||||
};
|
||||
|
||||
const ping = async () => {
|
||||
if (!auth) return; // Avoid race condition if ping is called before open
|
||||
|
||||
const { actorId, projectId } = auth;
|
||||
const key = KeyStorePrefixes.ActiveSSEConnections(projectId, actorId, id);
|
||||
|
||||
await redis.set(key, "1", "EX", 60);
|
||||
|
||||
stream.push("1");
|
||||
};
|
||||
|
||||
const close = () => {
|
||||
if (stream.closed) return;
|
||||
stream.push(null);
|
||||
stream.destroy();
|
||||
};
|
||||
|
||||
/**
|
||||
* Refreshes the connection's auth permissions
|
||||
* Must be called atleast once when connection is opened
|
||||
*/
|
||||
const refresh = async () => {
|
||||
try {
|
||||
auth = await options.getAuthInfo();
|
||||
await options.onAuthRefresh(auth);
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
send({
|
||||
type: "error",
|
||||
data: {
|
||||
...error
|
||||
}
|
||||
});
|
||||
return close();
|
||||
}
|
||||
stream.emit("error", error);
|
||||
}
|
||||
};
|
||||
|
||||
const abort = () => {
|
||||
try {
|
||||
control.abort();
|
||||
} catch (error) {
|
||||
logger.debug(error, "Error aborting SSE stream");
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
id,
|
||||
stream,
|
||||
open,
|
||||
send,
|
||||
ping,
|
||||
refresh,
|
||||
close,
|
||||
signal: control.signal,
|
||||
abort,
|
||||
matcher,
|
||||
get auth() {
|
||||
if (!auth) {
|
||||
throw new Error("Auth info not set");
|
||||
}
|
||||
|
||||
return auth;
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { ProjectType } from "@app/db/schemas";
|
||||
import { Event, EventType } from "@app/ee/services/audit-log/audit-log-types";
|
||||
|
||||
export enum TopicName {
|
||||
CoreServers = "infisical::core-servers"
|
||||
}
|
||||
|
||||
export enum BusEventName {
|
||||
CreateSecret = "secret:create",
|
||||
UpdateSecret = "secret:update",
|
||||
DeleteSecret = "secret:delete"
|
||||
}
|
||||
|
||||
type PublisableEventTypes =
|
||||
| EventType.CREATE_SECRET
|
||||
| EventType.CREATE_SECRETS
|
||||
| EventType.DELETE_SECRET
|
||||
| EventType.DELETE_SECRETS
|
||||
| EventType.UPDATE_SECRETS
|
||||
| EventType.UPDATE_SECRET;
|
||||
|
||||
export function toBusEventName(input: EventType) {
|
||||
switch (input) {
|
||||
case EventType.CREATE_SECRET:
|
||||
case EventType.CREATE_SECRETS:
|
||||
return BusEventName.CreateSecret;
|
||||
case EventType.UPDATE_SECRET:
|
||||
case EventType.UPDATE_SECRETS:
|
||||
return BusEventName.UpdateSecret;
|
||||
case EventType.DELETE_SECRET:
|
||||
case EventType.DELETE_SECRETS:
|
||||
return BusEventName.DeleteSecret;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const isBulkEvent = (event: Event): event is Extract<Event, { metadata: { secrets: Array<unknown> } }> => {
|
||||
return event.type.endsWith("-secrets"); // Feels so wrong
|
||||
};
|
||||
|
||||
export const toPublishableEvent = (event: Event) => {
|
||||
const name = toBusEventName(event.type);
|
||||
|
||||
if (!name) return null;
|
||||
|
||||
const e = event as Extract<Event, { type: PublisableEventTypes }>;
|
||||
|
||||
if (isBulkEvent(e)) {
|
||||
return {
|
||||
name,
|
||||
isBulk: true,
|
||||
data: {
|
||||
eventType: e.type,
|
||||
payload: e.metadata.secrets.map((s) => ({
|
||||
environment: e.metadata.environment,
|
||||
secretPath: e.metadata.secretPath,
|
||||
...s
|
||||
}))
|
||||
}
|
||||
} as const;
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
isBulk: false,
|
||||
data: {
|
||||
eventType: e.type,
|
||||
payload: {
|
||||
...e.metadata,
|
||||
environment: e.metadata.environment
|
||||
}
|
||||
}
|
||||
} as const;
|
||||
};
|
||||
|
||||
export const EventName = z.nativeEnum(BusEventName);
|
||||
|
||||
const EventSecretPayload = z.object({
|
||||
secretPath: z.string().optional(),
|
||||
secretId: z.string(),
|
||||
secretKey: z.string(),
|
||||
environment: z.string()
|
||||
});
|
||||
|
||||
export type EventSecret = z.infer<typeof EventSecretPayload>;
|
||||
|
||||
export const EventSchema = z.object({
|
||||
datacontenttype: z.literal("application/json").optional().default("application/json"),
|
||||
type: z.nativeEnum(ProjectType),
|
||||
source: z.string(),
|
||||
time: z
|
||||
.string()
|
||||
.optional()
|
||||
.default(() => new Date().toISOString()),
|
||||
data: z.discriminatedUnion("eventType", [
|
||||
z.object({
|
||||
specversion: z.number().optional().default(1),
|
||||
eventType: z.enum([EventType.CREATE_SECRET, EventType.UPDATE_SECRET, EventType.DELETE_SECRET]),
|
||||
payload: EventSecretPayload
|
||||
}),
|
||||
z.object({
|
||||
specversion: z.number().optional().default(1),
|
||||
eventType: z.enum([EventType.CREATE_SECRETS, EventType.UPDATE_SECRETS, EventType.DELETE_SECRETS]),
|
||||
payload: EventSecretPayload.array()
|
||||
})
|
||||
// Add more event types as needed
|
||||
])
|
||||
});
|
||||
|
||||
export type EventData = z.infer<typeof EventSchema>;
|
||||
|
||||
export const EventRegisterSchema = z.object({
|
||||
event: EventName,
|
||||
conditions: z
|
||||
.object({
|
||||
secretPath: z.string().optional().default("/"),
|
||||
environmentSlug: z.string()
|
||||
})
|
||||
.optional()
|
||||
});
|
||||
|
||||
export type RegisteredEvent = z.infer<typeof EventRegisterSchema>;
|
||||
@@ -59,7 +59,8 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
|
||||
secretScanning: false,
|
||||
enterpriseSecretSyncs: false,
|
||||
enterpriseAppConnections: false,
|
||||
fips: false
|
||||
fips: false,
|
||||
eventSubscriptions: false
|
||||
});
|
||||
|
||||
export const setupLicenseRequestWithStore = (
|
||||
|
||||
@@ -76,6 +76,7 @@ export type TFeatureSet = {
|
||||
enterpriseSecretSyncs: false;
|
||||
enterpriseAppConnections: false;
|
||||
fips: false;
|
||||
eventSubscriptions: false;
|
||||
};
|
||||
|
||||
export type TOrgPlansTableDTO = {
|
||||
|
||||
@@ -161,7 +161,8 @@ const buildAdminPermissionRules = () => {
|
||||
ProjectPermissionSecretActions.ReadValue,
|
||||
ProjectPermissionSecretActions.Create,
|
||||
ProjectPermissionSecretActions.Edit,
|
||||
ProjectPermissionSecretActions.Delete
|
||||
ProjectPermissionSecretActions.Delete,
|
||||
ProjectPermissionSecretActions.Subscribe
|
||||
],
|
||||
ProjectPermissionSub.Secrets
|
||||
);
|
||||
@@ -265,7 +266,8 @@ const buildMemberPermissionRules = () => {
|
||||
ProjectPermissionSecretActions.ReadValue,
|
||||
ProjectPermissionSecretActions.Edit,
|
||||
ProjectPermissionSecretActions.Create,
|
||||
ProjectPermissionSecretActions.Delete
|
||||
ProjectPermissionSecretActions.Delete,
|
||||
ProjectPermissionSecretActions.Subscribe
|
||||
],
|
||||
ProjectPermissionSub.Secrets
|
||||
);
|
||||
|
||||
@@ -36,7 +36,8 @@ export enum ProjectPermissionSecretActions {
|
||||
ReadValue = "readValue",
|
||||
Create = "create",
|
||||
Edit = "edit",
|
||||
Delete = "delete"
|
||||
Delete = "delete",
|
||||
Subscribe = "subscribe"
|
||||
}
|
||||
|
||||
export enum ProjectPermissionCmekActions {
|
||||
@@ -204,6 +205,7 @@ export type SecretSubjectFields = {
|
||||
secretPath: string;
|
||||
secretName?: string;
|
||||
secretTags?: string[];
|
||||
eventType?: string;
|
||||
};
|
||||
|
||||
export type SecretFolderSubjectFields = {
|
||||
@@ -483,7 +485,17 @@ const SecretConditionV2Schema = z
|
||||
.object({
|
||||
[PermissionConditionOperators.$IN]: PermissionConditionSchema[PermissionConditionOperators.$IN]
|
||||
})
|
||||
.partial()
|
||||
.partial(),
|
||||
eventType: z.union([
|
||||
z.string(),
|
||||
z
|
||||
.object({
|
||||
[PermissionConditionOperators.$EQ]: PermissionConditionSchema[PermissionConditionOperators.$EQ],
|
||||
[PermissionConditionOperators.$NEQ]: PermissionConditionSchema[PermissionConditionOperators.$NEQ],
|
||||
[PermissionConditionOperators.$IN]: PermissionConditionSchema[PermissionConditionOperators.$IN]
|
||||
})
|
||||
.partial()
|
||||
])
|
||||
})
|
||||
.partial();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user