mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-10-18 17:41:16 +00:00
feat: events system implementation (#4246)
* chore: save poc * chore: save wip * fix: undo cors * fix: impl changes * fix: PR changes * fix: mocks * fix: connection tracking and auth changes * fix: PR changes * fix: revert license * feat: frontend change * fix: revert docker compose.dev * fix: duplicate publisher connection * fix: pr changes * chore: move event impl to `ee` * fix: lint errors * fix: check length of events * fix: static permissions matching * fix: secretPath * fix: remove source prefix in bus event name * fix: license check
This commit is contained in:
Sid
@@ -0,0 +1,83 @@
|
||||
import Redis from "ioredis";
|
||||
import { z } from "zod";
|
||||
|
||||
import { logger } from "@app/lib/logger";
|
||||
|
||||
import { EventSchema, TopicName } from "./types";
|
||||
|
||||
export const eventBusFactory = (redis: Redis) => {
|
||||
const publisher = redis.duplicate();
|
||||
// Duplicate the publisher to create a subscriber.
|
||||
// This is necessary because Redis does not allow a single connection to both publish and subscribe.
|
||||
const subscriber = publisher.duplicate();
|
||||
|
||||
const init = async (topics: TopicName[] = Object.values(TopicName)) => {
|
||||
subscriber.on("error", (e) => {
|
||||
logger.error(e, "Event Bus subscriber error");
|
||||
});
|
||||
|
||||
publisher.on("error", (e) => {
|
||||
logger.error(e, "Event Bus publisher error");
|
||||
});
|
||||
|
||||
await subscriber.subscribe(...topics);
|
||||
};
|
||||
|
||||
/**
|
||||
* Publishes an event to the specified topic.
|
||||
* @param topic - The topic to publish the event to.
|
||||
* @param event - The event data to publish.
|
||||
*/
|
||||
const publish = async <T extends z.input<typeof EventSchema>>(topic: TopicName, event: T) => {
|
||||
const json = JSON.stringify(event);
|
||||
|
||||
return publisher.publish(topic, json, (err) => {
|
||||
if (err) {
|
||||
return logger.error(err, `Error publishing to channel ${topic}`);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* @param fn - The function to call when a message is received.
|
||||
* It should accept the parsed event data as an argument.
|
||||
* @template T - The type of the event data, which should match the schema defined in EventSchema.
|
||||
* @returns A function that can be called to unsubscribe from the event bus.
|
||||
*/
|
||||
const subscribe = <T extends z.infer<typeof EventSchema>>(fn: (data: T) => Promise<void> | void) => {
|
||||
// Not using async await cause redis client's `on` method does not expect async listeners.
|
||||
const listener = (channel: string, message: string) => {
|
||||
try {
|
||||
const parsed = JSON.parse(message) as T;
|
||||
const thenable = fn(parsed);
|
||||
|
||||
// If the function returns a Promise, catch any errors that occur during processing.
|
||||
if (thenable instanceof Promise) {
|
||||
thenable.catch((error) => {
|
||||
logger.error(error, `Error processing message from channel ${channel}`);
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(error, `Error parsing message data from channel ${channel}`);
|
||||
}
|
||||
};
|
||||
subscriber.on("message", listener);
|
||||
|
||||
return () => {
|
||||
subscriber.off("message", listener);
|
||||
};
|
||||
};
|
||||
|
||||
const close = async () => {
|
||||
try {
|
||||
await publisher.quit();
|
||||
await subscriber.quit();
|
||||
} catch (error) {
|
||||
logger.error(error, "Error closing event bus connections");
|
||||
}
|
||||
};
|
||||
|
||||
return { init, publish, subscribe, close };
|
||||
};
|
||||
|
||||
export type TEventBusService = ReturnType<typeof eventBusFactory>;
|
||||
@@ -0,0 +1,164 @@
|
||||
/* eslint-disable no-continue */
|
||||
import { subject } from "@casl/ability";
|
||||
import Redis from "ioredis";
|
||||
|
||||
import { KeyStorePrefixes } from "@app/keystore/keystore";
|
||||
import { logger } from "@app/lib/logger";
|
||||
|
||||
import { TEventBusService } from "./event-bus-service";
|
||||
import { createEventStreamClient, EventStreamClient, IEventStreamClientOpts } from "./event-sse-stream";
|
||||
import { EventData, RegisteredEvent, toBusEventName } from "./types";
|
||||
|
||||
const AUTH_REFRESH_INTERVAL = 60 * 1000;
|
||||
const HEART_BEAT_INTERVAL = 15 * 1000;
|
||||
|
||||
export const sseServiceFactory = (bus: TEventBusService, redis: Redis) => {
|
||||
let heartbeatInterval: NodeJS.Timeout | null = null;
|
||||
|
||||
const clients = new Set<EventStreamClient>();
|
||||
|
||||
heartbeatInterval = setInterval(() => {
|
||||
for (const client of clients) {
|
||||
if (client.stream.closed) continue;
|
||||
void client.ping();
|
||||
}
|
||||
}, HEART_BEAT_INTERVAL);
|
||||
|
||||
const refreshInterval = setInterval(() => {
|
||||
for (const client of clients) {
|
||||
if (client.stream.closed) continue;
|
||||
void client.refresh();
|
||||
}
|
||||
}, AUTH_REFRESH_INTERVAL);
|
||||
|
||||
const removeActiveConnection = async (projectId: string, identityId: string, connectionId: string) => {
|
||||
const set = KeyStorePrefixes.ActiveSSEConnectionsSet(projectId, identityId);
|
||||
const key = KeyStorePrefixes.ActiveSSEConnections(projectId, identityId, connectionId);
|
||||
|
||||
await Promise.all([redis.lrem(set, 0, connectionId), redis.del(key)]);
|
||||
};
|
||||
|
||||
const getActiveConnectionsCount = async (projectId: string, identityId: string) => {
|
||||
const set = KeyStorePrefixes.ActiveSSEConnectionsSet(projectId, identityId);
|
||||
const connections = await redis.lrange(set, 0, -1);
|
||||
|
||||
if (connections.length === 0) {
|
||||
return 0; // No active connections
|
||||
}
|
||||
|
||||
const keys = connections.map((c) => KeyStorePrefixes.ActiveSSEConnections(projectId, identityId, c));
|
||||
|
||||
const values = await redis.mget(...keys);
|
||||
|
||||
// eslint-disable-next-line no-plusplus
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
if (values[i] === null) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await removeActiveConnection(projectId, identityId, connections[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return redis.llen(set);
|
||||
};
|
||||
|
||||
const onDisconnect = async (client: EventStreamClient) => {
|
||||
try {
|
||||
client.close();
|
||||
clients.delete(client);
|
||||
await removeActiveConnection(client.auth.projectId, client.auth.actorId, client.id);
|
||||
} catch (error) {
|
||||
logger.error(error, "Error during SSE stream disconnection");
|
||||
}
|
||||
};
|
||||
|
||||
function filterEventsForClient(client: EventStreamClient, event: EventData, registered: RegisteredEvent[]) {
|
||||
const eventType = toBusEventName(event.data.eventType);
|
||||
const match = registered.find((r) => r.event === eventType);
|
||||
if (!match) return;
|
||||
|
||||
const item = event.data.payload;
|
||||
|
||||
if (Array.isArray(item)) {
|
||||
if (item.length === 0) return;
|
||||
|
||||
const baseSubject = {
|
||||
eventType,
|
||||
environment: undefined as string | undefined,
|
||||
secretPath: undefined as string | undefined
|
||||
};
|
||||
|
||||
const filtered = item.filter((ev) => {
|
||||
baseSubject.secretPath = ev.secretPath ?? "/";
|
||||
baseSubject.environment = ev.environment;
|
||||
|
||||
return client.matcher.can("subscribe", subject(event.type, baseSubject));
|
||||
});
|
||||
|
||||
if (filtered.length === 0) return;
|
||||
|
||||
return client.send({
|
||||
...event,
|
||||
data: {
|
||||
...event.data,
|
||||
payload: filtered
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// For single item
|
||||
const baseSubject = {
|
||||
eventType,
|
||||
secretPath: item.secretPath ?? "/",
|
||||
environment: item.environment
|
||||
};
|
||||
|
||||
if (client.matcher.can("subscribe", subject(event.type, baseSubject))) {
|
||||
client.send(event);
|
||||
}
|
||||
}
|
||||
|
||||
const subscribe = async (
|
||||
opts: IEventStreamClientOpts & {
|
||||
onClose?: () => void;
|
||||
}
|
||||
) => {
|
||||
const client = createEventStreamClient(redis, opts);
|
||||
|
||||
// Set up event listener on event bus
|
||||
const unsubscribe = bus.subscribe((event) => {
|
||||
if (event.type !== opts.type) return;
|
||||
filterEventsForClient(client, event, opts.registered);
|
||||
});
|
||||
|
||||
client.stream.on("close", () => {
|
||||
unsubscribe();
|
||||
void onDisconnect(client); // This will never throw
|
||||
});
|
||||
|
||||
await client.open();
|
||||
|
||||
clients.add(client);
|
||||
|
||||
return client;
|
||||
};
|
||||
|
||||
const close = () => {
|
||||
if (heartbeatInterval) {
|
||||
clearInterval(heartbeatInterval);
|
||||
}
|
||||
|
||||
if (refreshInterval) {
|
||||
clearInterval(refreshInterval);
|
||||
}
|
||||
|
||||
for (const client of clients) {
|
||||
client.close();
|
||||
}
|
||||
|
||||
clients.clear();
|
||||
};
|
||||
|
||||
return { subscribe, close, getActiveConnectionsCount };
|
||||
};
|
||||
|
||||
export type TServerSentEventsService = ReturnType<typeof sseServiceFactory>;
|
||||
@@ -0,0 +1,178 @@
|
||||
/* eslint-disable no-underscore-dangle */
|
||||
import { Readable } from "node:stream";
|
||||
|
||||
import { MongoAbility, PureAbility } from "@casl/ability";
|
||||
import { MongoQuery } from "@ucast/mongo2js";
|
||||
import Redis from "ioredis";
|
||||
import { nanoid } from "nanoid";
|
||||
|
||||
import { ProjectType } from "@app/db/schemas";
|
||||
import { ProjectPermissionSet } from "@app/ee/services/permission/project-permission";
|
||||
import { KeyStorePrefixes } from "@app/keystore/keystore";
|
||||
import { conditionsMatcher } from "@app/lib/casl";
|
||||
import { logger } from "@app/lib/logger";
|
||||
|
||||
import { EventData, RegisteredEvent } from "./types";
|
||||
|
||||
export const getServerSentEventsHeaders = () =>
|
||||
({
|
||||
"Cache-Control": "no-cache",
|
||||
"Content-Type": "text/event-stream",
|
||||
Connection: "keep-alive",
|
||||
"X-Accel-Buffering": "no"
|
||||
}) as const;
|
||||
|
||||
type TAuthInfo = {
|
||||
actorId: string;
|
||||
projectId: string;
|
||||
permission: MongoAbility<ProjectPermissionSet, MongoQuery>;
|
||||
};
|
||||
|
||||
export interface IEventStreamClientOpts {
|
||||
type: ProjectType;
|
||||
registered: RegisteredEvent[];
|
||||
onAuthRefresh: (info: TAuthInfo) => Promise<void> | void;
|
||||
getAuthInfo: () => Promise<TAuthInfo> | TAuthInfo;
|
||||
}
|
||||
|
||||
interface EventMessage {
|
||||
time?: string | number;
|
||||
type: string;
|
||||
data?: unknown;
|
||||
}
|
||||
|
||||
function serializeSseEvent(chunk: EventMessage): string {
|
||||
let payload = "";
|
||||
|
||||
if (chunk.time) payload += `id: ${chunk.time}\n`;
|
||||
if (chunk.type) payload += `event: ${chunk.type}\n`;
|
||||
if (chunk.data) payload += `data: ${JSON.stringify(chunk)}\n`;
|
||||
|
||||
return `${payload}\n`;
|
||||
}
|
||||
|
||||
export type EventStreamClient = {
|
||||
id: string;
|
||||
stream: Readable;
|
||||
open: () => Promise<void>;
|
||||
send: (data: EventMessage | EventData) => void;
|
||||
ping: () => Promise<void>;
|
||||
refresh: () => Promise<void>;
|
||||
close: () => void;
|
||||
get auth(): TAuthInfo;
|
||||
signal: AbortSignal;
|
||||
abort: () => void;
|
||||
matcher: PureAbility;
|
||||
};
|
||||
|
||||
export function createEventStreamClient(redis: Redis, options: IEventStreamClientOpts): EventStreamClient {
|
||||
const rules = options.registered.map((r) => ({
|
||||
subject: options.type,
|
||||
action: "subscribe",
|
||||
conditions: {
|
||||
eventType: r.event,
|
||||
secretPath: r.conditions?.secretPath ?? "/",
|
||||
environment: r.conditions?.environmentSlug
|
||||
}
|
||||
}));
|
||||
|
||||
const id = `sse-${nanoid()}`;
|
||||
const control = new AbortController();
|
||||
const matcher = new PureAbility(rules, { conditionsMatcher });
|
||||
|
||||
let auth: TAuthInfo | undefined;
|
||||
|
||||
const stream = new Readable({
|
||||
objectMode: true
|
||||
});
|
||||
|
||||
// We will manually push data to the stream
|
||||
stream._read = () => {};
|
||||
|
||||
const send = (data: EventMessage | EventData) => {
|
||||
const chunk = serializeSseEvent(data);
|
||||
if (!stream.push(chunk)) {
|
||||
logger.debug("Backpressure detected: dropped manual event");
|
||||
}
|
||||
};
|
||||
|
||||
stream.on("error", (error: Error) => stream.destroy(error));
|
||||
|
||||
const open = async () => {
|
||||
auth = await options.getAuthInfo();
|
||||
await options.onAuthRefresh(auth);
|
||||
|
||||
const { actorId, projectId } = auth;
|
||||
const set = KeyStorePrefixes.ActiveSSEConnectionsSet(projectId, actorId);
|
||||
const key = KeyStorePrefixes.ActiveSSEConnections(projectId, actorId, id);
|
||||
|
||||
await Promise.all([redis.rpush(set, id), redis.set(key, "1", "EX", 60)]);
|
||||
};
|
||||
|
||||
const ping = async () => {
|
||||
if (!auth) return; // Avoid race condition if ping is called before open
|
||||
|
||||
const { actorId, projectId } = auth;
|
||||
const key = KeyStorePrefixes.ActiveSSEConnections(projectId, actorId, id);
|
||||
|
||||
await redis.set(key, "1", "EX", 60);
|
||||
|
||||
stream.push("1");
|
||||
};
|
||||
|
||||
const close = () => {
|
||||
if (stream.closed) return;
|
||||
stream.push(null);
|
||||
stream.destroy();
|
||||
};
|
||||
|
||||
/**
|
||||
* Refreshes the connection's auth permissions
|
||||
* Must be called atleast once when connection is opened
|
||||
*/
|
||||
const refresh = async () => {
|
||||
try {
|
||||
auth = await options.getAuthInfo();
|
||||
await options.onAuthRefresh(auth);
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
send({
|
||||
type: "error",
|
||||
data: {
|
||||
...error
|
||||
}
|
||||
});
|
||||
return close();
|
||||
}
|
||||
stream.emit("error", error);
|
||||
}
|
||||
};
|
||||
|
||||
const abort = () => {
|
||||
try {
|
||||
control.abort();
|
||||
} catch (error) {
|
||||
logger.debug(error, "Error aborting SSE stream");
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
id,
|
||||
stream,
|
||||
open,
|
||||
send,
|
||||
ping,
|
||||
refresh,
|
||||
close,
|
||||
signal: control.signal,
|
||||
abort,
|
||||
matcher,
|
||||
get auth() {
|
||||
if (!auth) {
|
||||
throw new Error("Auth info not set");
|
||||
}
|
||||
|
||||
return auth;
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { ProjectType } from "@app/db/schemas";
|
||||
import { Event, EventType } from "@app/ee/services/audit-log/audit-log-types";
|
||||
|
||||
export enum TopicName {
|
||||
CoreServers = "infisical::core-servers"
|
||||
}
|
||||
|
||||
export enum BusEventName {
|
||||
CreateSecret = "secret:create",
|
||||
UpdateSecret = "secret:update",
|
||||
DeleteSecret = "secret:delete"
|
||||
}
|
||||
|
||||
type PublisableEventTypes =
|
||||
| EventType.CREATE_SECRET
|
||||
| EventType.CREATE_SECRETS
|
||||
| EventType.DELETE_SECRET
|
||||
| EventType.DELETE_SECRETS
|
||||
| EventType.UPDATE_SECRETS
|
||||
| EventType.UPDATE_SECRET;
|
||||
|
||||
export function toBusEventName(input: EventType) {
|
||||
switch (input) {
|
||||
case EventType.CREATE_SECRET:
|
||||
case EventType.CREATE_SECRETS:
|
||||
return BusEventName.CreateSecret;
|
||||
case EventType.UPDATE_SECRET:
|
||||
case EventType.UPDATE_SECRETS:
|
||||
return BusEventName.UpdateSecret;
|
||||
case EventType.DELETE_SECRET:
|
||||
case EventType.DELETE_SECRETS:
|
||||
return BusEventName.DeleteSecret;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const isBulkEvent = (event: Event): event is Extract<Event, { metadata: { secrets: Array<unknown> } }> => {
|
||||
return event.type.endsWith("-secrets"); // Feels so wrong
|
||||
};
|
||||
|
||||
export const toPublishableEvent = (event: Event) => {
|
||||
const name = toBusEventName(event.type);
|
||||
|
||||
if (!name) return null;
|
||||
|
||||
const e = event as Extract<Event, { type: PublisableEventTypes }>;
|
||||
|
||||
if (isBulkEvent(e)) {
|
||||
return {
|
||||
name,
|
||||
isBulk: true,
|
||||
data: {
|
||||
eventType: e.type,
|
||||
payload: e.metadata.secrets.map((s) => ({
|
||||
environment: e.metadata.environment,
|
||||
secretPath: e.metadata.secretPath,
|
||||
...s
|
||||
}))
|
||||
}
|
||||
} as const;
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
isBulk: false,
|
||||
data: {
|
||||
eventType: e.type,
|
||||
payload: {
|
||||
...e.metadata,
|
||||
environment: e.metadata.environment
|
||||
}
|
||||
}
|
||||
} as const;
|
||||
};
|
||||
|
||||
export const EventName = z.nativeEnum(BusEventName);
|
||||
|
||||
const EventSecretPayload = z.object({
|
||||
secretPath: z.string().optional(),
|
||||
secretId: z.string(),
|
||||
secretKey: z.string(),
|
||||
environment: z.string()
|
||||
});
|
||||
|
||||
export type EventSecret = z.infer<typeof EventSecretPayload>;
|
||||
|
||||
export const EventSchema = z.object({
|
||||
datacontenttype: z.literal("application/json").optional().default("application/json"),
|
||||
type: z.nativeEnum(ProjectType),
|
||||
source: z.string(),
|
||||
time: z
|
||||
.string()
|
||||
.optional()
|
||||
.default(() => new Date().toISOString()),
|
||||
data: z.discriminatedUnion("eventType", [
|
||||
z.object({
|
||||
specversion: z.number().optional().default(1),
|
||||
eventType: z.enum([EventType.CREATE_SECRET, EventType.UPDATE_SECRET, EventType.DELETE_SECRET]),
|
||||
payload: EventSecretPayload
|
||||
}),
|
||||
z.object({
|
||||
specversion: z.number().optional().default(1),
|
||||
eventType: z.enum([EventType.CREATE_SECRETS, EventType.UPDATE_SECRETS, EventType.DELETE_SECRETS]),
|
||||
payload: EventSecretPayload.array()
|
||||
})
|
||||
// Add more event types as needed
|
||||
])
|
||||
});
|
||||
|
||||
export type EventData = z.infer<typeof EventSchema>;
|
||||
|
||||
export const EventRegisterSchema = z.object({
|
||||
event: EventName,
|
||||
conditions: z
|
||||
.object({
|
||||
secretPath: z.string().optional().default("/"),
|
||||
environmentSlug: z.string()
|
||||
})
|
||||
.optional()
|
||||
});
|
||||
|
||||
export type RegisteredEvent = z.infer<typeof EventRegisterSchema>;
|
||||
Reference in New Issue
Block a user