import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq"; import Redis from "ioredis"; import { SecretKeyEncoding } from "@app/db/schemas"; import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types"; import { TScanFullRepoEventPayload, TScanPushEventPayload } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types"; import { TSyncSecretsDTO } from "@app/services/secret/secret-types"; export enum QueueName { SecretRotation = "secret-rotation", SecretReminder = "secret-reminder", AuditLog = "audit-log", // TODO(akhilmhdh): This will get removed later. For now this is kept to stop the repeatable queue AuditLogPrune = "audit-log-prune", DailyResourceCleanUp = "daily-resource-cleanup", TelemetryInstanceStats = "telemtry-self-hosted-stats", IntegrationSync = "sync-integrations", SecretWebhook = "secret-webhook", SecretFullRepoScan = "secret-full-repo-scan", SecretPushEventScan = "secret-push-event-scan", UpgradeProjectToGhost = "upgrade-project-to-ghost", DynamicSecretRevocation = "dynamic-secret-revocation", SecretReplication = "secret-replication", SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication } export enum QueueJobs { SecretReminder = "secret-reminder-job", SecretRotation = "secret-rotation-job", AuditLog = "audit-log-job", // TODO(akhilmhdh): This will get removed later. For now this is kept to stop the repeatable queue AuditLogPrune = "audit-log-prune-job", DailyResourceCleanUp = "daily-resource-cleanup-job", SecWebhook = "secret-webhook-trigger", TelemetryInstanceStats = "telemetry-self-hosted-stats", IntegrationSync = "secret-integration-pull", SecretScan = "secret-scan", UpgradeProjectToGhost = "upgrade-project-to-ghost-job", DynamicSecretRevocation = "dynamic-secret-revocation", DynamicSecretPruning = "dynamic-secret-pruning", SecretReplication = "secret-replication", SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication } export type TQueueJobTypes = { [QueueName.SecretReminder]: { payload: { projectId: string; secretId: string; repeatDays: number; note: string | undefined | null; }; name: QueueJobs.SecretReminder; }; [QueueName.SecretRotation]: { payload: { rotationId: string }; name: QueueJobs.SecretRotation; }; [QueueName.AuditLog]: { name: QueueJobs.AuditLog; payload: TCreateAuditLogDTO; }; [QueueName.DailyResourceCleanUp]: { name: QueueJobs.DailyResourceCleanUp; payload: undefined; }; [QueueName.AuditLogPrune]: { name: QueueJobs.AuditLogPrune; payload: undefined; }; [QueueName.SecretWebhook]: { name: QueueJobs.SecWebhook; payload: { projectId: string; environment: string; secretPath: string; depth?: number }; }; [QueueName.IntegrationSync]: { name: QueueJobs.IntegrationSync; payload: { projectId: string; environment: string; secretPath: string; depth?: number; deDupeQueue?: Record; }; }; [QueueName.SecretFullRepoScan]: { name: QueueJobs.SecretScan; payload: TScanFullRepoEventPayload; }; [QueueName.SecretPushEventScan]: { name: QueueJobs.SecretScan; payload: TScanPushEventPayload }; [QueueName.UpgradeProjectToGhost]: { name: QueueJobs.UpgradeProjectToGhost; payload: { projectId: string; startedByUserId: string; encryptedPrivateKey: { encryptedKey: string; encryptedKeyIv: string; encryptedKeyTag: string; keyEncoding: SecretKeyEncoding; }; }; }; [QueueName.TelemetryInstanceStats]: { name: QueueJobs.TelemetryInstanceStats; payload: undefined; }; [QueueName.DynamicSecretRevocation]: | { name: QueueJobs.DynamicSecretRevocation; payload: { leaseId: string; }; } | { name: QueueJobs.DynamicSecretPruning; payload: { dynamicSecretCfgId: string; }; }; [QueueName.SecretReplication]: { name: QueueJobs.SecretReplication; payload: TSyncSecretsDTO; }; [QueueName.SecretSync]: { name: QueueJobs.SecretSync; payload: TSyncSecretsDTO; }; }; export type TQueueServiceFactory = ReturnType; export const queueServiceFactory = (redisUrl: string) => { const connection = new Redis(redisUrl, { maxRetriesPerRequest: null }); const queueContainer = {} as Record< QueueName, Queue >; const workerContainer = {} as Record< QueueName, Worker >; const start = ( name: T, jobFn: (job: Job, token?: string) => Promise, queueSettings: Omit = {} ) => { if (queueContainer[name]) { throw new Error(`${name} queue is already initialized`); } queueContainer[name] = new Queue(name as string, { ...queueSettings, connection }); workerContainer[name] = new Worker(name, jobFn, { ...queueSettings, connection }); }; const listen = < T extends QueueName, U extends keyof WorkerListener >( name: T, event: U, listener: WorkerListener[U] ) => { const worker = workerContainer[name]; worker.on(event, listener); }; const queue = async ( name: T, job: TQueueJobTypes[T]["name"], data: TQueueJobTypes[T]["payload"], opts?: JobsOptions & { jobId?: string } ) => { const q = queueContainer[name]; await q.add(job, data, opts); }; const stopRepeatableJob = async ( name: T, job: TQueueJobTypes[T]["name"], repeatOpt: RepeatOptions, jobId?: string ) => { const q = queueContainer[name]; if (q) { return q.removeRepeatable(job, repeatOpt, jobId); } }; const stopRepeatableJobByJobId = async (name: T, jobId: string) => { const q = queueContainer[name]; const job = await q.getJob(jobId); if (!job) return true; if (!job.repeatJobKey) return true; return q.removeRepeatableByKey(job.repeatJobKey); }; const stopJobById = async (name: T, jobId: string) => { const q = queueContainer[name]; const job = await q.getJob(jobId); return job?.remove().catch(() => undefined); }; const clearQueue = async (name: QueueName) => { const q = queueContainer[name]; await q.drain(); }; const shutdown = async () => { await Promise.all(Object.values(workerContainer).map((worker) => worker.close())); }; return { start, listen, queue, shutdown, stopRepeatableJob, stopRepeatableJobByJobId, clearQueue, stopJobById }; };