Merge remote-tracking branch 'origin' into ssh-certs

backend/src/ee/services/audit-log/audit-log-queue.ts

@@ -1,6 +1,7 @@
 import { RawAxiosRequestHeaders } from "axios";
 
 import { SecretKeyEncoding } from "@app/db/schemas";
+import { getConfig } from "@app/lib/config/env";
 import { request } from "@app/lib/config/request";
 import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption";
 import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
@@ -20,27 +21,130 @@ type TAuditLogQueueServiceFactoryDep = {
   licenseService: Pick<TLicenseServiceFactory, "getPlan">;
 };
 
-export type TAuditLogQueueServiceFactory = ReturnType<typeof auditLogQueueServiceFactory>;
+export type TAuditLogQueueServiceFactory = Awaited<ReturnType<typeof auditLogQueueServiceFactory>>;
 
 // keep this timeout 5s it must be fast because else the queue will take time to finish
 // audit log is a crowded queue thus needs to be fast
 export const AUDIT_LOG_STREAM_TIMEOUT = 5 * 1000;
-export const auditLogQueueServiceFactory = ({
+
+export const auditLogQueueServiceFactory = async ({
   auditLogDAL,
   queueService,
   projectDAL,
   licenseService,
   auditLogStreamDAL
 }: TAuditLogQueueServiceFactoryDep) => {
+  const appCfg = getConfig();
+
   const pushToLog = async (data: TCreateAuditLogDTO) => {
-    await queueService.queue(QueueName.AuditLog, QueueJobs.AuditLog, data, {
-      removeOnFail: {
-        count: 3
-      },
-      removeOnComplete: true
-    });
+    if (appCfg.USE_PG_QUEUE && appCfg.SHOULD_INIT_PG_QUEUE) {
+      await queueService.queuePg<QueueName.AuditLog>(QueueJobs.AuditLog, data, {
+        retryLimit: 10,
+        retryBackoff: true
+      });
+    } else {
+      await queueService.queue<QueueName.AuditLog>(QueueName.AuditLog, QueueJobs.AuditLog, data, {
+        removeOnFail: {
+          count: 3
+        },
+        removeOnComplete: true
+      });
+    }
   };
 
+  if (appCfg.SHOULD_INIT_PG_QUEUE) {
+    await queueService.startPg<QueueName.AuditLog>(
+      QueueJobs.AuditLog,
+      async ([job]) => {
+        const { actor, event, ipAddress, projectId, userAgent, userAgentType } = job.data;
+        let { orgId } = job.data;
+        const MS_IN_DAY = 24 * 60 * 60 * 1000;
+        let project;
+
+        if (!orgId) {
+          // it will never be undefined for both org and project id
+          // TODO(akhilmhdh): use caching here in dal to avoid db calls
+          project = await projectDAL.findById(projectId as string);
+          orgId = project.orgId;
+        }
+
+        const plan = await licenseService.getPlan(orgId);
+        if (plan.auditLogsRetentionDays === 0) {
+          // skip inserting if audit log retention is 0 meaning its not supported
+          return;
+        }
+
+        // For project actions, set TTL to project-level audit log retention config
+        // This condition ensures that the plan's audit log retention days cannot be bypassed
+        const ttlInDays =
+          project?.auditLogsRetentionDays && project.auditLogsRetentionDays < plan.auditLogsRetentionDays
+            ? project.auditLogsRetentionDays
+            : plan.auditLogsRetentionDays;
+
+        const ttl = ttlInDays * MS_IN_DAY;
+
+        const auditLog = await auditLogDAL.create({
+          actor: actor.type,
+          actorMetadata: actor.metadata,
+          userAgent,
+          projectId,
+          projectName: project?.name,
+          ipAddress,
+          orgId,
+          eventType: event.type,
+          expiresAt: new Date(Date.now() + ttl),
+          eventMetadata: event.metadata,
+          userAgentType
+        });
+
+        const logStreams = orgId ? await auditLogStreamDAL.find({ orgId }) : [];
+        await Promise.allSettled(
+          logStreams.map(
+            async ({
+              url,
+              encryptedHeadersTag,
+              encryptedHeadersIV,
+              encryptedHeadersKeyEncoding,
+              encryptedHeadersCiphertext
+            }) => {
+              const streamHeaders =
+                encryptedHeadersIV && encryptedHeadersCiphertext && encryptedHeadersTag
+                  ? (JSON.parse(
+                      infisicalSymmetricDecrypt({
+                        keyEncoding: encryptedHeadersKeyEncoding as SecretKeyEncoding,
+                        iv: encryptedHeadersIV,
+                        tag: encryptedHeadersTag,
+                        ciphertext: encryptedHeadersCiphertext
+                      })
+                    ) as LogStreamHeaders[])
+                  : [];
+
+              const headers: RawAxiosRequestHeaders = { "Content-Type": "application/json" };
+
+              if (streamHeaders.length)
+                streamHeaders.forEach(({ key, value }) => {
+                  headers[key] = value;
+                });
+
+              return request.post(url, auditLog, {
+                headers,
+                // request timeout
+                timeout: AUDIT_LOG_STREAM_TIMEOUT,
+                // connection timeout
+                signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
+              });
+            }
+          )
+        );
+      },
+      {
+        batchSize: 1,
+        workerCount: 30,
+        pollingIntervalSeconds: 0.5
+      }
+    );
+  }
+
   queueService.start(QueueName.AuditLog, async (job) => {
     const { actor, event, ipAddress, projectId, userAgent, userAgentType } = job.data;
     let { orgId } = job.data;

backend/src/ee/services/audit-log/audit-log-types.ts

@@ -63,6 +63,7 @@ export enum EventType {
   DELETE_SECRETS = "delete-secrets",
   GET_WORKSPACE_KEY = "get-workspace-key",
   AUTHORIZE_INTEGRATION = "authorize-integration",
+  UPDATE_INTEGRATION_AUTH = "update-integration-auth",
   UNAUTHORIZE_INTEGRATION = "unauthorize-integration",
   CREATE_INTEGRATION = "create-integration",
   DELETE_INTEGRATION = "delete-integration",
@@ -371,6 +372,13 @@ interface AuthorizeIntegrationEvent {
   };
 }
 
+interface UpdateIntegrationAuthEvent {
+  type: EventType.UPDATE_INTEGRATION_AUTH;
+  metadata: {
+    integration: string;
+  };
+}
+
 interface UnauthorizeIntegrationEvent {
   type: EventType.UNAUTHORIZE_INTEGRATION;
   metadata: {
@@ -1805,6 +1813,7 @@ export type Event =
   | DeleteSecretBatchEvent
   | GetWorkspaceKeyEvent
   | AuthorizeIntegrationEvent
+  | UpdateIntegrationAuthEvent
   | UnauthorizeIntegrationEvent
   | CreateIntegrationEvent
   | DeleteIntegrationEvent