Merge pull request #3848 from Infisical/improve-audit-log-streams

improve audit log streams: add backend logs + DD source

backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts

@@ -0,0 +1,21 @@
+export function providerSpecificPayload(url: string) {
+  const { hostname } = new URL(url);
+
+  const payload: Record<string, string> = {};
+
+  switch (hostname) {
+    case "http-intake.logs.datadoghq.com":
+    case "http-intake.logs.us3.datadoghq.com":
+    case "http-intake.logs.us5.datadoghq.com":
+    case "http-intake.logs.datadoghq.eu":
+    case "http-intake.logs.ap1.datadoghq.com":
+    case "http-intake.logs.ddog-gov.com":
+      payload.ddsource = "infisical";
+      payload.service = "audit-logs";
+      break;
+    default:
+      break;
+  }
+
+  return payload;
+}

backend/src/ee/services/audit-log-stream/audit-log-stream-service.ts

@@ -13,6 +13,7 @@ import { TLicenseServiceFactory } from "../license/license-service";
 import { OrgPermissionActions, OrgPermissionSubjects } from "../permission/org-permission";
 import { TPermissionServiceFactory } from "../permission/permission-service-types";
 import { TAuditLogStreamDALFactory } from "./audit-log-stream-dal";
+import { providerSpecificPayload } from "./audit-log-stream-fns";
 import { LogStreamHeaders, TAuditLogStreamServiceFactory } from "./audit-log-stream-types";
 
 type TAuditLogStreamServiceFactoryDep = {
@@ -69,10 +70,11 @@ export const auditLogStreamServiceFactory = ({
       headers.forEach(({ key, value }) => {
         streamHeaders[key] = value;
       });
+
     await request
       .post(
         url,
-        { ping: "ok" },
+        { ...providerSpecificPayload(url), ping: "ok" },
         {
           headers: streamHeaders,
           // request timeout
@@ -137,7 +139,7 @@ export const auditLogStreamServiceFactory = ({
     await request
       .post(
         url || logStream.url,
-        { ping: "ok" },
+        { ...providerSpecificPayload(url || logStream.url), ping: "ok" },
         {
           headers: streamHeaders,
           // request timeout

backend/src/ee/services/audit-log/audit-log-queue.ts

@@ -1,13 +1,15 @@
-import { RawAxiosRequestHeaders } from "axios";
+import { AxiosError, RawAxiosRequestHeaders } from "axios";
 
 import { SecretKeyEncoding } from "@app/db/schemas";
 import { getConfig } from "@app/lib/config/env";
 import { request } from "@app/lib/config/request";
 import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption";
+import { logger } from "@app/lib/logger";
 import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
 
 import { TAuditLogStreamDALFactory } from "../audit-log-stream/audit-log-stream-dal";
+import { providerSpecificPayload } from "../audit-log-stream/audit-log-stream-fns";
 import { LogStreamHeaders } from "../audit-log-stream/audit-log-stream-types";
 import { TLicenseServiceFactory } from "../license/license-service";
 import { TAuditLogDALFactory } from "./audit-log-dal";
@@ -128,13 +130,29 @@ export const auditLogQueueServiceFactory = async ({
                   headers[key] = value;
                 });
 
-              return request.post(url, auditLog, {
+              try {
-                headers,
+                logger.info(`Streaming audit log [url=${url}] for org [orgId=${orgId}]`);
-                // request timeout
+                const response = await request.post(
-                timeout: AUDIT_LOG_STREAM_TIMEOUT,
+                  url,
-                // connection timeout
+                  { ...providerSpecificPayload(url), ...auditLog },
-                signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
+                  {
-              });
+                    headers,
+                    // request timeout
+                    timeout: AUDIT_LOG_STREAM_TIMEOUT,
+                    // connection timeout
+                    signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
+                  }
+                );
+                logger.info(
+                  `Successfully streamed audit log [url=${url}] for org [orgId=${orgId}] [response=${JSON.stringify(response.data)}]`
+                );
+                return response;
+              } catch (error) {
+                logger.error(
+                  `Failed to stream audit log [url=${url}] for org [orgId=${orgId}] [error=${(error as AxiosError).message}]`
+                );
+                return error;
+              }
             }
           )
         );
@@ -218,13 +236,29 @@ export const auditLogQueueServiceFactory = async ({
               headers[key] = value;
             });
 
-          return request.post(url, auditLog, {
+          try {
-            headers,
+            logger.info(`Streaming audit log [url=${url}] for org [orgId=${orgId}]`);
-            // request timeout
+            const response = await request.post(
-            timeout: AUDIT_LOG_STREAM_TIMEOUT,
+              url,
-            // connection timeout
+              { ...providerSpecificPayload(url), ...auditLog },
-            signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
+              {
-          });
+                headers,
+                // request timeout
+                timeout: AUDIT_LOG_STREAM_TIMEOUT,
+                // connection timeout
+                signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
+              }
+            );
+            logger.info(
+              `Successfully streamed audit log [url=${url}] for org [orgId=${orgId}] [response=${JSON.stringify(response.data)}]`
+            );
+            return response;
+          } catch (error) {
+            logger.error(
+              `Failed to stream audit log [url=${url}] for org [orgId=${orgId}] [error=${(error as AxiosError).message}]`
+            );
+            return error;
+          }
         }
       )
     );