From 4d4953e95a063043bcb00e931a50a0689bf186e0 Mon Sep 17 00:00:00 2001 From: x032205 Date: Tue, 24 Jun 2025 12:35:49 -0400 Subject: [PATCH 1/5] improve audit log streams: add backend logs + DD source --- .../audit-log-stream/audit-log-stream-fns.ts | 10 +++ .../audit-log-stream-service.ts | 6 +- .../ee/services/audit-log/audit-log-queue.ts | 64 ++++++++++++++----- 3 files changed, 63 insertions(+), 17 deletions(-) create mode 100644 backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts diff --git a/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts b/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts new file mode 100644 index 000000000..cce8f2a58 --- /dev/null +++ b/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts @@ -0,0 +1,10 @@ +export function providerSpecificPayload(url: string) { + const payload: Record = {}; + + // If URL is related to DataDog, add a "ddsource: infisical" entry + if (url.includes("datadoghq")) { + payload.ddsource = "infisical"; + } + + return payload; +} diff --git a/backend/src/ee/services/audit-log-stream/audit-log-stream-service.ts b/backend/src/ee/services/audit-log-stream/audit-log-stream-service.ts index 72207aca1..65e49bdea 100644 --- a/backend/src/ee/services/audit-log-stream/audit-log-stream-service.ts +++ b/backend/src/ee/services/audit-log-stream/audit-log-stream-service.ts @@ -13,6 +13,7 @@ import { TLicenseServiceFactory } from "../license/license-service"; import { OrgPermissionActions, OrgPermissionSubjects } from "../permission/org-permission"; import { TPermissionServiceFactory } from "../permission/permission-service-types"; import { TAuditLogStreamDALFactory } from "./audit-log-stream-dal"; +import { providerSpecificPayload } from "./audit-log-stream-fns"; import { LogStreamHeaders, TAuditLogStreamServiceFactory } from "./audit-log-stream-types"; type TAuditLogStreamServiceFactoryDep = { @@ -69,10 +70,11 @@ export const auditLogStreamServiceFactory = ({ headers.forEach(({ key, value }) => { streamHeaders[key] = value; }); + await request .post( url, - { ping: "ok" }, + { ...providerSpecificPayload(url), ping: "ok" }, { headers: streamHeaders, // request timeout @@ -137,7 +139,7 @@ export const auditLogStreamServiceFactory = ({ await request .post( url || logStream.url, - { ping: "ok" }, + { ...providerSpecificPayload(url || logStream.url), ping: "ok" }, { headers: streamHeaders, // request timeout diff --git a/backend/src/ee/services/audit-log/audit-log-queue.ts b/backend/src/ee/services/audit-log/audit-log-queue.ts index 0f774e911..03af17285 100644 --- a/backend/src/ee/services/audit-log/audit-log-queue.ts +++ b/backend/src/ee/services/audit-log/audit-log-queue.ts @@ -1,13 +1,15 @@ -import { RawAxiosRequestHeaders } from "axios"; +import { AxiosError, RawAxiosRequestHeaders } from "axios"; import { SecretKeyEncoding } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; import { request } from "@app/lib/config/request"; import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; +import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TAuditLogStreamDALFactory } from "../audit-log-stream/audit-log-stream-dal"; +import { providerSpecificPayload } from "../audit-log-stream/audit-log-stream-fns"; import { LogStreamHeaders } from "../audit-log-stream/audit-log-stream-types"; import { TLicenseServiceFactory } from "../license/license-service"; import { TAuditLogDALFactory } from "./audit-log-dal"; @@ -128,13 +130,29 @@ export const auditLogQueueServiceFactory = async ({ headers[key] = value; }); - return request.post(url, auditLog, { - headers, - // request timeout - timeout: AUDIT_LOG_STREAM_TIMEOUT, - // connection timeout - signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT) - }); + try { + const response = await request.post( + url, + { ...providerSpecificPayload(url), ...auditLog }, + { + headers, + // request timeout + timeout: AUDIT_LOG_STREAM_TIMEOUT, + // connection timeout + signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT) + } + ); + logger.info( + `Successfully streamed audit log to ${url} for org ${orgId}. Response body: ${JSON.stringify(response.data)}` + ); + return response; + } catch (error) { + logger.error( + `Failed to stream audit log to ${url} for org ${orgId}. Error:`, + (error as AxiosError).message + ); + return error; + } } ) ); @@ -218,13 +236,29 @@ export const auditLogQueueServiceFactory = async ({ headers[key] = value; }); - return request.post(url, auditLog, { - headers, - // request timeout - timeout: AUDIT_LOG_STREAM_TIMEOUT, - // connection timeout - signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT) - }); + try { + const response = await request.post( + url, + { ...providerSpecificPayload(url), ...auditLog }, + { + headers, + // request timeout + timeout: AUDIT_LOG_STREAM_TIMEOUT, + // connection timeout + signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT) + } + ); + logger.info( + `Successfully streamed audit log to ${url} for org ${orgId}. Response body: ${JSON.stringify(response.data)}` + ); + return response; + } catch (error) { + logger.error( + `Failed to stream audit log to ${url} for org ${orgId}. Error:`, + (error as AxiosError).message + ); + return error; + } } ) ); From c17df7e951ead47f5ea9d640a9b57b69b60530ed Mon Sep 17 00:00:00 2001 From: x032205 Date: Tue, 24 Jun 2025 12:44:16 -0400 Subject: [PATCH 2/5] Improve URL detection --- .../audit-log-stream/audit-log-stream-fns.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts b/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts index cce8f2a58..3b0eb412b 100644 --- a/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts +++ b/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts @@ -1,8 +1,20 @@ +// Specific DataDog log intake hostnames +const dataDogHostnames = [ + "http-intake.logs.datadoghq.com", + "http-intake.logs.us3.datadoghq.com", + "http-intake.logs.us5.datadoghq.com", + "http-intake.logs.datadoghq.eu", + "http-intake.logs.ap1.datadoghq.com", + "http-intake.logs.ddog-gov.com" +]; + export function providerSpecificPayload(url: string) { const payload: Record = {}; - // If URL is related to DataDog, add a "ddsource: infisical" entry - if (url.includes("datadoghq")) { + // If URL is for DataDog, add a "ddsource: infisical" entry + if ( + dataDogHostnames.some((hostname) => url.startsWith(`https://${hostname}`) || url.startsWith(`http://${hostname}`)) + ) { payload.ddsource = "infisical"; } From af245b1f168adc06e124084669064d1a69fffa45 Mon Sep 17 00:00:00 2001 From: x032205 Date: Tue, 24 Jun 2025 14:22:26 -0400 Subject: [PATCH 3/5] Add "service: audit-logs" entry for DataDog --- backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts b/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts index 3b0eb412b..ba0434982 100644 --- a/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts +++ b/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts @@ -16,6 +16,7 @@ export function providerSpecificPayload(url: string) { dataDogHostnames.some((hostname) => url.startsWith(`https://${hostname}`) || url.startsWith(`http://${hostname}`)) ) { payload.ddsource = "infisical"; + payload.service = "audit-logs"; } return payload; From 6566393e219792cc47276af00bb74db9180f6b33 Mon Sep 17 00:00:00 2001 From: x032205 Date: Tue, 24 Jun 2025 14:39:46 -0400 Subject: [PATCH 4/5] Review fixes --- .../audit-log-stream/audit-log-stream-fns.ts | 30 +++++++++---------- .../ee/services/audit-log/audit-log-queue.ts | 10 +++---- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts b/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts index ba0434982..dc93f238e 100644 --- a/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts +++ b/backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts @@ -1,22 +1,20 @@ -// Specific DataDog log intake hostnames -const dataDogHostnames = [ - "http-intake.logs.datadoghq.com", - "http-intake.logs.us3.datadoghq.com", - "http-intake.logs.us5.datadoghq.com", - "http-intake.logs.datadoghq.eu", - "http-intake.logs.ap1.datadoghq.com", - "http-intake.logs.ddog-gov.com" -]; - export function providerSpecificPayload(url: string) { + const { hostname } = new URL(url); + const payload: Record = {}; - // If URL is for DataDog, add a "ddsource: infisical" entry - if ( - dataDogHostnames.some((hostname) => url.startsWith(`https://${hostname}`) || url.startsWith(`http://${hostname}`)) - ) { - payload.ddsource = "infisical"; - payload.service = "audit-logs"; + switch (hostname) { + case "http-intake.logs.datadoghq.com": + case "http-intake.logs.us3.datadoghq.com": + case "http-intake.logs.us5.datadoghq.com": + case "http-intake.logs.datadoghq.eu": + case "http-intake.logs.ap1.datadoghq.com": + case "http-intake.logs.ddog-gov.com": + payload.ddsource = "infisical"; + payload.service = "audit-logs"; + break; + default: + break; } return payload; diff --git a/backend/src/ee/services/audit-log/audit-log-queue.ts b/backend/src/ee/services/audit-log/audit-log-queue.ts index 03af17285..c240bc6ec 100644 --- a/backend/src/ee/services/audit-log/audit-log-queue.ts +++ b/backend/src/ee/services/audit-log/audit-log-queue.ts @@ -143,13 +143,12 @@ export const auditLogQueueServiceFactory = async ({ } ); logger.info( - `Successfully streamed audit log to ${url} for org ${orgId}. Response body: ${JSON.stringify(response.data)}` + `Successfully streamed audit log [url=${url}] for org [orgId=${orgId}] [response=${JSON.stringify(response.data)}]` ); return response; } catch (error) { logger.error( - `Failed to stream audit log to ${url} for org ${orgId}. Error:`, - (error as AxiosError).message + `Failed to stream audit log [url=${url}] for org [orgId=${orgId}] [error=${(error as AxiosError).message}]` ); return error; } @@ -249,13 +248,12 @@ export const auditLogQueueServiceFactory = async ({ } ); logger.info( - `Successfully streamed audit log to ${url} for org ${orgId}. Response body: ${JSON.stringify(response.data)}` + `Successfully streamed audit log [url=${url}] for org [orgId=${orgId}] [response=${JSON.stringify(response.data)}]` ); return response; } catch (error) { logger.error( - `Failed to stream audit log to ${url} for org ${orgId}. Error:`, - (error as AxiosError).message + `Failed to stream audit log [url=${url}] for org [orgId=${orgId}] [error=${(error as AxiosError).message}]` ); return error; } From 7f9c9be2c8be04c9c1b23002c8665522991ef350 Mon Sep 17 00:00:00 2001 From: x032205 Date: Tue, 24 Jun 2025 22:00:45 -0400 Subject: [PATCH 5/5] review fix --- backend/src/ee/services/audit-log/audit-log-queue.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/ee/services/audit-log/audit-log-queue.ts b/backend/src/ee/services/audit-log/audit-log-queue.ts index c240bc6ec..e812cb7ea 100644 --- a/backend/src/ee/services/audit-log/audit-log-queue.ts +++ b/backend/src/ee/services/audit-log/audit-log-queue.ts @@ -131,6 +131,7 @@ export const auditLogQueueServiceFactory = async ({ }); try { + logger.info(`Streaming audit log [url=${url}] for org [orgId=${orgId}]`); const response = await request.post( url, { ...providerSpecificPayload(url), ...auditLog }, @@ -236,6 +237,7 @@ export const auditLogQueueServiceFactory = async ({ }); try { + logger.info(`Streaming audit log [url=${url}] for org [orgId=${orgId}]`); const response = await request.post( url, { ...providerSpecificPayload(url), ...auditLog },