Merge pull request #2387 from akhilmhdh/docs/fluent-bit-log-stream

feat: added doc for audit log stream via fluentbit

backend/src/ee/services/audit-log-stream/audit-log-stream-service.ts

@@ -2,10 +2,11 @@ import { ForbiddenError } from "@casl/ability";
 import { RawAxiosRequestHeaders } from "axios";
 
 import { SecretKeyEncoding } from "@app/db/schemas";
+import { getConfig } from "@app/lib/config/env";
 import { request } from "@app/lib/config/request";
 import { infisicalSymmetricDecrypt, infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
 import { BadRequestError } from "@app/lib/errors";
-import { validateLocalIps } from "@app/lib/validator";
+import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
 
 import { AUDIT_LOG_STREAM_TIMEOUT } from "../audit-log/audit-log-queue";
 import { TLicenseServiceFactory } from "../license/license-service";
@@ -44,6 +45,7 @@ export const auditLogStreamServiceFactory = ({
   }: TCreateAuditLogStreamDTO) => {
     if (!actorOrgId) throw new BadRequestError({ message: "Missing org id from token" });
 
+    const appCfg = getConfig();
     const plan = await licenseService.getPlan(actorOrgId);
     if (!plan.auditLogStreams)
       throw new BadRequestError({
@@ -59,7 +61,9 @@ export const auditLogStreamServiceFactory = ({
     );
     ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Settings);
 
-    validateLocalIps(url);
+    if (appCfg.isCloud) {
+      blockLocalAndPrivateIpAddresses(url);
+    }
 
     const totalStreams = await auditLogStreamDAL.find({ orgId: actorOrgId });
     if (totalStreams.length >= plan.auditLogStreamLimit) {
@@ -131,7 +135,8 @@ export const auditLogStreamServiceFactory = ({
     const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
     ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Settings);
 
-    if (url) validateLocalIps(url);
+    const appCfg = getConfig();
+    if (url && appCfg.isCloud) blockLocalAndPrivateIpAddresses(url);
 
     // testing connection first
     const streamHeaders: RawAxiosRequestHeaders = { "Content-Type": "application/json" };

backend/src/lib/validator/index.ts

@@ -1,2 +1,2 @@
 export { isDisposableEmail } from "./validate-email";
-export { validateLocalIps } from "./validate-url";
+export { blockLocalAndPrivateIpAddresses } from "./validate-url";

backend/src/lib/validator/validate-url.ts

@@ -1,7 +1,7 @@
 import { getConfig } from "../config/env";
 import { BadRequestError } from "../errors";
 
-export const validateLocalIps = (url: string) => {
+export const blockLocalAndPrivateIpAddresses = (url: string) => {
   const validUrl = new URL(url);
   const appCfg = getConfig();
   // on cloud local ips are not allowed

docs/documentation/platform/audit-log-streams/audit-log-streams-with-fluentbit.mdx

@@ -0,0 +1,61 @@
+---
+title: "Stream to Non-HTTP providers"
+description: "How to stream Infisical Audit Logs to Non-HTTP log providers"
+---
+
+<Info>
+    Audit log streams is a paid feature.
+   
+    If you're using Infisical Cloud, then it is available under the **Enterprise Tier**. If you're self-hosting Infisical,
+    then you should contact team@infisical.com to purchase an enterprise license to use it.
+</Info>
+
+This guide will demonstrate how you can send Infisical Audit log streams to storage solutions that do not support direct HTTP-based ingestion, such as AWS S3.
+To achieve this, you will learn how you can use a log collector like Fluent Bit to capture and forward logs from Infisical to non-HTTP storage options. 
+In this pattern, Fluent Bit acts as an intermediary, accepting HTTP log streams from Infisical and transforming them into a format that can be sent to your desired storage provider.
+
+## Overview
+
+Log collectors are tools used to collect, analyze, transform, and send logs to storage. 
+For the purposes of this guide, we will use [Fluent Bit](https://fluentbit.io) as our log collector and send logs from Infisical to AWS S3. 
+However, this is just a example and you can use any log collector of your choice. 
+
+## Deploy Fluent Bit
+
+You can deploy Fluent Bit in one of two ways:
+1. As a sidecar to your self-hosted Infisical instance
+2. As a standalone service in any deployment/compute service (e.g., AWS EC2, ECS, or GCP Compute Engine)
+
+To view all deployment methods, visit the [Fluent Bit Getting Started guide](https://docs.fluentbit.io/manual/installation/getting-started-with-fluent-bit).
+
+## Configure Fluent Bit
+
+To set up Fluent Bit, you'll need to provide a configuration file that establishes an HTTP listener and configures an output to send JSON data to your chosen storage solution.
+
+The following Fluent Bit configuration sets up an HTTP listener on port `8888` and sends logs to AWS S3:
+
+```ini
+[SERVICE]
+    Flush        1
+    Log_Level    info
+    Daemon       off
+
+[INPUT]
+    Name   http
+    Listen 0.0.0.0
+    Port   8888
+
+[OUTPUT]
+    Name                         s3
+    Match                        *
+    bucket                       my-bucket
+    region                       us-west-2
+    total_file_size              50M
+    use_put_object               Off
+    compression                  gzip
+    s3_key_format                /$TAG/%Y/%m/%d/%H_%M_%S.gz
+```
+### Connecting Infisical Audit Log Stream
+
+Once Fluent Bit is set up and configured, you can point the Infisical [audit log stream](/documentation/platform/audit-log-streams/audit-log-streams) to Fluent Bit's HTTP listener, which will then forward the logs to your chosen provider. 
+Using this pattern, you are able to send Infisical Audit logs to various providers that do not support HTTP based log ingestion by default. 

docs/documentation/platform/audit-logs.mdx

@@ -1,5 +1,5 @@
 ---
-title: "Audit Logs"
+title: "Overview"
 description: "Track evert event action performed within Infisical projects."
 ---
 

docs/mint.json

@@ -130,11 +130,18 @@
             "documentation/platform/access-controls/temporary-access",
             "documentation/platform/access-controls/access-requests",
             "documentation/platform/pr-workflows",
-            "documentation/platform/audit-logs",
             "documentation/platform/audit-log-streams",
             "documentation/platform/groups"
           ]
         },
+        {
+          "group": "Audit Logs",
+          "pages": [
+            "documentation/platform/audit-logs",
+            "documentation/platform/audit-log-streams/audit-log-streams",
+            "documentation/platform/audit-log-streams/audit-log-streams-with-fluentbit"
+          ]
+        },
         {
           "group": "Secret Rotation",
           "pages": [