Merge pull request #4500 from Infisical/ENG-3674

feat(audit-log-stream): cribl support

backend/src/ee/routes/v1/audit-log-stream-routers/audit-log-stream-router.ts

@@ -1,5 +1,9 @@
 import { z } from "zod";
 
+import {
+  CriblProviderListItemSchema,
+  SanitizedCriblProviderSchema
+} from "@app/ee/services/audit-log-stream/cribl/cribl-provider-schemas";
 import {
   CustomProviderListItemSchema,
   SanitizedCustomProviderSchema
@@ -19,13 +23,15 @@ import { AuthMode } from "@app/services/auth/auth-type";
 const SanitizedAuditLogStreamSchema = z.union([
   SanitizedCustomProviderSchema,
   SanitizedDatadogProviderSchema,
-  SanitizedSplunkProviderSchema
+  SanitizedSplunkProviderSchema,
+  SanitizedCriblProviderSchema
 ]);
 
 const ProviderOptionsSchema = z.discriminatedUnion("provider", [
   CustomProviderListItemSchema,
   DatadogProviderListItemSchema,
-  SplunkProviderListItemSchema
+  SplunkProviderListItemSchema,
+  CriblProviderListItemSchema
 ]);
 
 export const registerAuditLogStreamRouter = async (server: FastifyZodProvider) => {

backend/src/ee/routes/v1/audit-log-stream-routers/index.ts

@@ -1,4 +1,9 @@
 import { LogProvider } from "@app/ee/services/audit-log-stream/audit-log-stream-enums";
+import {
+  CreateCriblProviderLogStreamSchema,
+  SanitizedCriblProviderSchema,
+  UpdateCriblProviderLogStreamSchema
+} from "@app/ee/services/audit-log-stream/cribl/cribl-provider-schemas";
 import {
   CreateCustomProviderLogStreamSchema,
   SanitizedCustomProviderSchema,
@@ -47,5 +52,14 @@ export const AUDIT_LOG_STREAM_REGISTER_ROUTER_MAP: Record<LogProvider, (server:
         createSchema: CreateSplunkProviderLogStreamSchema,
         updateSchema: UpdateSplunkProviderLogStreamSchema
       });
+    },
+    [LogProvider.Cribl]: async (server: FastifyZodProvider) => {
+      registerAuditLogStreamEndpoints({
+        server,
+        provider: LogProvider.Cribl,
+        sanitizedResponseSchema: SanitizedCriblProviderSchema,
+        createSchema: CreateCriblProviderLogStreamSchema,
+        updateSchema: UpdateCriblProviderLogStreamSchema
+      });
     }
   };

backend/src/ee/routes/v2/index.ts

@@ -7,9 +7,9 @@ import {
   SECRET_SCANNING_REGISTER_ROUTER_MAP
 } from "@app/ee/routes/v2/secret-scanning-v2-routers";
 
+import { registerGatewayV2Router } from "./gateway-router";
 import { registerIdentityProjectAdditionalPrivilegeRouter } from "./identity-project-additional-privilege-router";
 import { registerProjectRoleRouter } from "./project-role-router";
-import { registerGatewayV2Router } from "./gateway-router";
 
 export const registerV2EERoutes = async (server: FastifyZodProvider) => {
   // org role starts with organization

backend/src/ee/services/audit-log-stream/audit-log-stream-enums.ts

@@ -1,5 +1,6 @@
 export enum LogProvider {
+  Cribl = "cribl",
+  Custom = "custom",
   Datadog = "datadog",
-  Splunk = "splunk",
+  Splunk = "splunk"
-  Custom = "custom"
 }

backend/src/ee/services/audit-log-stream/audit-log-stream-factory.ts

@@ -1,5 +1,6 @@
 import { LogProvider } from "./audit-log-stream-enums";
 import { TAuditLogStreamCredentials, TLogStreamFactory } from "./audit-log-stream-types";
+import { CriblProviderFactory } from "./cribl/cribl-provider-factory";
 import { CustomProviderFactory } from "./custom/custom-provider-factory";
 import { DatadogProviderFactory } from "./datadog/datadog-provider-factory";
 import { SplunkProviderFactory } from "./splunk/splunk-provider-factory";
@@ -9,5 +10,6 @@ type TLogStreamFactoryImplementation = TLogStreamFactory<TAuditLogStreamCredenti
 export const LOG_STREAM_FACTORY_MAP: Record<LogProvider, TLogStreamFactoryImplementation> = {
   [LogProvider.Datadog]: DatadogProviderFactory as TLogStreamFactoryImplementation,
   [LogProvider.Splunk]: SplunkProviderFactory as TLogStreamFactoryImplementation,
-  [LogProvider.Custom]: CustomProviderFactory as TLogStreamFactoryImplementation
+  [LogProvider.Custom]: CustomProviderFactory as TLogStreamFactoryImplementation,
+  [LogProvider.Cribl]: CriblProviderFactory as TLogStreamFactoryImplementation
 };

backend/src/ee/services/audit-log-stream/audit-log-stream-fns.ts

@@ -3,14 +3,18 @@ import { TKmsServiceFactory } from "@app/services/kms/kms-service";
 import { KmsDataKey } from "@app/services/kms/kms-types";
 
 import { TAuditLogStream, TAuditLogStreamCredentials } from "./audit-log-stream-types";
+import { getCriblProviderListItem } from "./cribl/cribl-provider-fns";
 import { getCustomProviderListItem } from "./custom/custom-provider-fns";
 import { getDatadogProviderListItem } from "./datadog/datadog-provider-fns";
 import { getSplunkProviderListItem } from "./splunk/splunk-provider-fns";
 
 export const listProviderOptions = () => {
-  return [getDatadogProviderListItem(), getSplunkProviderListItem(), getCustomProviderListItem()].sort((a, b) =>
+  return [
-    a.name.localeCompare(b.name)
+    getDatadogProviderListItem(),
-  );
+    getSplunkProviderListItem(),
+    getCustomProviderListItem(),
+    getCriblProviderListItem()
+  ].sort((a, b) => a.name.localeCompare(b.name));
 };
 
 export const encryptLogStreamCredentials = async ({

backend/src/ee/services/audit-log-stream/audit-log-stream-types.ts

@@ -1,16 +1,18 @@
 import { TAuditLogs } from "@app/db/schemas";
 
 import { LogProvider } from "./audit-log-stream-enums";
+import { TCriblProvider, TCriblProviderCredentials } from "./cribl/cribl-provider-types";
 import { TCustomProvider, TCustomProviderCredentials } from "./custom/custom-provider-types";
 import { TDatadogProvider, TDatadogProviderCredentials } from "./datadog/datadog-provider-types";
 import { TSplunkProvider, TSplunkProviderCredentials } from "./splunk/splunk-provider-types";
 
-export type TAuditLogStream = TDatadogProvider | TSplunkProvider | TCustomProvider;
+export type TAuditLogStream = TDatadogProvider | TSplunkProvider | TCustomProvider | TCriblProvider;
 
 export type TAuditLogStreamCredentials =
   | TDatadogProviderCredentials
   | TSplunkProviderCredentials
-  | TCustomProviderCredentials;
+  | TCustomProviderCredentials
+  | TCriblProviderCredentials;
 
 export type TCreateAuditLogStreamDTO = {
   provider: LogProvider;

backend/src/ee/services/audit-log-stream/cribl/cribl-provider-factory.ts

@@ -0,0 +1,58 @@
+import { RawAxiosRequestHeaders } from "axios";
+
+import { request } from "@app/lib/config/request";
+import { BadRequestError } from "@app/lib/errors";
+import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
+
+import { AUDIT_LOG_STREAM_TIMEOUT } from "../../audit-log/audit-log-queue";
+import { TLogStreamFactoryStreamLog, TLogStreamFactoryValidateCredentials } from "../audit-log-stream-types";
+import { TCriblProviderCredentials } from "./cribl-provider-types";
+
+export const CriblProviderFactory = () => {
+  const validateCredentials: TLogStreamFactoryValidateCredentials<TCriblProviderCredentials> = async ({
+    credentials
+  }) => {
+    const { url, token } = credentials;
+
+    await blockLocalAndPrivateIpAddresses(url);
+
+    const streamHeaders: RawAxiosRequestHeaders = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${token}`
+    };
+
+    await request
+      .post(url, JSON.stringify({ ping: "ok" }), {
+        headers: streamHeaders,
+        timeout: AUDIT_LOG_STREAM_TIMEOUT,
+        signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
+      })
+      .catch((err) => {
+        throw new BadRequestError({ message: `Failed to connect with Cribl: ${(err as Error)?.message}` });
+      });
+
+    return credentials;
+  };
+
+  const streamLog: TLogStreamFactoryStreamLog<TCriblProviderCredentials> = async ({ credentials, auditLog }) => {
+    const { url, token } = credentials;
+
+    await blockLocalAndPrivateIpAddresses(url);
+
+    const streamHeaders: RawAxiosRequestHeaders = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${token}`
+    };
+
+    await request.post(url, JSON.stringify(auditLog), {
+      headers: streamHeaders,
+      timeout: AUDIT_LOG_STREAM_TIMEOUT,
+      signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
+    });
+  };
+
+  return {
+    validateCredentials,
+    streamLog
+  };
+};

backend/src/ee/services/audit-log-stream/cribl/cribl-provider-fns.ts

@@ -0,0 +1,8 @@
+import { LogProvider } from "../audit-log-stream-enums";
+
+export const getCriblProviderListItem = () => {
+  return {
+    name: "Cribl" as const,
+    provider: LogProvider.Cribl as const
+  };
+};

backend/src/ee/services/audit-log-stream/cribl/cribl-provider-schemas.ts

@@ -0,0 +1,34 @@
+import { z } from "zod";
+
+import { LogProvider } from "../audit-log-stream-enums";
+import { BaseProviderSchema } from "../audit-log-stream-schemas";
+
+export const CriblProviderCredentialsSchema = z.object({
+  url: z.string().url().trim().min(1).max(255),
+  token: z.string().trim().min(21).max(255)
+});
+
+const BaseCriblProviderSchema = BaseProviderSchema.extend({ provider: z.literal(LogProvider.Cribl) });
+
+export const CriblProviderSchema = BaseCriblProviderSchema.extend({
+  credentials: CriblProviderCredentialsSchema
+});
+
+export const SanitizedCriblProviderSchema = BaseCriblProviderSchema.extend({
+  credentials: CriblProviderCredentialsSchema.pick({
+    url: true
+  })
+});
+
+export const CriblProviderListItemSchema = z.object({
+  name: z.literal("Cribl"),
+  provider: z.literal(LogProvider.Cribl)
+});
+
+export const CreateCriblProviderLogStreamSchema = z.object({
+  credentials: CriblProviderCredentialsSchema
+});
+
+export const UpdateCriblProviderLogStreamSchema = z.object({
+  credentials: CriblProviderCredentialsSchema
+});

backend/src/ee/services/audit-log-stream/cribl/cribl-provider-types.ts

@@ -0,0 +1,7 @@
+import { z } from "zod";
+
+import { CriblProviderCredentialsSchema, CriblProviderSchema } from "./cribl-provider-schemas";
+
+export type TCriblProvider = z.infer<typeof CriblProviderSchema>;
+
+export type TCriblProviderCredentials = z.infer<typeof CriblProviderCredentialsSchema>;

backend/src/ee/services/dynamic-secret/providers/sql-database.ts

@@ -1,6 +1,6 @@
 import handlebars from "handlebars";
-import RE2 from "re2";
 import knex from "knex";
+import RE2 from "re2";
 import { z } from "zod";
 
 import { crypto } from "@app/lib/crypto/cryptography";

docs/documentation/platform/audit-log-streams/audit-log-streams.mdx

@@ -72,6 +72,48 @@ Infisical Audit Log Streaming enables you to transmit your organization's audit
       		  </Step>
         </Steps>
     </Accordion>
+    <Accordion title="Cribl">
+        Stream Infisical audit logs to Cribl Stream for centralized processing and routing. Infisical supports Cribl as a provider for seamless integration.
+
+        <Steps>
+            <Step title="Create Infisical Data Source">
+                In Cribl Stream, navigate to **Worker Groups** and select your Worker Group. Take note of the **Ingress Address** for later steps.
+
+        				![cribl ingress address](/images/platform/audit-log-streams/cribl-ingress-address.png)
+
+                Within your Worker Group, navigate to **Data > Sources > HTTP** and click **Add Source**.
+
+        				![cribl add source](/images/platform/audit-log-streams/cribl-add-source.png)
+
+                Configure the **Input ID**, **Port**, and **Cribl HTTP event API** path (e.g., `/infisical`). Then, generate an **Auth Token**.
+
+                You can optionally configure TLS in the **TLS Settings** tab and add a pipeline in the **Pre-Processing** tab.
+
+                <Warning>
+                    Ensure that you're using a port that's open on your instance.
+                </Warning>
+
+        				![cribl general settings](/images/platform/audit-log-streams/cribl-general-settings.png)
+
+                Once you've configured the Data Source, click **Save** and deploy your changes.
+        		</Step>
+            <Step title="Create Audit Log Stream on Infisical">
+                On Infisical, create a new audit log stream and select the **Cribl** provider option.
+
+                Input the following credentials:
+                - **Cribl Stream URL**: Your HTTP source endpoint composed of `http://<ingress-address>:<port>/<http-event-api-path>/_bulk`
+                - **Cribl Stream Token**: The authentication token from Step 1
+
+                <Info>
+                    If you configured TLS for your Data Source, use the `https://` protocol.
+                </Info>
+
+        				![cribl details](/images/platform/audit-log-streams/cribl-details.png)
+
+                Once you're finished, click **Create Log Stream**.
+        		</Step>
+        </Steps>
+    </Accordion>
     <Accordion title="Datadog">
         You can stream to Datadog using the **Datadog** provider log stream.
 

frontend/src/helpers/auditLogStreams.ts

@@ -8,6 +8,7 @@ export const AUDIT_LOG_STREAM_PROVIDER_MAP: Record<
   LogProvider,
   { name: string; image?: string; icon?: IconDefinition; size?: number }
 > = {
+  [LogProvider.Cribl]: { name: "Cribl", image: "Cribl.png", size: 60 },
   [LogProvider.Custom]: { name: "Custom", icon: faCode },
   [LogProvider.Datadog]: { name: "Datadog", image: "Datadog.png" },
   [LogProvider.Splunk]: { name: "Splunk", image: "Splunk.png", size: 65 }
@@ -19,8 +20,8 @@ export function getProviderUrl(
 ) {
   switch (logStream.provider) {
     case LogProvider.Custom:
-      return logStream.credentials.url;
     case LogProvider.Datadog:
+    case LogProvider.Cribl:
       return logStream.credentials.url;
     case LogProvider.Splunk:
       return `https://${logStream.credentials.hostname}:8088/services/collector/event`;

frontend/src/hooks/api/auditLogStreams/enums.ts

@@ -1,5 +1,6 @@
 export enum LogProvider {
+  Cribl = "cribl",
+  Custom = "custom",
   Datadog = "datadog",
-  Splunk = "splunk",
+  Splunk = "splunk"
-  Custom = "custom"
 }

frontend/src/hooks/api/auditLogStreams/types/index.ts

@@ -1,4 +1,5 @@
 import { LogProvider } from "../enums";
+import { TCriblProviderLogStream } from "./providers/cribl-provider";
 import { TCustomProviderLogStream } from "./providers/custom-provider";
 import { TDatadogProviderLogStream } from "./providers/datadog-provider";
 import { TSplunkProviderLogStream } from "./providers/splunk-provider";
@@ -6,9 +7,11 @@ import { TSplunkProviderLogStream } from "./providers/splunk-provider";
 export type TAuditLogStream =
   | TCustomProviderLogStream
   | TDatadogProviderLogStream
-  | TSplunkProviderLogStream;
+  | TSplunkProviderLogStream
+  | TCriblProviderLogStream;
 
 export type TAuditLogStreamProviderMap = {
+  [LogProvider.Cribl]: TCriblProviderLogStream;
   [LogProvider.Custom]: TCustomProviderLogStream;
   [LogProvider.Datadog]: TDatadogProviderLogStream;
   [LogProvider.Splunk]: TSplunkProviderLogStream;

frontend/src/hooks/api/auditLogStreams/types/providers/cribl-provider.ts

@@ -0,0 +1,10 @@
+import { LogProvider } from "../../enums";
+import { TRootProviderLogStream } from "./root-provider";
+
+export type TCriblProviderLogStream = TRootProviderLogStream & {
+  provider: LogProvider.Cribl;
+  credentials: {
+    url: string;
+    token: string;
+  };
+};

frontend/src/pages/organization/SettingsPage/components/AuditLogStreamTab/AuditLogStreamForm/AuditLogStreamForm.tsx

@@ -6,6 +6,7 @@ import { TAuditLogStream } from "@app/hooks/api/types";
 import { DiscriminativePick } from "@app/types";
 
 import { AuditLogStreamHeader } from "../components/AuditLogStreamHeader";
+import { CriblProviderAuditLogStreamForm } from "./CriblProviderAuditLogStreamForm";
 import { CustomProviderAuditLogStreamForm } from "./CustomProviderAuditLogStreamForm";
 import { DatadogProviderAuditLogStreamForm } from "./DatadogProviderAuditLogStreamForm";
 import { SplunkProviderAuditLogStreamForm } from "./SplunkProviderAuditLogStreamForm";
@@ -44,6 +45,8 @@ const CreateForm = ({ provider, onComplete }: CreateFormProps) => {
   };
 
   switch (provider) {
+    case LogProvider.Cribl:
+      return <CriblProviderAuditLogStreamForm onSubmit={onSubmit} />;
     case LogProvider.Custom:
       return <CustomProviderAuditLogStreamForm onSubmit={onSubmit} />;
     case LogProvider.Datadog:
@@ -83,6 +86,10 @@ const UpdateForm = ({ auditLogStream, onComplete }: UpdateFormProps) => {
   };
 
   switch (auditLogStream.provider) {
+    case LogProvider.Cribl:
+      return (
+        <CriblProviderAuditLogStreamForm onSubmit={onSubmit} auditLogStream={auditLogStream} />
+      );
     case LogProvider.Custom:
       return (
         <CustomProviderAuditLogStreamForm onSubmit={onSubmit} auditLogStream={auditLogStream} />

frontend/src/pages/organization/SettingsPage/components/AuditLogStreamTab/AuditLogStreamForm/CriblProviderAuditLogStreamForm.tsx

@@ -0,0 +1,108 @@
+import { Controller, FormProvider, useForm } from "react-hook-form";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { z } from "zod";
+
+import { Button, FormControl, Input, ModalClose, SecretInput } from "@app/components/v2";
+import { LogProvider } from "@app/hooks/api/auditLogStreams/enums";
+import { TCriblProviderLogStream } from "@app/hooks/api/auditLogStreams/types/providers/cribl-provider";
+
+type Props = {
+  auditLogStream?: TCriblProviderLogStream;
+  onSubmit: (formData: FormData) => void;
+};
+
+const formSchema = z.object({
+  provider: z.literal(LogProvider.Cribl),
+  credentials: z.object({
+    url: z.string().url().trim().min(1).max(255),
+    token: z.string().trim().min(21).max(255)
+  })
+});
+
+type FormData = z.infer<typeof formSchema>;
+
+export const CriblProviderAuditLogStreamForm = ({ auditLogStream, onSubmit }: Props) => {
+  const isUpdate = Boolean(auditLogStream);
+
+  const form = useForm<FormData>({
+    resolver: zodResolver(formSchema),
+    defaultValues: auditLogStream ?? {
+      provider: LogProvider.Cribl
+    }
+  });
+
+  const {
+    handleSubmit,
+    control,
+    formState: { isSubmitting, isDirty }
+  } = form;
+
+  return (
+    <FormProvider {...form}>
+      <form onSubmit={handleSubmit(onSubmit)}>
+        <Controller
+          name="credentials.url"
+          control={control}
+          shouldUnregister
+          render={({ field, fieldState: { error } }) => (
+            <FormControl
+              errorText={error?.message}
+              isError={Boolean(error?.message)}
+              label="Cribl Stream URL"
+              tooltipText={
+                <>
+                  To derive your Stream URL: Obtain your Cribl hostname (e.g. cribl.example.com),
+                  Infisical HTTP data source port (e.g. 20000), and HTTP event API path (e.g.
+                  /infisical).
+                  <br />
+                  <br />
+                  If your Infisical Data Source has TLS enabled, then use the https protocol.
+                </>
+              }
+            >
+              <Input
+                {...field}
+                placeholder="http://default.main.example.cribl.cloud:20000/infisical/_bulk"
+              />
+            </FormControl>
+          )}
+        />
+        <Controller
+          name="credentials.token"
+          control={control}
+          shouldUnregister
+          render={({ field: { value, onChange }, fieldState: { error } }) => (
+            <FormControl
+              errorText={error?.message}
+              isError={Boolean(error?.message)}
+              label="Cribl Stream Token"
+            >
+              <SecretInput
+                containerClassName="text-gray-400 group-focus-within:!border-primary-400/50 border border-mineshaft-500 bg-mineshaft-900 px-2.5 py-1.5"
+                value={value}
+                onChange={(e) => onChange(e.target.value)}
+              />
+            </FormControl>
+          )}
+        />
+        <div className="mt-8 flex items-center">
+          <Button
+            className="mr-4"
+            size="sm"
+            type="submit"
+            colorSchema="secondary"
+            isLoading={isSubmitting}
+            isDisabled={isSubmitting || !isDirty}
+          >
+            {isUpdate ? "Update Credentials" : "Create Log Stream"}
+          </Button>
+          <ModalClose asChild>
+            <Button colorSchema="secondary" variant="plain">
+              Cancel
+            </Button>
+          </ModalClose>
+        </div>
+      </form>
+    </FormProvider>
+  );
+};