Merge remote-tracking branch 'origin' into groups-phase-2c

backend/src/ee/services/audit-log-stream/audit-log-stream-dal.ts

@@ -0,0 +1,11 @@
+import { TDbClient } from "@app/db";
+import { TableName } from "@app/db/schemas";
+import { ormify } from "@app/lib/knex";
+
+export type TAuditLogStreamDALFactory = ReturnType<typeof auditLogStreamDALFactory>;
+
+export const auditLogStreamDALFactory = (db: TDbClient) => {
+  const orm = ormify(db, TableName.AuditLogStream);
+
+  return orm;
+};

backend/src/ee/services/audit-log-stream/audit-log-stream-service.ts

@@ -0,0 +1,233 @@
+import { ForbiddenError } from "@casl/ability";
+import { RawAxiosRequestHeaders } from "axios";
+
+import { SecretKeyEncoding } from "@app/db/schemas";
+import { request } from "@app/lib/config/request";
+import { infisicalSymmetricDecrypt, infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
+import { BadRequestError } from "@app/lib/errors";
+import { validateLocalIps } from "@app/lib/validator";
+
+import { AUDIT_LOG_STREAM_TIMEOUT } from "../audit-log/audit-log-queue";
+import { TLicenseServiceFactory } from "../license/license-service";
+import { OrgPermissionActions, OrgPermissionSubjects } from "../permission/org-permission";
+import { TPermissionServiceFactory } from "../permission/permission-service";
+import { TAuditLogStreamDALFactory } from "./audit-log-stream-dal";
+import {
+  LogStreamHeaders,
+  TCreateAuditLogStreamDTO,
+  TDeleteAuditLogStreamDTO,
+  TGetDetailsAuditLogStreamDTO,
+  TListAuditLogStreamDTO,
+  TUpdateAuditLogStreamDTO
+} from "./audit-log-stream-types";
+
+type TAuditLogStreamServiceFactoryDep = {
+  auditLogStreamDAL: TAuditLogStreamDALFactory;
+  permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
+  licenseService: Pick<TLicenseServiceFactory, "getPlan">;
+};
+
+export type TAuditLogStreamServiceFactory = ReturnType<typeof auditLogStreamServiceFactory>;
+
+export const auditLogStreamServiceFactory = ({
+  auditLogStreamDAL,
+  permissionService,
+  licenseService
+}: TAuditLogStreamServiceFactoryDep) => {
+  const create = async ({
+    url,
+    actor,
+    headers = [],
+    actorId,
+    actorOrgId,
+    actorAuthMethod
+  }: TCreateAuditLogStreamDTO) => {
+    if (!actorOrgId) throw new BadRequestError({ message: "Missing org id from token" });
+
+    const plan = await licenseService.getPlan(actorOrgId);
+    if (!plan.auditLogStreams)
+      throw new BadRequestError({
+        message: "Failed to create audit log streams due to plan restriction. Upgrade plan to create group."
+      });
+
+    const { permission } = await permissionService.getOrgPermission(
+      actor,
+      actorId,
+      actorOrgId,
+      actorAuthMethod,
+      actorOrgId
+    );
+    ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Settings);
+
+    validateLocalIps(url);
+
+    const totalStreams = await auditLogStreamDAL.find({ orgId: actorOrgId });
+    if (totalStreams.length >= plan.auditLogStreamLimit) {
+      throw new BadRequestError({
+        message:
+          "Failed to create audit log streams due to plan limit reached. Kindly contact Infisical to add more streams."
+      });
+    }
+
+    // testing connection first
+    const streamHeaders: RawAxiosRequestHeaders = { "Content-Type": "application/json" };
+    if (headers.length)
+      headers.forEach(({ key, value }) => {
+        streamHeaders[key] = value;
+      });
+    await request
+      .post(
+        url,
+        { ping: "ok" },
+        {
+          headers: streamHeaders,
+          // request timeout
+          timeout: AUDIT_LOG_STREAM_TIMEOUT,
+          // connection timeout
+          signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
+        }
+      )
+      .catch((err) => {
+        throw new Error(`Failed to connect with the source ${(err as Error)?.message}`);
+      });
+    const encryptedHeaders = headers ? infisicalSymmetricEncypt(JSON.stringify(headers)) : undefined;
+    const logStream = await auditLogStreamDAL.create({
+      orgId: actorOrgId,
+      url,
+      ...(encryptedHeaders
+        ? {
+            encryptedHeadersCiphertext: encryptedHeaders.ciphertext,
+            encryptedHeadersIV: encryptedHeaders.iv,
+            encryptedHeadersTag: encryptedHeaders.tag,
+            encryptedHeadersAlgorithm: encryptedHeaders.algorithm,
+            encryptedHeadersKeyEncoding: encryptedHeaders.encoding
+          }
+        : {})
+    });
+    return logStream;
+  };
+
+  const updateById = async ({
+    id,
+    url,
+    actor,
+    headers = [],
+    actorId,
+    actorOrgId,
+    actorAuthMethod
+  }: TUpdateAuditLogStreamDTO) => {
+    if (!actorOrgId) throw new BadRequestError({ message: "Missing org id from token" });
+
+    const plan = await licenseService.getPlan(actorOrgId);
+    if (!plan.auditLogStreams)
+      throw new BadRequestError({
+        message: "Failed to update audit log streams due to plan restriction. Upgrade plan to create group."
+      });
+
+    const logStream = await auditLogStreamDAL.findById(id);
+    if (!logStream) throw new BadRequestError({ message: "Audit log stream not found" });
+
+    const { orgId } = logStream;
+    const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
+    ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Settings);
+
+    if (url) validateLocalIps(url);
+
+    // testing connection first
+    const streamHeaders: RawAxiosRequestHeaders = { "Content-Type": "application/json" };
+    if (headers.length)
+      headers.forEach(({ key, value }) => {
+        streamHeaders[key] = value;
+      });
+
+    await request
+      .post(
+        url || logStream.url,
+        { ping: "ok" },
+        {
+          headers: streamHeaders,
+          // request timeout
+          timeout: AUDIT_LOG_STREAM_TIMEOUT,
+          // connection timeout
+          signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
+        }
+      )
+      .catch((err) => {
+        throw new Error(`Failed to connect with the source ${(err as Error)?.message}`);
+      });
+
+    const encryptedHeaders = headers ? infisicalSymmetricEncypt(JSON.stringify(headers)) : undefined;
+    const updatedLogStream = await auditLogStreamDAL.updateById(id, {
+      url,
+      ...(encryptedHeaders
+        ? {
+            encryptedHeadersCiphertext: encryptedHeaders.ciphertext,
+            encryptedHeadersIV: encryptedHeaders.iv,
+            encryptedHeadersTag: encryptedHeaders.tag,
+            encryptedHeadersAlgorithm: encryptedHeaders.algorithm,
+            encryptedHeadersKeyEncoding: encryptedHeaders.encoding
+          }
+        : {})
+    });
+    return updatedLogStream;
+  };
+
+  const deleteById = async ({ id, actor, actorId, actorOrgId, actorAuthMethod }: TDeleteAuditLogStreamDTO) => {
+    if (!actorOrgId) throw new BadRequestError({ message: "Missing org id from token" });
+
+    const logStream = await auditLogStreamDAL.findById(id);
+    if (!logStream) throw new BadRequestError({ message: "Audit log stream not found" });
+
+    const { orgId } = logStream;
+    const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
+    ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Delete, OrgPermissionSubjects.Settings);
+
+    const deletedLogStream = await auditLogStreamDAL.deleteById(id);
+    return deletedLogStream;
+  };
+
+  const getById = async ({ id, actor, actorId, actorOrgId, actorAuthMethod }: TGetDetailsAuditLogStreamDTO) => {
+    const logStream = await auditLogStreamDAL.findById(id);
+    if (!logStream) throw new BadRequestError({ message: "Audit log stream not found" });
+
+    const { orgId } = logStream;
+    const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
+    ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.Settings);
+
+    const headers =
+      logStream?.encryptedHeadersCiphertext && logStream?.encryptedHeadersIV && logStream?.encryptedHeadersTag
+        ? (JSON.parse(
+            infisicalSymmetricDecrypt({
+              tag: logStream.encryptedHeadersTag,
+              iv: logStream.encryptedHeadersIV,
+              ciphertext: logStream.encryptedHeadersCiphertext,
+              keyEncoding: logStream.encryptedHeadersKeyEncoding as SecretKeyEncoding
+            })
+          ) as LogStreamHeaders[])
+        : undefined;
+
+    return { ...logStream, headers };
+  };
+
+  const list = async ({ actor, actorId, actorOrgId, actorAuthMethod }: TListAuditLogStreamDTO) => {
+    const { permission } = await permissionService.getOrgPermission(
+      actor,
+      actorId,
+      actorOrgId,
+      actorAuthMethod,
+      actorOrgId
+    );
+    ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.Settings);
+
+    const logStreams = await auditLogStreamDAL.find({ orgId: actorOrgId });
+    return logStreams;
+  };
+
+  return {
+    create,
+    updateById,
+    deleteById,
+    getById,
+    list
+  };
+};

backend/src/ee/services/audit-log-stream/audit-log-stream-types.ts

@@ -0,0 +1,27 @@
+import { TOrgPermission } from "@app/lib/types";
+
+export type LogStreamHeaders = {
+  key: string;
+  value: string;
+};
+
+export type TCreateAuditLogStreamDTO = Omit<TOrgPermission, "orgId"> & {
+  url: string;
+  headers?: LogStreamHeaders[];
+};
+
+export type TUpdateAuditLogStreamDTO = Omit<TOrgPermission, "orgId"> & {
+  id: string;
+  url?: string;
+  headers?: LogStreamHeaders[];
+};
+
+export type TDeleteAuditLogStreamDTO = Omit<TOrgPermission, "orgId"> & {
+  id: string;
+};
+
+export type TListAuditLogStreamDTO = Omit<TOrgPermission, "orgId">;
+
+export type TGetDetailsAuditLogStreamDTO = Omit<TOrgPermission, "orgId"> & {
+  id: string;
+};

backend/src/ee/services/audit-log/audit-log-queue.ts

@@ -1,13 +1,21 @@
+import { RawAxiosRequestHeaders } from "axios";
+
+import { SecretKeyEncoding } from "@app/db/schemas";
+import { request } from "@app/lib/config/request";
+import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption";
 import { logger } from "@app/lib/logger";
 import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
 
+import { TAuditLogStreamDALFactory } from "../audit-log-stream/audit-log-stream-dal";
+import { LogStreamHeaders } from "../audit-log-stream/audit-log-stream-types";
 import { TLicenseServiceFactory } from "../license/license-service";
 import { TAuditLogDALFactory } from "./audit-log-dal";
 import { TCreateAuditLogDTO } from "./audit-log-types";
 
 type TAuditLogQueueServiceFactoryDep = {
   auditLogDAL: TAuditLogDALFactory;
+  auditLogStreamDAL: Pick<TAuditLogStreamDALFactory, "find">;
   queueService: TQueueServiceFactory;
   projectDAL: Pick<TProjectDALFactory, "findById">;
   licenseService: Pick<TLicenseServiceFactory, "getPlan">;
@@ -15,11 +23,15 @@ type TAuditLogQueueServiceFactoryDep = {
 
 export type TAuditLogQueueServiceFactory = ReturnType<typeof auditLogQueueServiceFactory>;
 
+// keep this timeout 5s it must be fast because else the queue will take time to finish
+// audit log is a crowded queue thus needs to be fast
+export const AUDIT_LOG_STREAM_TIMEOUT = 5 * 1000;
 export const auditLogQueueServiceFactory = ({
   auditLogDAL,
   queueService,
   projectDAL,
-  licenseService
+  licenseService,
+  auditLogStreamDAL
 }: TAuditLogQueueServiceFactoryDep) => {
   const pushToLog = async (data: TCreateAuditLogDTO) => {
     await queueService.queue(QueueName.AuditLog, QueueJobs.AuditLog, data, {
@@ -47,7 +59,7 @@ export const auditLogQueueServiceFactory = ({
     // skip inserting if audit log retention is 0 meaning its not supported
     if (ttl === 0) return;
 
-    await auditLogDAL.create({
+    const auditLog = await auditLogDAL.create({
       actor: actor.type,
       actorMetadata: actor.metadata,
       userAgent,
@@ -59,6 +71,46 @@ export const auditLogQueueServiceFactory = ({
       eventMetadata: event.metadata,
       userAgentType
     });
+
+    const logStreams = orgId ? await auditLogStreamDAL.find({ orgId }) : [];
+    await Promise.allSettled(
+      logStreams.map(
+        async ({
+          url,
+          encryptedHeadersTag,
+          encryptedHeadersIV,
+          encryptedHeadersKeyEncoding,
+          encryptedHeadersCiphertext
+        }) => {
+          const streamHeaders =
+            encryptedHeadersIV && encryptedHeadersCiphertext && encryptedHeadersTag
+              ? (JSON.parse(
+                  infisicalSymmetricDecrypt({
+                    keyEncoding: encryptedHeadersKeyEncoding as SecretKeyEncoding,
+                    iv: encryptedHeadersIV,
+                    tag: encryptedHeadersTag,
+                    ciphertext: encryptedHeadersCiphertext
+                  })
+                ) as LogStreamHeaders[])
+              : [];
+
+          const headers: RawAxiosRequestHeaders = { "Content-Type": "application/json" };
+
+          if (streamHeaders.length)
+            streamHeaders.forEach(({ key, value }) => {
+              headers[key] = value;
+            });
+
+          return request.post(url, auditLog, {
+            headers,
+            // request timeout
+            timeout: AUDIT_LOG_STREAM_TIMEOUT,
+            // connection timeout
+            signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
+          });
+        }
+      )
+    );
   });
 
   queueService.start(QueueName.AuditLogPrune, async () => {

backend/src/ee/services/license/licence-fns.ts

@@ -24,6 +24,8 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
   customAlerts: false,
   auditLogs: false,
   auditLogsRetentionDays: 0,
+  auditLogStreams: false,
+  auditLogStreamLimit: 3,
   samlSSO: false,
   scim: false,
   ldap: false,

backend/src/ee/services/license/license-types.ts

@@ -40,6 +40,8 @@ export type TFeatureSet = {
   customAlerts: false;
   auditLogs: false;
   auditLogsRetentionDays: 0;
+  auditLogStreams: false;
+  auditLogStreamLimit: 3;
   samlSSO: false;
   scim: false;
   ldap: false;