feat(server): audit log streams services and api routes

This commit is contained in:
Akhil Mohan
2024-04-23 20:51:57 +05:30
parent 69aa20e35c
commit f2b1f3f0e7
14 changed files with 546 additions and 3 deletions

View File

@@ -0,0 +1,199 @@
import { z } from "zod";
import { AUDIT_LOG_STREAMS } from "@app/lib/api-docs";
import { readLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { SanitizedAuditLogStreamSchema } from "@app/server/routes/sanitizedSchemas";
import { AuthMode } from "@app/services/auth/auth-type";
export const registerAuditLogStreamRouter = async (server: FastifyZodProvider) => {
server.route({
method: "POST",
url: "/",
config: {
rateLimit: readLimit
},
schema: {
description: "Create an Audit Log Stream.",
security: [
{
bearerAuth: []
}
],
body: z.object({
projectSlug: z.string().min(1).describe(AUDIT_LOG_STREAMS.CREATE.projectSlug),
url: z.string().min(1).describe(AUDIT_LOG_STREAMS.CREATE.url),
token: z.string().optional().describe(AUDIT_LOG_STREAMS.CREATE.token)
}),
response: {
200: z.object({
auditLogStream: SanitizedAuditLogStreamSchema
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const auditLogStream = await server.services.auditLogStream.create({
actorId: req.permission.id,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod,
projectSlug: req.body.projectSlug,
url: req.body.url,
token: req.body.token
});
return { auditLogStream };
}
});
server.route({
method: "PATCH",
url: "/:id",
config: {
rateLimit: readLimit
},
schema: {
description: "Update an Audit Log Stream by ID.",
security: [
{
bearerAuth: []
}
],
params: z.object({
id: z.string().describe(AUDIT_LOG_STREAMS.UPDATE.id)
}),
body: z.object({
url: z.string().optional().describe(AUDIT_LOG_STREAMS.UPDATE.url),
token: z.string().optional().describe(AUDIT_LOG_STREAMS.UPDATE.token)
}),
response: {
200: z.object({
auditLogStream: SanitizedAuditLogStreamSchema
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const auditLogStream = await server.services.auditLogStream.updateById({
actorId: req.permission.id,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod,
id: req.params.id,
url: req.body.url,
token: req.body.token
});
return { auditLogStream };
}
});
server.route({
method: "DELETE",
url: "/:id",
config: {
rateLimit: readLimit
},
schema: {
description: "Delete an Audit Log Stream by ID.",
security: [
{
bearerAuth: []
}
],
params: z.object({
id: z.string().describe(AUDIT_LOG_STREAMS.DELETE.id)
}),
response: {
200: z.object({
auditLogStream: SanitizedAuditLogStreamSchema
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const auditLogStream = await server.services.auditLogStream.deleteById({
actorId: req.permission.id,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod,
id: req.params.id
});
return { auditLogStream };
}
});
server.route({
method: "GET",
url: "/:id",
config: {
rateLimit: readLimit
},
schema: {
description: "Get an Audit Log Stream by ID.",
security: [
{
bearerAuth: []
}
],
params: z.object({
id: z.string().describe(AUDIT_LOG_STREAMS.GET_BY_ID.id)
}),
response: {
200: z.object({
auditLogStream: SanitizedAuditLogStreamSchema.extend({ token: z.string().optional() })
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const auditLogStream = await server.services.auditLogStream.getById({
actorId: req.permission.id,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod,
id: req.params.id
});
return { auditLogStream };
}
});
server.route({
method: "GET",
url: "/",
config: {
rateLimit: readLimit
},
schema: {
description: "List Audit Log Streams.",
security: [
{
bearerAuth: []
}
],
querystring: z.object({
projectSlug: z.string().describe(AUDIT_LOG_STREAMS.LIST.projectSlug)
}),
response: {
200: z.object({
auditLogStreams: SanitizedAuditLogStreamSchema.array()
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const auditLogStreams = await server.services.auditLogStream.list({
actorId: req.permission.id,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod,
projectSlug: req.query.projectSlug
});
return { auditLogStreams };
}
});
};

View File

@@ -1,3 +1,4 @@
import { registerAuditLogStreamRouter } from "./audit-log-stream-router";
import { registerDynamicSecretLeaseRouter } from "./dynamic-secret-lease-router";
import { registerDynamicSecretRouter } from "./dynamic-secret-router";
import { registerGroupRouter } from "./group-router";
@@ -55,6 +56,7 @@ export const registerV1EERoutes = async (server: FastifyZodProvider) => {
await server.register(registerSecretRotationRouter, { prefix: "/secret-rotations" });
await server.register(registerSecretVersionRouter, { prefix: "/secret" });
await server.register(registerGroupRouter, { prefix: "/groups" });
await server.register(registerAuditLogStreamRouter, { prefix: "/audit-log-streams" });
await server.register(
async (privilegeRouter) => {
await privilegeRouter.register(registerUserAdditionalPrivilegeRouter, { prefix: "/users" });

View File

@@ -0,0 +1,11 @@
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { ormify } from "@app/lib/knex";
export type TAuditLogStreamDALFactory = ReturnType<typeof auditLogStreamDALFactory>;
export const auditLogStreamDALFactory = (db: TDbClient) => {
const orm = ormify(db, TableName.AuditLogStream);
return orm;
};

View File

@@ -0,0 +1,205 @@
import { ForbiddenError } from "@casl/ability";
import { SecretKeyEncoding } from "@app/db/schemas";
import { infisicalSymmetricDecrypt, infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
import { BadRequestError } from "@app/lib/errors";
import { validateLocalIps } from "@app/lib/validator";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TLicenseServiceFactory } from "../license/license-service";
import { TPermissionServiceFactory } from "../permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "../permission/project-permission";
import { TAuditLogStreamDALFactory } from "./audit-log-stream-dal";
import {
TCreateAuditLogStreamDTO,
TDeleteAuditLogStreamDTO,
TGetDetailsAuditLogStreamDTO,
TListAuditLogStreamDTO,
TUpdateAuditLogStreamDTO
} from "./audit-log-stream-types";
type TAuditLogStreamServiceFactoryDep = {
auditLogStreamDAL: TAuditLogStreamDALFactory;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
projectDAL: Pick<TProjectDALFactory, "findProjectBySlug">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
};
export type TAuditLogStreamServiceFactory = ReturnType<typeof auditLogStreamServiceFactory>;
export const auditLogStreamServiceFactory = ({
auditLogStreamDAL,
permissionService,
projectDAL,
licenseService
}: TAuditLogStreamServiceFactoryDep) => {
const create = async ({
projectSlug,
url,
actor,
token,
actorId,
actorOrgId,
actorAuthMethod
}: TCreateAuditLogStreamDTO) => {
const plan = await licenseService.getPlan(actorOrgId);
if (!plan.auditLogStreams)
throw new BadRequestError({
message: "Failed to create audit log streams due to plan restriction. Upgrade plan to create group."
});
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new BadRequestError({ message: "Project not found" });
const projectId = project.id;
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Settings);
validateLocalIps(url);
const totalStreams = await auditLogStreamDAL.find({ projectId });
if (totalStreams.length >= plan.auditLogStreamLimit) {
throw new BadRequestError({
message:
"Failed to create audit log streams due to plan limit reached. Kindly contact Infisical to add more streams."
});
}
const encryptedToken = token ? infisicalSymmetricEncypt(token) : undefined;
const logStream = await auditLogStreamDAL.create({
projectId,
url,
...(encryptedToken
? {
encryptedTokenCiphertext: encryptedToken.ciphertext,
encryptedTokenIV: encryptedToken.iv,
encryptedTokenTag: encryptedToken.tag,
encryptedTokenAlgorithm: encryptedToken.algorithm,
encryptedTokenKeyEncoding: encryptedToken.encoding
}
: {})
});
return logStream;
};
const updateById = async ({
id,
url,
actor,
token,
actorId,
actorOrgId,
actorAuthMethod
}: TUpdateAuditLogStreamDTO) => {
const plan = await licenseService.getPlan(actorOrgId);
if (!plan.auditLogStreams)
throw new BadRequestError({
message: "Failed to update audit log streams due to plan restriction. Upgrade plan to create group."
});
const logStream = await auditLogStreamDAL.findById(id);
if (!logStream) throw new BadRequestError({ message: "Audit log stream not found" });
const { projectId } = logStream;
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.Settings);
if (url) validateLocalIps(url);
const encryptedToken = token ? infisicalSymmetricEncypt(token) : undefined;
const updatedLogStream = await auditLogStreamDAL.updateById(id, {
projectId,
url,
...(encryptedToken
? {
encryptedTokenCiphertext: encryptedToken.ciphertext,
encryptedTokenIV: encryptedToken.iv,
encryptedTokenTag: encryptedToken.tag,
encryptedTokenAlgorithm: encryptedToken.algorithm,
encryptedTokenKeyEncoding: encryptedToken.encoding
}
: {})
});
return updatedLogStream;
};
const deleteById = async ({ id, actor, actorId, actorOrgId, actorAuthMethod }: TDeleteAuditLogStreamDTO) => {
const logStream = await auditLogStreamDAL.findById(id);
if (!logStream) throw new BadRequestError({ message: "Audit log stream not found" });
const { projectId } = logStream;
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.Settings);
const deletedLogStream = await auditLogStreamDAL.deleteById(id);
return deletedLogStream;
};
const getById = async ({ id, actor, actorId, actorOrgId, actorAuthMethod }: TGetDetailsAuditLogStreamDTO) => {
const logStream = await auditLogStreamDAL.findById(id);
if (!logStream) throw new BadRequestError({ message: "Audit log stream not found" });
const { projectId } = logStream;
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Settings);
const token =
logStream?.encryptedTokenCiphertext && logStream?.encryptedTokenIV && logStream?.encryptedTokenTag
? infisicalSymmetricDecrypt({
tag: logStream.encryptedTokenTag,
iv: logStream.encryptedTokenIV,
ciphertext: logStream.encryptedTokenCiphertext,
keyEncoding: logStream.encryptedTokenKeyEncoding as SecretKeyEncoding
})
: undefined;
return { ...logStream, token };
};
const list = async ({ projectSlug, actor, actorId, actorOrgId, actorAuthMethod }: TListAuditLogStreamDTO) => {
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new BadRequestError({ message: "Project not found" });
const projectId = project.id;
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Settings);
const logStreams = await auditLogStreamDAL.find({ projectId });
return logStreams;
};
return {
create,
updateById,
deleteById,
getById,
list
};
};

View File

@@ -0,0 +1,25 @@
import { TProjectPermission } from "@app/lib/types";
export type TCreateAuditLogStreamDTO = Omit<TProjectPermission, "projectId"> & {
projectSlug: string;
url: string;
token?: string;
};
export type TUpdateAuditLogStreamDTO = Omit<TProjectPermission, "projectId"> & {
id: string;
url?: string;
token?: string;
};
export type TDeleteAuditLogStreamDTO = Omit<TProjectPermission, "projectId"> & {
id: string;
};
export type TListAuditLogStreamDTO = Omit<TProjectPermission, "projectId"> & {
projectSlug: string;
};
export type TGetDetailsAuditLogStreamDTO = Omit<TProjectPermission, "projectId"> & {
id: string;
};

View File

@@ -1,13 +1,20 @@
import { RawAxiosRequestHeaders } from "axios";
import { SecretKeyEncoding } from "@app/db/schemas";
import { request } from "@app/lib/config/request";
import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TAuditLogStreamDALFactory } from "../audit-log-stream/audit-log-stream-dal";
import { TLicenseServiceFactory } from "../license/license-service";
import { TAuditLogDALFactory } from "./audit-log-dal";
import { TCreateAuditLogDTO } from "./audit-log-types";
type TAuditLogQueueServiceFactoryDep = {
auditLogDAL: TAuditLogDALFactory;
auditLogStreamDAL: Pick<TAuditLogStreamDALFactory, "find">;
queueService: TQueueServiceFactory;
projectDAL: Pick<TProjectDALFactory, "findById">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
@@ -15,11 +22,15 @@ type TAuditLogQueueServiceFactoryDep = {
export type TAuditLogQueueServiceFactory = ReturnType<typeof auditLogQueueServiceFactory>;
// keep this timeout 5s it must be fast because else the queue will take time to finish
// audit log is a crowded queue thus needs to be fast
const AUDIT_LOG_STREAM_TIMEOUT = 5 * 1000;
export const auditLogQueueServiceFactory = ({
auditLogDAL,
queueService,
projectDAL,
licenseService
licenseService,
auditLogStreamDAL
}: TAuditLogQueueServiceFactoryDep) => {
const pushToLog = async (data: TCreateAuditLogDTO) => {
await queueService.queue(QueueName.AuditLog, QueueJobs.AuditLog, data, {
@@ -47,7 +58,7 @@ export const auditLogQueueServiceFactory = ({
// skip inserting if audit log retention is 0 meaning its not supported
if (ttl === 0) return;
await auditLogDAL.create({
const auditLog = await auditLogDAL.create({
actor: actor.type,
actorMetadata: actor.metadata,
userAgent,
@@ -59,6 +70,32 @@ export const auditLogQueueServiceFactory = ({
eventMetadata: event.metadata,
userAgentType
});
const logStreams = await auditLogStreamDAL.find({ projectId });
await Promise.allSettled(
logStreams.map(
async ({ url, encryptedTokenTag, encryptedTokenIV, encryptedTokenKeyEncoding, encryptedTokenCiphertext }) => {
const token =
encryptedTokenIV && encryptedTokenCiphertext && encryptedTokenTag
? infisicalSymmetricDecrypt({
keyEncoding: encryptedTokenKeyEncoding as SecretKeyEncoding,
iv: encryptedTokenIV,
tag: encryptedTokenTag,
ciphertext: encryptedTokenCiphertext
})
: undefined;
const headers: RawAxiosRequestHeaders = { "Content-Type": "application/json" };
if (token) headers.Authorization = `Bearer ${token}`;
return request.post(url, auditLog, {
headers,
// request timeout
timeout: AUDIT_LOG_STREAM_TIMEOUT,
// connection timeout
signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
});
}
)
);
});
queueService.start(QueueName.AuditLogPrune, async () => {

View File

@@ -24,6 +24,8 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
customAlerts: false,
auditLogs: false,
auditLogsRetentionDays: 0,
auditLogStreams: false,
auditLogStreamLimit: 3,
samlSSO: false,
scim: false,
ldap: false,

View File

@@ -40,6 +40,8 @@ export type TFeatureSet = {
customAlerts: false;
auditLogs: false;
auditLogsRetentionDays: 0;
auditLogStreams: false;
auditLogStreamLimit: 3;
samlSSO: false;
scim: false;
ldap: false;

View File

@@ -614,3 +614,25 @@ export const INTEGRATION = {
integrationId: "The ID of the integration object."
}
};
export const AUDIT_LOG_STREAMS = {
CREATE: {
projectSlug: "The slug of the project to create audit log stream.",
url: "The socket URL to push logs to.",
token: "Authentication token for the external provider used for identification."
},
UPDATE: {
id: "The ID of the audit log stream to update.",
url: "The socket URL to push logs to.",
token: "Authentication token for the external provider used for identification."
},
DELETE: {
id: "The ID of the audit log stream to delete."
},
LIST: {
projectSlug: "The slug of the project to list audit log streams."
},
GET_BY_ID: {
id: "The ID of the audit log stream to get details."
}
};

View File

@@ -119,6 +119,7 @@ const envSchema = z
})
.transform((data) => ({
...data,
isCloud: Boolean(data.LICENSE_SERVER_KEY),
isSmtpConfigured: Boolean(data.SMTP_HOST),
isRedisConfigured: Boolean(data.REDIS_URL),
isDevelopmentMode: data.NODE_ENV === "development",

View File

@@ -1 +1,2 @@
export { isDisposableEmail } from "./validate-email";
export { validateLocalIps } from "./validate-url";

View File

@@ -0,0 +1,18 @@
import { getConfig } from "../config/env";
import { BadRequestError } from "../errors";
export const validateLocalIps = (url: string) => {
const validUrl = new URL(url);
const appCfg = getConfig();
// on cloud local ips are not allowed
if (
appCfg.isCloud &&
(validUrl.host === "host.docker.internal" ||
validUrl.host.match(/^10\.\d+\.\d+\.\d+/) ||
validUrl.host.match(/^192\.168\.\d+\.\d+/))
)
throw new BadRequestError({ message: "Local IPs not allowed as URL" });
if (validUrl.host === "localhost" || validUrl.host === "127.0.0.1")
throw new BadRequestError({ message: "Localhost not allowed" });
};

View File

@@ -5,6 +5,8 @@ import { registerV1EERoutes } from "@app/ee/routes/v1";
import { auditLogDALFactory } from "@app/ee/services/audit-log/audit-log-dal";
import { auditLogQueueServiceFactory } from "@app/ee/services/audit-log/audit-log-queue";
import { auditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service";
import { auditLogStreamDALFactory } from "@app/ee/services/audit-log-stream/audit-log-stream-dal";
import { auditLogStreamServiceFactory } from "@app/ee/services/audit-log-stream/audit-log-stream-service";
import { dynamicSecretDALFactory } from "@app/ee/services/dynamic-secret/dynamic-secret-dal";
import { dynamicSecretServiceFactory } from "@app/ee/services/dynamic-secret/dynamic-secret-service";
import { buildDynamicSecretProviders } from "@app/ee/services/dynamic-secret/providers";
@@ -193,6 +195,7 @@ export const registerRoutes = async (
const identityUaClientSecretDAL = identityUaClientSecretDALFactory(db);
const auditLogDAL = auditLogDALFactory(db);
const auditLogStreamDAL = auditLogStreamDALFactory(db);
const trustedIpDAL = trustedIpDALFactory(db);
const telemetryDAL = telemetryDALFactory(db);
@@ -243,9 +246,16 @@ export const registerRoutes = async (
auditLogDAL,
queueService,
projectDAL,
licenseService
licenseService,
auditLogStreamDAL
});
const auditLogService = auditLogServiceFactory({ auditLogDAL, permissionService, auditLogQueue });
const auditLogStreamService = auditLogStreamServiceFactory({
projectDAL,
licenseService,
permissionService,
auditLogStreamDAL
});
const sapService = secretApprovalPolicyServiceFactory({
projectMembershipDAL,
projectEnvDAL,
@@ -715,6 +725,7 @@ export const registerRoutes = async (
saml: samlService,
ldap: ldapService,
auditLog: auditLogService,
auditLogStream: auditLogStreamService,
secretScanning: secretScanningService,
license: licenseService,
trustedIp: trustedIpService,

View File

@@ -69,3 +69,10 @@ export const SanitizedDynamicSecretSchema = DynamicSecretsSchema.omit({
keyEncoding: true,
algorithm: true
});
export const SanitizedAuditLogStreamSchema = z.object({
id: z.string(),
url: z.string(),
createdAt: z.date(),
updatedAt: z.date()
});