From 560f8d4a9bb296c0a7e4d4da3e30979d1e3c3abc Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Wed, 31 Jan 2024 22:07:49 +0530 Subject: [PATCH] feat: added audit log prune, resolved env update and pino file transport on prod --- backend/e2e-test/mocks/queue.ts | 4 +- .../ee/services/audit-log/audit-log-dal.ts | 56 +++++++++++------- .../ee/services/audit-log/audit-log-queue.ts | 58 ++++++++++++++++++- backend/src/lib/fn/dates.ts | 0 backend/src/lib/logger/logger.ts | 17 ++++-- backend/src/queue/queue-service.ts | 27 ++++++++- backend/src/server/routes/index.ts | 1 + .../project-env/project-env-service.ts | 2 +- docker-compose.pg.yml | 52 ++++++++--------- 9 files changed, 161 insertions(+), 56 deletions(-) create mode 100644 backend/src/lib/fn/dates.ts diff --git a/backend/e2e-test/mocks/queue.ts b/backend/e2e-test/mocks/queue.ts index f39e98738..c694979db 100644 --- a/backend/e2e-test/mocks/queue.ts +++ b/backend/e2e-test/mocks/queue.ts @@ -16,9 +16,11 @@ export const mockQueue = (): TQueueServiceFactory => { queues[name] = jobFn; workers[name] = jobFn; }, - listen: async (name, event) => { + listen: (name, event) => { events[name] = event; }, + clearQueue: async () => {}, + stopJobById: async () => {}, stopRepeatableJobByJobId: async () => true }; }; diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index c1368880d..aa9b5b4ab 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -2,6 +2,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { TableName } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; import { ormify, stripUndefinedInWhere } from "@app/lib/knex"; export type TAuditLogDALFactory = ReturnType; @@ -25,27 +26,42 @@ export const auditLogDALFactory = (db: TDbClient) => { { orgId, projectId, userAgentType, startDate, endDate, limit = 20, offset = 0, actor, eventType }: TFindQuery, tx?: Knex ) => { - const sqlQuery = (tx || db)(TableName.AuditLog) - .where( - stripUndefinedInWhere({ - projectId, - orgId, - eventType, - actor, - userAgentType - }) - ) - .limit(limit) - .offset(offset); - if (startDate) { - void sqlQuery.where("createdAt", ">=", startDate); + try { + const sqlQuery = (tx || db)(TableName.AuditLog) + .where( + stripUndefinedInWhere({ + projectId, + orgId, + eventType, + actor, + userAgentType + }) + ) + .limit(limit) + .offset(offset); + if (startDate) { + void sqlQuery.where("createdAt", ">=", startDate); + } + if (endDate) { + void sqlQuery.where("createdAt", "<=", endDate); + } + const docs = await sqlQuery; + return docs; + } catch (error) { + throw new DatabaseError({ error }); } - if (endDate) { - void sqlQuery.where("createdAt", "<=", endDate); - } - const docs = await sqlQuery; - return docs; }; - return { ...auditLogOrm, find }; + // delete all audit log that have expired + const pruneAuditLog = async (tx?: Knex) => { + try { + const today = new Date(); + const docs = await (tx || db)(TableName.AuditLog).where("expiresAt", "<", today).del(); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "PruneAuditLog" }); + } + }; + + return { ...auditLogOrm, pruneAuditLog, find }; }; diff --git a/backend/src/ee/services/audit-log/audit-log-queue.ts b/backend/src/ee/services/audit-log/audit-log-queue.ts index bcdf92920..670292da1 100644 --- a/backend/src/ee/services/audit-log/audit-log-queue.ts +++ b/backend/src/ee/services/audit-log/audit-log-queue.ts @@ -1,3 +1,4 @@ +import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { TProjectDALFactory } from "@app/services/project/project-dal"; @@ -14,6 +15,19 @@ type TAuditLogQueueServiceFactoryDep = { export type TAuditLogQueueServiceFactory = ReturnType; +const getTimeDiffForNextAuditLogPrune = (mills: number) => { + const today = new Date(mills); + // Get UTC midnight timestamp for today + const nextUtcMidnight = new Date(Date.UTC(today.getFullYear(), today.getMonth(), today.getDate(), 0, 0, 0)); + + // Check if we have already passed UTC midnight today + if (today.getTime() >= nextUtcMidnight.getTime()) { + // Add one day to get the timestamp for tomorrow's UTC midnight + nextUtcMidnight.setDate(nextUtcMidnight.getDate() + 1); + } + return nextUtcMidnight.getTime(); +}; + export const auditLogQueueServiceFactory = ({ auditLogDAL, queueService, @@ -43,6 +57,8 @@ export const auditLogQueueServiceFactory = ({ const plan = await licenseService.getPlan(orgId); const ttl = plan.auditLogsRetentionDays * MS_IN_DAY; + // skip inserting if audit log retension is 0 meaning its not supported + if (ttl === 0) return; await auditLogDAL.create({ actor: actor.type, actorMetadata: actor.metadata, @@ -57,7 +73,47 @@ export const auditLogQueueServiceFactory = ({ }); }); + queueService.start( + QueueName.AuditLogPrune, + async () => { + logger.info("Started audit log pruning"); + await auditLogDAL.pruneAuditLog(); + // calculate next utc time delay + // const nextPruneTime = getTimeDiffForNextAuditLogPrune(); + // await queueService.stopJobById(QueueName.AuditLogPrune, "audit-log-prune"); + logger.info("Finished audit log pruning"); + }, + { + settings: { + repeatStrategy: getTimeDiffForNextAuditLogPrune + } + } + ); + + // we are not using repeat because we want to run the in a predictable time of midnight UTC + // repeat has only cron job and every so we do the repeat manually + const startAuditLogPruneJob = async () => { + // clear previous job + await queueService.stopRepeatableJob( + QueueName.AuditLogPrune, + QueueJobs.AuditLogPrune, + {}, + QueueName.AuditLogPrune // just a job id + ); + await queueService.queue(QueueName.AuditLogPrune, QueueJobs.AuditLogPrune, undefined, { + delay: 5000, + jobId: QueueName.AuditLogPrune, + repeat: {} + }); + }; + + queueService.listen(QueueName.AuditLogPrune, "error", (err) => { + logger.error("Audit log pruning failed"); + logger.error(err); + }); + return { - pushToLog + pushToLog, + startAuditLogPruneJob }; }; diff --git a/backend/src/lib/fn/dates.ts b/backend/src/lib/fn/dates.ts new file mode 100644 index 000000000..e69de29bb diff --git a/backend/src/lib/logger/logger.ts b/backend/src/lib/logger/logger.ts index 67e200c8a..1cf4d11b8 100644 --- a/backend/src/lib/logger/logger.ts +++ b/backend/src/lib/logger/logger.ts @@ -26,14 +26,23 @@ const loggerConfig = z.object({ AWS_CLOUDWATCH_LOG_REGION: z.string().default("us-east-1"), AWS_CLOUDWATCH_LOG_ACCESS_KEY_ID: z.string().min(1).optional(), AWS_CLOUDWATCH_LOG_ACCESS_KEY_SECRET: z.string().min(1).optional(), - AWS_CLOUDWATCH_LOG_INTERVAL: z.coerce.number().default(1000) + AWS_CLOUDWATCH_LOG_INTERVAL: z.coerce.number().default(1000), + NODE_ENV: z.enum(["development", "test", "production"]).default("production") }); export const initLogger = async () => { - const targets: pino.TransportMultiOptions["targets"][number][] = [ - { level: "info", target: "pino/file", options: {} } - ]; const cfg = loggerConfig.parse(process.env); + const targets: pino.TransportMultiOptions["targets"][number][] = [ + { + level: "info", + target: "pino/file", + options: { + destination: cfg.NODE_ENV === "development" ? 1 : "/var/log/infisical", + mkdir: true + } + } + ]; + if (cfg.AWS_CLOUDWATCH_LOG_ACCESS_KEY_ID && cfg.AWS_CLOUDWATCH_LOG_ACCESS_KEY_SECRET) { targets.push({ target: "@serdnam/pino-cloudwatch-transport", diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index c46b60ca9..58c829549 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -1,4 +1,4 @@ -import { Job, JobsOptions, Queue, RepeatOptions, Worker, WorkerListener } from "bullmq"; +import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq"; import Redis from "ioredis"; import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types"; @@ -11,6 +11,7 @@ export enum QueueName { SecretRotation = "secret-rotation", SecretReminder = "secret-reminder", AuditLog = "audit-log", + AuditLogPrune = "audit-log-prune", IntegrationSync = "sync-integrations", SecretWebhook = "secret-webhook", SecretFullRepoScan = "secret-full-repo-scan", @@ -21,6 +22,7 @@ export enum QueueJobs { SecretReminder = "secret-reminder-job", SecretRotation = "secret-rotation-job", AuditLog = "audit-log-job", + AuditLogPrune = "audit-log-prune-job", SecWebhook = "secret-webhook-trigger", IntegrationSync = "secret-integration-pull", SecretScan = "secret-scan" @@ -45,6 +47,10 @@ export type TQueueJobTypes = { name: QueueJobs.AuditLog; payload: TCreateAuditLogDTO; }; + [QueueName.AuditLogPrune]: { + name: QueueJobs.AuditLogPrune; + payload: undefined; + }; [QueueName.SecretWebhook]: { name: QueueJobs.SecWebhook; payload: { projectId: string; environment: string; secretPath: string }; @@ -74,16 +80,20 @@ export const queueServiceFactory = (redisUrl: string) => { const start = ( name: T, - jobFn: (job: Job) => Promise + jobFn: (job: Job) => Promise, + queueSettings: Omit = {} ) => { if (queueContainer[name]) { throw new Error(`${name} queue is already initialized`); } queueContainer[name] = new Queue(name as string, { + ...queueSettings, connection }); + workerContainer[name] = new Worker(name, jobFn, { + ...queueSettings, connection }); }; @@ -129,9 +139,20 @@ export const queueServiceFactory = (redisUrl: string) => { return q.removeRepeatableByKey(job.repeatJobKey); }; + const stopJobById = async (name: T, jobId: string) => { + const q = queueContainer[name]; + const job = await q.getJob(jobId); + return job?.remove().catch(() => undefined); + }; + + const clearQueue = async (name: QueueName) => { + const q = queueContainer[name]; + await q.drain(); + }; + const shutdown = async () => { await Promise.all(Object.values(workerContainer).map((worker) => worker.close())); }; - return { start, listen, queue, shutdown, stopRepeatableJob, stopRepeatableJobByJobId }; + return { start, listen, queue, shutdown, stopRepeatableJob, stopRepeatableJobByJobId, clearQueue, stopJobById }; }; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 113060b65..fe79a6f11 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -444,6 +444,7 @@ export const registerRoutes = async ( }); await superAdminService.initServerCfg(); + await auditLogQueue.startAuditLogPruneJob(); // setup the communication with license key server await licenseService.init(); // inject all services diff --git a/backend/src/services/project-env/project-env-service.ts b/backend/src/services/project-env/project-env-service.ts index c430548cf..49657204e 100644 --- a/backend/src/services/project-env/project-env-service.ts +++ b/backend/src/services/project-env/project-env-service.ts @@ -67,7 +67,7 @@ export const projectEnvServiceFactory = ({ if (!oldEnv) throw new BadRequestError({ message: "Environment not found" }); if (slug) { - const existingEnv = await projectEnvDAL.findOne({ slug }); + const existingEnv = await projectEnvDAL.findOne({ slug, projectId }); if (existingEnv && existingEnv.id !== id) { throw new BadRequestError({ message: "Environment with slug already exist", diff --git a/docker-compose.pg.yml b/docker-compose.pg.yml index 353b6177b..62d0cc6ac 100644 --- a/docker-compose.pg.yml +++ b/docker-compose.pg.yml @@ -96,32 +96,32 @@ services: - 1025:1025 # SMTP server - 8025:8025 # Web UI - mongo: - image: mongo - container_name: infisical-dev-mongo - restart: always - env_file: .env - environment: - - MONGO_INITDB_ROOT_USERNAME=root - - MONGO_INITDB_ROOT_PASSWORD=example - volumes: - - mongo-data:/data/db - ports: - - 27017:27017 - - mongo-express: - container_name: infisical-dev-mongo-express - image: mongo-express - restart: always - depends_on: - - mongo - env_file: .env - environment: - - ME_CONFIG_MONGODB_ADMINUSERNAME=root - - ME_CONFIG_MONGODB_ADMINPASSWORD=example - - ME_CONFIG_MONGODB_URL=mongodb://root:example@mongo:27017/ - ports: - - 8081:8081 + # mongo: + # image: mongo + # container_name: infisical-dev-mongo + # restart: always + # env_file: .env + # environment: + # - MONGO_INITDB_ROOT_USERNAME=root + # - MONGO_INITDB_ROOT_PASSWORD=example + # volumes: + # - mongo-data:/data/db + # ports: + # - 27017:27017 + # + # mongo-express: + # container_name: infisical-dev-mongo-express + # image: mongo-express + # restart: always + # depends_on: + # - mongo + # env_file: .env + # environment: + # - ME_CONFIG_MONGODB_ADMINUSERNAME=root + # - ME_CONFIG_MONGODB_ADMINPASSWORD=example + # - ME_CONFIG_MONGODB_URL=mongodb://root:example@mongo:27017/ + # ports: + # - 8081:8081 volumes: postgres-data: