feat: added audit log prune, resolved env update and pino file transport on prod

backend/src/ee/services/audit-log/audit-log-dal.ts

@@ -2,6 +2,7 @@ import { Knex } from "knex";
 
 import { TDbClient } from "@app/db";
 import { TableName } from "@app/db/schemas";
+import { DatabaseError } from "@app/lib/errors";
 import { ormify, stripUndefinedInWhere } from "@app/lib/knex";
 
 export type TAuditLogDALFactory = ReturnType<typeof auditLogDALFactory>;
@@ -25,27 +26,42 @@ export const auditLogDALFactory = (db: TDbClient) => {
     { orgId, projectId, userAgentType, startDate, endDate, limit = 20, offset = 0, actor, eventType }: TFindQuery,
     tx?: Knex
   ) => {
-    const sqlQuery = (tx || db)(TableName.AuditLog)
-      .where(
-        stripUndefinedInWhere({
-          projectId,
-          orgId,
-          eventType,
-          actor,
-          userAgentType
-        })
-      )
-      .limit(limit)
-      .offset(offset);
-    if (startDate) {
-      void sqlQuery.where("createdAt", ">=", startDate);
+    try {
+      const sqlQuery = (tx || db)(TableName.AuditLog)
+        .where(
+          stripUndefinedInWhere({
+            projectId,
+            orgId,
+            eventType,
+            actor,
+            userAgentType
+          })
+        )
+        .limit(limit)
+        .offset(offset);
+      if (startDate) {
+        void sqlQuery.where("createdAt", ">=", startDate);
+      }
+      if (endDate) {
+        void sqlQuery.where("createdAt", "<=", endDate);
+      }
+      const docs = await sqlQuery;
+      return docs;
+    } catch (error) {
+      throw new DatabaseError({ error });
     }
-    if (endDate) {
-      void sqlQuery.where("createdAt", "<=", endDate);
-    }
-    const docs = await sqlQuery;
-    return docs;
   };
 
-  return { ...auditLogOrm, find };
+  // delete all audit log that have expired
+  const pruneAuditLog = async (tx?: Knex) => {
+    try {
+      const today = new Date();
+      const docs = await (tx || db)(TableName.AuditLog).where("expiresAt", "<", today).del();
+      return docs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "PruneAuditLog" });
+    }
+  };
+
+  return { ...auditLogOrm, pruneAuditLog, find };
 };

backend/src/ee/services/audit-log/audit-log-queue.ts

@@ -1,3 +1,4 @@
+import { logger } from "@app/lib/logger";
 import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
 
@@ -14,6 +15,19 @@ type TAuditLogQueueServiceFactoryDep = {
 
 export type TAuditLogQueueServiceFactory = ReturnType<typeof auditLogQueueServiceFactory>;
 
+const getTimeDiffForNextAuditLogPrune = (mills: number) => {
+  const today = new Date(mills);
+  // Get UTC midnight timestamp for today
+  const nextUtcMidnight = new Date(Date.UTC(today.getFullYear(), today.getMonth(), today.getDate(), 0, 0, 0));
+
+  // Check if we have already passed UTC midnight today
+  if (today.getTime() >= nextUtcMidnight.getTime()) {
+    // Add one day to get the timestamp for tomorrow's UTC midnight
+    nextUtcMidnight.setDate(nextUtcMidnight.getDate() + 1);
+  }
+  return nextUtcMidnight.getTime();
+};
+
 export const auditLogQueueServiceFactory = ({
   auditLogDAL,
   queueService,
@@ -43,6 +57,8 @@ export const auditLogQueueServiceFactory = ({
 
     const plan = await licenseService.getPlan(orgId);
     const ttl = plan.auditLogsRetentionDays * MS_IN_DAY;
+    // skip inserting if audit log retension is 0 meaning its not supported
+    if (ttl === 0) return;
     await auditLogDAL.create({
       actor: actor.type,
       actorMetadata: actor.metadata,
@@ -57,7 +73,47 @@ export const auditLogQueueServiceFactory = ({
     });
   });
 
+  queueService.start(
+    QueueName.AuditLogPrune,
+    async () => {
+      logger.info("Started audit log pruning");
+      await auditLogDAL.pruneAuditLog();
+      // calculate next utc time delay
+      // const nextPruneTime = getTimeDiffForNextAuditLogPrune();
+      // await queueService.stopJobById(QueueName.AuditLogPrune, "audit-log-prune");
+      logger.info("Finished audit log pruning");
+    },
+    {
+      settings: {
+        repeatStrategy: getTimeDiffForNextAuditLogPrune
+      }
+    }
+  );
+
+  // we are not using repeat because we want to run the in a predictable time of midnight UTC
+  // repeat has only cron job and every so we do the repeat manually
+  const startAuditLogPruneJob = async () => {
+    // clear previous job
+    await queueService.stopRepeatableJob(
+      QueueName.AuditLogPrune,
+      QueueJobs.AuditLogPrune,
+      {},
+      QueueName.AuditLogPrune // just a job id
+    );
+    await queueService.queue(QueueName.AuditLogPrune, QueueJobs.AuditLogPrune, undefined, {
+      delay: 5000,
+      jobId: QueueName.AuditLogPrune,
+      repeat: {}
+    });
+  };
+
+  queueService.listen(QueueName.AuditLogPrune, "error", (err) => {
+    logger.error("Audit log pruning failed");
+    logger.error(err);
+  });
+
   return {
-    pushToLog
+    pushToLog,
+    startAuditLogPruneJob
   };
 };