feat: added audit log prune, resolved env update and pino file transport on prod

backend/src/ee/services/audit-log/audit-log-queue.ts

@@ -1,3 +1,4 @@
+import { logger } from "@app/lib/logger";
 import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
 
@@ -14,6 +15,19 @@ type TAuditLogQueueServiceFactoryDep = {
 
 export type TAuditLogQueueServiceFactory = ReturnType<typeof auditLogQueueServiceFactory>;
 
+const getTimeDiffForNextAuditLogPrune = (mills: number) => {
+  const today = new Date(mills);
+  // Get UTC midnight timestamp for today
+  const nextUtcMidnight = new Date(Date.UTC(today.getFullYear(), today.getMonth(), today.getDate(), 0, 0, 0));
+
+  // Check if we have already passed UTC midnight today
+  if (today.getTime() >= nextUtcMidnight.getTime()) {
+    // Add one day to get the timestamp for tomorrow's UTC midnight
+    nextUtcMidnight.setDate(nextUtcMidnight.getDate() + 1);
+  }
+  return nextUtcMidnight.getTime();
+};
+
 export const auditLogQueueServiceFactory = ({
   auditLogDAL,
   queueService,
@@ -43,6 +57,8 @@ export const auditLogQueueServiceFactory = ({
 
     const plan = await licenseService.getPlan(orgId);
     const ttl = plan.auditLogsRetentionDays * MS_IN_DAY;
+    // skip inserting if audit log retension is 0 meaning its not supported
+    if (ttl === 0) return;
     await auditLogDAL.create({
       actor: actor.type,
       actorMetadata: actor.metadata,
@@ -57,7 +73,47 @@ export const auditLogQueueServiceFactory = ({
     });
   });
 
+  queueService.start(
+    QueueName.AuditLogPrune,
+    async () => {
+      logger.info("Started audit log pruning");
+      await auditLogDAL.pruneAuditLog();
+      // calculate next utc time delay
+      // const nextPruneTime = getTimeDiffForNextAuditLogPrune();
+      // await queueService.stopJobById(QueueName.AuditLogPrune, "audit-log-prune");
+      logger.info("Finished audit log pruning");
+    },
+    {
+      settings: {
+        repeatStrategy: getTimeDiffForNextAuditLogPrune
+      }
+    }
+  );
+
+  // we are not using repeat because we want to run the in a predictable time of midnight UTC
+  // repeat has only cron job and every so we do the repeat manually
+  const startAuditLogPruneJob = async () => {
+    // clear previous job
+    await queueService.stopRepeatableJob(
+      QueueName.AuditLogPrune,
+      QueueJobs.AuditLogPrune,
+      {},
+      QueueName.AuditLogPrune // just a job id
+    );
+    await queueService.queue(QueueName.AuditLogPrune, QueueJobs.AuditLogPrune, undefined, {
+      delay: 5000,
+      jobId: QueueName.AuditLogPrune,
+      repeat: {}
+    });
+  };
+
+  queueService.listen(QueueName.AuditLogPrune, "error", (err) => {
+    logger.error("Audit log pruning failed");
+    logger.error(err);
+  });
+
   return {
-    pushToLog
+    pushToLog,
+    startAuditLogPruneJob
   };
 };