From 5342c856968ad2c4773dc6aadd217a3c7f18c90e Mon Sep 17 00:00:00 2001
From: = <akhilmhdh@gmail.com>
Date: Thu, 4 Jul 2024 13:26:11 +0530
Subject: [PATCH 1/2] feat: changed audit log deletion to batched process

---
 .../ee/services/audit-log/audit-log-dal.ts    | 32 +++++++++++++++----
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts
index ffb7de4f3..e42736565 100644
--- a/backend/src/ee/services/audit-log/audit-log-dal.ts
+++ b/backend/src/ee/services/audit-log/audit-log-dal.ts
@@ -4,6 +4,7 @@ import { TDbClient } from "@app/db";
 import { TableName } from "@app/db/schemas";
 import { DatabaseError } from "@app/lib/errors";
 import { ormify, stripUndefinedInWhere } from "@app/lib/knex";
+import { logger } from "@app/lib/logger";
 
 export type TAuditLogDALFactory = ReturnType<typeof auditLogDALFactory>;
 
@@ -55,13 +56,30 @@ export const auditLogDALFactory = (db: TDbClient) => {
 
   // delete all audit log that have expired
   const pruneAuditLog = async (tx?: Knex) => {
-    try {
-      const today = new Date();
-      const docs = await (tx || db)(TableName.AuditLog).where("expiresAt", "<", today).del();
-      return docs;
-    } catch (error) {
-      throw new DatabaseError({ error, name: "PruneAuditLog" });
-    }
+    const AUDIT_LOG_PRUNE_BATCH_SIZE = 10000;
+    const MAX_RETRY_ON_FAILURE = 3;
+
+    const today = new Date();
+    let deletedAuditLogIds: { id: string }[] = [];
+    let numberOfRetryOnFailure = 0;
+
+    do {
+      try {
+        const findExpiredLogSubQuery = (tx || db)(TableName.AuditLog)
+          .where("expiresAt", "<", today)
+          .select("id")
+          .limit(AUDIT_LOG_PRUNE_BATCH_SIZE);
+        // eslint-disable-next-line no-await-in-loop
+        deletedAuditLogIds = await (tx || db)(TableName.AuditLog)
+          .whereIn("id", findExpiredLogSubQuery)
+          .del()
+          .returning("id");
+        numberOfRetryOnFailure = 0; // reset
+      } catch (error) {
+        numberOfRetryOnFailure += 1;
+        logger.error(error, "Failed to delete audit log on pruning");
+      }
+    } while (deletedAuditLogIds.length > 0 && numberOfRetryOnFailure < MAX_RETRY_ON_FAILURE);
   };
 
   return { ...auditLogOrm, pruneAuditLog, find };

From f4d9c6140465fbc3f0a1c9bfa940f5b8626d654b Mon Sep 17 00:00:00 2001
From: = <akhilmhdh@gmail.com>
Date: Thu, 4 Jul 2024 13:59:15 +0530
Subject: [PATCH 2/2] feat: added a pause in between as breather for db delete

---
 backend/src/ee/services/audit-log/audit-log-dal.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts
index e42736565..316cf34a5 100644
--- a/backend/src/ee/services/audit-log/audit-log-dal.ts
+++ b/backend/src/ee/services/audit-log/audit-log-dal.ts
@@ -75,6 +75,10 @@ export const auditLogDALFactory = (db: TDbClient) => {
           .del()
           .returning("id");
         numberOfRetryOnFailure = 0; // reset
+        // eslint-disable-next-line no-await-in-loop
+        await new Promise((resolve) => {
+          setTimeout(resolve, 100); // time to breathe for db
+        });
       } catch (error) {
         numberOfRetryOnFailure += 1;
         logger.error(error, "Failed to delete audit log on pruning");