misc: finalized structure

backend/e2e-test/mocks/queue.ts

@@ -10,12 +10,15 @@ export const mockQueue = (): TQueueServiceFactory => {
     queue: async (name, jobData) => {
       job[name] = jobData;
     },
+    queuePg: async () => {},
+    initialize: async () => {},
     shutdown: async () => undefined,
     stopRepeatableJob: async () => true,
     start: (name, jobFn) => {
       queues[name] = jobFn;
       workers[name] = jobFn;
     },
+    startPg: async () => {},
     listen: (name, event) => {
       events[name] = event;
     },

backend/src/ee/services/audit-log/audit-log-queue.ts

@@ -34,8 +34,9 @@ export const auditLogQueueServiceFactory = async ({
   licenseService,
   auditLogStreamDAL
 }: TAuditLogQueueServiceFactoryDep) => {
-  const pushToLog = async (data: TCreateAuditLogDTO) => {
   const appCfg = getConfig();
+
+  const pushToLog = async (data: TCreateAuditLogDTO) => {
     if (appCfg.USE_PG_QUEUE) {
       await queueService.queuePg<QueueName.AuditLog>(QueueJobs.AuditLog, data, {
         retryLimit: 10,
@@ -51,6 +52,7 @@ export const auditLogQueueServiceFactory = async ({
     }
   };
 
+  if (appCfg.USE_PG_QUEUE) {
     await queueService.startPg<QueueName.AuditLog>(
       QueueJobs.AuditLog,
       async ([job]) => {
@@ -141,6 +143,7 @@ export const auditLogQueueServiceFactory = async ({
         pollingIntervalSeconds: 0.5
       }
     );
+  }
 
   queueService.start(QueueName.AuditLog, async (job) => {
     const { actor, event, ipAddress, projectId, userAgent, userAgentType } = job.data;

backend/src/main.ts

@@ -57,11 +57,7 @@ const run = async () => {
   const smtp = smtpServiceFactory(formatSmtpConfig());
 
   const queue = queueServiceFactory(appCfg.REDIS_URL, appCfg.DB_CONNECTION_URI);
-
-  if (appCfg.USE_PG_QUEUE) {
-    logger.info("Initializing PG queue...");
   await queue.initialize();
-  }
 
   const keyStore = keyStoreFactory(appCfg.REDIS_URL);
 

backend/src/queue/queue-service.ts

@@ -8,6 +8,7 @@ import {
   TScanFullRepoEventPayload,
   TScanPushEventPayload
 } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
+import { getConfig } from "@app/lib/config/env";
 import { logger } from "@app/lib/logger";
 import {
   TFailedIntegrationSyncEmailsPayload,
@@ -195,8 +196,8 @@ export const queueServiceFactory = (redisUrl: string, dbConnectionUrl: string) =
 
   const pgBoss = new PgBoss({
     connectionString: dbConnectionUrl,
-    archiveCompletedAfterSeconds: 30,
-    maintenanceIntervalSeconds: 30,
+    archiveCompletedAfterSeconds: 60,
+    archiveFailedAfterSeconds: 1000, // we want to keep failed jobs for a longer time so that it can be retried
     deleteAfterSeconds: 30
   });
 
@@ -208,11 +209,15 @@ export const queueServiceFactory = (redisUrl: string, dbConnectionUrl: string) =
   >;
 
   const initialize = async () => {
+    const appCfg = getConfig();
+    if (appCfg.USE_PG_QUEUE) {
+      logger.info("Initializing PG queue...");
       await pgBoss.start();
 
       pgBoss.on("error", (error) => {
         logger.error(error, "pg-queue error");
       });
+    }
   };
 
   const start = <T extends QueueName>(