feat(telemetry): add batch events and groups logic

backend/src/services/telemetry/telemetry-queue.ts

@@ -7,13 +7,18 @@ import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
 
 import { getServerCfg } from "../super-admin/super-admin-service";
 import { TTelemetryDALFactory } from "./telemetry-dal";
-import { TELEMETRY_SECRET_OPERATIONS_KEY, TELEMETRY_SECRET_PROCESSED_KEY } from "./telemetry-service";
+import {
+  TELEMETRY_SECRET_OPERATIONS_KEY,
+  TELEMETRY_SECRET_PROCESSED_KEY,
+  TTelemetryServiceFactory
+} from "./telemetry-service";
 import { PostHogEventTypes } from "./telemetry-types";
 
 type TTelemetryQueueServiceFactoryDep = {
   queueService: TQueueServiceFactory;
   keyStore: Pick<TKeyStoreFactory, "getItem" | "deleteItem">;
   telemetryDAL: TTelemetryDALFactory;
+  telemetryService: TTelemetryServiceFactory;
 };
 
 export type TTelemetryQueueServiceFactory = ReturnType<typeof telemetryQueueServiceFactory>;
@@ -21,7 +26,8 @@ export type TTelemetryQueueServiceFactory = ReturnType<typeof telemetryQueueServ
 export const telemetryQueueServiceFactory = ({
   queueService,
   keyStore,
-  telemetryDAL
+  telemetryDAL,
+  telemetryService
 }: TTelemetryQueueServiceFactoryDep) => {
   const appCfg = getConfig();
   const postHog =
@@ -48,6 +54,14 @@ export const telemetryQueueServiceFactory = ({
     await keyStore.deleteItem(TELEMETRY_SECRET_OPERATIONS_KEY);
   });
 
+  queueService.start(QueueName.TelemetryAggregatedEvents, async () => {
+    try {
+      await telemetryService.processAggregatedEvents();
+    } catch (error) {
+      logger.error(error, "Failed to process aggregated telemetry events");
+    }
+  });
+
   // every day at midnight a telemetry job executes on self-hosted instances
   // this sends some telemetry information like instance id secrets operated etc
   const startTelemetryCheck = async () => {
@@ -60,11 +74,26 @@ export const telemetryQueueServiceFactory = ({
       { pattern: "0 0 * * *", utc: true },
       QueueName.TelemetryInstanceStats // just a job id
     );
+
+    // clear previous aggregated events job
+    await queueService.stopRepeatableJob(
+      QueueName.TelemetryAggregatedEvents,
+      QueueJobs.TelemetryAggregatedEvents,
+      { pattern: "*/2 * * * *", utc: true },
+      QueueName.TelemetryAggregatedEvents // just a job id
+    );
+
     if (postHog) {
       await queueService.queue(QueueName.TelemetryInstanceStats, QueueJobs.TelemetryInstanceStats, undefined, {
         jobId: QueueName.TelemetryInstanceStats,
         repeat: { pattern: "0 0 * * *", utc: true }
       });
+
+      // Start aggregated events job (runs every five minutes)
+      await queueService.queue(QueueName.TelemetryAggregatedEvents, QueueJobs.TelemetryAggregatedEvents, undefined, {
+        jobId: QueueName.TelemetryAggregatedEvents,
+        repeat: { pattern: "*/2 * * * *", utc: true }
+      });
     }
   };
 
@@ -72,6 +101,10 @@ export const telemetryQueueServiceFactory = ({
     logger.error(err?.failedReason, `${QueueName.TelemetryInstanceStats}: failed`);
   });
 
+  queueService.listen(QueueName.TelemetryAggregatedEvents, "failed", (err) => {
+    logger.error(err?.failedReason, `${QueueName.TelemetryAggregatedEvents}: failed`);
+  });
+
   return {
     startTelemetryCheck
   };