feat(telemetry): add batch events and groups logic

backend/src/keystore/keystore.ts

@@ -72,6 +72,7 @@ type TWaitTillReady = {
 export type TKeyStoreFactory = {
   setItem: (key: string, value: string | number | Buffer, prefix?: string) => Promise<"OK">;
   getItem: (key: string, prefix?: string) => Promise<string | null>;
+  getItems: (keys: string[], prefix?: string) => Promise<(string | null)[]>;
   setExpiry: (key: string, expiryInSeconds: number) => Promise<number>;
   setItemWithExpiry: (
     key: string,
@@ -80,6 +81,7 @@ export type TKeyStoreFactory = {
     prefix?: string
   ) => Promise<"OK">;
   deleteItem: (key: string) => Promise<number>;
+  deleteItemsByKeyIn: (keys: string[]) => Promise<number>;
   deleteItems: (arg: TDeleteItems) => Promise<number>;
   incrementBy: (key: string, value: number) => Promise<number>;
   acquireLock(
@@ -88,6 +90,7 @@ export type TKeyStoreFactory = {
     settings?: Partial<Settings>
   ): Promise<{ release: () => Promise<ExecutionResult> }>;
   waitTillReady: ({ key, waitingCb, keyCheckCb, waitIteration, delay, jitter }: TWaitTillReady) => Promise<void>;
+  getKeysByPattern: (pattern: string, limit?: number) => Promise<string[]>;
 };
 
 export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFactory => {
@@ -99,6 +102,9 @@ export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFac
 
   const getItem = async (key: string, prefix?: string) => redis.get(prefix ? `${prefix}:${key}` : key);
 
+  const getItems = async (keys: string[], prefix?: string) =>
+    redis.mget(keys.map((key) => (prefix ? `${prefix}:${key}` : key)));
+
   const setItemWithExpiry = async (
     key: string,
     expiryInSeconds: number | string,
@@ -108,6 +114,8 @@ export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFac
 
   const deleteItem = async (key: string) => redis.del(key);
 
+  const deleteItemsByKeyIn = async (keys: string[]) => redis.del(keys);
+
   const deleteItems = async ({ pattern, batchSize = 500, delay = 1500, jitter = 200 }: TDeleteItems) => {
     let cursor = "0";
     let totalDeleted = 0;
@@ -163,6 +171,24 @@ export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFac
     }
   };
 
+  const getKeysByPattern = async (pattern: string, limit?: number) => {
+    let cursor = "0";
+    const allKeys: string[] = [];
+
+    do {
+      // eslint-disable-next-line no-await-in-loop
+      const [nextCursor, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 1000);
+      cursor = nextCursor;
+      allKeys.push(...keys);
+
+      if (limit && allKeys.length >= limit) {
+        return allKeys.slice(0, limit);
+      }
+    } while (cursor !== "0");
+
+    return allKeys;
+  };
+
   return {
     setItem,
     getItem,
@@ -174,6 +200,9 @@ export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFac
     acquireLock(resources: string[], duration: number, settings?: Partial<Settings>) {
       return redisLock.acquire(resources, duration, settings);
     },
-    waitTillReady
+    waitTillReady,
+    getKeysByPattern,
+    deleteItemsByKeyIn,
+    getItems
   };
 };

backend/src/keystore/memory.ts

@@ -59,6 +59,27 @@ export const inMemoryKeyStore = (): TKeyStoreFactory => {
         release: () => {}
       }) as Promise<Lock>;
     },
-    waitTillReady: async () => {}
+    waitTillReady: async () => {},
+    getKeysByPattern: async (pattern) => {
+      const regex = new RE2(`^${pattern.replace(/[-[\]/{}()+?.\\^$|]/g, "\\$&").replace(/\*/g, ".*")}$`);
+      const keys = Object.keys(store);
+      return keys.filter((key) => regex.test(key));
+    },
+    deleteItemsByKeyIn: async (keys) => {
+      for (const key of keys) {
+        delete store[key];
+      }
+      return keys.length;
+    },
+    getItems: async (keys) => {
+      const values = keys.map((key) => {
+        const value = store[key];
+        if (typeof value === "string") {
+          return value;
+        }
+        return null;
+      });
+      return values;
+    }
   };
 };