Merge pull request #3866 from Infisical/feat/posthogEventBatch

feat(telemetry): Add aggregated events and groups to posthog

backend/src/keystore/keystore.ts

@@ -73,6 +73,7 @@ type TWaitTillReady = {
 export type TKeyStoreFactory = {
   setItem: (key: string, value: string | number | Buffer, prefix?: string) => Promise<"OK">;
   getItem: (key: string, prefix?: string) => Promise<string | null>;
+  getItems: (keys: string[], prefix?: string) => Promise<(string | null)[]>;
   setExpiry: (key: string, expiryInSeconds: number) => Promise<number>;
   setItemWithExpiry: (
     key: string,
@@ -81,6 +82,7 @@ export type TKeyStoreFactory = {
     prefix?: string
   ) => Promise<"OK">;
   deleteItem: (key: string) => Promise<number>;
+  deleteItemsByKeyIn: (keys: string[]) => Promise<number>;
   deleteItems: (arg: TDeleteItems) => Promise<number>;
   incrementBy: (key: string, value: number) => Promise<number>;
   acquireLock(
@@ -89,6 +91,7 @@ export type TKeyStoreFactory = {
     settings?: Partial<Settings>
   ): Promise<{ release: () => Promise<ExecutionResult> }>;
   waitTillReady: ({ key, waitingCb, keyCheckCb, waitIteration, delay, jitter }: TWaitTillReady) => Promise<void>;
+  getKeysByPattern: (pattern: string, limit?: number) => Promise<string[]>;
 };
 
 export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFactory => {
@@ -100,6 +103,9 @@ export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFac
 
   const getItem = async (key: string, prefix?: string) => redis.get(prefix ? `${prefix}:${key}` : key);
 
+  const getItems = async (keys: string[], prefix?: string) =>
+    redis.mget(keys.map((key) => (prefix ? `${prefix}:${key}` : key)));
+
   const setItemWithExpiry = async (
     key: string,
     expiryInSeconds: number | string,
@@ -109,6 +115,11 @@ export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFac
 
   const deleteItem = async (key: string) => redis.del(key);
 
+  const deleteItemsByKeyIn = async (keys: string[]) => {
+    if (keys.length === 0) return 0;
+    return redis.del(keys);
+  };
+
   const deleteItems = async ({ pattern, batchSize = 500, delay = 1500, jitter = 200 }: TDeleteItems) => {
     let cursor = "0";
     let totalDeleted = 0;
@@ -164,6 +175,24 @@ export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFac
     }
   };
 
+  const getKeysByPattern = async (pattern: string, limit?: number) => {
+    let cursor = "0";
+    const allKeys: string[] = [];
+
+    do {
+      // eslint-disable-next-line no-await-in-loop
+      const [nextCursor, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 1000);
+      cursor = nextCursor;
+      allKeys.push(...keys);
+
+      if (limit && allKeys.length >= limit) {
+        return allKeys.slice(0, limit);
+      }
+    } while (cursor !== "0");
+
+    return allKeys;
+  };
+
   return {
     setItem,
     getItem,
@@ -175,6 +204,9 @@ export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFac
     acquireLock(resources: string[], duration: number, settings?: Partial<Settings>) {
       return redisLock.acquire(resources, duration, settings);
     },
-    waitTillReady
+    waitTillReady,
+    getKeysByPattern,
+    deleteItemsByKeyIn,
+    getItems
   };
 };

backend/src/keystore/memory.ts

@@ -8,6 +8,8 @@ import { TKeyStoreFactory } from "./keystore";
 
 export const inMemoryKeyStore = (): TKeyStoreFactory => {
   const store: Record<string, string | number | Buffer> = {};
+  const getRegex = (pattern: string) =>
+    new RE2(`^${pattern.replace(/[-[\]/{}()+?.\\^$|]/g, "\\$&").replace(/\*/g, ".*")}$`);
 
   return {
     setItem: async (key, value) => {
@@ -24,7 +26,7 @@ export const inMemoryKeyStore = (): TKeyStoreFactory => {
       return 1;
     },
     deleteItems: async ({ pattern, batchSize = 500, delay = 1500, jitter = 200 }) => {
-      const regex = new RE2(`^${pattern.replace(/[-[\]/{}()+?.\\^$|]/g, "\\$&").replace(/\*/g, ".*")}$`);
+      const regex = getRegex(pattern);
       let totalDeleted = 0;
       const keys = Object.keys(store);
 
@@ -59,6 +61,27 @@ export const inMemoryKeyStore = (): TKeyStoreFactory => {
         release: () => {}
       }) as Promise<Lock>;
     },
-    waitTillReady: async () => {}
+    waitTillReady: async () => {},
+    getKeysByPattern: async (pattern) => {
+      const regex = getRegex(pattern);
+      const keys = Object.keys(store);
+      return keys.filter((key) => regex.test(key));
+    },
+    deleteItemsByKeyIn: async (keys) => {
+      for (const key of keys) {
+        delete store[key];
+      }
+      return keys.length;
+    },
+    getItems: async (keys) => {
+      const values = keys.map((key) => {
+        const value = store[key];
+        if (typeof value === "string") {
+          return value;
+        }
+        return null;
+      });
+      return values;
+    }
   };
 };