feat(server): added telemetry queue for self hosted to upload instance stats to posthog on midnight

backend/e2e-test/mocks/keystore.ts

@@ -22,6 +22,9 @@ export const mockKeyStore = (): TKeyStoreFactory => {
         return value;
       }
       return null;
+    },
+    incrementBy: async () => {
+      return 1;
     }
   };
 };

backend/src/ee/services/license/license-service.ts

@@ -505,6 +505,9 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }:
     get isValidLicense() {
       return isValidLicense;
     },
+    getInstanceType() {
+      return instanceType;
+    },
     getPlan,
     updateSubscriptionOrgMemberCount,
     refreshPlan,

backend/src/keystore/keystore.ts

@@ -14,5 +14,7 @@ export const keyStoreFactory = (redisUrl: string) => {
 
   const deleteItem = async (key: string) => redis.del(key);
 
-  return { setItem, getItem, setItemWithExpiry, deleteItem };
+  const incrementBy = async (key: string, value: number) => redis.incrby(key, value);
+
+  return { setItem, getItem, setItemWithExpiry, deleteItem, incrementBy };
 };

backend/src/queue/queue-service.ts

@@ -13,6 +13,7 @@ export enum QueueName {
   SecretReminder = "secret-reminder",
   AuditLog = "audit-log",
   AuditLogPrune = "audit-log-prune",
+  TelemetryInstanceStats = "telemtry-self-hosted-stats",
   IntegrationSync = "sync-integrations",
   SecretWebhook = "secret-webhook",
   SecretFullRepoScan = "secret-full-repo-scan",
@@ -26,6 +27,7 @@ export enum QueueJobs {
   AuditLog = "audit-log-job",
   AuditLogPrune = "audit-log-prune-job",
   SecWebhook = "secret-webhook-trigger",
+  TelemetryInstanceStats = "telemetry-self-hosted-stats",
   IntegrationSync = "secret-integration-pull",
   SecretScan = "secret-scan",
   UpgradeProjectToGhost = "upgrade-project-to-ghost-job"
@@ -67,7 +69,6 @@ export type TQueueJobTypes = {
     payload: TScanFullRepoEventPayload;
   };
   [QueueName.SecretPushEventScan]: { name: QueueJobs.SecretScan; payload: TScanPushEventPayload };
-
   [QueueName.UpgradeProjectToGhost]: {
     name: QueueJobs.UpgradeProjectToGhost;
     payload: {
@@ -81,6 +82,10 @@ export type TQueueJobTypes = {
       };
     };
   };
+  [QueueName.TelemetryInstanceStats]: {
+    name: QueueJobs.TelemetryInstanceStats;
+    payload: undefined;
+  };
 };
 
 export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;

backend/src/server/routes/index.ts

@@ -97,6 +97,8 @@ import { serviceTokenServiceFactory } from "@app/services/service-token/service-
 import { TSmtpService } from "@app/services/smtp/smtp-service";
 import { superAdminDALFactory } from "@app/services/super-admin/super-admin-dal";
 import { getServerCfg, superAdminServiceFactory } from "@app/services/super-admin/super-admin-service";
+import { telemetryDALFactory } from "@app/services/telemetry/telemetry-dal";
+import { telemetryQueueServiceFactory } from "@app/services/telemetry/telemetry-queue";
 import { telemetryServiceFactory } from "@app/services/telemetry/telemetry-service";
 import { userDALFactory } from "@app/services/user/user-dal";
 import { userServiceFactory } from "@app/services/user/user-service";
@@ -165,6 +167,7 @@ export const registerRoutes = async (
   const auditLogDAL = auditLogDALFactory(db);
   const trustedIpDAL = trustedIpDALFactory(db);
   const scimDAL = scimDALFactory(db);
+  const telemetryDAL = telemetryDALFactory(db);
 
   // ee db layer ops
   const permissionDAL = permissionDALFactory(db);
@@ -232,7 +235,16 @@ export const registerRoutes = async (
     smtpService
   });
 
-  const telemetryService = telemetryServiceFactory();
+  const telemetryService = telemetryServiceFactory({
+    keyStore,
+    licenseService
+  });
+  const telemetryQueue = telemetryQueueServiceFactory({
+    keyStore,
+    telemetryDAL,
+    queueService
+  });
+
   const tokenService = tokenServiceFactory({ tokenDAL: authTokenDAL, userDAL });
   const userService = userServiceFactory({ userDAL });
   const loginService = authLoginServiceFactory({ userDAL, smtpService, tokenService });
@@ -498,9 +510,13 @@ export const registerRoutes = async (
   });
 
   await superAdminService.initServerCfg();
-  await auditLogQueue.startAuditLogPruneJob();
+  //
   // setup the communication with license key server
   await licenseService.init();
+
+  await auditLogQueue.startAuditLogPruneJob();
+  await telemetryQueue.startTelemetryCheck();
+
   // inject all services
   server.decorate<FastifyZodProvider["services"]>("services", {
     login: loginService,

backend/src/services/telemetry/telemetry-dal.ts

@@ -0,0 +1,39 @@
+import { TDbClient } from "@app/db";
+import { TableName } from "@app/db/schemas";
+import { DatabaseError } from "@app/lib/errors";
+
+export type TTelemetryDALFactory = ReturnType<typeof telemetryDALFactory>;
+
+export const telemetryDALFactory = (db: TDbClient) => {
+  const getTelemetryInstanceStats = async () => {
+    try {
+      const userCount = (await db(TableName.Users).where({ isGhost: false }).count().first())?.count as string;
+      const users = parseInt(userCount || "0", 10);
+
+      const identityCount = (await db(TableName.Identity).count().first())?.count as string;
+      const identities = parseInt(identityCount || "0", 10);
+
+      const projectCount = (await db(TableName.Project).count().first())?.count as string;
+      const projects = parseInt(projectCount || "0", 10);
+
+      const secretCount = (await db(TableName.Secret).count().first())?.count as string;
+      const secrets = parseInt(secretCount || "0", 10);
+
+      const organizationNames = await db(TableName.Organization).select("name");
+      const organizations = organizationNames.length;
+
+      return {
+        users,
+        identities,
+        projects,
+        secrets,
+        organizations,
+        organizationNames: organizationNames.map(({ name }) => name)
+      };
+    } catch (error) {
+      throw new DatabaseError({ error, name: "TelemtryInstanceStats" });
+    }
+  };
+
+  return { getTelemetryInstanceStats };
+};

backend/src/services/telemetry/telemetry-queue.ts

@@ -0,0 +1,78 @@
+import { PostHog } from "posthog-node";
+
+import { TKeyStoreFactory } from "@app/keystore/keystore";
+import { getConfig } from "@app/lib/config/env";
+import { logger } from "@app/lib/logger";
+import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
+
+import { getServerCfg } from "../super-admin/super-admin-service";
+import { TTelemetryDALFactory } from "./telemetry-dal";
+import { TELEMETRY_SECRET_OPERATONS_KEY, TELEMETRY_SECRET_PROCESSED_KEY } from "./telemetry-service";
+import { PostHogEventTypes } from "./telemetry-types";
+
+type TTelemetryQueueServiceFactoryDep = {
+  queueService: TQueueServiceFactory;
+  keyStore: Pick<TKeyStoreFactory, "getItem" | "deleteItem">;
+  telemetryDAL: TTelemetryDALFactory;
+};
+
+export type TTelemetryQueueServiceFactory = ReturnType<typeof telemetryQueueServiceFactory>;
+
+export const telemetryQueueServiceFactory = ({
+  queueService,
+  keyStore,
+  telemetryDAL
+}: TTelemetryQueueServiceFactoryDep) => {
+  const appCfg = getConfig();
+  const postHog =
+    appCfg.isProductionMode && appCfg.TELEMETRY_ENABLED
+      ? new PostHog(appCfg.POSTHOG_PROJECT_API_KEY, { host: appCfg.POSTHOG_HOST })
+      : undefined;
+
+  queueService.start(QueueName.TelemetryInstanceStats, async () => {
+    const { instanceId } = await getServerCfg();
+    const telemtryStats = await telemetryDAL.getTelemetryInstanceStats();
+    // parse the redis values into integer
+    const numberOfSecretOperations = parseInt((await keyStore.getItem(TELEMETRY_SECRET_OPERATONS_KEY)) || "0", 10);
+    const numberOfSecretProcessed = parseInt((await keyStore.getItem(TELEMETRY_SECRET_PROCESSED_KEY)) || "0", 10);
+    const stats = { ...telemtryStats, numberOfSecretProcessed, numberOfSecretOperations };
+
+    // send to postHog
+    postHog?.capture({
+      event: PostHogEventTypes.TelemetryInstanceStats,
+      distinctId: instanceId,
+      properties: stats
+    });
+    // reset the stats
+    await keyStore.deleteItem(TELEMETRY_SECRET_PROCESSED_KEY);
+    await keyStore.deleteItem(TELEMETRY_SECRET_OPERATONS_KEY);
+  });
+
+  // every day at midnight a telemetry job executes on self hosted
+  // this sends some telemetry information like instance id secrets operated etc
+  const startTelemetryCheck = async () => {
+    // this is a fast way to check its cloud or not
+    if (appCfg.LICENSE_SERVER_KEY) return;
+    // clear previous job
+    await queueService.stopRepeatableJob(
+      QueueName.TelemetryInstanceStats,
+      QueueJobs.TelemetryInstanceStats,
+      { pattern: "0 0 * * *", utc: true },
+      QueueName.TelemetryInstanceStats // just a job id
+    );
+    if (postHog) {
+      await queueService.queue(QueueName.TelemetryInstanceStats, QueueJobs.TelemetryInstanceStats, undefined, {
+        jobId: QueueName.TelemetryInstanceStats,
+        repeat: { pattern: "0 0 * * *", utc: true }
+      });
+    }
+  };
+
+  queueService.listen(QueueName.TelemetryInstanceStats, "failed", (err) => {
+    logger.error(err?.failedReason, `${QueueName.TelemetryInstanceStats}: failed`);
+  });
+
+  return {
+    startTelemetryCheck
+  };
+};

backend/src/services/telemetry/telemetry-service.ts

@@ -1,15 +1,24 @@
 import { PostHog } from "posthog-node";
 
+import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
+import { InstanceType } from "@app/ee/services/license/license-types";
+import { TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig } from "@app/lib/config/env";
 import { request } from "@app/lib/config/request";
 import { logger } from "@app/lib/logger";
 
-import { TPostHogEvent } from "./telemetry-types";
+import { PostHogEventTypes, TPostHogEvent, TSecretModifiedEvent } from "./telemetry-types";
+
+export const TELEMETRY_SECRET_PROCESSED_KEY = "telemtry-secret-processed";
+export const TELEMETRY_SECRET_OPERATONS_KEY = "telemtry-secret-operations";
 
 export type TTelemetryServiceFactory = ReturnType<typeof telemetryServiceFactory>;
+export type TTelemetryServiceFactoryDep = {
+  keyStore: Pick<TKeyStoreFactory, "getItem" | "incrementBy">;
+  licenseService: Pick<TLicenseServiceFactory, "getInstanceType">;
+};
 
-// type TTelemetryServiceFactoryDep = {};
-export const telemetryServiceFactory = () => {
+export const telemetryServiceFactory = ({ keyStore, licenseService }: TTelemetryServiceFactoryDep) => {
   const appCfg = getConfig();
 
   if (appCfg.isProductionMode && !appCfg.TELEMETRY_ENABLED) {
@@ -51,13 +60,33 @@ To opt into telemetry, you can set "TELEMETRY_ENABLED=true" within the environme
     }
   };
 
-  const sendPostHogEvents = (event: TPostHogEvent) => {
+  const sendPostHogEvents = async (event: TPostHogEvent) => {
     if (postHog) {
-      postHog.capture({
-        event: event.event,
-        distinctId: event.distinctId,
-        properties: event.properties
-      });
+      const instanceType = licenseService.getInstanceType();
+      // capture posthog only when its cloud or signup event happens in self hosted
+      if (instanceType === InstanceType.Cloud || event.event === PostHogEventTypes.UserSignedUp) {
+        postHog.capture({
+          event: event.event,
+          distinctId: event.distinctId,
+          properties: event.properties
+        });
+        return;
+      }
+
+      if (
+        [
+          PostHogEventTypes.SecretPulled,
+          PostHogEventTypes.SecretCreated,
+          PostHogEventTypes.SecretDeleted,
+          PostHogEventTypes.SecretUpdated
+        ].includes(event.event)
+      ) {
+        await keyStore.incrementBy(
+          TELEMETRY_SECRET_PROCESSED_KEY,
+          (event as TSecretModifiedEvent).properties.numberOfSecrets
+        );
+        await keyStore.incrementBy(TELEMETRY_SECRET_OPERATONS_KEY, 1);
+      }
     }
   };
 

backend/src/services/telemetry/telemetry-types.ts

@@ -12,7 +12,8 @@ export enum PostHogEventTypes {
   ProjectCreated = "Project Created",
   IntegrationCreated = "Integration Created",
   MachineIdentityCreated = "Machine Identity Created",
-  UserOrgInvitation = "User Org Invitation"
+  UserOrgInvitation = "User Org Invitation",
+  TelemetryInstanceStats = "Self Hosted Instance Stats"
 }
 
 export type TSecretModifiedEvent = {
@@ -101,6 +102,20 @@ export type TUserOrgInvitedEvent = {
   };
 };
 
+export type TTelemetryInstanceStatsEvent = {
+  event: PostHogEventTypes.TelemetryInstanceStats;
+  properties: {
+    users: number;
+    identities: number;
+    projects: number;
+    secrets: number;
+    organizations: number;
+    organizationNames: number;
+    numberOfSecretOperations: number;
+    numberOfSecretProcessed: number;
+  };
+};
+
 export type TPostHogEvent = { distinctId: string } & (
   | TSecretModifiedEvent
   | TAdminInitEvent
@@ -110,4 +125,5 @@ export type TPostHogEvent = { distinctId: string } & (
   | TMachineIdentityCreatedEvent
   | TIntegrationCreatedEvent
   | TProjectCreateEvent
+  | TTelemetryInstanceStatsEvent
 );