diff --git a/backend/e2e-test/mocks/keystore.ts b/backend/e2e-test/mocks/keystore.ts index 7a1bab2b3..c85244129 100644 --- a/backend/e2e-test/mocks/keystore.ts +++ b/backend/e2e-test/mocks/keystore.ts @@ -22,6 +22,9 @@ export const mockKeyStore = (): TKeyStoreFactory => { return value; } return null; + }, + incrementBy: async () => { + return 1; } }; }; diff --git a/backend/src/ee/services/license/license-service.ts b/backend/src/ee/services/license/license-service.ts index 6d97b537c..b0bc11e31 100644 --- a/backend/src/ee/services/license/license-service.ts +++ b/backend/src/ee/services/license/license-service.ts @@ -505,6 +505,9 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }: get isValidLicense() { return isValidLicense; }, + getInstanceType() { + return instanceType; + }, getPlan, updateSubscriptionOrgMemberCount, refreshPlan, diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 9458501b1..5e2c3aab3 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -14,5 +14,7 @@ export const keyStoreFactory = (redisUrl: string) => { const deleteItem = async (key: string) => redis.del(key); - return { setItem, getItem, setItemWithExpiry, deleteItem }; + const incrementBy = async (key: string, value: number) => redis.incrby(key, value); + + return { setItem, getItem, setItemWithExpiry, deleteItem, incrementBy }; }; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index e11e0b590..45c135b77 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -13,6 +13,7 @@ export enum QueueName { SecretReminder = "secret-reminder", AuditLog = "audit-log", AuditLogPrune = "audit-log-prune", + TelemetryInstanceStats = "telemtry-self-hosted-stats", IntegrationSync = "sync-integrations", SecretWebhook = "secret-webhook", SecretFullRepoScan = "secret-full-repo-scan", @@ -26,6 +27,7 @@ export enum QueueJobs { AuditLog = "audit-log-job", AuditLogPrune = "audit-log-prune-job", SecWebhook = "secret-webhook-trigger", + TelemetryInstanceStats = "telemetry-self-hosted-stats", IntegrationSync = "secret-integration-pull", SecretScan = "secret-scan", UpgradeProjectToGhost = "upgrade-project-to-ghost-job" @@ -67,7 +69,6 @@ export type TQueueJobTypes = { payload: TScanFullRepoEventPayload; }; [QueueName.SecretPushEventScan]: { name: QueueJobs.SecretScan; payload: TScanPushEventPayload }; - [QueueName.UpgradeProjectToGhost]: { name: QueueJobs.UpgradeProjectToGhost; payload: { @@ -81,6 +82,10 @@ export type TQueueJobTypes = { }; }; }; + [QueueName.TelemetryInstanceStats]: { + name: QueueJobs.TelemetryInstanceStats; + payload: undefined; + }; }; export type TQueueServiceFactory = ReturnType; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index e9c35b1d7..a21825633 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -97,6 +97,8 @@ import { serviceTokenServiceFactory } from "@app/services/service-token/service- import { TSmtpService } from "@app/services/smtp/smtp-service"; import { superAdminDALFactory } from "@app/services/super-admin/super-admin-dal"; import { getServerCfg, superAdminServiceFactory } from "@app/services/super-admin/super-admin-service"; +import { telemetryDALFactory } from "@app/services/telemetry/telemetry-dal"; +import { telemetryQueueServiceFactory } from "@app/services/telemetry/telemetry-queue"; import { telemetryServiceFactory } from "@app/services/telemetry/telemetry-service"; import { userDALFactory } from "@app/services/user/user-dal"; import { userServiceFactory } from "@app/services/user/user-service"; @@ -165,6 +167,7 @@ export const registerRoutes = async ( const auditLogDAL = auditLogDALFactory(db); const trustedIpDAL = trustedIpDALFactory(db); const scimDAL = scimDALFactory(db); + const telemetryDAL = telemetryDALFactory(db); // ee db layer ops const permissionDAL = permissionDALFactory(db); @@ -232,7 +235,16 @@ export const registerRoutes = async ( smtpService }); - const telemetryService = telemetryServiceFactory(); + const telemetryService = telemetryServiceFactory({ + keyStore, + licenseService + }); + const telemetryQueue = telemetryQueueServiceFactory({ + keyStore, + telemetryDAL, + queueService + }); + const tokenService = tokenServiceFactory({ tokenDAL: authTokenDAL, userDAL }); const userService = userServiceFactory({ userDAL }); const loginService = authLoginServiceFactory({ userDAL, smtpService, tokenService }); @@ -498,9 +510,13 @@ export const registerRoutes = async ( }); await superAdminService.initServerCfg(); - await auditLogQueue.startAuditLogPruneJob(); + // // setup the communication with license key server await licenseService.init(); + + await auditLogQueue.startAuditLogPruneJob(); + await telemetryQueue.startTelemetryCheck(); + // inject all services server.decorate("services", { login: loginService, diff --git a/backend/src/services/telemetry/telemetry-dal.ts b/backend/src/services/telemetry/telemetry-dal.ts new file mode 100644 index 000000000..9fac4f1ef --- /dev/null +++ b/backend/src/services/telemetry/telemetry-dal.ts @@ -0,0 +1,39 @@ +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; + +export type TTelemetryDALFactory = ReturnType; + +export const telemetryDALFactory = (db: TDbClient) => { + const getTelemetryInstanceStats = async () => { + try { + const userCount = (await db(TableName.Users).where({ isGhost: false }).count().first())?.count as string; + const users = parseInt(userCount || "0", 10); + + const identityCount = (await db(TableName.Identity).count().first())?.count as string; + const identities = parseInt(identityCount || "0", 10); + + const projectCount = (await db(TableName.Project).count().first())?.count as string; + const projects = parseInt(projectCount || "0", 10); + + const secretCount = (await db(TableName.Secret).count().first())?.count as string; + const secrets = parseInt(secretCount || "0", 10); + + const organizationNames = await db(TableName.Organization).select("name"); + const organizations = organizationNames.length; + + return { + users, + identities, + projects, + secrets, + organizations, + organizationNames: organizationNames.map(({ name }) => name) + }; + } catch (error) { + throw new DatabaseError({ error, name: "TelemtryInstanceStats" }); + } + }; + + return { getTelemetryInstanceStats }; +}; diff --git a/backend/src/services/telemetry/telemetry-queue.ts b/backend/src/services/telemetry/telemetry-queue.ts new file mode 100644 index 000000000..0a79aa9cc --- /dev/null +++ b/backend/src/services/telemetry/telemetry-queue.ts @@ -0,0 +1,78 @@ +import { PostHog } from "posthog-node"; + +import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { getConfig } from "@app/lib/config/env"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { getServerCfg } from "../super-admin/super-admin-service"; +import { TTelemetryDALFactory } from "./telemetry-dal"; +import { TELEMETRY_SECRET_OPERATONS_KEY, TELEMETRY_SECRET_PROCESSED_KEY } from "./telemetry-service"; +import { PostHogEventTypes } from "./telemetry-types"; + +type TTelemetryQueueServiceFactoryDep = { + queueService: TQueueServiceFactory; + keyStore: Pick; + telemetryDAL: TTelemetryDALFactory; +}; + +export type TTelemetryQueueServiceFactory = ReturnType; + +export const telemetryQueueServiceFactory = ({ + queueService, + keyStore, + telemetryDAL +}: TTelemetryQueueServiceFactoryDep) => { + const appCfg = getConfig(); + const postHog = + appCfg.isProductionMode && appCfg.TELEMETRY_ENABLED + ? new PostHog(appCfg.POSTHOG_PROJECT_API_KEY, { host: appCfg.POSTHOG_HOST }) + : undefined; + + queueService.start(QueueName.TelemetryInstanceStats, async () => { + const { instanceId } = await getServerCfg(); + const telemtryStats = await telemetryDAL.getTelemetryInstanceStats(); + // parse the redis values into integer + const numberOfSecretOperations = parseInt((await keyStore.getItem(TELEMETRY_SECRET_OPERATONS_KEY)) || "0", 10); + const numberOfSecretProcessed = parseInt((await keyStore.getItem(TELEMETRY_SECRET_PROCESSED_KEY)) || "0", 10); + const stats = { ...telemtryStats, numberOfSecretProcessed, numberOfSecretOperations }; + + // send to postHog + postHog?.capture({ + event: PostHogEventTypes.TelemetryInstanceStats, + distinctId: instanceId, + properties: stats + }); + // reset the stats + await keyStore.deleteItem(TELEMETRY_SECRET_PROCESSED_KEY); + await keyStore.deleteItem(TELEMETRY_SECRET_OPERATONS_KEY); + }); + + // every day at midnight a telemetry job executes on self hosted + // this sends some telemetry information like instance id secrets operated etc + const startTelemetryCheck = async () => { + // this is a fast way to check its cloud or not + if (appCfg.LICENSE_SERVER_KEY) return; + // clear previous job + await queueService.stopRepeatableJob( + QueueName.TelemetryInstanceStats, + QueueJobs.TelemetryInstanceStats, + { pattern: "0 0 * * *", utc: true }, + QueueName.TelemetryInstanceStats // just a job id + ); + if (postHog) { + await queueService.queue(QueueName.TelemetryInstanceStats, QueueJobs.TelemetryInstanceStats, undefined, { + jobId: QueueName.TelemetryInstanceStats, + repeat: { pattern: "0 0 * * *", utc: true } + }); + } + }; + + queueService.listen(QueueName.TelemetryInstanceStats, "failed", (err) => { + logger.error(err?.failedReason, `${QueueName.TelemetryInstanceStats}: failed`); + }); + + return { + startTelemetryCheck + }; +}; diff --git a/backend/src/services/telemetry/telemetry-service.ts b/backend/src/services/telemetry/telemetry-service.ts index a1a4b78f7..ebaa61444 100644 --- a/backend/src/services/telemetry/telemetry-service.ts +++ b/backend/src/services/telemetry/telemetry-service.ts @@ -1,15 +1,24 @@ import { PostHog } from "posthog-node"; +import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; +import { InstanceType } from "@app/ee/services/license/license-types"; +import { TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { request } from "@app/lib/config/request"; import { logger } from "@app/lib/logger"; -import { TPostHogEvent } from "./telemetry-types"; +import { PostHogEventTypes, TPostHogEvent, TSecretModifiedEvent } from "./telemetry-types"; + +export const TELEMETRY_SECRET_PROCESSED_KEY = "telemtry-secret-processed"; +export const TELEMETRY_SECRET_OPERATONS_KEY = "telemtry-secret-operations"; export type TTelemetryServiceFactory = ReturnType; +export type TTelemetryServiceFactoryDep = { + keyStore: Pick; + licenseService: Pick; +}; -// type TTelemetryServiceFactoryDep = {}; -export const telemetryServiceFactory = () => { +export const telemetryServiceFactory = ({ keyStore, licenseService }: TTelemetryServiceFactoryDep) => { const appCfg = getConfig(); if (appCfg.isProductionMode && !appCfg.TELEMETRY_ENABLED) { @@ -51,13 +60,33 @@ To opt into telemetry, you can set "TELEMETRY_ENABLED=true" within the environme } }; - const sendPostHogEvents = (event: TPostHogEvent) => { + const sendPostHogEvents = async (event: TPostHogEvent) => { if (postHog) { - postHog.capture({ - event: event.event, - distinctId: event.distinctId, - properties: event.properties - }); + const instanceType = licenseService.getInstanceType(); + // capture posthog only when its cloud or signup event happens in self hosted + if (instanceType === InstanceType.Cloud || event.event === PostHogEventTypes.UserSignedUp) { + postHog.capture({ + event: event.event, + distinctId: event.distinctId, + properties: event.properties + }); + return; + } + + if ( + [ + PostHogEventTypes.SecretPulled, + PostHogEventTypes.SecretCreated, + PostHogEventTypes.SecretDeleted, + PostHogEventTypes.SecretUpdated + ].includes(event.event) + ) { + await keyStore.incrementBy( + TELEMETRY_SECRET_PROCESSED_KEY, + (event as TSecretModifiedEvent).properties.numberOfSecrets + ); + await keyStore.incrementBy(TELEMETRY_SECRET_OPERATONS_KEY, 1); + } } }; diff --git a/backend/src/services/telemetry/telemetry-types.ts b/backend/src/services/telemetry/telemetry-types.ts index 0f92da4da..9feb56b14 100644 --- a/backend/src/services/telemetry/telemetry-types.ts +++ b/backend/src/services/telemetry/telemetry-types.ts @@ -12,7 +12,8 @@ export enum PostHogEventTypes { ProjectCreated = "Project Created", IntegrationCreated = "Integration Created", MachineIdentityCreated = "Machine Identity Created", - UserOrgInvitation = "User Org Invitation" + UserOrgInvitation = "User Org Invitation", + TelemetryInstanceStats = "Self Hosted Instance Stats" } export type TSecretModifiedEvent = { @@ -101,6 +102,20 @@ export type TUserOrgInvitedEvent = { }; }; +export type TTelemetryInstanceStatsEvent = { + event: PostHogEventTypes.TelemetryInstanceStats; + properties: { + users: number; + identities: number; + projects: number; + secrets: number; + organizations: number; + organizationNames: number; + numberOfSecretOperations: number; + numberOfSecretProcessed: number; + }; +}; + export type TPostHogEvent = { distinctId: string } & ( | TSecretModifiedEvent | TAdminInitEvent @@ -110,4 +125,5 @@ export type TPostHogEvent = { distinctId: string } & ( | TMachineIdentityCreatedEvent | TIntegrationCreatedEvent | TProjectCreateEvent + | TTelemetryInstanceStatsEvent );