feat(infisical-pg): added telemtry service

This commit is contained in:
Akhil Mohan
2024-01-25 21:10:32 +05:30
parent ca858f8e13
commit 9677836b76
15 changed files with 418 additions and 7 deletions

View File

@@ -1,2 +1,3 @@
vitest-environment-infisical.ts
vitest.config.ts
.eslintrc.js

View File

@@ -13,7 +13,7 @@ module.exports = {
tsconfigRootDir: __dirname
},
rules: {
"@typescript-eslint/no-empty-function": "off",
// "@typescript-eslint/no-empty-function": "off",
"consistent-return": "off", // my style
"import/order": "off", // for simple-import-order
"import/prefer-default-export": "off", // why

View File

@@ -55,6 +55,7 @@
"pg": "^8.11.3",
"picomatch": "^3.0.1",
"pino": "^8.16.2",
"posthog-node": "^3.6.0",
"probot": "^12.3.3",
"smee-client": "^2.0.0",
"tweetnacl": "^1.0.3",
@@ -10100,6 +10101,18 @@
"integrity": "sha512-VdlZoocy5lCP0c/t66xAfclglEapXPCIVhqqJRncYpvbCgImF0w67aPKfbqUMr72tO2k5q0TdTZwCLjPTI6C9g==",
"dev": true
},
"node_modules/posthog-node": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-3.6.0.tgz",
"integrity": "sha512-N/4//SIQR4fhwbHnDdJ2rQCYdu9wo0EVPK4lVgZswp5R/E42RKlpuO6ZfPsBl+Bcg06OYiOd/WR/jLV90FCoSw==",
"dependencies": {
"axios": "^1.6.2",
"rusha": "^0.8.14"
},
"engines": {
"node": ">=15.0.0"
}
},
"node_modules/prelude-ls": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
@@ -11129,6 +11142,11 @@
"queue-microtask": "^1.2.2"
}
},
"node_modules/rusha": {
"version": "0.8.14",
"resolved": "https://registry.npmjs.org/rusha/-/rusha-0.8.14.tgz",
"integrity": "sha512-cLgakCUf6PedEu15t8kbsjnwIFFR2D4RfL+W3iWFJ4iac7z4B0ZI8fxy4R3J956kAI68HclCFGL8MPoUVC3qVA=="
},
"node_modules/safe-array-concat": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.0.1.tgz",

View File

@@ -112,6 +112,7 @@
"pg": "^8.11.3",
"picomatch": "^3.0.1",
"pino": "^8.16.2",
"posthog-node": "^3.6.0",
"probot": "^12.3.3",
"smee-client": "^2.0.0",
"tweetnacl": "^1.0.3",

View File

@@ -40,6 +40,7 @@ import { TSecretImportServiceFactory } from "@app/services/secret-import/secret-
import { TSecretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service";
import { TServiceTokenServiceFactory } from "@app/services/service-token/service-token-service";
import { TSuperAdminServiceFactory } from "@app/services/super-admin/super-admin-service";
import { TTelemetryServiceFactory } from "@app/services/telemetry/telemetry-service";
import { TUserDALFactory } from "@app/services/user/user-dal";
import { TUserServiceFactory } from "@app/services/user/user-service";
import { TWebhookServiceFactory } from "@app/services/webhook/webhook-service";
@@ -107,6 +108,7 @@ declare module "fastify" {
license: TLicenseServiceFactory;
trustedIp: TTrustedIpServiceFactory;
secretBlindIndex: TSecretBlindIndexServiceFactory;
telemetry: TTelemetryServiceFactory;
};
// this is exclusive use for middlewares in which we need to inject data
// everywhere else access using service layer

View File

@@ -12,6 +12,8 @@ import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TTelemetryServiceFactory } from "@app/services/telemetry/telemetry-service";
import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
import { TSecretRotationDALFactory } from "../secret-rotation-dal";
import { rotationTemplates } from "../templates";
@@ -41,6 +43,7 @@ type TSecretRotationQueueFactoryDep = {
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
secretDAL: Pick<TSecretDALFactory, "bulkUpdate" | "find">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany" | "findLatestVersionMany">;
telemetryService: Pick<TTelemetryServiceFactory, "sendPostHogEvents">;
};
// These error should stop the repeatable job and ask user to reconfigure rotation
@@ -61,7 +64,8 @@ export const secretRotationQueueFactory = ({
secretRotationDAL,
projectBotService,
secretDAL,
secretVersionDAL
secretVersionDAL,
telemetryService
}: TSecretRotationQueueFactoryDep) => {
const addToQueue = async (rotationId: string, interval: number) => {
const appCfg = getConfig();
@@ -262,6 +266,18 @@ export const secretRotationQueueFactory = ({
tx
);
});
telemetryService.sendPostHogEvents({
event: PostHogEventTypes.SecretRotated,
distinctId: "",
properties: {
numberOfSecrets: encryptedSecrets.length,
environment: secretRotation.environment.slug,
folderId: "",
workspaceId: secretRotation.projectId
}
});
logger.info("Finished rotating: rotation id: ", rotationId);
} catch (error) {
logger.error(error);

View File

@@ -6,6 +6,8 @@ import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TOrgDALFactory } from "@app/services/org/org-dal";
import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
import { TTelemetryServiceFactory } from "@app/services/telemetry/telemetry-service";
import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
import { TSecretScanningDALFactory } from "../secret-scanning-dal";
import {
@@ -23,6 +25,7 @@ type TSecretScanningQueueFactoryDep = {
secretScanningDAL: TSecretScanningDALFactory;
smtpService: Pick<TSmtpService, "sendMail">;
orgMembershipDAL: Pick<TOrgDALFactory, "findMembership">;
telemetryService: Pick<TTelemetryServiceFactory, "sendPostHogEvents">;
};
export type TSecretScanningQueueFactory = ReturnType<typeof secretScanningQueueFactory>;
@@ -31,7 +34,8 @@ export const secretScanningQueueFactory = ({
queueService,
secretScanningDAL,
smtpService,
orgMembershipDAL: orgMemberDAL,
telemetryService,
orgMembershipDAL: orgMemberDAL
}: TSecretScanningQueueFactoryDep) => {
const startFullRepoScan = async (payload: TScanFullRepoEventPayload) => {
await queueService.queue(QueueName.SecretFullRepoScan, QueueJobs.SecretScan, payload, {
@@ -160,6 +164,14 @@ export const secretScanningQueueFactory = ({
}
});
}
telemetryService.sendPostHogEvents({
event: PostHogEventTypes.SecretScannerPush,
distinctId: repository.fullName,
properties: {
numberOfRisks: Object.keys(allFindingsByFingerprint).length
}
});
});
queueService.start(QueueName.SecretFullRepoScan, async (job) => {
@@ -220,6 +232,14 @@ export const secretScanningQueueFactory = ({
}
});
}
telemetryService.sendPostHogEvents({
event: PostHogEventTypes.SecretScannerFull,
distinctId: repository.fullName,
properties: {
numberOfRisks: findings.length
}
});
});
queueService.listen(QueueName.SecretPushEventScan, "failed", (job, err) => {

View File

@@ -35,6 +35,13 @@ const envSchema = z
.min(32)
.default("#5VihU%rbXHcHwWwCot5L3vyPsx$7dWYw^iGk!EJg2bC*f$PD$%KCqx^R@#^LSEf"),
SITE_URL: zpStr(z.string().optional()),
// Telemetry
TELEMETRY_ENABLED: zodStrBool.default("true"),
POSTHOG_HOST: zpStr(z.string().optional().default("https://app.posthog.com")),
POSTHOG_PROJECT_API_KEY: zpStr(
z.string().optional().default("phc_nSin8j5q2zdhpFDI1ETmFNUIuTG4DwKVyIigrY10XiE")
),
LOOPS_API_KEY: zpStr(z.string().optional()),
// jwt options
AUTH_SECRET: zpStr(z.string()).default(process.env.JWT_AUTH_SECRET), // for those still using old JWT_AUTH_SECRET
JWT_AUTH_LIFETIME: zpStr(z.string().default("10d")),

View File

@@ -93,6 +93,7 @@ import { serviceTokenServiceFactory } from "@app/services/service-token/service-
import { TSmtpService } from "@app/services/smtp/smtp-service";
import { superAdminDALFactory } from "@app/services/super-admin/super-admin-dal";
import { getServerCfg, superAdminServiceFactory } from "@app/services/super-admin/super-admin-service";
import { telemetryServiceFactory } from "@app/services/telemetry/telemetry-service";
import { userDALFactory } from "@app/services/user/user-dal";
import { userServiceFactory } from "@app/services/user/user-service";
import { webhookDALFactory } from "@app/services/webhook/webhook-dal";
@@ -214,6 +215,7 @@ export const registerRoutes = async (
licenseService
});
const telemetryService = telemetryServiceFactory();
const tokenService = tokenServiceFactory({ tokenDAL: authTokenDAL, userDAL });
const userService = userServiceFactory({ userDAL });
const loginService = authLoginServiceFactory({ userDAL, smtpService, tokenService });
@@ -254,6 +256,7 @@ export const registerRoutes = async (
const apiKeyService = apiKeyServiceFactory({ apiKeyDAL, userDAL });
const secretScanningQueue = secretScanningQueueFactory({
telemetryService,
smtpService,
secretScanningDAL,
queueService,
@@ -389,6 +392,7 @@ export const registerRoutes = async (
secretQueueService
});
const secretRotationQueue = secretRotationQueueFactory({
telemetryService,
secretRotationDAL,
queue: queueService,
secretDAL,
@@ -483,7 +487,8 @@ export const registerRoutes = async (
secretScanning: secretScanningService,
license: licenseService,
trustedIp: trustedIpService,
secretBlindIndex: secretBlindIndexService
secretBlindIndex: secretBlindIndexService,
telemetry: telemetryService
});
server.decorate<FastifyZodProvider["store"]>("store", {

View File

@@ -7,6 +7,7 @@ import { verifySuperAdmin } from "@app/server/plugins/auth/superAdmin";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
import { getServerCfg } from "@app/services/super-admin/super-admin-service";
import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
export const registerAdminRouter = async (server: FastifyZodProvider) => {
server.route({
@@ -86,6 +87,16 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
userAgent: req.headers["user-agent"] || ""
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.AdminInit,
distinctId: user.user.email,
properties: {
email: user.user.email,
lastName: user.user.lastName || "",
firstName: user.user.firstName || ""
}
});
res.setCookie("jid", token.refresh, {
httpOnly: true,
path: "/",

View File

@@ -1,3 +1,4 @@
import { FastifyRequest } from "fastify";
import picomatch from "picomatch";
import { z } from "zod";
@@ -13,9 +14,23 @@ import { CommitType } from "@app/ee/services/secret-approval-request/secret-appr
import { BadRequestError } from "@app/lib/errors";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { ActorType, AuthMode } from "@app/services/auth/auth-type";
import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
import { secretRawSchema } from "../sanitizedSchemas";
const getDistinctId = (req: FastifyRequest) => {
if (req.auth.actor === ActorType.USER) {
return req.auth.user.email;
}
if (req.auth.actor === ActorType.IDENTITY) {
return `identity-${req.auth.identityId}`;
}
if (req.auth.actor === ActorType.SERVICE) {
return `service-token-${req.auth.serviceToken.id}`;
}
return "unknown-auth-data";
};
export const registerSecretRouter = async (server: FastifyZodProvider) => {
server.route({
url: "/raw",
@@ -88,6 +103,18 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
}
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretPulled,
distinctId: getDistinctId(req),
properties: {
numberOfSecrets: secrets.length,
workspaceId,
environment,
secretPath: req.query.secretPath,
...req.auditLogInfo
}
});
return { secrets, imports };
}
});
@@ -163,6 +190,18 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
}
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretPulled,
distinctId: getDistinctId(req),
properties: {
numberOfSecrets: 1,
workspaceId,
environment,
secretPath: req.query.secretPath,
...req.auditLogInfo
}
});
return { secret };
}
});
@@ -217,7 +256,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
event: {
type: EventType.CREATE_SECRET,
metadata: {
environment: req.body.environment,
environment: req.body.environment,
secretPath: req.body.secretPath,
secretId: secret.id,
secretKey: req.params.secretName,
@@ -225,6 +264,20 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
}
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretCreated,
distinctId: getDistinctId(req),
properties: {
numberOfSecrets: 1,
workspaceId: req.body.workspaceId,
environment: req.body.environment,
secretPath: req.body.secretPath,
...req.auditLogInfo
}
});
return { secret };
}
});
@@ -285,6 +338,19 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
}
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretUpdated,
distinctId: getDistinctId(req),
properties: {
numberOfSecrets: 1,
workspaceId: req.body.workspaceId,
environment: req.body.environment,
secretPath: req.body.secretPath,
...req.auditLogInfo
}
});
return { secret };
}
});
@@ -339,6 +405,20 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
}
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretDeleted,
distinctId: getDistinctId(req),
properties: {
numberOfSecrets: 1,
workspaceId: req.body.workspaceId,
environment: req.body.environment,
secretPath: req.body.secretPath,
...req.auditLogInfo
}
});
return { secret };
}
});
@@ -411,6 +491,19 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretPulled,
distinctId: getDistinctId(req),
properties: {
numberOfSecrets: secrets.length,
workspaceId: req.query.workspaceId,
environment: req.query.environment,
secretPath: req.query.secretPath,
...req.auditLogInfo
}
});
return { secrets, imports };
}
});
@@ -472,6 +565,19 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
}
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretPulled,
distinctId: getDistinctId(req),
properties: {
numberOfSecrets: 1,
workspaceId: req.query.workspaceId,
environment: req.query.environment,
secretPath: req.query.secretPath,
...req.auditLogInfo
}
});
return { secret };
}
});
@@ -583,6 +689,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
}
});
return { approval };
}
}
@@ -621,6 +728,20 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
}
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretCreated,
distinctId: getDistinctId(req),
properties: {
numberOfSecrets: 1,
workspaceId: req.body.workspaceId,
environment: req.body.environment,
secretPath: req.body.secretPath,
...req.auditLogInfo
}
});
return { secret };
}
});
@@ -788,6 +909,19 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
}
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretUpdated,
distinctId: getDistinctId(req),
properties: {
numberOfSecrets: 1,
workspaceId: req.body.workspaceId,
environment: req.body.environment,
secretPath: req.body.secretPath,
...req.auditLogInfo
}
});
return { secret };
}
});
@@ -892,6 +1026,19 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
}
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretDeleted,
distinctId: getDistinctId(req),
properties: {
numberOfSecrets: 1,
workspaceId: req.body.workspaceId,
environment: req.body.environment,
secretPath: req.body.secretPath,
...req.auditLogInfo
}
});
return { secret };
}
});
@@ -1005,6 +1152,19 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
}
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretCreated,
distinctId: getDistinctId(req),
properties: {
numberOfSecrets: secrets.length,
workspaceId: req.body.workspaceId,
environment: req.body.environment,
secretPath: req.body.secretPath,
...req.auditLogInfo
}
});
return { secrets };
}
});
@@ -1117,6 +1277,19 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
}
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretUpdated,
distinctId: getDistinctId(req),
properties: {
numberOfSecrets: secrets.length,
workspaceId: req.body.workspaceId,
environment: req.body.environment,
secretPath: req.body.secretPath,
...req.auditLogInfo
}
});
return { secrets };
}
});
@@ -1217,6 +1390,19 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
}
});
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretDeleted,
distinctId: getDistinctId(req),
properties: {
numberOfSecrets: secrets.length,
workspaceId: req.body.workspaceId,
environment: req.body.environment,
secretPath: req.body.secretPath,
...req.auditLogInfo
}
});
return { secrets };
}
});

View File

@@ -3,6 +3,7 @@ import { z } from "zod";
import { UsersSchema } from "@app/db/schemas";
import { getConfig } from "@app/lib/config/env";
import { authRateLimit } from "@app/server/config/rateLimiter";
import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
export const registerSignupRouter = async (server: FastifyZodProvider) => {
server.route({
@@ -100,13 +101,27 @@ export const registerSignupRouter = async (server: FastifyZodProvider) => {
authorization: req.headers.authorization as string
});
server.services.telemetry.sendLoopsEvent(
user.email,
user.firstName || "",
user.lastName || ""
);
server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.UserSignedUp,
distinctId: user.email,
properties: {
email: user.email,
attributionSource: req.body.attributionSource
}
});
res.setCookie("jid", refreshToken, {
httpOnly: true,
path: "/",
sameSite: "strict",
secure: appCfg.HTTPS_ENABLED
});
// TODO(akhilmhdh-pg): add telemetry service
return { message: "Successfully set up account", user, token: accessToken };
}

View File

@@ -0,0 +1,68 @@
import { PostHog } from "posthog-node";
import { getConfig } from "@app/lib/config/env";
import { request } from "@app/lib/config/request";
import { logger } from "@app/lib/logger";
import { TPostHogEvent } from "./telemetry-types";
export type TTelemetryServiceFactory = ReturnType<typeof telemetryServiceFactory>;
// type TTelemetryServiceFactoryDep = {};
export const telemetryServiceFactory = () => {
const appCfg = getConfig();
if (appCfg.isProductionMode && !appCfg.TELEMETRY_ENABLED) {
// eslint-disable-next-line
console.log(`
To improve, Infisical collects telemetry data about general usage.
This helps us understand how the product is doing and guide our product development to create the best possible platform; it also helps us demonstrate growth as we support Infisical as open-source software.
To opt into telemetry, you can set "TELEMETRY_ENABLED=true" within the environment variables.
`);
}
const postHog =
appCfg.isProductionMode && appCfg.TELEMETRY_ENABLED
? new PostHog(appCfg.POSTHOG_PROJECT_API_KEY, { host: appCfg.POSTHOG_HOST })
: undefined;
// used for email marketting email sending purpose
const sendLoopsEvent = async (email: string, firstName?: string, lastName?: string) => {
if (appCfg.isProductionMode && appCfg.LOOPS_API_KEY) {
try {
await request.post(
"https://app.loops.so/api/v1/events/send",
{
eventName: "Sign Up",
email,
firstName,
lastName
},
{
headers: {
Accept: "application/json",
Authorization: `Bearer ${appCfg.LOOPS_API_KEY}`
}
}
);
} catch (error) {
logger.error(error);
}
}
};
const sendPostHogEvents = async (event: TPostHogEvent) => {
if (postHog) {
postHog.capture({
event: event.event,
distinctId: event.distinctId,
properties: event.properties
});
}
};
return {
sendLoopsEvent,
sendPostHogEvents
};
};

View File

@@ -0,0 +1,61 @@
export enum PostHogEventTypes {
SecretPush = "secrets pushed",
SecretPulled = "secrets pulled",
SecretCreated = "secrets added",
SecretUpdated = "secrets modified",
SecretDeleted = "secrets deleted",
AdminInit = "admin initialization",
UserSignedUp = "User Signed Up",
SecretRotated = "secrets rotated",
SecretScannerFull = "historical cloud secret scan",
SecretScannerPush = "cloud secret scan"
}
export type TSecretModifiedEvent = {
event:
| PostHogEventTypes.SecretPush
| PostHogEventTypes.SecretRotated
| PostHogEventTypes.SecretPulled
| PostHogEventTypes.SecretCreated
| PostHogEventTypes.SecretUpdated
| PostHogEventTypes.SecretDeleted;
properties: {
numberOfSecrets: number;
environment: string;
workspaceId: string;
secretPath: string;
channel?: string;
userAgent?: string;
};
};
export type TAdminInitEvent = {
event: PostHogEventTypes.AdminInit;
properties: {
email: string;
firstName: string;
lastName: string;
};
};
export type TUserSignedUpEvent = {
event: PostHogEventTypes.UserSignedUp;
properties: {
email: string;
attributionSource?: string;
};
};
export type TSecretScannerEvent = {
event: PostHogEventTypes.SecretScannerFull | PostHogEventTypes.SecretScannerPush;
properties: {
numberOfRisks: number;
};
};
export type TPostHogEvent = { distinctId: string } & (
| TSecretModifiedEvent
| TAdminInitEvent
| TUserSignedUpEvent
| TSecretScannerEvent
);

View File

@@ -27,6 +27,6 @@
"@server/*": ["./src/server/*"]
}
},
"include": ["src/**/*", "scripts/**/*", "e2e-test/**/*", ".eslintrc.js"],
"include": ["src/**/*", "scripts/**/*", "e2e-test/**/*"],
"exclude": ["node_modules"]
}