From 7d8b399102fbfe05262578805b9c742f4f78e5ab Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Mon, 26 Feb 2024 15:04:28 +0530 Subject: [PATCH 1/7] feat(server): added keystore and made server cfg fetch from keystore to avoid db calls --- backend/e2e-test/mocks/keystore.ts | 19 ++++++++++++ backend/e2e-test/vitest-environment-knex.ts | 4 ++- backend/src/cache/redis.ts | 6 ---- backend/src/keystore/keystore.ts | 13 ++++++++ backend/src/main.ts | 4 ++- backend/src/server/app.ts | 6 ++-- backend/src/server/routes/index.ts | 11 +++++-- backend/src/server/routes/v1/admin-router.ts | 2 +- .../super-admin/super-admin-service.ts | 30 +++++++++++++++---- 9 files changed, 77 insertions(+), 18 deletions(-) create mode 100644 backend/e2e-test/mocks/keystore.ts delete mode 100644 backend/src/cache/redis.ts create mode 100644 backend/src/keystore/keystore.ts diff --git a/backend/e2e-test/mocks/keystore.ts b/backend/e2e-test/mocks/keystore.ts new file mode 100644 index 000000000..f3ce008ed --- /dev/null +++ b/backend/e2e-test/mocks/keystore.ts @@ -0,0 +1,19 @@ +import { TKeyStoreFactory } from "@app/keystore/keystore"; + +export const mockKeyStore = (): TKeyStoreFactory => { + const store: Record = {}; + + return { + setItem: async (key, value) => { + store[key] = value; + return "OK"; + }, + getItem: async (key) => { + const value = store[key]; + if (typeof value === "string") { + return value; + } + return null; + } + }; +}; diff --git a/backend/e2e-test/vitest-environment-knex.ts b/backend/e2e-test/vitest-environment-knex.ts index afb1bbabe..c1c750225 100644 --- a/backend/e2e-test/vitest-environment-knex.ts +++ b/backend/e2e-test/vitest-environment-knex.ts @@ -14,6 +14,7 @@ import { AuthTokenType } from "@app/services/auth/auth-type"; import { mockQueue } from "./mocks/queue"; import { mockSmtpServer } from "./mocks/smtp"; +import { mockKeyStore } from "./mocks/keystore"; dotenv.config({ path: path.join(__dirname, "../../.env.test"), debug: true }); export default { @@ -41,7 +42,8 @@ export default { await db.seed.run(); const smtp = mockSmtpServer(); const queue = mockQueue(); - const server = await main({ db, smtp, logger, queue }); + const keyStore = mockKeyStore(); + const server = await main({ db, smtp, logger, queue, keyStore }); // @ts-expect-error type globalThis.testServer = server; // @ts-expect-error type diff --git a/backend/src/cache/redis.ts b/backend/src/cache/redis.ts deleted file mode 100644 index 4e856fac1..000000000 --- a/backend/src/cache/redis.ts +++ /dev/null @@ -1,6 +0,0 @@ -import Redis from "ioredis"; - -export const initRedisConnection = (redisUrl: string) => { - const redis = new Redis(redisUrl); - return redis; -}; diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts new file mode 100644 index 000000000..99b781b24 --- /dev/null +++ b/backend/src/keystore/keystore.ts @@ -0,0 +1,13 @@ +import { Redis } from "ioredis"; + +export type TKeyStoreFactory = ReturnType; + +export const keyStoreFactory = (redisUrl: string) => { + const redis = new Redis(redisUrl); + + const setItem = async (key: string, value: string | number | Buffer) => redis.set(key, value); + + const getItem = async (key: string) => redis.get(key); + + return { setItem, getItem }; +}; diff --git a/backend/src/main.ts b/backend/src/main.ts index fab576d3b..86681ef33 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -1,6 +1,7 @@ import dotenv from "dotenv"; import { initDbConnection } from "./db"; +import { keyStoreFactory } from "./keystore/keystore"; import { formatSmtpConfig, initEnvConfig } from "./lib/config/env"; import { initLogger } from "./lib/logger"; import { queueServiceFactory } from "./queue"; @@ -19,8 +20,9 @@ const run = async () => { const smtp = smtpServiceFactory(formatSmtpConfig()); const queue = queueServiceFactory(appCfg.REDIS_URL); + const keyStore = keyStoreFactory(appCfg.REDIS_URL); - const server = await main({ db, smtp, logger, queue }); + const server = await main({ db, smtp, logger, queue, keyStore }); const bootstrap = await bootstrapCheck({ db }); // eslint-disable-next-line process.on("SIGINT", async () => { diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index 969d2fe43..556a88d7c 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -14,6 +14,7 @@ import fasitfy from "fastify"; import { Knex } from "knex"; import { Logger } from "pino"; +import { TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { TQueueServiceFactory } from "@app/queue"; import { TSmtpService } from "@app/services/smtp/smtp-service"; @@ -31,10 +32,11 @@ type TMain = { smtp: TSmtpService; logger?: Logger; queue: TQueueServiceFactory; + keyStore: TKeyStoreFactory; }; // Run the server! -export const main = async ({ db, smtp, logger, queue }: TMain) => { +export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { const appCfg = getConfig(); const server = fasitfy({ logger: appCfg.NODE_ENV === "test" ? false : logger, @@ -70,7 +72,7 @@ export const main = async ({ db, smtp, logger, queue }: TMain) => { } await server.register(helmet, { contentSecurityPolicy: false }); - await server.register(registerRoutes, { smtp, queue, db }); + await server.register(registerRoutes, { smtp, queue, db, keyStore }); if (appCfg.isProductionMode) { await server.register(registerExternalNextjs, { diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 7366c7041..e9c35b1d7 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -34,6 +34,7 @@ import { snapshotFolderDALFactory } from "@app/ee/services/secret-snapshot/snaps import { snapshotSecretDALFactory } from "@app/ee/services/secret-snapshot/snapshot-secret-dal"; import { trustedIpDALFactory } from "@app/ee/services/trusted-ip/trusted-ip-dal"; import { trustedIpServiceFactory } from "@app/ee/services/trusted-ip/trusted-ip-service"; +import { TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { TQueueServiceFactory } from "@app/queue"; import { apiKeyDALFactory } from "@app/services/api-key/api-key-dal"; @@ -112,7 +113,12 @@ import { registerV3Routes } from "./v3"; export const registerRoutes = async ( server: FastifyZodProvider, - { db, smtp: smtpService, queue: queueService }: { db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory } + { + db, + smtp: smtpService, + queue: queueService, + keyStore + }: { db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory; keyStore: TKeyStoreFactory } ) => { await server.register(registerSecretScannerGhApp, { prefix: "/ss-webhook" }); @@ -263,7 +269,8 @@ export const registerRoutes = async ( userDAL, authService: loginService, serverCfgDAL: superAdminDAL, - orgService + orgService, + keyStore }); const apiKeyService = apiKeyServiceFactory({ apiKeyDAL, userDAL }); diff --git a/backend/src/server/routes/v1/admin-router.ts b/backend/src/server/routes/v1/admin-router.ts index b89e3dc4c..6eec355a8 100644 --- a/backend/src/server/routes/v1/admin-router.ts +++ b/backend/src/server/routes/v1/admin-router.ts @@ -16,7 +16,7 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => { schema: { response: { 200: z.object({ - config: SuperAdminSchema + config: SuperAdminSchema.omit({ createdAt: true, updatedAt: true }) }) } }, diff --git a/backend/src/services/super-admin/super-admin-service.ts b/backend/src/services/super-admin/super-admin-service.ts index 64dac73f2..98491ba82 100644 --- a/backend/src/services/super-admin/super-admin-service.ts +++ b/backend/src/services/super-admin/super-admin-service.ts @@ -1,4 +1,5 @@ import { TSuperAdmin, TSuperAdminUpdate } from "@app/db/schemas"; +import { TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { BadRequestError } from "@app/lib/errors"; @@ -14,6 +15,7 @@ type TSuperAdminServiceFactoryDep = { userDAL: TUserDALFactory; authService: Pick; orgService: Pick; + keyStore: Pick; }; export type TSuperAdminServiceFactory = ReturnType; @@ -21,26 +23,44 @@ export type TSuperAdminServiceFactory = ReturnType Promise; +const ADMIN_CONFIG_KEY = "infisical-admin-cfg"; export const superAdminServiceFactory = ({ serverCfgDAL, userDAL, authService, - orgService + orgService, + keyStore }: TSuperAdminServiceFactoryDep) => { const initServerCfg = async () => { // TODO(akhilmhdh): bad pattern time less change this later to me itself - getServerCfg = () => serverCfgDAL.findOne({}); + getServerCfg = async () => { + const config = await keyStore.getItem(ADMIN_CONFIG_KEY); + if (!config) { + const serverCfg = await serverCfgDAL.findOne({}); + await keyStore.setItem(ADMIN_CONFIG_KEY, JSON.stringify(serverCfg)); + return serverCfg; + } + const keyStoreServerCfg = JSON.parse(config) as TSuperAdmin; + return { + ...keyStoreServerCfg, + // this is to allow admin router to work + createdAt: new Date(keyStoreServerCfg.createdAt), + updatedAt: new Date(keyStoreServerCfg.updatedAt) + }; + }; - const serverCfg = await serverCfgDAL.findOne({}); + const serverCfg = await getServerCfg(); if (serverCfg) return; + const newCfg = await serverCfgDAL.create({ initialized: false, allowSignUp: true }); return newCfg; }; const updateServerCfg = async (data: TSuperAdminUpdate) => { const serverCfg = await getServerCfg(); - const cfg = await serverCfgDAL.updateById(serverCfg.id, data); - return cfg; + const updatedServerCfg = await serverCfgDAL.updateById(serverCfg.id, data); + await keyStore.setItem(ADMIN_CONFIG_KEY, JSON.stringify(updatedServerCfg)); + return updatedServerCfg; }; const adminSignUp = async ({ From 8aaed739d5d6c205a4c6ee73930e9c1a64027cb9 Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Mon, 26 Feb 2024 17:58:32 +0530 Subject: [PATCH 2/7] feat(server): resolved a possible race condition on replication on frest first boot up and fixed making values optional on create rows for generate schema --- backend/e2e-test/mocks/keystore.ts | 8 ++++++ backend/scripts/generate-schema-types.ts | 10 +++++--- .../migrations/20240226094411_instance-id.ts | 21 ++++++++++++++++ backend/src/db/schemas/api-keys.ts | 4 +-- backend/src/db/schemas/audit-logs.ts | 4 +-- backend/src/db/schemas/auth-token-sessions.ts | 4 +-- backend/src/db/schemas/auth-tokens.ts | 4 +-- backend/src/db/schemas/backup-private-key.ts | 4 +-- .../db/schemas/git-app-install-sessions.ts | 4 +-- backend/src/db/schemas/git-app-org.ts | 4 +-- backend/src/db/schemas/identities.ts | 4 +-- .../src/db/schemas/identity-access-tokens.ts | 4 +-- .../db/schemas/identity-org-memberships.ts | 6 +++-- .../schemas/identity-project-memberships.ts | 9 +++++-- .../db/schemas/identity-ua-client-secrets.ts | 6 +++-- .../db/schemas/identity-universal-auths.ts | 6 +++-- backend/src/db/schemas/incident-contacts.ts | 4 +-- backend/src/db/schemas/integration-auths.ts | 4 +-- backend/src/db/schemas/integrations.ts | 4 +-- backend/src/db/schemas/org-bots.ts | 4 +-- backend/src/db/schemas/org-memberships.ts | 4 +-- backend/src/db/schemas/org-roles.ts | 4 +-- backend/src/db/schemas/organizations.ts | 4 +-- backend/src/db/schemas/project-bots.ts | 4 +-- .../src/db/schemas/project-environments.ts | 4 +-- backend/src/db/schemas/project-keys.ts | 4 +-- backend/src/db/schemas/project-memberships.ts | 4 +-- backend/src/db/schemas/project-roles.ts | 4 +-- backend/src/db/schemas/projects.ts | 4 +-- backend/src/db/schemas/saml-configs.ts | 4 +-- backend/src/db/schemas/scim-tokens.ts | 4 +-- .../secret-approval-policies-approvers.ts | 9 +++++-- .../db/schemas/secret-approval-policies.ts | 6 +++-- .../secret-approval-request-secret-tags.ts | 9 +++++-- .../secret-approval-requests-reviewers.ts | 9 +++++-- .../secret-approval-requests-secrets.ts | 9 +++++-- .../db/schemas/secret-approval-requests.ts | 6 +++-- .../src/db/schemas/secret-blind-indexes.ts | 4 +-- .../src/db/schemas/secret-folder-versions.ts | 4 +-- backend/src/db/schemas/secret-folders.ts | 4 +-- backend/src/db/schemas/secret-imports.ts | 4 +-- .../src/db/schemas/secret-rotation-outputs.ts | 4 +-- backend/src/db/schemas/secret-rotations.ts | 4 +-- .../db/schemas/secret-scanning-git-risks.ts | 6 +++-- .../src/db/schemas/secret-snapshot-folders.ts | 4 +-- .../src/db/schemas/secret-snapshot-secrets.ts | 4 +-- backend/src/db/schemas/secret-snapshots.ts | 4 +-- backend/src/db/schemas/secret-tag-junction.ts | 4 +-- backend/src/db/schemas/secret-tags.ts | 4 +-- .../db/schemas/secret-version-tag-junction.ts | 6 +++-- backend/src/db/schemas/secret-versions.ts | 4 +-- backend/src/db/schemas/secrets.ts | 4 +-- backend/src/db/schemas/service-tokens.ts | 4 +-- backend/src/db/schemas/super-admin.ts | 7 +++--- backend/src/db/schemas/trusted-ips.ts | 4 +-- backend/src/db/schemas/user-actions.ts | 4 +-- .../src/db/schemas/user-encryption-keys.ts | 4 +-- backend/src/db/schemas/users.ts | 4 +-- backend/src/db/schemas/webhooks.ts | 4 +-- backend/src/keystore/keystore.ts | 7 +++++- .../super-admin/super-admin-service.ts | 25 +++++++++++++------ 61 files changed, 211 insertions(+), 126 deletions(-) create mode 100644 backend/src/db/migrations/20240226094411_instance-id.ts diff --git a/backend/e2e-test/mocks/keystore.ts b/backend/e2e-test/mocks/keystore.ts index f3ce008ed..7a1bab2b3 100644 --- a/backend/e2e-test/mocks/keystore.ts +++ b/backend/e2e-test/mocks/keystore.ts @@ -8,6 +8,14 @@ export const mockKeyStore = (): TKeyStoreFactory => { store[key] = value; return "OK"; }, + setItemWithExpiry: async (key, value) => { + store[key] = value; + return "OK"; + }, + deleteItem: async (key) => { + delete store[key]; + return 1; + }, getItem: async (key) => { const value = store[key]; if (typeof value === "string") { diff --git a/backend/scripts/generate-schema-types.ts b/backend/scripts/generate-schema-types.ts index 68330613c..8c913991f 100644 --- a/backend/scripts/generate-schema-types.ts +++ b/backend/scripts/generate-schema-types.ts @@ -44,7 +44,7 @@ const getZodDefaultValue = (type: unknown, value: string | number | boolean | Ob if (!value || value === "null") return; switch (type) { case "uuid": - return; + return `.default("00000000-0000-0000-0000-000000000000")`; case "character varying": { if (value === "gen_random_uuid()") return; if (typeof value === "string" && value.includes("::")) { @@ -100,7 +100,8 @@ const main = async () => { const columnName = columnNames[colNum]; const colInfo = columns[columnName]; let ztype = getZodPrimitiveType(colInfo.type); - if (colInfo.defaultValue) { + // don't put optional on id + if (colInfo.defaultValue && columnName !== "id") { const { defaultValue } = colInfo; const zSchema = getZodDefaultValue(colInfo.type, defaultValue); if (zSchema) { @@ -120,6 +121,7 @@ const main = async () => { .split("_") .reduce((prev, curr) => prev + `${curr.at(0)?.toUpperCase()}${curr.slice(1).toLowerCase()}`, ""); + // the insert and update are changed to zod input type to use default cases writeFileSync( path.join(__dirname, "../src/db/schemas", `${dashcase}.ts`), `// Code generated by automation script, DO NOT EDIT. @@ -134,8 +136,8 @@ import { TImmutableDBKeys } from "./models"; export const ${pascalCase}Schema = z.object({${schema}}); export type T${pascalCase} = z.infer; -export type T${pascalCase}Insert = Omit; -export type T${pascalCase}Update = Partial>; +export type T${pascalCase}Insert = Omit, TImmutableDBKeys>; +export type T${pascalCase}Update = Partial, TImmutableDBKeys>>; ` ); } diff --git a/backend/src/db/migrations/20240226094411_instance-id.ts b/backend/src/db/migrations/20240226094411_instance-id.ts new file mode 100644 index 000000000..ae88bb3fd --- /dev/null +++ b/backend/src/db/migrations/20240226094411_instance-id.ts @@ -0,0 +1,21 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +const ADMIN_CONFIG_UUID = "00000000-0000-0000-0000-000000000000"; + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable(TableName.SuperAdmin, (t) => { + t.uuid("instanceId").notNullable().defaultTo(knex.fn.uuid()); + }); + // this is updated to avoid race condition on replication + // eslint-disable-next-line + // @ts-ignore + await knex(TableName.SuperAdmin).update({ id: ADMIN_CONFIG_UUID }).whereNotNull("id").limit(1); +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable(TableName.SuperAdmin, (t) => { + t.dropColumn("instanceId"); + }); +} diff --git a/backend/src/db/schemas/api-keys.ts b/backend/src/db/schemas/api-keys.ts index ff29a54e1..cf836fd88 100644 --- a/backend/src/db/schemas/api-keys.ts +++ b/backend/src/db/schemas/api-keys.ts @@ -19,5 +19,5 @@ export const ApiKeysSchema = z.object({ }); export type TApiKeys = z.infer; -export type TApiKeysInsert = Omit; -export type TApiKeysUpdate = Partial>; +export type TApiKeysInsert = Omit, TImmutableDBKeys>; +export type TApiKeysUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/audit-logs.ts b/backend/src/db/schemas/audit-logs.ts index f7143bb57..b8906698b 100644 --- a/backend/src/db/schemas/audit-logs.ts +++ b/backend/src/db/schemas/audit-logs.ts @@ -24,5 +24,5 @@ export const AuditLogsSchema = z.object({ }); export type TAuditLogs = z.infer; -export type TAuditLogsInsert = Omit; -export type TAuditLogsUpdate = Partial>; +export type TAuditLogsInsert = Omit, TImmutableDBKeys>; +export type TAuditLogsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/auth-token-sessions.ts b/backend/src/db/schemas/auth-token-sessions.ts index 46ed7c201..3a9376c83 100644 --- a/backend/src/db/schemas/auth-token-sessions.ts +++ b/backend/src/db/schemas/auth-token-sessions.ts @@ -20,5 +20,5 @@ export const AuthTokenSessionsSchema = z.object({ }); export type TAuthTokenSessions = z.infer; -export type TAuthTokenSessionsInsert = Omit; -export type TAuthTokenSessionsUpdate = Partial>; +export type TAuthTokenSessionsInsert = Omit, TImmutableDBKeys>; +export type TAuthTokenSessionsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/auth-tokens.ts b/backend/src/db/schemas/auth-tokens.ts index 9ae8eed44..dd8563b85 100644 --- a/backend/src/db/schemas/auth-tokens.ts +++ b/backend/src/db/schemas/auth-tokens.ts @@ -21,5 +21,5 @@ export const AuthTokensSchema = z.object({ }); export type TAuthTokens = z.infer; -export type TAuthTokensInsert = Omit; -export type TAuthTokensUpdate = Partial>; +export type TAuthTokensInsert = Omit, TImmutableDBKeys>; +export type TAuthTokensUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/backup-private-key.ts b/backend/src/db/schemas/backup-private-key.ts index 5930bbd4a..5a2148aa1 100644 --- a/backend/src/db/schemas/backup-private-key.ts +++ b/backend/src/db/schemas/backup-private-key.ts @@ -22,5 +22,5 @@ export const BackupPrivateKeySchema = z.object({ }); export type TBackupPrivateKey = z.infer; -export type TBackupPrivateKeyInsert = Omit; -export type TBackupPrivateKeyUpdate = Partial>; +export type TBackupPrivateKeyInsert = Omit, TImmutableDBKeys>; +export type TBackupPrivateKeyUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/git-app-install-sessions.ts b/backend/src/db/schemas/git-app-install-sessions.ts index 6c6db40ea..986ae9d8e 100644 --- a/backend/src/db/schemas/git-app-install-sessions.ts +++ b/backend/src/db/schemas/git-app-install-sessions.ts @@ -17,5 +17,5 @@ export const GitAppInstallSessionsSchema = z.object({ }); export type TGitAppInstallSessions = z.infer; -export type TGitAppInstallSessionsInsert = Omit; -export type TGitAppInstallSessionsUpdate = Partial>; +export type TGitAppInstallSessionsInsert = Omit, TImmutableDBKeys>; +export type TGitAppInstallSessionsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/git-app-org.ts b/backend/src/db/schemas/git-app-org.ts index 57e0d474a..627df6b4c 100644 --- a/backend/src/db/schemas/git-app-org.ts +++ b/backend/src/db/schemas/git-app-org.ts @@ -17,5 +17,5 @@ export const GitAppOrgSchema = z.object({ }); export type TGitAppOrg = z.infer; -export type TGitAppOrgInsert = Omit; -export type TGitAppOrgUpdate = Partial>; +export type TGitAppOrgInsert = Omit, TImmutableDBKeys>; +export type TGitAppOrgUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/identities.ts b/backend/src/db/schemas/identities.ts index 005adf025..adf3a6ef2 100644 --- a/backend/src/db/schemas/identities.ts +++ b/backend/src/db/schemas/identities.ts @@ -16,5 +16,5 @@ export const IdentitiesSchema = z.object({ }); export type TIdentities = z.infer; -export type TIdentitiesInsert = Omit; -export type TIdentitiesUpdate = Partial>; +export type TIdentitiesInsert = Omit, TImmutableDBKeys>; +export type TIdentitiesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/identity-access-tokens.ts b/backend/src/db/schemas/identity-access-tokens.ts index cbd71e5c5..18dbb8193 100644 --- a/backend/src/db/schemas/identity-access-tokens.ts +++ b/backend/src/db/schemas/identity-access-tokens.ts @@ -23,5 +23,5 @@ export const IdentityAccessTokensSchema = z.object({ }); export type TIdentityAccessTokens = z.infer; -export type TIdentityAccessTokensInsert = Omit; -export type TIdentityAccessTokensUpdate = Partial>; +export type TIdentityAccessTokensInsert = Omit, TImmutableDBKeys>; +export type TIdentityAccessTokensUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/identity-org-memberships.ts b/backend/src/db/schemas/identity-org-memberships.ts index 647ec7124..2f29c52e4 100644 --- a/backend/src/db/schemas/identity-org-memberships.ts +++ b/backend/src/db/schemas/identity-org-memberships.ts @@ -18,5 +18,7 @@ export const IdentityOrgMembershipsSchema = z.object({ }); export type TIdentityOrgMemberships = z.infer; -export type TIdentityOrgMembershipsInsert = Omit; -export type TIdentityOrgMembershipsUpdate = Partial>; +export type TIdentityOrgMembershipsInsert = Omit, TImmutableDBKeys>; +export type TIdentityOrgMembershipsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/identity-project-memberships.ts b/backend/src/db/schemas/identity-project-memberships.ts index 866324c8b..276c9581e 100644 --- a/backend/src/db/schemas/identity-project-memberships.ts +++ b/backend/src/db/schemas/identity-project-memberships.ts @@ -18,5 +18,10 @@ export const IdentityProjectMembershipsSchema = z.object({ }); export type TIdentityProjectMemberships = z.infer; -export type TIdentityProjectMembershipsInsert = Omit; -export type TIdentityProjectMembershipsUpdate = Partial>; +export type TIdentityProjectMembershipsInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TIdentityProjectMembershipsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/identity-ua-client-secrets.ts b/backend/src/db/schemas/identity-ua-client-secrets.ts index 60f8d862f..bd549ca5b 100644 --- a/backend/src/db/schemas/identity-ua-client-secrets.ts +++ b/backend/src/db/schemas/identity-ua-client-secrets.ts @@ -23,5 +23,7 @@ export const IdentityUaClientSecretsSchema = z.object({ }); export type TIdentityUaClientSecrets = z.infer; -export type TIdentityUaClientSecretsInsert = Omit; -export type TIdentityUaClientSecretsUpdate = Partial>; +export type TIdentityUaClientSecretsInsert = Omit, TImmutableDBKeys>; +export type TIdentityUaClientSecretsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/identity-universal-auths.ts b/backend/src/db/schemas/identity-universal-auths.ts index 5a8f0c7ec..eeec2f666 100644 --- a/backend/src/db/schemas/identity-universal-auths.ts +++ b/backend/src/db/schemas/identity-universal-auths.ts @@ -21,5 +21,7 @@ export const IdentityUniversalAuthsSchema = z.object({ }); export type TIdentityUniversalAuths = z.infer; -export type TIdentityUniversalAuthsInsert = Omit; -export type TIdentityUniversalAuthsUpdate = Partial>; +export type TIdentityUniversalAuthsInsert = Omit, TImmutableDBKeys>; +export type TIdentityUniversalAuthsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/incident-contacts.ts b/backend/src/db/schemas/incident-contacts.ts index 431bf05ab..23c8503b0 100644 --- a/backend/src/db/schemas/incident-contacts.ts +++ b/backend/src/db/schemas/incident-contacts.ts @@ -16,5 +16,5 @@ export const IncidentContactsSchema = z.object({ }); export type TIncidentContacts = z.infer; -export type TIncidentContactsInsert = Omit; -export type TIncidentContactsUpdate = Partial>; +export type TIncidentContactsInsert = Omit, TImmutableDBKeys>; +export type TIncidentContactsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/integration-auths.ts b/backend/src/db/schemas/integration-auths.ts index db602c0af..185beae36 100644 --- a/backend/src/db/schemas/integration-auths.ts +++ b/backend/src/db/schemas/integration-auths.ts @@ -33,5 +33,5 @@ export const IntegrationAuthsSchema = z.object({ }); export type TIntegrationAuths = z.infer; -export type TIntegrationAuthsInsert = Omit; -export type TIntegrationAuthsUpdate = Partial>; +export type TIntegrationAuthsInsert = Omit, TImmutableDBKeys>; +export type TIntegrationAuthsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/integrations.ts b/backend/src/db/schemas/integrations.ts index 62f73d190..cf8c88154 100644 --- a/backend/src/db/schemas/integrations.ts +++ b/backend/src/db/schemas/integrations.ts @@ -31,5 +31,5 @@ export const IntegrationsSchema = z.object({ }); export type TIntegrations = z.infer; -export type TIntegrationsInsert = Omit; -export type TIntegrationsUpdate = Partial>; +export type TIntegrationsInsert = Omit, TImmutableDBKeys>; +export type TIntegrationsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/org-bots.ts b/backend/src/db/schemas/org-bots.ts index b328f1aaf..77be907ec 100644 --- a/backend/src/db/schemas/org-bots.ts +++ b/backend/src/db/schemas/org-bots.ts @@ -27,5 +27,5 @@ export const OrgBotsSchema = z.object({ }); export type TOrgBots = z.infer; -export type TOrgBotsInsert = Omit; -export type TOrgBotsUpdate = Partial>; +export type TOrgBotsInsert = Omit, TImmutableDBKeys>; +export type TOrgBotsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/org-memberships.ts b/backend/src/db/schemas/org-memberships.ts index b2fffa117..585addb7c 100644 --- a/backend/src/db/schemas/org-memberships.ts +++ b/backend/src/db/schemas/org-memberships.ts @@ -20,5 +20,5 @@ export const OrgMembershipsSchema = z.object({ }); export type TOrgMemberships = z.infer; -export type TOrgMembershipsInsert = Omit; -export type TOrgMembershipsUpdate = Partial>; +export type TOrgMembershipsInsert = Omit, TImmutableDBKeys>; +export type TOrgMembershipsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/org-roles.ts b/backend/src/db/schemas/org-roles.ts index 72b582f96..ca01c6574 100644 --- a/backend/src/db/schemas/org-roles.ts +++ b/backend/src/db/schemas/org-roles.ts @@ -19,5 +19,5 @@ export const OrgRolesSchema = z.object({ }); export type TOrgRoles = z.infer; -export type TOrgRolesInsert = Omit; -export type TOrgRolesUpdate = Partial>; +export type TOrgRolesInsert = Omit, TImmutableDBKeys>; +export type TOrgRolesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/organizations.ts b/backend/src/db/schemas/organizations.ts index a91b93ea5..f2933af86 100644 --- a/backend/src/db/schemas/organizations.ts +++ b/backend/src/db/schemas/organizations.ts @@ -19,5 +19,5 @@ export const OrganizationsSchema = z.object({ }); export type TOrganizations = z.infer; -export type TOrganizationsInsert = Omit; -export type TOrganizationsUpdate = Partial>; +export type TOrganizationsInsert = Omit, TImmutableDBKeys>; +export type TOrganizationsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/project-bots.ts b/backend/src/db/schemas/project-bots.ts index c68576943..1fa59eb78 100644 --- a/backend/src/db/schemas/project-bots.ts +++ b/backend/src/db/schemas/project-bots.ts @@ -26,5 +26,5 @@ export const ProjectBotsSchema = z.object({ }); export type TProjectBots = z.infer; -export type TProjectBotsInsert = Omit; -export type TProjectBotsUpdate = Partial>; +export type TProjectBotsInsert = Omit, TImmutableDBKeys>; +export type TProjectBotsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/project-environments.ts b/backend/src/db/schemas/project-environments.ts index 8b95dbba0..76556b7e9 100644 --- a/backend/src/db/schemas/project-environments.ts +++ b/backend/src/db/schemas/project-environments.ts @@ -18,5 +18,5 @@ export const ProjectEnvironmentsSchema = z.object({ }); export type TProjectEnvironments = z.infer; -export type TProjectEnvironmentsInsert = Omit; -export type TProjectEnvironmentsUpdate = Partial>; +export type TProjectEnvironmentsInsert = Omit, TImmutableDBKeys>; +export type TProjectEnvironmentsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/project-keys.ts b/backend/src/db/schemas/project-keys.ts index 720cd79bf..924918b12 100644 --- a/backend/src/db/schemas/project-keys.ts +++ b/backend/src/db/schemas/project-keys.ts @@ -19,5 +19,5 @@ export const ProjectKeysSchema = z.object({ }); export type TProjectKeys = z.infer; -export type TProjectKeysInsert = Omit; -export type TProjectKeysUpdate = Partial>; +export type TProjectKeysInsert = Omit, TImmutableDBKeys>; +export type TProjectKeysUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/project-memberships.ts b/backend/src/db/schemas/project-memberships.ts index b9f191a84..8576a318e 100644 --- a/backend/src/db/schemas/project-memberships.ts +++ b/backend/src/db/schemas/project-memberships.ts @@ -18,5 +18,5 @@ export const ProjectMembershipsSchema = z.object({ }); export type TProjectMemberships = z.infer; -export type TProjectMembershipsInsert = Omit; -export type TProjectMembershipsUpdate = Partial>; +export type TProjectMembershipsInsert = Omit, TImmutableDBKeys>; +export type TProjectMembershipsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/project-roles.ts b/backend/src/db/schemas/project-roles.ts index 1946ab5e1..e10f6fd4c 100644 --- a/backend/src/db/schemas/project-roles.ts +++ b/backend/src/db/schemas/project-roles.ts @@ -19,5 +19,5 @@ export const ProjectRolesSchema = z.object({ }); export type TProjectRoles = z.infer; -export type TProjectRolesInsert = Omit; -export type TProjectRolesUpdate = Partial>; +export type TProjectRolesInsert = Omit, TImmutableDBKeys>; +export type TProjectRolesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index 3834d6d58..3965e24c0 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -20,5 +20,5 @@ export const ProjectsSchema = z.object({ }); export type TProjects = z.infer; -export type TProjectsInsert = Omit; -export type TProjectsUpdate = Partial>; +export type TProjectsInsert = Omit, TImmutableDBKeys>; +export type TProjectsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/saml-configs.ts b/backend/src/db/schemas/saml-configs.ts index 6891d8add..67171469a 100644 --- a/backend/src/db/schemas/saml-configs.ts +++ b/backend/src/db/schemas/saml-configs.ts @@ -27,5 +27,5 @@ export const SamlConfigsSchema = z.object({ }); export type TSamlConfigs = z.infer; -export type TSamlConfigsInsert = Omit; -export type TSamlConfigsUpdate = Partial>; +export type TSamlConfigsInsert = Omit, TImmutableDBKeys>; +export type TSamlConfigsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/scim-tokens.ts b/backend/src/db/schemas/scim-tokens.ts index 593e4f7d9..ab6e10d27 100644 --- a/backend/src/db/schemas/scim-tokens.ts +++ b/backend/src/db/schemas/scim-tokens.ts @@ -17,5 +17,5 @@ export const ScimTokensSchema = z.object({ }); export type TScimTokens = z.infer; -export type TScimTokensInsert = Omit; -export type TScimTokensUpdate = Partial>; +export type TScimTokensInsert = Omit, TImmutableDBKeys>; +export type TScimTokensUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-approval-policies-approvers.ts b/backend/src/db/schemas/secret-approval-policies-approvers.ts index 503299d30..12a3119e6 100644 --- a/backend/src/db/schemas/secret-approval-policies-approvers.ts +++ b/backend/src/db/schemas/secret-approval-policies-approvers.ts @@ -16,5 +16,10 @@ export const SecretApprovalPoliciesApproversSchema = z.object({ }); export type TSecretApprovalPoliciesApprovers = z.infer; -export type TSecretApprovalPoliciesApproversInsert = Omit; -export type TSecretApprovalPoliciesApproversUpdate = Partial>; +export type TSecretApprovalPoliciesApproversInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TSecretApprovalPoliciesApproversUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-approval-policies.ts b/backend/src/db/schemas/secret-approval-policies.ts index 6c331f1b1..d907ef1e0 100644 --- a/backend/src/db/schemas/secret-approval-policies.ts +++ b/backend/src/db/schemas/secret-approval-policies.ts @@ -18,5 +18,7 @@ export const SecretApprovalPoliciesSchema = z.object({ }); export type TSecretApprovalPolicies = z.infer; -export type TSecretApprovalPoliciesInsert = Omit; -export type TSecretApprovalPoliciesUpdate = Partial>; +export type TSecretApprovalPoliciesInsert = Omit, TImmutableDBKeys>; +export type TSecretApprovalPoliciesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-approval-request-secret-tags.ts b/backend/src/db/schemas/secret-approval-request-secret-tags.ts index f5e7ba632..2851321e2 100644 --- a/backend/src/db/schemas/secret-approval-request-secret-tags.ts +++ b/backend/src/db/schemas/secret-approval-request-secret-tags.ts @@ -16,5 +16,10 @@ export const SecretApprovalRequestSecretTagsSchema = z.object({ }); export type TSecretApprovalRequestSecretTags = z.infer; -export type TSecretApprovalRequestSecretTagsInsert = Omit; -export type TSecretApprovalRequestSecretTagsUpdate = Partial>; +export type TSecretApprovalRequestSecretTagsInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TSecretApprovalRequestSecretTagsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-approval-requests-reviewers.ts b/backend/src/db/schemas/secret-approval-requests-reviewers.ts index a3657f1f9..f3ff88047 100644 --- a/backend/src/db/schemas/secret-approval-requests-reviewers.ts +++ b/backend/src/db/schemas/secret-approval-requests-reviewers.ts @@ -17,5 +17,10 @@ export const SecretApprovalRequestsReviewersSchema = z.object({ }); export type TSecretApprovalRequestsReviewers = z.infer; -export type TSecretApprovalRequestsReviewersInsert = Omit; -export type TSecretApprovalRequestsReviewersUpdate = Partial>; +export type TSecretApprovalRequestsReviewersInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TSecretApprovalRequestsReviewersUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-approval-requests-secrets.ts b/backend/src/db/schemas/secret-approval-requests-secrets.ts index 810a4f2cf..b795b47b4 100644 --- a/backend/src/db/schemas/secret-approval-requests-secrets.ts +++ b/backend/src/db/schemas/secret-approval-requests-secrets.ts @@ -35,5 +35,10 @@ export const SecretApprovalRequestsSecretsSchema = z.object({ }); export type TSecretApprovalRequestsSecrets = z.infer; -export type TSecretApprovalRequestsSecretsInsert = Omit; -export type TSecretApprovalRequestsSecretsUpdate = Partial>; +export type TSecretApprovalRequestsSecretsInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TSecretApprovalRequestsSecretsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-approval-requests.ts b/backend/src/db/schemas/secret-approval-requests.ts index 590c283f5..6ee97fbb6 100644 --- a/backend/src/db/schemas/secret-approval-requests.ts +++ b/backend/src/db/schemas/secret-approval-requests.ts @@ -22,5 +22,7 @@ export const SecretApprovalRequestsSchema = z.object({ }); export type TSecretApprovalRequests = z.infer; -export type TSecretApprovalRequestsInsert = Omit; -export type TSecretApprovalRequestsUpdate = Partial>; +export type TSecretApprovalRequestsInsert = Omit, TImmutableDBKeys>; +export type TSecretApprovalRequestsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-blind-indexes.ts b/backend/src/db/schemas/secret-blind-indexes.ts index fa919babd..474e0aa54 100644 --- a/backend/src/db/schemas/secret-blind-indexes.ts +++ b/backend/src/db/schemas/secret-blind-indexes.ts @@ -20,5 +20,5 @@ export const SecretBlindIndexesSchema = z.object({ }); export type TSecretBlindIndexes = z.infer; -export type TSecretBlindIndexesInsert = Omit; -export type TSecretBlindIndexesUpdate = Partial>; +export type TSecretBlindIndexesInsert = Omit, TImmutableDBKeys>; +export type TSecretBlindIndexesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-folder-versions.ts b/backend/src/db/schemas/secret-folder-versions.ts index 8c550d065..8bef6e83f 100644 --- a/backend/src/db/schemas/secret-folder-versions.ts +++ b/backend/src/db/schemas/secret-folder-versions.ts @@ -18,5 +18,5 @@ export const SecretFolderVersionsSchema = z.object({ }); export type TSecretFolderVersions = z.infer; -export type TSecretFolderVersionsInsert = Omit; -export type TSecretFolderVersionsUpdate = Partial>; +export type TSecretFolderVersionsInsert = Omit, TImmutableDBKeys>; +export type TSecretFolderVersionsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-folders.ts b/backend/src/db/schemas/secret-folders.ts index 648238b0a..0f9684d0e 100644 --- a/backend/src/db/schemas/secret-folders.ts +++ b/backend/src/db/schemas/secret-folders.ts @@ -18,5 +18,5 @@ export const SecretFoldersSchema = z.object({ }); export type TSecretFolders = z.infer; -export type TSecretFoldersInsert = Omit; -export type TSecretFoldersUpdate = Partial>; +export type TSecretFoldersInsert = Omit, TImmutableDBKeys>; +export type TSecretFoldersUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-imports.ts b/backend/src/db/schemas/secret-imports.ts index 9c1ee905f..9d42d8da5 100644 --- a/backend/src/db/schemas/secret-imports.ts +++ b/backend/src/db/schemas/secret-imports.ts @@ -19,5 +19,5 @@ export const SecretImportsSchema = z.object({ }); export type TSecretImports = z.infer; -export type TSecretImportsInsert = Omit; -export type TSecretImportsUpdate = Partial>; +export type TSecretImportsInsert = Omit, TImmutableDBKeys>; +export type TSecretImportsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-rotation-outputs.ts b/backend/src/db/schemas/secret-rotation-outputs.ts index 3b594365a..3ac5c2c9e 100644 --- a/backend/src/db/schemas/secret-rotation-outputs.ts +++ b/backend/src/db/schemas/secret-rotation-outputs.ts @@ -15,5 +15,5 @@ export const SecretRotationOutputsSchema = z.object({ }); export type TSecretRotationOutputs = z.infer; -export type TSecretRotationOutputsInsert = Omit; -export type TSecretRotationOutputsUpdate = Partial>; +export type TSecretRotationOutputsInsert = Omit, TImmutableDBKeys>; +export type TSecretRotationOutputsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-rotations.ts b/backend/src/db/schemas/secret-rotations.ts index 4c65712fa..b491edc46 100644 --- a/backend/src/db/schemas/secret-rotations.ts +++ b/backend/src/db/schemas/secret-rotations.ts @@ -26,5 +26,5 @@ export const SecretRotationsSchema = z.object({ }); export type TSecretRotations = z.infer; -export type TSecretRotationsInsert = Omit; -export type TSecretRotationsUpdate = Partial>; +export type TSecretRotationsInsert = Omit, TImmutableDBKeys>; +export type TSecretRotationsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-scanning-git-risks.ts b/backend/src/db/schemas/secret-scanning-git-risks.ts index 85cfcd376..08ba690e4 100644 --- a/backend/src/db/schemas/secret-scanning-git-risks.ts +++ b/backend/src/db/schemas/secret-scanning-git-risks.ts @@ -42,5 +42,7 @@ export const SecretScanningGitRisksSchema = z.object({ }); export type TSecretScanningGitRisks = z.infer; -export type TSecretScanningGitRisksInsert = Omit; -export type TSecretScanningGitRisksUpdate = Partial>; +export type TSecretScanningGitRisksInsert = Omit, TImmutableDBKeys>; +export type TSecretScanningGitRisksUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-snapshot-folders.ts b/backend/src/db/schemas/secret-snapshot-folders.ts index acf11ab0a..3e2853cbe 100644 --- a/backend/src/db/schemas/secret-snapshot-folders.ts +++ b/backend/src/db/schemas/secret-snapshot-folders.ts @@ -17,5 +17,5 @@ export const SecretSnapshotFoldersSchema = z.object({ }); export type TSecretSnapshotFolders = z.infer; -export type TSecretSnapshotFoldersInsert = Omit; -export type TSecretSnapshotFoldersUpdate = Partial>; +export type TSecretSnapshotFoldersInsert = Omit, TImmutableDBKeys>; +export type TSecretSnapshotFoldersUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-snapshot-secrets.ts b/backend/src/db/schemas/secret-snapshot-secrets.ts index 6a83d1155..121c5d56e 100644 --- a/backend/src/db/schemas/secret-snapshot-secrets.ts +++ b/backend/src/db/schemas/secret-snapshot-secrets.ts @@ -17,5 +17,5 @@ export const SecretSnapshotSecretsSchema = z.object({ }); export type TSecretSnapshotSecrets = z.infer; -export type TSecretSnapshotSecretsInsert = Omit; -export type TSecretSnapshotSecretsUpdate = Partial>; +export type TSecretSnapshotSecretsInsert = Omit, TImmutableDBKeys>; +export type TSecretSnapshotSecretsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-snapshots.ts b/backend/src/db/schemas/secret-snapshots.ts index ed255cb77..7f070075f 100644 --- a/backend/src/db/schemas/secret-snapshots.ts +++ b/backend/src/db/schemas/secret-snapshots.ts @@ -17,5 +17,5 @@ export const SecretSnapshotsSchema = z.object({ }); export type TSecretSnapshots = z.infer; -export type TSecretSnapshotsInsert = Omit; -export type TSecretSnapshotsUpdate = Partial>; +export type TSecretSnapshotsInsert = Omit, TImmutableDBKeys>; +export type TSecretSnapshotsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-tag-junction.ts b/backend/src/db/schemas/secret-tag-junction.ts index 1d25574c5..d14384fab 100644 --- a/backend/src/db/schemas/secret-tag-junction.ts +++ b/backend/src/db/schemas/secret-tag-junction.ts @@ -14,5 +14,5 @@ export const SecretTagJunctionSchema = z.object({ }); export type TSecretTagJunction = z.infer; -export type TSecretTagJunctionInsert = Omit; -export type TSecretTagJunctionUpdate = Partial>; +export type TSecretTagJunctionInsert = Omit, TImmutableDBKeys>; +export type TSecretTagJunctionUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-tags.ts b/backend/src/db/schemas/secret-tags.ts index 622c29bd3..f94e1e262 100644 --- a/backend/src/db/schemas/secret-tags.ts +++ b/backend/src/db/schemas/secret-tags.ts @@ -19,5 +19,5 @@ export const SecretTagsSchema = z.object({ }); export type TSecretTags = z.infer; -export type TSecretTagsInsert = Omit; -export type TSecretTagsUpdate = Partial>; +export type TSecretTagsInsert = Omit, TImmutableDBKeys>; +export type TSecretTagsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-version-tag-junction.ts b/backend/src/db/schemas/secret-version-tag-junction.ts index 2c9a24fee..b36e28c72 100644 --- a/backend/src/db/schemas/secret-version-tag-junction.ts +++ b/backend/src/db/schemas/secret-version-tag-junction.ts @@ -14,5 +14,7 @@ export const SecretVersionTagJunctionSchema = z.object({ }); export type TSecretVersionTagJunction = z.infer; -export type TSecretVersionTagJunctionInsert = Omit; -export type TSecretVersionTagJunctionUpdate = Partial>; +export type TSecretVersionTagJunctionInsert = Omit, TImmutableDBKeys>; +export type TSecretVersionTagJunctionUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-versions.ts b/backend/src/db/schemas/secret-versions.ts index d1675e3d2..d60db9b75 100644 --- a/backend/src/db/schemas/secret-versions.ts +++ b/backend/src/db/schemas/secret-versions.ts @@ -36,5 +36,5 @@ export const SecretVersionsSchema = z.object({ }); export type TSecretVersions = z.infer; -export type TSecretVersionsInsert = Omit; -export type TSecretVersionsUpdate = Partial>; +export type TSecretVersionsInsert = Omit, TImmutableDBKeys>; +export type TSecretVersionsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secrets.ts b/backend/src/db/schemas/secrets.ts index 3fe5ad8d8..f261c40bb 100644 --- a/backend/src/db/schemas/secrets.ts +++ b/backend/src/db/schemas/secrets.ts @@ -34,5 +34,5 @@ export const SecretsSchema = z.object({ }); export type TSecrets = z.infer; -export type TSecretsInsert = Omit; -export type TSecretsUpdate = Partial>; +export type TSecretsInsert = Omit, TImmutableDBKeys>; +export type TSecretsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/service-tokens.ts b/backend/src/db/schemas/service-tokens.ts index 24720f3e4..720c8fd6f 100644 --- a/backend/src/db/schemas/service-tokens.ts +++ b/backend/src/db/schemas/service-tokens.ts @@ -25,5 +25,5 @@ export const ServiceTokensSchema = z.object({ }); export type TServiceTokens = z.infer; -export type TServiceTokensInsert = Omit; -export type TServiceTokensUpdate = Partial>; +export type TServiceTokensInsert = Omit, TImmutableDBKeys>; +export type TServiceTokensUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/super-admin.ts b/backend/src/db/schemas/super-admin.ts index b4631195b..958fed0ab 100644 --- a/backend/src/db/schemas/super-admin.ts +++ b/backend/src/db/schemas/super-admin.ts @@ -13,9 +13,10 @@ export const SuperAdminSchema = z.object({ allowSignUp: z.boolean().default(true).nullable().optional(), createdAt: z.date(), updatedAt: z.date(), - allowedSignUpDomain: z.string().nullable().optional() + allowedSignUpDomain: z.string().nullable().optional(), + instanceId: z.string().uuid().default("00000000-0000-0000-0000-000000000000") }); export type TSuperAdmin = z.infer; -export type TSuperAdminInsert = Omit; -export type TSuperAdminUpdate = Partial>; +export type TSuperAdminInsert = Omit, TImmutableDBKeys>; +export type TSuperAdminUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/trusted-ips.ts b/backend/src/db/schemas/trusted-ips.ts index 6d9018db3..f9973a843 100644 --- a/backend/src/db/schemas/trusted-ips.ts +++ b/backend/src/db/schemas/trusted-ips.ts @@ -20,5 +20,5 @@ export const TrustedIpsSchema = z.object({ }); export type TTrustedIps = z.infer; -export type TTrustedIpsInsert = Omit; -export type TTrustedIpsUpdate = Partial>; +export type TTrustedIpsInsert = Omit, TImmutableDBKeys>; +export type TTrustedIpsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/user-actions.ts b/backend/src/db/schemas/user-actions.ts index eaa03ba98..89d269847 100644 --- a/backend/src/db/schemas/user-actions.ts +++ b/backend/src/db/schemas/user-actions.ts @@ -16,5 +16,5 @@ export const UserActionsSchema = z.object({ }); export type TUserActions = z.infer; -export type TUserActionsInsert = Omit; -export type TUserActionsUpdate = Partial>; +export type TUserActionsInsert = Omit, TImmutableDBKeys>; +export type TUserActionsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/user-encryption-keys.ts b/backend/src/db/schemas/user-encryption-keys.ts index 8f35b09fe..693b73b4c 100644 --- a/backend/src/db/schemas/user-encryption-keys.ts +++ b/backend/src/db/schemas/user-encryption-keys.ts @@ -25,5 +25,5 @@ export const UserEncryptionKeysSchema = z.object({ }); export type TUserEncryptionKeys = z.infer; -export type TUserEncryptionKeysInsert = Omit; -export type TUserEncryptionKeysUpdate = Partial>; +export type TUserEncryptionKeysInsert = Omit, TImmutableDBKeys>; +export type TUserEncryptionKeysUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/users.ts b/backend/src/db/schemas/users.ts index cdbb5607a..8f9758b4f 100644 --- a/backend/src/db/schemas/users.ts +++ b/backend/src/db/schemas/users.ts @@ -24,5 +24,5 @@ export const UsersSchema = z.object({ }); export type TUsers = z.infer; -export type TUsersInsert = Omit; -export type TUsersUpdate = Partial>; +export type TUsersInsert = Omit, TImmutableDBKeys>; +export type TUsersUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/webhooks.ts b/backend/src/db/schemas/webhooks.ts index 7abfb3772..44aa8c5da 100644 --- a/backend/src/db/schemas/webhooks.ts +++ b/backend/src/db/schemas/webhooks.ts @@ -25,5 +25,5 @@ export const WebhooksSchema = z.object({ }); export type TWebhooks = z.infer; -export type TWebhooksInsert = Omit; -export type TWebhooksUpdate = Partial>; +export type TWebhooksInsert = Omit, TImmutableDBKeys>; +export type TWebhooksUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 99b781b24..9458501b1 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -9,5 +9,10 @@ export const keyStoreFactory = (redisUrl: string) => { const getItem = async (key: string) => redis.get(key); - return { setItem, getItem }; + const setItemWithExpiry = async (key: string, exp: number | string, value: string | number | Buffer) => + redis.setex(key, exp, value); + + const deleteItem = async (key: string) => redis.del(key); + + return { setItem, getItem, setItemWithExpiry, deleteItem }; }; diff --git a/backend/src/services/super-admin/super-admin-service.ts b/backend/src/services/super-admin/super-admin-service.ts index 98491ba82..2a6a2c75e 100644 --- a/backend/src/services/super-admin/super-admin-service.ts +++ b/backend/src/services/super-admin/super-admin-service.ts @@ -15,7 +15,7 @@ type TSuperAdminServiceFactoryDep = { userDAL: TUserDALFactory; authService: Pick; orgService: Pick; - keyStore: Pick; + keyStore: Pick; }; export type TSuperAdminServiceFactory = ReturnType; @@ -24,6 +24,9 @@ export type TSuperAdminServiceFactory = ReturnType Promise; const ADMIN_CONFIG_KEY = "infisical-admin-cfg"; +const ADMIN_CONFIG_KEY_EXP = 60; // 60s +const ADMIN_CONFIG_DB_UUID = "00000000-0000-0000-0000-000000000000"; + export const superAdminServiceFactory = ({ serverCfgDAL, userDAL, @@ -35,11 +38,15 @@ export const superAdminServiceFactory = ({ // TODO(akhilmhdh): bad pattern time less change this later to me itself getServerCfg = async () => { const config = await keyStore.getItem(ADMIN_CONFIG_KEY); + // missing in keystore means fetch from db if (!config) { - const serverCfg = await serverCfgDAL.findOne({}); - await keyStore.setItem(ADMIN_CONFIG_KEY, JSON.stringify(serverCfg)); + const serverCfg = await serverCfgDAL.findById(ADMIN_CONFIG_DB_UUID); + if (serverCfg) { + await keyStore.setItemWithExpiry(ADMIN_CONFIG_KEY, ADMIN_CONFIG_KEY_EXP, JSON.stringify(serverCfg)); // insert it back to keystore + } return serverCfg; } + const keyStoreServerCfg = JSON.parse(config) as TSuperAdmin; return { ...keyStoreServerCfg, @@ -49,17 +56,19 @@ export const superAdminServiceFactory = ({ }; }; - const serverCfg = await getServerCfg(); + // reset on initialized + await keyStore.deleteItem(ADMIN_CONFIG_KEY); + const serverCfg = await serverCfgDAL.findById(ADMIN_CONFIG_DB_UUID); if (serverCfg) return; - const newCfg = await serverCfgDAL.create({ initialized: false, allowSignUp: true }); + // @ts-expect-error id is kept as fixed for idempotence and to avoid race condition + const newCfg = await serverCfgDAL.create({ initialized: false, allowSignUp: true, id: ADMIN_CONFIG_DB_UUID }); return newCfg; }; const updateServerCfg = async (data: TSuperAdminUpdate) => { - const serverCfg = await getServerCfg(); - const updatedServerCfg = await serverCfgDAL.updateById(serverCfg.id, data); - await keyStore.setItem(ADMIN_CONFIG_KEY, JSON.stringify(updatedServerCfg)); + const updatedServerCfg = await serverCfgDAL.updateById(ADMIN_CONFIG_DB_UUID, data); + await keyStore.setItemWithExpiry(ADMIN_CONFIG_KEY, ADMIN_CONFIG_KEY_EXP, JSON.stringify(updatedServerCfg)); return updatedServerCfg; }; From d64d935d7d11adb99bacd9ed7eb0db57911544e8 Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Mon, 26 Feb 2024 19:48:24 +0530 Subject: [PATCH 3/7] feat(server): added telemetry queue for self hosted to upload instance stats to posthog on midnight --- backend/e2e-test/mocks/keystore.ts | 3 + .../ee/services/license/license-service.ts | 3 + backend/src/keystore/keystore.ts | 4 +- backend/src/queue/queue-service.ts | 7 +- backend/src/server/routes/index.ts | 20 ++++- .../src/services/telemetry/telemetry-dal.ts | 39 ++++++++++ .../src/services/telemetry/telemetry-queue.ts | 78 +++++++++++++++++++ .../services/telemetry/telemetry-service.ts | 47 ++++++++--- .../src/services/telemetry/telemetry-types.ts | 18 ++++- 9 files changed, 205 insertions(+), 14 deletions(-) create mode 100644 backend/src/services/telemetry/telemetry-dal.ts create mode 100644 backend/src/services/telemetry/telemetry-queue.ts diff --git a/backend/e2e-test/mocks/keystore.ts b/backend/e2e-test/mocks/keystore.ts index 7a1bab2b3..c85244129 100644 --- a/backend/e2e-test/mocks/keystore.ts +++ b/backend/e2e-test/mocks/keystore.ts @@ -22,6 +22,9 @@ export const mockKeyStore = (): TKeyStoreFactory => { return value; } return null; + }, + incrementBy: async () => { + return 1; } }; }; diff --git a/backend/src/ee/services/license/license-service.ts b/backend/src/ee/services/license/license-service.ts index 6d97b537c..b0bc11e31 100644 --- a/backend/src/ee/services/license/license-service.ts +++ b/backend/src/ee/services/license/license-service.ts @@ -505,6 +505,9 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }: get isValidLicense() { return isValidLicense; }, + getInstanceType() { + return instanceType; + }, getPlan, updateSubscriptionOrgMemberCount, refreshPlan, diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 9458501b1..5e2c3aab3 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -14,5 +14,7 @@ export const keyStoreFactory = (redisUrl: string) => { const deleteItem = async (key: string) => redis.del(key); - return { setItem, getItem, setItemWithExpiry, deleteItem }; + const incrementBy = async (key: string, value: number) => redis.incrby(key, value); + + return { setItem, getItem, setItemWithExpiry, deleteItem, incrementBy }; }; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index e11e0b590..45c135b77 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -13,6 +13,7 @@ export enum QueueName { SecretReminder = "secret-reminder", AuditLog = "audit-log", AuditLogPrune = "audit-log-prune", + TelemetryInstanceStats = "telemtry-self-hosted-stats", IntegrationSync = "sync-integrations", SecretWebhook = "secret-webhook", SecretFullRepoScan = "secret-full-repo-scan", @@ -26,6 +27,7 @@ export enum QueueJobs { AuditLog = "audit-log-job", AuditLogPrune = "audit-log-prune-job", SecWebhook = "secret-webhook-trigger", + TelemetryInstanceStats = "telemetry-self-hosted-stats", IntegrationSync = "secret-integration-pull", SecretScan = "secret-scan", UpgradeProjectToGhost = "upgrade-project-to-ghost-job" @@ -67,7 +69,6 @@ export type TQueueJobTypes = { payload: TScanFullRepoEventPayload; }; [QueueName.SecretPushEventScan]: { name: QueueJobs.SecretScan; payload: TScanPushEventPayload }; - [QueueName.UpgradeProjectToGhost]: { name: QueueJobs.UpgradeProjectToGhost; payload: { @@ -81,6 +82,10 @@ export type TQueueJobTypes = { }; }; }; + [QueueName.TelemetryInstanceStats]: { + name: QueueJobs.TelemetryInstanceStats; + payload: undefined; + }; }; export type TQueueServiceFactory = ReturnType; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index e9c35b1d7..a21825633 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -97,6 +97,8 @@ import { serviceTokenServiceFactory } from "@app/services/service-token/service- import { TSmtpService } from "@app/services/smtp/smtp-service"; import { superAdminDALFactory } from "@app/services/super-admin/super-admin-dal"; import { getServerCfg, superAdminServiceFactory } from "@app/services/super-admin/super-admin-service"; +import { telemetryDALFactory } from "@app/services/telemetry/telemetry-dal"; +import { telemetryQueueServiceFactory } from "@app/services/telemetry/telemetry-queue"; import { telemetryServiceFactory } from "@app/services/telemetry/telemetry-service"; import { userDALFactory } from "@app/services/user/user-dal"; import { userServiceFactory } from "@app/services/user/user-service"; @@ -165,6 +167,7 @@ export const registerRoutes = async ( const auditLogDAL = auditLogDALFactory(db); const trustedIpDAL = trustedIpDALFactory(db); const scimDAL = scimDALFactory(db); + const telemetryDAL = telemetryDALFactory(db); // ee db layer ops const permissionDAL = permissionDALFactory(db); @@ -232,7 +235,16 @@ export const registerRoutes = async ( smtpService }); - const telemetryService = telemetryServiceFactory(); + const telemetryService = telemetryServiceFactory({ + keyStore, + licenseService + }); + const telemetryQueue = telemetryQueueServiceFactory({ + keyStore, + telemetryDAL, + queueService + }); + const tokenService = tokenServiceFactory({ tokenDAL: authTokenDAL, userDAL }); const userService = userServiceFactory({ userDAL }); const loginService = authLoginServiceFactory({ userDAL, smtpService, tokenService }); @@ -498,9 +510,13 @@ export const registerRoutes = async ( }); await superAdminService.initServerCfg(); - await auditLogQueue.startAuditLogPruneJob(); + // // setup the communication with license key server await licenseService.init(); + + await auditLogQueue.startAuditLogPruneJob(); + await telemetryQueue.startTelemetryCheck(); + // inject all services server.decorate("services", { login: loginService, diff --git a/backend/src/services/telemetry/telemetry-dal.ts b/backend/src/services/telemetry/telemetry-dal.ts new file mode 100644 index 000000000..9fac4f1ef --- /dev/null +++ b/backend/src/services/telemetry/telemetry-dal.ts @@ -0,0 +1,39 @@ +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; + +export type TTelemetryDALFactory = ReturnType; + +export const telemetryDALFactory = (db: TDbClient) => { + const getTelemetryInstanceStats = async () => { + try { + const userCount = (await db(TableName.Users).where({ isGhost: false }).count().first())?.count as string; + const users = parseInt(userCount || "0", 10); + + const identityCount = (await db(TableName.Identity).count().first())?.count as string; + const identities = parseInt(identityCount || "0", 10); + + const projectCount = (await db(TableName.Project).count().first())?.count as string; + const projects = parseInt(projectCount || "0", 10); + + const secretCount = (await db(TableName.Secret).count().first())?.count as string; + const secrets = parseInt(secretCount || "0", 10); + + const organizationNames = await db(TableName.Organization).select("name"); + const organizations = organizationNames.length; + + return { + users, + identities, + projects, + secrets, + organizations, + organizationNames: organizationNames.map(({ name }) => name) + }; + } catch (error) { + throw new DatabaseError({ error, name: "TelemtryInstanceStats" }); + } + }; + + return { getTelemetryInstanceStats }; +}; diff --git a/backend/src/services/telemetry/telemetry-queue.ts b/backend/src/services/telemetry/telemetry-queue.ts new file mode 100644 index 000000000..0a79aa9cc --- /dev/null +++ b/backend/src/services/telemetry/telemetry-queue.ts @@ -0,0 +1,78 @@ +import { PostHog } from "posthog-node"; + +import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { getConfig } from "@app/lib/config/env"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { getServerCfg } from "../super-admin/super-admin-service"; +import { TTelemetryDALFactory } from "./telemetry-dal"; +import { TELEMETRY_SECRET_OPERATONS_KEY, TELEMETRY_SECRET_PROCESSED_KEY } from "./telemetry-service"; +import { PostHogEventTypes } from "./telemetry-types"; + +type TTelemetryQueueServiceFactoryDep = { + queueService: TQueueServiceFactory; + keyStore: Pick; + telemetryDAL: TTelemetryDALFactory; +}; + +export type TTelemetryQueueServiceFactory = ReturnType; + +export const telemetryQueueServiceFactory = ({ + queueService, + keyStore, + telemetryDAL +}: TTelemetryQueueServiceFactoryDep) => { + const appCfg = getConfig(); + const postHog = + appCfg.isProductionMode && appCfg.TELEMETRY_ENABLED + ? new PostHog(appCfg.POSTHOG_PROJECT_API_KEY, { host: appCfg.POSTHOG_HOST }) + : undefined; + + queueService.start(QueueName.TelemetryInstanceStats, async () => { + const { instanceId } = await getServerCfg(); + const telemtryStats = await telemetryDAL.getTelemetryInstanceStats(); + // parse the redis values into integer + const numberOfSecretOperations = parseInt((await keyStore.getItem(TELEMETRY_SECRET_OPERATONS_KEY)) || "0", 10); + const numberOfSecretProcessed = parseInt((await keyStore.getItem(TELEMETRY_SECRET_PROCESSED_KEY)) || "0", 10); + const stats = { ...telemtryStats, numberOfSecretProcessed, numberOfSecretOperations }; + + // send to postHog + postHog?.capture({ + event: PostHogEventTypes.TelemetryInstanceStats, + distinctId: instanceId, + properties: stats + }); + // reset the stats + await keyStore.deleteItem(TELEMETRY_SECRET_PROCESSED_KEY); + await keyStore.deleteItem(TELEMETRY_SECRET_OPERATONS_KEY); + }); + + // every day at midnight a telemetry job executes on self hosted + // this sends some telemetry information like instance id secrets operated etc + const startTelemetryCheck = async () => { + // this is a fast way to check its cloud or not + if (appCfg.LICENSE_SERVER_KEY) return; + // clear previous job + await queueService.stopRepeatableJob( + QueueName.TelemetryInstanceStats, + QueueJobs.TelemetryInstanceStats, + { pattern: "0 0 * * *", utc: true }, + QueueName.TelemetryInstanceStats // just a job id + ); + if (postHog) { + await queueService.queue(QueueName.TelemetryInstanceStats, QueueJobs.TelemetryInstanceStats, undefined, { + jobId: QueueName.TelemetryInstanceStats, + repeat: { pattern: "0 0 * * *", utc: true } + }); + } + }; + + queueService.listen(QueueName.TelemetryInstanceStats, "failed", (err) => { + logger.error(err?.failedReason, `${QueueName.TelemetryInstanceStats}: failed`); + }); + + return { + startTelemetryCheck + }; +}; diff --git a/backend/src/services/telemetry/telemetry-service.ts b/backend/src/services/telemetry/telemetry-service.ts index a1a4b78f7..ebaa61444 100644 --- a/backend/src/services/telemetry/telemetry-service.ts +++ b/backend/src/services/telemetry/telemetry-service.ts @@ -1,15 +1,24 @@ import { PostHog } from "posthog-node"; +import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; +import { InstanceType } from "@app/ee/services/license/license-types"; +import { TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { request } from "@app/lib/config/request"; import { logger } from "@app/lib/logger"; -import { TPostHogEvent } from "./telemetry-types"; +import { PostHogEventTypes, TPostHogEvent, TSecretModifiedEvent } from "./telemetry-types"; + +export const TELEMETRY_SECRET_PROCESSED_KEY = "telemtry-secret-processed"; +export const TELEMETRY_SECRET_OPERATONS_KEY = "telemtry-secret-operations"; export type TTelemetryServiceFactory = ReturnType; +export type TTelemetryServiceFactoryDep = { + keyStore: Pick; + licenseService: Pick; +}; -// type TTelemetryServiceFactoryDep = {}; -export const telemetryServiceFactory = () => { +export const telemetryServiceFactory = ({ keyStore, licenseService }: TTelemetryServiceFactoryDep) => { const appCfg = getConfig(); if (appCfg.isProductionMode && !appCfg.TELEMETRY_ENABLED) { @@ -51,13 +60,33 @@ To opt into telemetry, you can set "TELEMETRY_ENABLED=true" within the environme } }; - const sendPostHogEvents = (event: TPostHogEvent) => { + const sendPostHogEvents = async (event: TPostHogEvent) => { if (postHog) { - postHog.capture({ - event: event.event, - distinctId: event.distinctId, - properties: event.properties - }); + const instanceType = licenseService.getInstanceType(); + // capture posthog only when its cloud or signup event happens in self hosted + if (instanceType === InstanceType.Cloud || event.event === PostHogEventTypes.UserSignedUp) { + postHog.capture({ + event: event.event, + distinctId: event.distinctId, + properties: event.properties + }); + return; + } + + if ( + [ + PostHogEventTypes.SecretPulled, + PostHogEventTypes.SecretCreated, + PostHogEventTypes.SecretDeleted, + PostHogEventTypes.SecretUpdated + ].includes(event.event) + ) { + await keyStore.incrementBy( + TELEMETRY_SECRET_PROCESSED_KEY, + (event as TSecretModifiedEvent).properties.numberOfSecrets + ); + await keyStore.incrementBy(TELEMETRY_SECRET_OPERATONS_KEY, 1); + } } }; diff --git a/backend/src/services/telemetry/telemetry-types.ts b/backend/src/services/telemetry/telemetry-types.ts index 0f92da4da..9feb56b14 100644 --- a/backend/src/services/telemetry/telemetry-types.ts +++ b/backend/src/services/telemetry/telemetry-types.ts @@ -12,7 +12,8 @@ export enum PostHogEventTypes { ProjectCreated = "Project Created", IntegrationCreated = "Integration Created", MachineIdentityCreated = "Machine Identity Created", - UserOrgInvitation = "User Org Invitation" + UserOrgInvitation = "User Org Invitation", + TelemetryInstanceStats = "Self Hosted Instance Stats" } export type TSecretModifiedEvent = { @@ -101,6 +102,20 @@ export type TUserOrgInvitedEvent = { }; }; +export type TTelemetryInstanceStatsEvent = { + event: PostHogEventTypes.TelemetryInstanceStats; + properties: { + users: number; + identities: number; + projects: number; + secrets: number; + organizations: number; + organizationNames: number; + numberOfSecretOperations: number; + numberOfSecretProcessed: number; + }; +}; + export type TPostHogEvent = { distinctId: string } & ( | TSecretModifiedEvent | TAdminInitEvent @@ -110,4 +125,5 @@ export type TPostHogEvent = { distinctId: string } & ( | TMachineIdentityCreatedEvent | TIntegrationCreatedEvent | TProjectCreateEvent + | TTelemetryInstanceStatsEvent ); From 3b9ceff21c052918f43120b7fa9bfeee5e941aaa Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Mon, 26 Feb 2024 19:52:14 +0530 Subject: [PATCH 4/7] refactor(server): updated all telemetry send events to await as changed to async --- .../secret-rotation-queue.ts | 2 +- .../secret-scanning-queue.ts | 4 +-- backend/src/server/routes/v1/admin-router.ts | 2 +- .../src/server/routes/v1/identity-router.ts | 2 +- .../server/routes/v1/integration-router.ts | 2 +- .../src/server/routes/v1/invite-org-router.ts | 2 +- .../src/server/routes/v2/project-router.ts | 2 +- backend/src/server/routes/v3/secret-router.ts | 26 +++++++++---------- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index 9e69f0a8f..330f6beca 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -240,7 +240,7 @@ export const secretRotationQueueFactory = ({ ); }); - telemetryService.sendPostHogEvents({ + await telemetryService.sendPostHogEvents({ event: PostHogEventTypes.SecretRotated, distinctId: "", properties: { diff --git a/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts b/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts index aab8d1218..d8f4e647e 100644 --- a/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts +++ b/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts @@ -158,7 +158,7 @@ export const secretScanningQueueFactory = ({ }); } - telemetryService.sendPostHogEvents({ + await telemetryService.sendPostHogEvents({ event: PostHogEventTypes.SecretScannerPush, distinctId: repository.fullName, properties: { @@ -228,7 +228,7 @@ export const secretScanningQueueFactory = ({ }); } - telemetryService.sendPostHogEvents({ + await telemetryService.sendPostHogEvents({ event: PostHogEventTypes.SecretScannerFull, distinctId: repository.fullName, properties: { diff --git a/backend/src/server/routes/v1/admin-router.ts b/backend/src/server/routes/v1/admin-router.ts index 6eec355a8..38f948851 100644 --- a/backend/src/server/routes/v1/admin-router.ts +++ b/backend/src/server/routes/v1/admin-router.ts @@ -90,7 +90,7 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => { userAgent: req.headers["user-agent"] || "" }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.AdminInit, distinctId: user.user.email, properties: { diff --git a/backend/src/server/routes/v1/identity-router.ts b/backend/src/server/routes/v1/identity-router.ts index a03cde62f..0ec27a98b 100644 --- a/backend/src/server/routes/v1/identity-router.ts +++ b/backend/src/server/routes/v1/identity-router.ts @@ -51,7 +51,7 @@ export const registerIdentityRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.MachineIdentityCreated, distinctId: getTelemetryDistinctId(req), properties: { diff --git a/backend/src/server/routes/v1/integration-router.ts b/backend/src/server/routes/v1/integration-router.ts index 1ccf75e94..ec712a543 100644 --- a/backend/src/server/routes/v1/integration-router.ts +++ b/backend/src/server/routes/v1/integration-router.ts @@ -82,7 +82,7 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.IntegrationCreated, distinctId: getTelemetryDistinctId(req), properties: { diff --git a/backend/src/server/routes/v1/invite-org-router.ts b/backend/src/server/routes/v1/invite-org-router.ts index dd2fd8157..5956b53df 100644 --- a/backend/src/server/routes/v1/invite-org-router.ts +++ b/backend/src/server/routes/v1/invite-org-router.ts @@ -32,7 +32,7 @@ export const registerInviteOrgRouter = async (server: FastifyZodProvider) => { actorOrgId: req.permission.orgId }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.UserOrgInvitation, distinctId: getTelemetryDistinctId(req), properties: { diff --git a/backend/src/server/routes/v2/project-router.ts b/backend/src/server/routes/v2/project-router.ts index 62f4c7cf5..fe1254b2b 100644 --- a/backend/src/server/routes/v2/project-router.ts +++ b/backend/src/server/routes/v2/project-router.ts @@ -154,7 +154,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { slug: req.body.slug }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.ProjectCreated, distinctId: getTelemetryDistinctId(req), properties: { diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index cfe4a87d7..6b3dd6041 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -95,7 +95,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretPulled, distinctId: getTelemetryDistinctId(req), properties: { @@ -185,7 +185,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretPulled, distinctId: getTelemetryDistinctId(req), properties: { @@ -261,7 +261,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretCreated, distinctId: getTelemetryDistinctId(req), properties: { @@ -336,7 +336,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretUpdated, distinctId: getTelemetryDistinctId(req), properties: { @@ -406,7 +406,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretDeleted, distinctId: getTelemetryDistinctId(req), properties: { @@ -512,7 +512,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { (req.headers["user-agent"] !== "k8-operator" || shouldRecordK8Event); const approximateNumberTotalSecrets = secrets.length * 20; if (shouldCapture) { - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretPulled, distinctId: getTelemetryDistinctId(req), properties: { @@ -589,7 +589,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretPulled, distinctId: getTelemetryDistinctId(req), properties: { @@ -752,7 +752,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretCreated, distinctId: getTelemetryDistinctId(req), properties: { @@ -934,7 +934,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretUpdated, distinctId: getTelemetryDistinctId(req), properties: { @@ -1052,7 +1052,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretDeleted, distinctId: getTelemetryDistinctId(req), properties: { @@ -1172,7 +1172,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretCreated, distinctId: getTelemetryDistinctId(req), properties: { @@ -1292,7 +1292,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretUpdated, distinctId: getTelemetryDistinctId(req), properties: { @@ -1400,7 +1400,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretDeleted, distinctId: getTelemetryDistinctId(req), properties: { From fe497d87c0b632a74cd096d7b8908749b30a2240 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Tue, 27 Feb 2024 16:38:18 -0500 Subject: [PATCH 5/7] add INFISICAL_CLOUD env back from old backend --- backend/src/lib/config/env.ts | 7 +++++-- backend/src/services/telemetry/telemetry-queue.ts | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 4542c7fc3..655e402df 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -94,14 +94,17 @@ const envSchema = z SECRET_SCANNING_WEBHOOK_SECRET: zpStr(z.string().optional()), SECRET_SCANNING_GIT_APP_ID: zpStr(z.string().optional()), SECRET_SCANNING_PRIVATE_KEY: zpStr(z.string().optional()), - // LICENCE + // LICENSE LICENSE_SERVER_URL: zpStr(z.string().optional().default("https://portal.infisical.com")), LICENSE_SERVER_KEY: zpStr(z.string().optional()), LICENSE_KEY: zpStr(z.string().optional()), + + // GENERIC STANDALONE_MODE: z .enum(["true", "false"]) .transform((val) => val === "true") - .optional() + .optional(), + INFISICAL_CLOUD: zodStrBool.default("false") }) .transform((data) => ({ ...data, diff --git a/backend/src/services/telemetry/telemetry-queue.ts b/backend/src/services/telemetry/telemetry-queue.ts index 0a79aa9cc..1b855e6bb 100644 --- a/backend/src/services/telemetry/telemetry-queue.ts +++ b/backend/src/services/telemetry/telemetry-queue.ts @@ -52,7 +52,7 @@ export const telemetryQueueServiceFactory = ({ // this sends some telemetry information like instance id secrets operated etc const startTelemetryCheck = async () => { // this is a fast way to check its cloud or not - if (appCfg.LICENSE_SERVER_KEY) return; + if (appCfg.INFISICAL_CLOUD) return; // clear previous job await queueService.stopRepeatableJob( QueueName.TelemetryInstanceStats, From d6cde4818146787e49d3db4f78945cc6b9309127 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Tue, 27 Feb 2024 18:23:24 -0500 Subject: [PATCH 6/7] set posthog flush to zero and fix typos --- backend/src/services/telemetry/telemetry-queue.ts | 10 +++++----- backend/src/services/telemetry/telemetry-service.ts | 13 ++++++------- backend/src/services/telemetry/telemetry-types.ts | 2 +- frontend/package-lock.json | 2 +- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/backend/src/services/telemetry/telemetry-queue.ts b/backend/src/services/telemetry/telemetry-queue.ts index 1b855e6bb..02e906fe6 100644 --- a/backend/src/services/telemetry/telemetry-queue.ts +++ b/backend/src/services/telemetry/telemetry-queue.ts @@ -7,7 +7,7 @@ import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { getServerCfg } from "../super-admin/super-admin-service"; import { TTelemetryDALFactory } from "./telemetry-dal"; -import { TELEMETRY_SECRET_OPERATONS_KEY, TELEMETRY_SECRET_PROCESSED_KEY } from "./telemetry-service"; +import { TELEMETRY_SECRET_OPERATIONS_KEY, TELEMETRY_SECRET_PROCESSED_KEY } from "./telemetry-service"; import { PostHogEventTypes } from "./telemetry-types"; type TTelemetryQueueServiceFactoryDep = { @@ -26,16 +26,16 @@ export const telemetryQueueServiceFactory = ({ const appCfg = getConfig(); const postHog = appCfg.isProductionMode && appCfg.TELEMETRY_ENABLED - ? new PostHog(appCfg.POSTHOG_PROJECT_API_KEY, { host: appCfg.POSTHOG_HOST }) + ? new PostHog(appCfg.POSTHOG_PROJECT_API_KEY, { host: appCfg.POSTHOG_HOST, flushAt: 1, flushInterval: 0 }) : undefined; queueService.start(QueueName.TelemetryInstanceStats, async () => { const { instanceId } = await getServerCfg(); const telemtryStats = await telemetryDAL.getTelemetryInstanceStats(); // parse the redis values into integer - const numberOfSecretOperations = parseInt((await keyStore.getItem(TELEMETRY_SECRET_OPERATONS_KEY)) || "0", 10); + const numberOfSecretOperationsMade = parseInt((await keyStore.getItem(TELEMETRY_SECRET_OPERATIONS_KEY)) || "0", 10); const numberOfSecretProcessed = parseInt((await keyStore.getItem(TELEMETRY_SECRET_PROCESSED_KEY)) || "0", 10); - const stats = { ...telemtryStats, numberOfSecretProcessed, numberOfSecretOperations }; + const stats = { ...telemtryStats, numberOfSecretProcessed, numberOfSecretOperationsMade }; // send to postHog postHog?.capture({ @@ -45,7 +45,7 @@ export const telemetryQueueServiceFactory = ({ }); // reset the stats await keyStore.deleteItem(TELEMETRY_SECRET_PROCESSED_KEY); - await keyStore.deleteItem(TELEMETRY_SECRET_OPERATONS_KEY); + await keyStore.deleteItem(TELEMETRY_SECRET_OPERATIONS_KEY); }); // every day at midnight a telemetry job executes on self hosted diff --git a/backend/src/services/telemetry/telemetry-service.ts b/backend/src/services/telemetry/telemetry-service.ts index ebaa61444..9912e0101 100644 --- a/backend/src/services/telemetry/telemetry-service.ts +++ b/backend/src/services/telemetry/telemetry-service.ts @@ -9,8 +9,8 @@ import { logger } from "@app/lib/logger"; import { PostHogEventTypes, TPostHogEvent, TSecretModifiedEvent } from "./telemetry-types"; -export const TELEMETRY_SECRET_PROCESSED_KEY = "telemtry-secret-processed"; -export const TELEMETRY_SECRET_OPERATONS_KEY = "telemtry-secret-operations"; +export const TELEMETRY_SECRET_PROCESSED_KEY = "telemetry-secret-processed"; +export const TELEMETRY_SECRET_OPERATIONS_KEY = "telemetry-secret-operations"; export type TTelemetryServiceFactory = ReturnType; export type TTelemetryServiceFactoryDep = { @@ -30,10 +30,9 @@ To opt into telemetry, you can set "TELEMETRY_ENABLED=true" within the environme `); } - const postHog = - appCfg.isProductionMode && appCfg.TELEMETRY_ENABLED - ? new PostHog(appCfg.POSTHOG_PROJECT_API_KEY, { host: appCfg.POSTHOG_HOST }) - : undefined; + const postHog = appCfg.TELEMETRY_ENABLED + ? new PostHog(appCfg.POSTHOG_PROJECT_API_KEY, { host: appCfg.POSTHOG_HOST }) + : undefined; // used for email marketting email sending purpose const sendLoopsEvent = async (email: string, firstName?: string, lastName?: string) => { @@ -85,7 +84,7 @@ To opt into telemetry, you can set "TELEMETRY_ENABLED=true" within the environme TELEMETRY_SECRET_PROCESSED_KEY, (event as TSecretModifiedEvent).properties.numberOfSecrets ); - await keyStore.incrementBy(TELEMETRY_SECRET_OPERATONS_KEY, 1); + await keyStore.incrementBy(TELEMETRY_SECRET_OPERATIONS_KEY, 1); } } }; diff --git a/backend/src/services/telemetry/telemetry-types.ts b/backend/src/services/telemetry/telemetry-types.ts index 9feb56b14..947403783 100644 --- a/backend/src/services/telemetry/telemetry-types.ts +++ b/backend/src/services/telemetry/telemetry-types.ts @@ -111,7 +111,7 @@ export type TTelemetryInstanceStatsEvent = { secrets: number; organizations: number; organizationNames: number; - numberOfSecretOperations: number; + numberOfSecretOperationsMade: number; numberOfSecretProcessed: number; }; }; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0ab91f92f..458d4e115 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,5 +1,5 @@ { - "name": "npm-proj-1708687711895-0.8280111363176879xoEiUg", + "name": "frontend", "lockfileVersion": 3, "requires": true, "packages": { From a2a959cc325bf7a6e171c7c83eb1bc3cbef339a1 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Tue, 27 Feb 2024 18:26:15 -0500 Subject: [PATCH 7/7] disable telemetry for local dev by default --- docker-compose.dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f95810777..6c3a89a74 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -86,6 +86,7 @@ services: environment: - NODE_ENV=development - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable + - TELEMETRY_ENABLED=false volumes: - ./backend/src:/app/src