From 1b15cb4c35cd588930077a79e740c4cc7ff14c2e Mon Sep 17 00:00:00 2001 From: = Date: Thu, 6 Feb 2025 15:08:37 +0530 Subject: [PATCH] feat: updated kms init to use pgsql lock --- backend/src/db/migrations/utils/kms.ts | 105 ------------------------ backend/src/keystore/keystore.ts | 3 +- backend/src/services/kms/kms-service.ts | 73 ++++++---------- 3 files changed, 28 insertions(+), 153 deletions(-) delete mode 100644 backend/src/db/migrations/utils/kms.ts diff --git a/backend/src/db/migrations/utils/kms.ts b/backend/src/db/migrations/utils/kms.ts deleted file mode 100644 index 9ed090978..000000000 --- a/backend/src/db/migrations/utils/kms.ts +++ /dev/null @@ -1,105 +0,0 @@ -import slugify from "@sindresorhus/slugify"; -import { Knex } from "knex"; - -import { TableName } from "@app/db/schemas"; -import { randomSecureBytes } from "@app/lib/crypto"; -import { symmetricCipherService, SymmetricEncryption } from "@app/lib/crypto/cipher"; -import { alphaNumericNanoId } from "@app/lib/nanoid"; - -const getInstanceRootKey = async (knex: Knex) => { - const encryptionKey = process.env.ENCRYPTION_KEY || process.env.ROOT_ENCRYPTION_KEY; - // if root key its base64 encoded - const isBase64 = !process.env.ENCRYPTION_KEY; - if (!encryptionKey) throw new Error("ENCRYPTION_KEY variable needed for migration"); - const encryptionKeyBuffer = Buffer.from(encryptionKey, isBase64 ? "base64" : "utf8"); - - const KMS_ROOT_CONFIG_UUID = "00000000-0000-0000-0000-000000000000"; - const kmsRootConfig = await knex(TableName.KmsServerRootConfig).where({ id: KMS_ROOT_CONFIG_UUID }).first(); - const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); - if (kmsRootConfig) { - const decryptedRootKey = cipher.decrypt(kmsRootConfig.encryptedRootKey, encryptionKeyBuffer); - // set the flag so that other instancen nodes can start - return decryptedRootKey; - } - - const newRootKey = randomSecureBytes(32); - const encryptedRootKey = cipher.encrypt(newRootKey, encryptionKeyBuffer); - await knex(TableName.KmsServerRootConfig).insert({ - encryptedRootKey, - // eslint-disable-next-line - // @ts-ignore id is kept as fixed for idempotence and to avoid race condition - id: KMS_ROOT_CONFIG_UUID - }); - return encryptedRootKey; -}; - -export const getSecretManagerDataKey = async (knex: Knex, projectId: string) => { - const KMS_VERSION = "v01"; - const KMS_VERSION_BLOB_LENGTH = 3; - const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); - const project = await knex(TableName.Project).where({ id: projectId }).first(); - if (!project) throw new Error("Missing project id"); - - const ROOT_ENCRYPTION_KEY = await getInstanceRootKey(knex); - - let secretManagerKmsKey; - const projectSecretManagerKmsId = project?.kmsSecretManagerKeyId; - if (projectSecretManagerKmsId) { - const kmsDoc = await knex(TableName.KmsKey) - .leftJoin(TableName.InternalKms, `${TableName.KmsKey}.id`, `${TableName.InternalKms}.kmsKeyId`) - .where({ [`${TableName.KmsKey}.id` as "id"]: projectSecretManagerKmsId }) - .first(); - if (!kmsDoc) throw new Error("missing kms"); - secretManagerKmsKey = cipher.decrypt(kmsDoc.encryptedKey, ROOT_ENCRYPTION_KEY); - } else { - const [kmsDoc] = await knex(TableName.KmsKey) - .insert({ - name: slugify(alphaNumericNanoId(8).toLowerCase()), - orgId: project.orgId, - isReserved: false - }) - .returning("*"); - - secretManagerKmsKey = randomSecureBytes(32); - const encryptedKeyMaterial = cipher.encrypt(secretManagerKmsKey, ROOT_ENCRYPTION_KEY); - await knex(TableName.InternalKms).insert({ - version: 1, - encryptedKey: encryptedKeyMaterial, - encryptionAlgorithm: SymmetricEncryption.AES_GCM_256, - kmsKeyId: kmsDoc.id - }); - } - - const encryptedSecretManagerDataKey = project?.kmsSecretManagerEncryptedDataKey; - let dataKey: Buffer; - if (!encryptedSecretManagerDataKey) { - dataKey = randomSecureBytes(); - // the below versioning we do it automatically in kms service - const unversionedDataKey = cipher.encrypt(dataKey, secretManagerKmsKey); - const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3 - await knex(TableName.Project) - .where({ id: projectId }) - .update({ - kmsSecretManagerEncryptedDataKey: Buffer.concat([unversionedDataKey, versionBlob]) - }); - } else { - const cipherTextBlob = encryptedSecretManagerDataKey.subarray(0, -KMS_VERSION_BLOB_LENGTH); - dataKey = cipher.decrypt(cipherTextBlob, secretManagerKmsKey); - } - - return { - encryptor: ({ plainText }: { plainText: Buffer }) => { - const encryptedPlainTextBlob = cipher.encrypt(plainText, dataKey); - - // Buffer#1 encrypted text + Buffer#2 version number - const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3 - const cipherTextBlob = Buffer.concat([encryptedPlainTextBlob, versionBlob]); - return { cipherTextBlob }; - }, - decryptor: ({ cipherTextBlob: versionedCipherTextBlob }: { cipherTextBlob: Buffer }) => { - const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH); - const decryptedBlob = cipher.decrypt(cipherTextBlob, dataKey); - return decryptedBlob; - } - }; -}; diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 2214a3970..a5f3c24c0 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -4,7 +4,8 @@ import { Redlock, Settings } from "@app/lib/red-lock"; export enum PgSqlLock { BootUpMigration = 2023, - SuperAdminInit = 2024 + SuperAdminInit = 2024, + KmsRootKeyInit = 2025 } export type TKeyStoreFactory = ReturnType; diff --git a/backend/src/services/kms/kms-service.ts b/backend/src/services/kms/kms-service.ts index 7f8c8cf2c..2f4a0b6cf 100644 --- a/backend/src/services/kms/kms-service.ts +++ b/backend/src/services/kms/kms-service.ts @@ -12,7 +12,7 @@ import { TExternalKmsProviderFns } from "@app/ee/services/external-kms/providers/model"; import { THsmServiceFactory } from "@app/ee/services/hsm/hsm-service"; -import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore"; +import { KeyStorePrefixes, PgSqlLock, TKeyStoreFactory } from "@app/keystore/keystore"; import { TEnvConfig } from "@app/lib/config/env"; import { randomSecureBytes } from "@app/lib/crypto"; import { symmetricCipherService, SymmetricEncryption } from "@app/lib/crypto/cipher"; @@ -44,7 +44,7 @@ type TKmsServiceFactoryDep = { kmsDAL: TKmsKeyDALFactory; projectDAL: Pick; orgDAL: Pick; - kmsRootConfigDAL: Pick; + kmsRootConfigDAL: Pick; keyStore: Pick; internalKmsDAL: Pick; hsmService: THsmServiceFactory; @@ -53,9 +53,6 @@ type TKmsServiceFactoryDep = { export type TKmsServiceFactory = ReturnType; -const KMS_ROOT_CREATION_WAIT_KEY = "wait_till_ready_kms_root_key"; -const KMS_ROOT_CREATION_WAIT_TIME = 10; - // akhilmhdh: Don't edit this value. This is measured for blob concatination in kms const KMS_VERSION = "v01"; const KMS_VERSION_BLOB_LENGTH = 3; @@ -874,54 +871,36 @@ export const kmsServiceFactory = ({ return { id, name, orgId, isExternal }; }; - // akhilmhdh: a copy of this is made in migrations/utils/kms const startService = async () => { - const lock = await keyStore.acquireLock([`KMS_ROOT_CFG_LOCK`], 3000, { retryCount: 3 }).catch(() => null); - if (!lock) { - await keyStore.waitTillReady({ - key: KMS_ROOT_CREATION_WAIT_KEY, - keyCheckCb: (val) => val === "true", - waitingCb: () => logger.info("KMS. Waiting for leader to finish creation of KMS Root Key") + const kmsRootConfig = await kmsRootConfigDAL.transaction(async (tx) => { + await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.KmsRootKeyInit]); + // check if KMS root key was already generated and saved in DB + const existingRootConfig = await kmsRootConfigDAL.findById(KMS_ROOT_CONFIG_UUID, tx); + if (existingRootConfig) return existingRootConfig; + + logger.info("KMS: Generating new ROOT Key"); + const newRootKey = randomSecureBytes(32); + const encryptedRootKey = await $encryptRootKey(newRootKey, RootKeyEncryptionStrategy.Software).catch((err) => { + logger.error({ hsmEnabled: hsmService.isActive() }, "KMS: Failed to encrypt ROOT Key"); + throw err; }); - } - // check if KMS root key was already generated and saved in DB - const kmsRootConfig = await kmsRootConfigDAL.findById(KMS_ROOT_CONFIG_UUID); - - // case 1: a root key already exists in the DB - if (kmsRootConfig) { - if (lock) await lock.release(); - logger.info(`KMS: Encrypted ROOT Key found from DB. Decrypting. [strategy=${kmsRootConfig.encryptionStrategy}]`); - - const decryptedRootKey = await $decryptRootKey(kmsRootConfig); - - // set the flag so that other instance nodes can start - await keyStore.setItemWithExpiry(KMS_ROOT_CREATION_WAIT_KEY, KMS_ROOT_CREATION_WAIT_TIME, "true"); - logger.info("KMS: Loading ROOT Key into Memory."); - ROOT_ENCRYPTION_KEY = decryptedRootKey; - return; - } - - // case 2: no config is found, so we create a new root key with basic encryption - logger.info("KMS: Generating new ROOT Key"); - const newRootKey = randomSecureBytes(32); - const encryptedRootKey = await $encryptRootKey(newRootKey, RootKeyEncryptionStrategy.Software).catch((err) => { - logger.error({ hsmEnabled: hsmService.isActive() }, "KMS: Failed to encrypt ROOT Key"); - throw err; + const newRootConfig = await kmsRootConfigDAL.create( + { + // @ts-expect-error id is kept as fixed for idempotence and to avoid race condition + id: KMS_ROOT_CONFIG_UUID, + encryptedRootKey, + encryptionStrategy: RootKeyEncryptionStrategy.Software + }, + tx + ); + return newRootConfig; }); - await kmsRootConfigDAL.create({ - // @ts-expect-error id is kept as fixed for idempotence and to avoid race condition - id: KMS_ROOT_CONFIG_UUID, - encryptedRootKey, - encryptionStrategy: RootKeyEncryptionStrategy.Software - }); + const decryptedRootKey = await $decryptRootKey(kmsRootConfig); - // set the flag so that other instance nodes can start - await keyStore.setItemWithExpiry(KMS_ROOT_CREATION_WAIT_KEY, KMS_ROOT_CREATION_WAIT_TIME, "true"); - logger.info("KMS: Saved and loaded ROOT Key into memory"); - if (lock) await lock.release(); - ROOT_ENCRYPTION_KEY = newRootKey; + logger.info("KMS: Loading ROOT Key into Memory."); + ROOT_ENCRYPTION_KEY = decryptedRootKey; }; const updateEncryptionStrategy = async (strategy: RootKeyEncryptionStrategy) => {