feat: HSM support

backend/src/db/migrations/20241028134337_kms-root-cfg-hsm.ts

@@ -4,10 +4,12 @@ import { TableName } from "../schemas";
 
 export async function up(knex: Knex): Promise<void> {
   const hasEncryptionStrategy = await knex.schema.hasColumn(TableName.KmsServerRootConfig, "encryptionStrategy");
+  const hasExported = await knex.schema.hasColumn(TableName.KmsServerRootConfig, "exported");
   const hasTimestampsCol = await knex.schema.hasColumn(TableName.KmsServerRootConfig, "createdAt");
 
   await knex.schema.alterTable(TableName.KmsServerRootConfig, (t) => {
     if (!hasEncryptionStrategy) t.string("encryptionStrategy").defaultTo("BASIC");
+    if (!hasExported) t.boolean("exported").defaultTo(false);
     if (!hasTimestampsCol) t.timestamps(true, true, true);
   });
 }
@@ -15,9 +17,11 @@ export async function up(knex: Knex): Promise<void> {
 export async function down(knex: Knex): Promise<void> {
   const hasEncryptionStrategy = await knex.schema.hasColumn(TableName.KmsServerRootConfig, "encryptionStrategy");
   const hasTimestampsCol = await knex.schema.hasColumn(TableName.KmsServerRootConfig, "createdAt");
+  const hasExported = await knex.schema.hasColumn(TableName.KmsServerRootConfig, "exported");
 
   await knex.schema.alterTable(TableName.KmsServerRootConfig, (t) => {
     if (hasEncryptionStrategy) t.dropColumn("encryptionStrategy");
     if (hasTimestampsCol) t.dropTimestamps(true);
+    if (hasExported) t.dropColumn("exported");
   });
 }

backend/src/db/schemas/kms-root-config.ts

@@ -13,6 +13,7 @@ export const KmsRootConfigSchema = z.object({
   id: z.string().uuid(),
   encryptedRootKey: zodBuffer,
   encryptionStrategy: z.string(),
+  exported: z.boolean(),
   createdAt: z.date(),
   updatedAt: z.date()
 });

backend/src/lib/config/env.ts

@@ -176,7 +176,7 @@ const envSchema = z
         .string()
         .optional()
         .transform((val) => {
-          if (process.env.NODE_ENV === "development") return "/usr/local/lib/softhsm/libsofthsm2.so";
+          // if (process.env.NODE_ENV === "development") return "/usr/local/lib/softhsm/libsofthsm2.so";
           return val;
         })
     ),
@@ -201,6 +201,11 @@ const envSchema = z
     HSM_SLOT: z.coerce.number().optional().default(0),
     HSM_MECHANISM: zpStr(z.string().optional().default("AES_GCM"))
   })
+  // To ensure that basic encryption is always possible.
+  .refine(
+    (data) => data.ENCRYPTION_KEY != null || data.ROOT_ENCRYPTION_KEY != null,
+    "Either ENCRYPTION_KEY or ROOT_ENCRYPTION_KEY must be defined."
+  )
   .transform((data) => ({
     ...data,
 

backend/src/lib/crypto/index.ts

@@ -18,5 +18,6 @@ export {
   decryptSecrets,
   decryptSecretVersions
 } from "./secret-encryption";
+export { shamirsService } from "./shamirs";
 export { verifyOfflineLicense } from "./signing";
 export { generateSrpServerKey, srpCheckClientProof } from "./srp";

backend/src/lib/crypto/shamirs.ts

@@ -0,0 +1,38 @@
+import shamirs from "secrets.js-grempe";
+
+import { getConfig } from "../config/env";
+import { symmetricCipherService, SymmetricEncryption } from "./cipher";
+
+export const shamirsService = () => {
+  const $generateBasicEncryptionKey = () => {
+    const appCfg = getConfig();
+
+    const encryptionKey = appCfg.ENCRYPTION_KEY || appCfg.ROOT_ENCRYPTION_KEY;
+    const isBase64 = !appCfg.ENCRYPTION_KEY;
+    if (!encryptionKey)
+      throw new Error(
+        "Root encryption key not found for KMS service. Did you set the ENCRYPTION_KEY or ROOT_ENCRYPTION_KEY environment variables?"
+      );
+
+    return Buffer.from(encryptionKey, isBase64 ? "base64" : "utf8");
+  };
+
+  const share = (secretBuffer: Buffer, partsCount: number, thresholdCount: number) => {
+    const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+    const hexSecret = Buffer.from(cipher.encrypt(secretBuffer, $generateBasicEncryptionKey())).toString("hex");
+
+    const secretParts = shamirs.share(hexSecret, partsCount, thresholdCount);
+    return secretParts;
+  };
+
+  const combine = (parts: string[]) => {
+    const encryptedSecret = shamirs.combine(parts);
+
+    const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+    const decryptedSecret = cipher.decrypt(Buffer.from(encryptedSecret, "hex"), $generateBasicEncryptionKey());
+
+    return decryptedSecret;
+  };
+
+  return { share, combine };
+};

backend/src/server/routes/index.ts

@@ -572,6 +572,7 @@ export const registerRoutes = async (
     userDAL,
     authService: loginService,
     serverCfgDAL: superAdminDAL,
+    kmsRootConfigDAL,
     orgService,
     keyStore,
     licenseService,

backend/src/server/routes/v1/admin-router.ts

@@ -7,6 +7,7 @@ import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
 import { verifySuperAdmin } from "@app/server/plugins/auth/superAdmin";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
+import { RootKeyEncryptionStrategy } from "@app/services/kms/kms-types";
 import { getServerCfg } from "@app/services/super-admin/super-admin-service";
 import { LoginMethod } from "@app/services/super-admin/super-admin-types";
 import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
@@ -195,6 +196,107 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
     }
   });
 
+  server.route({
+    method: "POST",
+    url: "/kms-export",
+    config: {
+      rateLimit: writeLimit
+    },
+    schema: {
+      response: {
+        200: z.object({
+          secretParts: z.array(z.string())
+        })
+      }
+    },
+    onRequest: (req, res, done) => {
+      verifyAuth([AuthMode.JWT])(req, res, () => {
+        verifySuperAdmin(req, res, done);
+      });
+    },
+    handler: async () => {
+      const keyParts = await server.services.superAdmin.exportPlainKmsKey();
+
+      return {
+        secretParts: keyParts
+      };
+    }
+  });
+
+  server.route({
+    method: "POST",
+    url: "/kms-import",
+    config: {
+      rateLimit: writeLimit
+    },
+    schema: {
+      body: z.object({
+        secretParts: z.array(z.string())
+      })
+    },
+    onRequest: (req, res, done) => {
+      verifyAuth([AuthMode.JWT])(req, res, () => {
+        verifySuperAdmin(req, res, done);
+      });
+    },
+    handler: async (req) => {
+      await server.services.superAdmin.importPlainKmsKey(req.body.secretParts);
+    }
+  });
+
+  server.route({
+    method: "GET",
+    url: "/root-kms-config",
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      response: {
+        200: z.object({
+          strategies: z
+            .object({
+              strategy: z.nativeEnum(RootKeyEncryptionStrategy),
+              name: z.string(),
+              enabled: z.boolean()
+            })
+            .array(),
+          keyExported: z.boolean()
+        })
+      }
+    },
+    onRequest: (req, res, done) => {
+      verifyAuth([AuthMode.JWT])(req, res, () => {
+        verifySuperAdmin(req, res, done);
+      });
+    },
+
+    handler: async () => {
+      const encryptionDetails = await server.services.superAdmin.getConfiguredEncryptionStrategies();
+      return encryptionDetails;
+    }
+  });
+
+  server.route({
+    method: "POST",
+    url: "/encryption-strategies",
+    config: {
+      rateLimit: writeLimit
+    },
+    schema: {
+      body: z.object({
+        strategy: z.nativeEnum(RootKeyEncryptionStrategy)
+      })
+    },
+    onRequest: (req, res, done) => {
+      verifyAuth([AuthMode.JWT])(req, res, () => {
+        verifySuperAdmin(req, res, done);
+      });
+    },
+    handler: async (req) => {
+      await server.services.superAdmin.updateRootEncryptionStrategy(req.body.strategy);
+    }
+  });
+
   server.route({
     method: "POST",
     url: "/signup",

backend/src/services/kms/kms-fns.ts

@@ -1,5 +1,7 @@
 import { SymmetricEncryption } from "@app/lib/crypto/cipher";
 
+export const KMS_ROOT_CONFIG_UUID = "00000000-0000-0000-0000-000000000000";
+
 export const getByteLengthForAlgorithm = (encryptionAlgorithm: SymmetricEncryption) => {
   switch (encryptionAlgorithm) {
     case SymmetricEncryption.AES_GCM_128:

backend/src/services/kms/kms-service.ts

@@ -11,13 +11,13 @@ import {
 } from "@app/ee/services/external-kms/providers/model";
 import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig } from "@app/lib/config/env";
-import { randomSecureBytes } from "@app/lib/crypto";
+import { randomSecureBytes, shamirsService } from "@app/lib/crypto";
 import { symmetricCipherService, SymmetricEncryption } from "@app/lib/crypto/cipher";
 import { generateHash } from "@app/lib/crypto/encryption";
 import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors";
 import { logger } from "@app/lib/logger";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
-import { getByteLengthForAlgorithm } from "@app/services/kms/kms-fns";
+import { getByteLengthForAlgorithm, KMS_ROOT_CONFIG_UUID } from "@app/services/kms/kms-fns";
 
 import { THsmServiceFactory } from "../hsm/hsm-service";
 import { TOrgDALFactory } from "../org/org-dal";
@@ -50,8 +50,6 @@ type TKmsServiceFactoryDep = {
 
 export type TKmsServiceFactory = ReturnType<typeof kmsServiceFactory>;
 
-const KMS_ROOT_CONFIG_UUID = "00000000-0000-0000-0000-000000000000";
-
 const KMS_ROOT_CREATION_WAIT_KEY = "wait_till_ready_kms_root_key";
 const KMS_ROOT_CREATION_WAIT_TIME = 10;
 
@@ -614,7 +612,7 @@ export const kmsServiceFactory = ({
     }
   };
 
-  const $createBasicEncryptionKey = () => {
+  const $getBasicEncryptionKey = () => {
     const appCfg = getConfig();
 
     const encryptionKey = appCfg.ENCRYPTION_KEY || appCfg.ROOT_ENCRYPTION_KEY;
@@ -642,7 +640,7 @@ export const kmsServiceFactory = ({
     // case 2: root key is encrypted with basic encryption
     if (kmsRootConfig.encryptionStrategy === RootKeyEncryptionStrategy.Basic) {
       const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
-      const encryptionKeyBuffer = $createBasicEncryptionKey();
+      const encryptionKeyBuffer = $getBasicEncryptionKey();
 
       return cipher.decrypt(kmsRootConfig.encryptedRootKey, encryptionKeyBuffer);
     }
@@ -660,7 +658,7 @@ export const kmsServiceFactory = ({
 
     if (strategy === RootKeyEncryptionStrategy.Basic) {
       const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
-      const encryptionKeyBuffer = $createBasicEncryptionKey();
+      const encryptionKeyBuffer = $getBasicEncryptionKey();
 
       return cipher.encrypt(plainKeyBuffer, encryptionKeyBuffer);
     }
@@ -669,6 +667,31 @@ export const kmsServiceFactory = ({
     throw new Error(`Invalid root key encryption strategy: ${strategy}`);
   };
 
+  const exportRootEncryptionKeyParts = () => {
+    if (!ROOT_ENCRYPTION_KEY) {
+      throw new Error("Root encryption key not set");
+    }
+
+    const parts = shamirsService().share(ROOT_ENCRYPTION_KEY, 8, 4);
+
+    return parts;
+  };
+
+  const importRootEncryptionKey = async (parts: string[]) => {
+    const decryptedRootKey = shamirsService().combine(parts);
+
+    const encryptedRootKey = symmetricCipherService(SymmetricEncryption.AES_GCM_256).encrypt(
+      decryptedRootKey,
+      $getBasicEncryptionKey()
+    );
+
+    await kmsRootConfigDAL.updateById(KMS_ROOT_CONFIG_UUID, {
+      encryptedRootKey,
+      encryptionStrategy: RootKeyEncryptionStrategy.Basic
+    });
+    ROOT_ENCRYPTION_KEY = decryptedRootKey;
+  };
+
   // by keeping the decrypted data key in inner scope
   // none of the entities outside can interact directly or expose the data key
   // NOTICE: If changing here update migrations/utils/kms
@@ -853,9 +876,6 @@ export const kmsServiceFactory = ({
 
   // akhilmhdh: a copy of this is made in migrations/utils/kms
   const startService = async () => {
-    const appCfg = getConfig();
-    const hsmEnabled = hsmService.isActive();
-
     const lock = await keyStore.acquireLock([`KMS_ROOT_CFG_LOCK`], 3000, { retryCount: 3 }).catch(() => null);
     if (!lock) {
       await keyStore.waitTillReady({
@@ -875,34 +895,11 @@ export const kmsServiceFactory = ({
 
       const decryptedRootKey = await $decryptRootKey(kmsRootConfig).catch((err) => {
         logger.error(err, `KMS: Failed to decrypt ROOT Key [strategy=${kmsRootConfig.encryptionStrategy}]`);
-        throw err;
+        // We do not want to throw on startup. If the HSM has issues, this will throw an error, causing the entire API to shut down.
+        // If the API shuts down, the user will have no way to do recovery by importing their backup decryption key and rolling back to basic encryption.
+        return Buffer.alloc(0);
       });
 
-      const selectedEncryptionStrategy = appCfg.ROOT_KEY_ENCRYPTION_STRATEGY;
-
-      // case: the users selected encryption key strategy does not match the one in the DB
-      // in this case we need to re-encrypt the key with the selected strategy, and update the strategy and key in the DB
-      if (selectedEncryptionStrategy !== kmsRootConfig.encryptionStrategy) {
-        logger.info(
-          {
-            newStrategy: selectedEncryptionStrategy,
-            configuredStrategy: kmsRootConfig.encryptionStrategy
-          },
-          "KMS: Change in root encryption key strategy detected. Re-encrypting ROOT Key with selected strategy"
-        );
-        const encryptedRootKey = await $encryptRootKey(decryptedRootKey, selectedEncryptionStrategy);
-
-        if (!encryptedRootKey) {
-          logger.error("KMS: Failed to re-encrypt ROOT Key with selected strategy");
-          throw new Error("Failed to re-encrypt ROOT Key with selected strategy");
-        }
-
-        await kmsRootConfigDAL.updateById(KMS_ROOT_CONFIG_UUID, {
-          encryptedRootKey,
-          encryptionStrategy: selectedEncryptionStrategy
-        });
-      }
-
       // set the flag so that other instance nodes can start
       await keyStore.setItemWithExpiry(KMS_ROOT_CREATION_WAIT_KEY, KMS_ROOT_CREATION_WAIT_TIME, "true");
       logger.info("KMS: Loading ROOT Key into Memory.");
@@ -910,12 +907,11 @@ export const kmsServiceFactory = ({
       return;
     }
 
-    logger.info("KMS: Generating ROOT Key");
-    const selectedEncryptionStrategy = appCfg.ROOT_KEY_ENCRYPTION_STRATEGY;
-
+    // case 2: no config is found, so we create a new root key with basic encryption
+    logger.info("KMS: Generating new ROOT Key");
     const newRootKey = randomSecureBytes(32);
-    const encryptedRootKey = await $encryptRootKey(newRootKey, selectedEncryptionStrategy).catch((err) => {
-      logger.error({ hsmEnabled }, "KMS: Failed to encrypt ROOT Key");
+    const encryptedRootKey = await $encryptRootKey(newRootKey, RootKeyEncryptionStrategy.Basic).catch((err) => {
+      logger.error({ hsmEnabled: hsmService.isActive() }, "KMS: Failed to encrypt ROOT Key");
       throw err;
     });
 
@@ -923,7 +919,7 @@ export const kmsServiceFactory = ({
       // @ts-expect-error id is kept as fixed for idempotence and to avoid race condition
       id: KMS_ROOT_CONFIG_UUID,
       encryptedRootKey,
-      encryptionStrategy: selectedEncryptionStrategy
+      encryptionStrategy: RootKeyEncryptionStrategy.Basic
     });
 
     // set the flag so that other instance nodes can start
@@ -933,6 +929,32 @@ export const kmsServiceFactory = ({
     ROOT_ENCRYPTION_KEY = newRootKey;
   };
 
+  const updateEncryptionStrategy = async (strategy: RootKeyEncryptionStrategy) => {
+    const kmsRootConfig = await kmsRootConfigDAL.findById(KMS_ROOT_CONFIG_UUID);
+    if (!kmsRootConfig) {
+      throw new NotFoundError({ message: "KMS root config not found" });
+    }
+
+    if (kmsRootConfig.encryptionStrategy === strategy) {
+      return;
+    }
+
+    const decryptedRootKey = await $decryptRootKey(kmsRootConfig);
+    const encryptedRootKey = await $encryptRootKey(decryptedRootKey, strategy);
+
+    if (!encryptedRootKey) {
+      logger.error("KMS: Failed to re-encrypt ROOT Key with selected strategy");
+      throw new Error("Failed to re-encrypt ROOT Key with selected strategy");
+    }
+
+    await kmsRootConfigDAL.updateById(KMS_ROOT_CONFIG_UUID, {
+      encryptedRootKey,
+      encryptionStrategy: strategy
+    });
+
+    ROOT_ENCRYPTION_KEY = decryptedRootKey;
+  };
+
   return {
     startService,
     generateKmsKey,
@@ -944,11 +966,14 @@ export const kmsServiceFactory = ({
     encryptWithRootKey,
     decryptWithRootKey,
     getOrgKmsKeyId,
+    updateEncryptionStrategy,
     getProjectSecretManagerKmsKeyId,
     updateProjectSecretManagerKmsKey,
     getProjectKeyBackup,
     loadProjectKeyBackup,
     getKmsById,
-    createCipherPairWithDataKey
+    createCipherPairWithDataKey,
+    exportRootEncryptionKeyParts,
+    importRootEncryptionKey
   };
 };

backend/src/services/super-admin/super-admin-service.ts

@@ -10,7 +10,10 @@ import { BadRequestError, NotFoundError } from "@app/lib/errors";
 
 import { TAuthLoginFactory } from "../auth/auth-login-service";
 import { AuthMethod } from "../auth/auth-type";
+import { KMS_ROOT_CONFIG_UUID } from "../kms/kms-fns";
+import { TKmsRootConfigDALFactory } from "../kms/kms-root-config-dal";
 import { TKmsServiceFactory } from "../kms/kms-service";
+import { RootKeyEncryptionStrategy } from "../kms/kms-types";
 import { TOrgServiceFactory } from "../org/org-service";
 import { TUserDALFactory } from "../user/user-dal";
 import { TSuperAdminDALFactory } from "./super-admin-dal";
@@ -20,7 +23,15 @@ type TSuperAdminServiceFactoryDep = {
   serverCfgDAL: TSuperAdminDALFactory;
   userDAL: TUserDALFactory;
   authService: Pick<TAuthLoginFactory, "generateUserTokens">;
-  kmsService: Pick<TKmsServiceFactory, "encryptWithRootKey" | "decryptWithRootKey">;
+  kmsService: Pick<
+    TKmsServiceFactory,
+    | "encryptWithRootKey"
+    | "decryptWithRootKey"
+    | "exportRootEncryptionKeyParts"
+    | "importRootEncryptionKey"
+    | "updateEncryptionStrategy"
+  >;
+  kmsRootConfigDAL: TKmsRootConfigDALFactory;
   orgService: Pick<TOrgServiceFactory, "createOrganization">;
   keyStore: Pick<TKeyStoreFactory, "getItem" | "setItemWithExpiry" | "deleteItem">;
   licenseService: Pick<TLicenseServiceFactory, "onPremFeatures">;
@@ -47,6 +58,7 @@ export const superAdminServiceFactory = ({
   authService,
   orgService,
   keyStore,
+  kmsRootConfigDAL,
   kmsService,
   licenseService
 }: TSuperAdminServiceFactoryDep) => {
@@ -150,6 +162,35 @@ export const superAdminServiceFactory = ({
     return updatedServerCfg;
   };
 
+  const exportPlainKmsKey = async () => {
+    const kmsRootConfig = await kmsRootConfigDAL.findById(KMS_ROOT_CONFIG_UUID);
+
+    if (!kmsRootConfig) {
+      throw new NotFoundError({ name: "KmsRootConfig", message: "KMS root configuration not found" });
+    }
+
+    if (kmsRootConfig.exported) {
+      throw new BadRequestError({ name: "KmsRootConfig", message: "KMS root configuration already exported" });
+    }
+
+    await kmsRootConfigDAL.updateById(KMS_ROOT_CONFIG_UUID, { exported: true });
+    return kmsService.exportRootEncryptionKeyParts();
+  };
+
+  const importPlainKmsKey = async (secretParts: string[]) => {
+    const kmsRootConfig = await kmsRootConfigDAL.findById(KMS_ROOT_CONFIG_UUID);
+
+    if (!kmsRootConfig) {
+      throw new NotFoundError({ name: "KmsRootConfig", message: "KMS root configuration not found" });
+    }
+
+    if (!kmsRootConfig.exported) {
+      throw new BadRequestError({ name: "KmsRootConfig", message: "KMS root configuration was never exported" });
+    }
+
+    await kmsService.importRootEncryptionKey(secretParts);
+  };
+
   const adminSignUp = async ({
     lastName,
     firstName,
@@ -288,12 +329,69 @@ export const superAdminServiceFactory = ({
     };
   };
 
+  const getConfiguredEncryptionStrategies = async () => {
+    const appCfg = getConfig();
+
+    const kmsRootCfg = await kmsRootConfigDAL.findById(KMS_ROOT_CONFIG_UUID);
+
+    if (!kmsRootCfg) {
+      throw new NotFoundError({ name: "KmsRootConfig", message: "KMS root configuration not found" });
+    }
+
+    const selectedStrategy = kmsRootCfg.encryptionStrategy;
+    const enabledStrategies: { enabled: boolean; strategy: RootKeyEncryptionStrategy; name: string }[] = [];
+
+    if (appCfg.ROOT_ENCRYPTION_KEY || appCfg.ENCRYPTION_KEY) {
+      const basicStrategy = RootKeyEncryptionStrategy.Basic;
+
+      enabledStrategies.push({
+        name: "Regular Encryption",
+        enabled: selectedStrategy === basicStrategy,
+        strategy: basicStrategy
+      });
+    }
+    if (appCfg.isHsmConfigured) {
+      const hsmStrategy = RootKeyEncryptionStrategy.Hsm;
+
+      enabledStrategies.push({
+        name: "Hardware Security Module (HSM)",
+        enabled: selectedStrategy === hsmStrategy,
+        strategy: hsmStrategy
+      });
+    }
+
+    return {
+      strategies: enabledStrategies,
+      keyExported: kmsRootCfg.exported
+    };
+  };
+
+  const updateRootEncryptionStrategy = async (strategy: RootKeyEncryptionStrategy) => {
+    const configuredStrategies = await getConfiguredEncryptionStrategies();
+
+    const foundStrategy = configuredStrategies.strategies.find((s) => s.strategy === strategy);
+
+    if (!foundStrategy) {
+      throw new BadRequestError({ message: "Invalid encryption strategy" });
+    }
+
+    if (foundStrategy.enabled) {
+      throw new BadRequestError({ message: "The selected encryption strategy is already enabled" });
+    }
+
+    await kmsService.updateEncryptionStrategy(strategy);
+  };
+
   return {
     initServerCfg,
     updateServerCfg,
     adminSignUp,
     getUsers,
     deleteUser,
-    getAdminSlackConfig
+    getAdminSlackConfig,
+    updateRootEncryptionStrategy,
+    getConfiguredEncryptionStrategies,
+    exportPlainKmsKey,
+    importPlainKmsKey
   };
 };