feat: hsm improvements

backend/src/db/migrations/utils/services.ts

@@ -1,28 +1,24 @@
 import { Knex } from "knex";
 
-import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns";
+import { initializeHsmModule, isHsmActiveAndEnabled } from "@app/ee/services/hsm/hsm-fns";
 import { hsmServiceFactory } from "@app/ee/services/hsm/hsm-service";
+import { licenseDALFactory } from "@app/ee/services/license/license-dal";
+import { licenseServiceFactory } from "@app/ee/services/license/license-service";
+import { permissionDALFactory } from "@app/ee/services/permission/permission-dal";
+import { permissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import { TKeyStoreFactory } from "@app/keystore/keystore";
-import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal";
+import { BadRequestError } from "@app/lib/errors";
-import { folderCheckpointResourcesDALFactory } from "@app/services/folder-checkpoint-resources/folder-checkpoint-resources-dal";
-import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal";
-import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service";
-import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal";
-import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal";
-import { folderTreeCheckpointResourcesDALFactory } from "@app/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal";
 import { identityDALFactory } from "@app/services/identity/identity-dal";
+import { identityOrgDALFactory } from "@app/services/identity/identity-org-dal";
 import { internalKmsDALFactory } from "@app/services/kms/internal-kms-dal";
 import { kmskeyDALFactory } from "@app/services/kms/kms-key-dal";
 import { kmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal";
 import { kmsServiceFactory } from "@app/services/kms/kms-service";
+import { RootKeyEncryptionStrategy } from "@app/services/kms/kms-types";
 import { orgDALFactory } from "@app/services/org/org-dal";
 import { projectDALFactory } from "@app/services/project/project-dal";
-import { resourceMetadataDALFactory } from "@app/services/resource-metadata/resource-metadata-dal";
+import { roleDALFactory } from "@app/services/role/role-dal";
-import { secretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
+import { serviceTokenDALFactory } from "@app/services/service-token/service-token-dal";
-import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal";
-import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
-import { secretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
-import { secretVersionV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
 import { userDALFactory } from "@app/services/user/user-dal";
 
 import { TMigrationEnvConfig } from "./env-config";
@@ -34,20 +30,74 @@ type TDependencies = {
 };
 
 export const getMigrationEncryptionServices = async ({ envConfig, db, keyStore }: TDependencies) => {
-  // eslint-disable-next-line no-param-reassign
+  // ----- DAL dependencies -----
+  const orgDAL = orgDALFactory(db);
+  const licenseDAL = licenseDALFactory(db);
+  const permissionDAL = permissionDALFactory(db);
+  const projectDAL = projectDALFactory(db);
+  const roleDAL = roleDALFactory(db);
+  const userDAL = userDALFactory(db);
+  const identityDAL = identityDALFactory(db);
+  const serviceTokenDAL = serviceTokenDALFactory(db);
+  const identityOrgMembershipDAL = identityOrgDALFactory(db);
+  const kmsRootConfigDAL = kmsRootConfigDALFactory(db);
+  const kmsDAL = kmskeyDALFactory(db);
+  const internalKmsDAL = internalKmsDALFactory(db);
+
+  // ----- Service dependencies -----
+  const permissionService = permissionServiceFactory({
+    permissionDAL,
+    serviceTokenDAL,
+    projectDAL,
+    keyStore,
+    roleDAL,
+    userDAL,
+    identityDAL
+  });
+
+  const licenseService = licenseServiceFactory({
+    permissionService,
+    orgDAL,
+    licenseDAL,
+    keyStore,
+    identityOrgMembershipDAL,
+    projectDAL
+  });
+
+  // ----- HSM startup -----
+
   const hsmModule = initializeHsmModule(envConfig);
-  hsmModule.initialize();
 
   const hsmService = hsmServiceFactory({
     hsmModule: hsmModule.getModule(),
     envConfig
   });
 
-  const orgDAL = orgDALFactory(db);
+  hsmModule.initialize();
-  const kmsRootConfigDAL = kmsRootConfigDALFactory(db);
+  await hsmService.startService();
-  const kmsDAL = kmskeyDALFactory(db);
+
-  const internalKmsDAL = internalKmsDALFactory(db);
+  const hsmStatus = await isHsmActiveAndEnabled({
-  const projectDAL = projectDALFactory(db);
+    hsmService,
+    kmsRootConfigDAL,
+    licenseService
+  });
+
+  // if the encryption strategy is software - user needs to provide an encryption key
+  // if the encryption strategy is null AND the hsm is not configured - user needs to provide an encryption key
+  const needsEncryptionKey =
+    hsmStatus.rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.Software ||
+    (hsmStatus.rootKmsConfigEncryptionStrategy === null && !hsmStatus.isHsmConfigured);
+
+  if (needsEncryptionKey) {
+    if (!envConfig.ROOT_ENCRYPTION_KEY && !envConfig.ENCRYPTION_KEY) {
+      throw new BadRequestError({
+        message:
+          "Root KMS encryption strategy is set to software. Please set the ENCRYPTION_KEY environment variable and restart your deployment.\nYou can enable HSM encryption in the Server Console."
+      });
+    }
+  }
+
+  // ----- KMS startup -----
 
   const kmsService = kmsServiceFactory({
     kmsRootConfigDAL,
@@ -60,82 +110,7 @@ export const getMigrationEncryptionServices = async ({ envConfig, db, keyStore }
     envConfig
   });
 
-  await hsmService.startService();
+  await kmsService.startService(hsmStatus);
-  await kmsService.startService();
 
   return { kmsService };
 };
-
-export const getMigrationPITServices = async ({
-  db,
-  keyStore,
-  envConfig
-}: {
-  db: Knex;
-  keyStore: TKeyStoreFactory;
-  envConfig: TMigrationEnvConfig;
-}) => {
-  const projectDAL = projectDALFactory(db);
-  const folderCommitDAL = folderCommitDALFactory(db);
-  const folderCommitChangesDAL = folderCommitChangesDALFactory(db);
-  const folderCheckpointDAL = folderCheckpointDALFactory(db);
-  const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db);
-  const userDAL = userDALFactory(db);
-  const identityDAL = identityDALFactory(db);
-  const folderDAL = secretFolderDALFactory(db);
-  const folderVersionDAL = secretFolderVersionDALFactory(db);
-  const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db);
-  const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db);
-  const secretV2BridgeDAL = secretV2BridgeDALFactory({ db, keyStore });
-  const folderTreeCheckpointResourcesDAL = folderTreeCheckpointResourcesDALFactory(db);
-  const secretTagDAL = secretTagDALFactory(db);
-
-  const orgDAL = orgDALFactory(db);
-  const kmsRootConfigDAL = kmsRootConfigDALFactory(db);
-  const kmsDAL = kmskeyDALFactory(db);
-  const internalKmsDAL = internalKmsDALFactory(db);
-  const resourceMetadataDAL = resourceMetadataDALFactory(db);
-
-  const hsmModule = initializeHsmModule(envConfig);
-  hsmModule.initialize();
-
-  const hsmService = hsmServiceFactory({
-    hsmModule: hsmModule.getModule(),
-    envConfig
-  });
-
-  const kmsService = kmsServiceFactory({
-    kmsRootConfigDAL,
-    keyStore,
-    kmsDAL,
-    internalKmsDAL,
-    orgDAL,
-    projectDAL,
-    hsmService,
-    envConfig
-  });
-
-  await hsmService.startService();
-  await kmsService.startService();
-
-  const folderCommitService = folderCommitServiceFactory({
-    folderCommitDAL,
-    folderCommitChangesDAL,
-    folderCheckpointDAL,
-    folderTreeCheckpointDAL,
-    userDAL,
-    identityDAL,
-    folderDAL,
-    folderVersionDAL,
-    secretVersionV2BridgeDAL,
-    projectDAL,
-    folderCheckpointResourcesDAL,
-    secretV2BridgeDAL,
-    folderTreeCheckpointResourcesDAL,
-    kmsService,
-    secretTagDAL,
-    resourceMetadataDAL
-  });
-
-  return { folderCommitService };
-};

backend/src/ee/services/hsm/hsm-fns.ts

@@ -1,8 +1,14 @@
 import * as pkcs11js from "pkcs11js";
 
 import { TEnvConfig } from "@app/lib/config/env";
+import { BadRequestError } from "@app/lib/errors";
 import { logger } from "@app/lib/logger";
+import { KMS_ROOT_CONFIG_UUID } from "@app/services/kms/kms-fns";
+import { TKmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal";
+import { RootKeyEncryptionStrategy } from "@app/services/kms/kms-types";
 
+import { TLicenseServiceFactory } from "../license/license-service";
+import { THsmServiceFactory } from "./hsm-service";
 import { HsmModule } from "./hsm-types";
 
 export const initializeHsmModule = (envConfig: Pick<TEnvConfig, "isHsmConfigured" | "HSM_LIB_PATH">) => {
@@ -60,3 +66,32 @@ export const initializeHsmModule = (envConfig: Pick<TEnvConfig, "isHsmConfigured
     getModule
   };
 };
+
+export const isHsmActiveAndEnabled = async ({
+  hsmService,
+  kmsRootConfigDAL,
+  licenseService
+}: {
+  hsmService: Pick<THsmServiceFactory, "isActive">;
+  kmsRootConfigDAL: Pick<TKmsRootConfigDALFactory, "findById">;
+  licenseService: Pick<TLicenseServiceFactory, "onPremFeatures">;
+}) => {
+  const isHsmConfigured = await hsmService.isActive();
+
+  // null if the root kms config does not exist
+  let rootKmsConfigEncryptionStrategy: RootKeyEncryptionStrategy | null = null;
+
+  const rootKmsConfig = await kmsRootConfigDAL.findById(KMS_ROOT_CONFIG_UUID).catch(() => null);
+
+  rootKmsConfigEncryptionStrategy = rootKmsConfig?.encryptionStrategy as RootKeyEncryptionStrategy | null;
+  if (rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.HSM && !licenseService.onPremFeatures.hsm) {
+    throw new BadRequestError({
+      message: "Your license does not include HSM integration. Please upgrade to the Enterprise plan to use HSM."
+    });
+  }
+
+  return {
+    rootKmsConfigEncryptionStrategy,
+    isHsmConfigured
+  };
+};

backend/src/ee/services/hsm/hsm-types.ts

@@ -1,5 +1,7 @@
 import pkcs11js from "pkcs11js";
 
+import { RootKeyEncryptionStrategy } from "@app/services/kms/kms-types";
+
 export type HsmModule = {
   pkcs11: pkcs11js.PKCS11;
   isInitialized: boolean;
@@ -9,3 +11,8 @@ export enum HsmKeyType {
   AES = "AES",
   HMAC = "hmac"
 }
+
+export type THsmStatus = {
+  rootKmsConfigEncryptionStrategy: RootKeyEncryptionStrategy | null;
+  isHsmConfigured: boolean;
+};

backend/src/lib/crypto/cryptography/crypto.ts

@@ -258,6 +258,13 @@ const cryptographyFactory = () => {
         const rootEncryptionKey = appCfg.ROOT_ENCRYPTION_KEY;
         const encryptionKey = appCfg.ENCRYPTION_KEY;
 
+        // Sanity check
+        if (!rootEncryptionKey && !encryptionKey) {
+          throw new CryptographyError({
+            message: "Tried to encrypt with instance root encryption key, but no root encryption key is set."
+          });
+        }
+
         if (rootEncryptionKey) {
           const { iv, tag, ciphertext } = encrypt({
             plaintext: data,
@@ -303,6 +310,14 @@ const cryptographyFactory = () => {
         // the or gate is used used in migration
         const rootEncryptionKey = appCfg?.ROOT_ENCRYPTION_KEY || process.env.ROOT_ENCRYPTION_KEY;
         const encryptionKey = appCfg?.ENCRYPTION_KEY || process.env.ENCRYPTION_KEY;
+
+        // Sanity check
+        if (!rootEncryptionKey && !encryptionKey) {
+          throw new CryptographyError({
+            message: "Tried to decrypt with instance root encryption key, but no root encryption key is set."
+          });
+        }
+
         if (rootEncryptionKey && keyEncoding === SecretKeyEncoding.BASE64) {
           const data = symmetric().decrypt({
             key: rootEncryptionKey,

backend/src/server/routes/index.ts

@@ -46,6 +46,7 @@ import { githubOrgSyncServiceFactory } from "@app/ee/services/github-org-sync/gi
 import { groupDALFactory } from "@app/ee/services/group/group-dal";
 import { groupServiceFactory } from "@app/ee/services/group/group-service";
 import { userGroupMembershipDALFactory } from "@app/ee/services/group/user-group-membership-dal";
+import { isHsmActiveAndEnabled } from "@app/ee/services/hsm/hsm-fns";
 import { hsmServiceFactory } from "@app/ee/services/hsm/hsm-service";
 import { HsmModule } from "@app/ee/services/hsm/hsm-types";
 import { identityAuthTemplateDALFactory } from "@app/ee/services/identity-auth-template/identity-auth-template-dal";
@@ -137,6 +138,7 @@ import { keyValueStoreDALFactory } from "@app/keystore/key-value-store-dal";
 import { TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig, TEnvConfig } from "@app/lib/config/env";
 import { crypto } from "@app/lib/crypto/cryptography";
+import { BadRequestError } from "@app/lib/errors";
 import { logger } from "@app/lib/logger";
 import { TQueueServiceFactory } from "@app/queue";
 import { readLimit } from "@app/server/config/rateLimiter";
@@ -228,6 +230,7 @@ import { internalKmsDALFactory } from "@app/services/kms/internal-kms-dal";
 import { kmskeyDALFactory } from "@app/services/kms/kms-key-dal";
 import { kmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal";
 import { kmsServiceFactory } from "@app/services/kms/kms-service";
+import { RootKeyEncryptionStrategy } from "@app/services/kms/kms-types";
 import { membershipDALFactory } from "@app/services/membership/membership-dal";
 import { membershipRoleDALFactory } from "@app/services/membership/membership-role-dal";
 import { membershipGroupDALFactory } from "@app/services/membership-group/membership-group-dal";
@@ -2212,6 +2215,27 @@ export const registerRoutes = async (
   // Start HSM service if it's configured/enabled.
   await hsmService.startService();
 
+  const hsmStatus = await isHsmActiveAndEnabled({
+    hsmService,
+    kmsRootConfigDAL,
+    licenseService
+  });
+
+  // if the encryption strategy is software - user needs to provide an encryption key
+  // if the encryption strategy is null AND the hsm is not configured - user needs to provide an encryption key
+  const needsEncryptionKey =
+    hsmStatus.rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.Software ||
+    (hsmStatus.rootKmsConfigEncryptionStrategy === null && !hsmStatus.isHsmConfigured);
+
+  if (needsEncryptionKey) {
+    if (!envConfig.ROOT_ENCRYPTION_KEY && !envConfig.ENCRYPTION_KEY) {
+      throw new BadRequestError({
+        message:
+          "Root KMS encryption strategy is set to software. Please set the ENCRYPTION_KEY environment variable and restart your deployment.\nYou can enable HSM encryption in the Server Console."
+      });
+    }
+  }
+
   await telemetryQueue.startTelemetryCheck();
   await telemetryQueue.startAggregatedEventsJob();
   await dailyResourceCleanUp.init();
@@ -2221,7 +2245,7 @@ export const registerRoutes = async (
   await dailyReminderQueueService.startSecretReminderMigrationJob();
   await dailyExpiringPkiItemAlert.startSendingAlerts();
   await pkiSubscriberQueue.startDailyAutoRenewalJob();
-  await kmsService.startService();
+  await kmsService.startService(hsmStatus);
   await microsoftTeamsService.start();
   await dynamicSecretQueueService.init();
   await eventBusService.init();

backend/src/services/kms/kms-service.ts

@@ -12,6 +12,7 @@ import {
   TExternalKmsProviderFns
 } from "@app/ee/services/external-kms/providers/model";
 import { THsmServiceFactory } from "@app/ee/services/hsm/hsm-service";
+import { THsmStatus } from "@app/ee/services/hsm/hsm-types";
 import { KeyStorePrefixes, PgSqlLock, TKeyStoreFactory } from "@app/keystore/keystore";
 import { TEnvConfig } from "@app/lib/config/env";
 import { symmetricCipherService, SymmetricKeyAlgorithm } from "@app/lib/crypto/cipher";
@@ -1073,17 +1074,22 @@ export const kmsServiceFactory = ({
     return { id, name, orgId, isExternal };
   };
 
-  const startService = async () => {
+  const startService = async (hsmStatus: THsmStatus) => {
     const kmsRootConfig = await kmsRootConfigDAL.transaction(async (tx) => {
       await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.KmsRootKeyInit]);
       // check if KMS root key was already generated and saved in DB
       const existingRootConfig = await kmsRootConfigDAL.findById(KMS_ROOT_CONFIG_UUID);
       if (existingRootConfig) return existingRootConfig;
 
+      const isHsmActive = hsmStatus.isHsmConfigured;
+
       logger.info("KMS: Generating new ROOT Key");
       const newRootKey = crypto.randomBytes(32);
-      const encryptedRootKey = await $encryptRootKey(newRootKey, RootKeyEncryptionStrategy.Software).catch((err) => {
+
-        logger.error({ hsmEnabled: hsmService.isActive() }, "KMS: Failed to encrypt ROOT Key");
+      const encryptionStrategy = isHsmActive ? RootKeyEncryptionStrategy.HSM : RootKeyEncryptionStrategy.Software;
+
+      const encryptedRootKey = await $encryptRootKey(newRootKey, encryptionStrategy).catch((err) => {
+        logger.error({ hsmEnabled: isHsmActive, encryptionStrategy }, "KMS: Failed to encrypt ROOT Key");
         throw err;
       });
 
@@ -1091,7 +1097,7 @@ export const kmsServiceFactory = ({
         // @ts-expect-error id is kept as fixed for idempotence and to avoid race condition
         id: KMS_ROOT_CONFIG_UUID,
         encryptedRootKey,
-        encryptionStrategy: RootKeyEncryptionStrategy.Software
+        encryptionStrategy
       });
       return newRootConfig;
     });
@@ -1113,6 +1119,15 @@ export const kmsServiceFactory = ({
       return;
     }
 
+    if (strategy === RootKeyEncryptionStrategy.Software) {
+      if (!envConfig.ROOT_ENCRYPTION_KEY && !envConfig.ENCRYPTION_KEY) {
+        throw new BadRequestError({
+          message:
+            "Root KMS encryption strategy is set to software. Please set the ENCRYPTION_KEY environment variable and restart your deployment before trying to update the encryption strategy to software mode."
+        });
+      }
+    }
+
     const decryptedRootKey = await $decryptRootKey(kmsRootConfig);
     const encryptedRootKey = await $encryptRootKey(decryptedRootKey, strategy);