Revert "Root encrypted data to kms encryption"

backend/src/db/migrations/utils/env-config.ts

@@ -1,53 +0,0 @@
-import { z } from "zod";
-
-import { zpStr } from "@app/lib/zod";
-
-const envSchema = z
-  .object({
-    DB_CONNECTION_URI: zpStr(z.string().describe("Postgres database connection string")).default(
-      `postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`
-    ),
-    DB_ROOT_CERT: zpStr(z.string().describe("Postgres database base64-encoded CA cert").optional()),
-    DB_HOST: zpStr(z.string().describe("Postgres database host").optional()),
-    DB_PORT: zpStr(z.string().describe("Postgres database port").optional()).default("5432"),
-    DB_USER: zpStr(z.string().describe("Postgres database username").optional()),
-    DB_PASSWORD: zpStr(z.string().describe("Postgres database password").optional()),
-    DB_NAME: zpStr(z.string().describe("Postgres database name").optional()),
-    // TODO(akhilmhdh): will be changed to one
-    ENCRYPTION_KEY: zpStr(z.string().optional()),
-    ROOT_ENCRYPTION_KEY: zpStr(z.string().optional()),
-    // HSM
-    HSM_LIB_PATH: zpStr(z.string().optional()),
-    HSM_PIN: zpStr(z.string().optional()),
-    HSM_KEY_LABEL: zpStr(z.string().optional()),
-    HSM_SLOT: z.coerce.number().optional().default(0)
-  })
-  // To ensure that basic encryption is always possible.
-  .refine(
-    (data) => Boolean(data.ENCRYPTION_KEY) || Boolean(data.ROOT_ENCRYPTION_KEY),
-    "Either ENCRYPTION_KEY or ROOT_ENCRYPTION_KEY must be defined."
-  )
-  .transform((data) => ({
-    ...data,
-    isHsmConfigured:
-      Boolean(data.HSM_LIB_PATH) && Boolean(data.HSM_PIN) && Boolean(data.HSM_KEY_LABEL) && data.HSM_SLOT !== undefined
-  }));
-
-export type TMigrationEnvConfig = z.infer<typeof envSchema>;
-
-export const getMigrationEnvConfig = () => {
-  const parsedEnv = envSchema.safeParse(process.env);
-  if (!parsedEnv.success) {
-    // eslint-disable-next-line no-console
-    console.error("Invalid environment variables. Check the error below");
-    // eslint-disable-next-line no-console
-    console.error(
-      "Migration is now automatic at startup. Please remove this step from your workflow and start the application as normal."
-    );
-    // eslint-disable-next-line no-console
-    console.error(parsedEnv.error.issues);
-    process.exit(-1);
-  }
-
-  return Object.freeze(parsedEnv.data);
-};

backend/src/db/migrations/utils/kms.ts

@@ -0,0 +1,105 @@
+import slugify from "@sindresorhus/slugify";
+import { Knex } from "knex";
+
+import { TableName } from "@app/db/schemas";
+import { randomSecureBytes } from "@app/lib/crypto";
+import { symmetricCipherService, SymmetricEncryption } from "@app/lib/crypto/cipher";
+import { alphaNumericNanoId } from "@app/lib/nanoid";
+
+const getInstanceRootKey = async (knex: Knex) => {
+  const encryptionKey = process.env.ENCRYPTION_KEY || process.env.ROOT_ENCRYPTION_KEY;
+  // if root key its base64 encoded
+  const isBase64 = !process.env.ENCRYPTION_KEY;
+  if (!encryptionKey) throw new Error("ENCRYPTION_KEY variable needed for migration");
+  const encryptionKeyBuffer = Buffer.from(encryptionKey, isBase64 ? "base64" : "utf8");
+
+  const KMS_ROOT_CONFIG_UUID = "00000000-0000-0000-0000-000000000000";
+  const kmsRootConfig = await knex(TableName.KmsServerRootConfig).where({ id: KMS_ROOT_CONFIG_UUID }).first();
+  const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+  if (kmsRootConfig) {
+    const decryptedRootKey = cipher.decrypt(kmsRootConfig.encryptedRootKey, encryptionKeyBuffer);
+    // set the flag so that other instancen nodes can start
+    return decryptedRootKey;
+  }
+
+  const newRootKey = randomSecureBytes(32);
+  const encryptedRootKey = cipher.encrypt(newRootKey, encryptionKeyBuffer);
+  await knex(TableName.KmsServerRootConfig).insert({
+    encryptedRootKey,
+    // eslint-disable-next-line
+    // @ts-ignore id is kept as fixed for idempotence and to avoid race condition
+    id: KMS_ROOT_CONFIG_UUID
+  });
+  return encryptedRootKey;
+};
+
+export const getSecretManagerDataKey = async (knex: Knex, projectId: string) => {
+  const KMS_VERSION = "v01";
+  const KMS_VERSION_BLOB_LENGTH = 3;
+  const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+  const project = await knex(TableName.Project).where({ id: projectId }).first();
+  if (!project) throw new Error("Missing project id");
+
+  const ROOT_ENCRYPTION_KEY = await getInstanceRootKey(knex);
+
+  let secretManagerKmsKey;
+  const projectSecretManagerKmsId = project?.kmsSecretManagerKeyId;
+  if (projectSecretManagerKmsId) {
+    const kmsDoc = await knex(TableName.KmsKey)
+      .leftJoin(TableName.InternalKms, `${TableName.KmsKey}.id`, `${TableName.InternalKms}.kmsKeyId`)
+      .where({ [`${TableName.KmsKey}.id` as "id"]: projectSecretManagerKmsId })
+      .first();
+    if (!kmsDoc) throw new Error("missing kms");
+    secretManagerKmsKey = cipher.decrypt(kmsDoc.encryptedKey, ROOT_ENCRYPTION_KEY);
+  } else {
+    const [kmsDoc] = await knex(TableName.KmsKey)
+      .insert({
+        name: slugify(alphaNumericNanoId(8).toLowerCase()),
+        orgId: project.orgId,
+        isReserved: false
+      })
+      .returning("*");
+
+    secretManagerKmsKey = randomSecureBytes(32);
+    const encryptedKeyMaterial = cipher.encrypt(secretManagerKmsKey, ROOT_ENCRYPTION_KEY);
+    await knex(TableName.InternalKms).insert({
+      version: 1,
+      encryptedKey: encryptedKeyMaterial,
+      encryptionAlgorithm: SymmetricEncryption.AES_GCM_256,
+      kmsKeyId: kmsDoc.id
+    });
+  }
+
+  const encryptedSecretManagerDataKey = project?.kmsSecretManagerEncryptedDataKey;
+  let dataKey: Buffer;
+  if (!encryptedSecretManagerDataKey) {
+    dataKey = randomSecureBytes();
+    // the below versioning we do it automatically in kms service
+    const unversionedDataKey = cipher.encrypt(dataKey, secretManagerKmsKey);
+    const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3
+    await knex(TableName.Project)
+      .where({ id: projectId })
+      .update({
+        kmsSecretManagerEncryptedDataKey: Buffer.concat([unversionedDataKey, versionBlob])
+      });
+  } else {
+    const cipherTextBlob = encryptedSecretManagerDataKey.subarray(0, -KMS_VERSION_BLOB_LENGTH);
+    dataKey = cipher.decrypt(cipherTextBlob, secretManagerKmsKey);
+  }
+
+  return {
+    encryptor: ({ plainText }: { plainText: Buffer }) => {
+      const encryptedPlainTextBlob = cipher.encrypt(plainText, dataKey);
+
+      // Buffer#1 encrypted text + Buffer#2 version number
+      const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3
+      const cipherTextBlob = Buffer.concat([encryptedPlainTextBlob, versionBlob]);
+      return { cipherTextBlob };
+    },
+    decryptor: ({ cipherTextBlob: versionedCipherTextBlob }: { cipherTextBlob: Buffer }) => {
+      const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH);
+      const decryptedBlob = cipher.decrypt(cipherTextBlob, dataKey);
+      return decryptedBlob;
+    }
+  };
+};

backend/src/db/migrations/utils/ring-buffer.ts

@@ -1,19 +0,0 @@
-export const createCircularCache = <T>(bufferSize = 10) => {
-  const bufferItems: { id: string; item: T }[] = [];
-  let bufferIndex = 0;
-
-  const push = (id: string, item: T) => {
-    if (bufferItems.length < bufferSize) {
-      bufferItems.push({ id, item });
-    } else {
-      bufferItems[bufferIndex] = { id, item };
-    }
-    bufferIndex = (bufferIndex + 1) % bufferSize;
-  };
-
-  const getItem = (id: string) => {
-    return bufferItems.find((i) => i.id === id)?.item;
-  };
-
-  return { push, getItem };
-};

backend/src/db/migrations/utils/services.ts

@@ -1,52 +0,0 @@
-import { Knex } from "knex";
-
-import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns";
-import { hsmServiceFactory } from "@app/ee/services/hsm/hsm-service";
-import { TKeyStoreFactory } from "@app/keystore/keystore";
-import { internalKmsDALFactory } from "@app/services/kms/internal-kms-dal";
-import { kmskeyDALFactory } from "@app/services/kms/kms-key-dal";
-import { kmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal";
-import { kmsServiceFactory } from "@app/services/kms/kms-service";
-import { orgDALFactory } from "@app/services/org/org-dal";
-import { projectDALFactory } from "@app/services/project/project-dal";
-
-import { TMigrationEnvConfig } from "./env-config";
-
-type TDependencies = {
-  envConfig: TMigrationEnvConfig;
-  db: Knex;
-  keyStore: TKeyStoreFactory;
-};
-
-export const getMigrationEncryptionServices = async ({ envConfig, db, keyStore }: TDependencies) => {
-  // eslint-disable-next-line no-param-reassign
-  const hsmModule = initializeHsmModule(envConfig);
-  hsmModule.initialize();
-
-  const hsmService = hsmServiceFactory({
-    hsmModule: hsmModule.getModule(),
-    envConfig
-  });
-
-  const orgDAL = orgDALFactory(db);
-  const kmsRootConfigDAL = kmsRootConfigDALFactory(db);
-  const kmsDAL = kmskeyDALFactory(db);
-  const internalKmsDAL = internalKmsDALFactory(db);
-  const projectDAL = projectDALFactory(db);
-
-  const kmsService = kmsServiceFactory({
-    kmsRootConfigDAL,
-    keyStore,
-    kmsDAL,
-    internalKmsDAL,
-    orgDAL,
-    projectDAL,
-    hsmService,
-    envConfig
-  });
-
-  await hsmService.startService();
-  await kmsService.startService();
-
-  return { kmsService };
-};