feat: kms service changes for db change

backend/src/@types/knex.d.ts

@@ -59,6 +59,9 @@ import {
   TDynamicSecrets,
   TDynamicSecretsInsert,
   TDynamicSecretsUpdate,
+  TExternalKms,
+  TExternalKmsInsert,
+  TExternalKmsUpdate,
   TGitAppInstallSessions,
   TGitAppInstallSessionsInsert,
   TGitAppInstallSessionsUpdate,
@@ -122,6 +125,9 @@ import {
   TIntegrations,
   TIntegrationsInsert,
   TIntegrationsUpdate,
+  TInternalKms,
+  TInternalKmsInsert,
+  TInternalKmsUpdate,
   TKmsKeys,
   TKmsKeysInsert,
   TKmsKeysUpdate,
@@ -648,6 +654,8 @@ declare module "knex/types/tables" {
       TKmsRootConfigInsert,
       TKmsRootConfigUpdate
     >;
+    [TableName.InternalKms]: KnexOriginal.CompositeTableType<TInternalKms, TInternalKmsInsert, TInternalKmsUpdate>;
+    [TableName.ExternalKms]: KnexOriginal.CompositeTableType<TExternalKms, TExternalKmsInsert, TExternalKmsUpdate>;
     [TableName.KmsKey]: KnexOriginal.CompositeTableType<TKmsKeys, TKmsKeysInsert, TKmsKeysUpdate>;
     [TableName.KmsKeyVersion]: KnexOriginal.CompositeTableType<
       TKmsKeyVersions,

backend/src/services/kms/internal-kms-dal.ts

@@ -0,0 +1,10 @@
+import { TDbClient } from "@app/db";
+import { TableName } from "@app/db/schemas";
+import { ormify } from "@app/lib/knex";
+
+export type TInternalKmsDALFactory = ReturnType<typeof internalKmsDALFactory>;
+
+export const internalKmsDALFactory = (db: TDbClient) => {
+  const internalKmsOrm = ormify(db, TableName.InternalKms);
+  return internalKmsOrm;
+};

backend/src/services/kms/kms-dal.ts

@@ -1,10 +0,0 @@
-import { TDbClient } from "@app/db";
-import { TableName } from "@app/db/schemas";
-import { ormify } from "@app/lib/knex";
-
-export type TKmsDALFactory = ReturnType<typeof kmsDALFactory>;
-
-export const kmsDALFactory = (db: TDbClient) => {
-  const kmsOrm = ormify(db, TableName.KmsKey);
-  return kmsOrm;
-};

backend/src/services/kms/kms-key-dal.ts

@@ -0,0 +1,64 @@
+import { Knex } from "knex";
+
+import { TDbClient } from "@app/db";
+import { KmsKeysSchema, TableName } from "@app/db/schemas";
+import { DatabaseError } from "@app/lib/errors";
+import { ormify, selectAllTableCols } from "@app/lib/knex";
+
+export type TKmsKeyDALFactory = ReturnType<typeof kmskeyDALFactory>;
+
+export const kmskeyDALFactory = (db: TDbClient) => {
+  const kmsOrm = ormify(db, TableName.KmsKey);
+
+  const findByIdWithAssociatedKms = async (id: string, tx?: Knex) => {
+    try {
+      const result = await (tx || db.replicaNode())(TableName.KmsKey)
+        .where({ [`${TableName.KmsKey}.id` as "id"]: id })
+        .leftJoin(TableName.InternalKms, `${TableName.KmsKey}.id`, `${TableName.InternalKms}.kmsKeyId`)
+        .leftJoin(TableName.ExternalKms, `${TableName.KmsKey}.id`, `${TableName.ExternalKms}.kmsKeyId`)
+        .first()
+        .select(selectAllTableCols(TableName.KmsKey))
+        .select(
+          db.ref("id").withSchema(TableName.InternalKms).as("internalKmsId"),
+          db.ref("encryptedKey").withSchema(TableName.InternalKms).as("internalKmsEncryptedKey"),
+          db.ref("encryptionAlgorithm").withSchema(TableName.InternalKms).as("internalKmsEncryptionAlgorithm"),
+          db.ref("version").withSchema(TableName.InternalKms).as("internalKmsVersion"),
+          db.ref("id").withSchema(TableName.InternalKms).as("internalKmsId")
+        )
+        .select(
+          db.ref("id").withSchema(TableName.ExternalKms).as("externalKmsId"),
+          db.ref("provider").withSchema(TableName.ExternalKms).as("externalKmsProvider"),
+          db.ref("encryptedProviderInputs").withSchema(TableName.ExternalKms).as("externalKmsEncryptedProviderInput"),
+          db.ref("status").withSchema(TableName.ExternalKms).as("externalKmsStatus"),
+          db.ref("statusDetails").withSchema(TableName.ExternalKms).as("externalKmsStatusDetails")
+        );
+
+      const data = {
+        ...KmsKeysSchema.parse(result),
+        isExternal: Boolean(result?.externalKmsId),
+        externalKms: result?.externalKmsId
+          ? {
+              id: result.externalKmsId,
+              provider: result.externalKmsProvider,
+              encryptedProviderInput: result.externalKmsEncryptedProviderInput,
+              status: result.externalKmsStatus,
+              statusDetails: result.externalKmsStatusDetails
+            }
+          : undefined,
+        internalKms: result?.internalKmsId
+          ? {
+              id: result.internalKmsId,
+              encryptedKey: result.internalKmsEncryptedKey,
+              encryptionAlgorithm: result.internalKmsEncryptionAlgorithm,
+              version: result.internalKmsVersion
+            }
+          : undefined
+      };
+      return data;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find by id" });
+    }
+  };
+
+  return { ...kmsOrm, findByIdWithAssociatedKms };
+};

backend/src/services/kms/kms-service.ts

@@ -1,18 +1,28 @@
+import slugify from "@sindresorhus/slugify";
+import { Knex } from "knex";
+
 import { TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig } from "@app/lib/config/env";
 import { randomSecureBytes } from "@app/lib/crypto";
 import { symmetricCipherService, SymmetricEncryption } from "@app/lib/crypto/cipher";
 import { BadRequestError } from "@app/lib/errors";
 import { logger } from "@app/lib/logger";
+import { alphaNumericNanoId } from "@app/lib/nanoid";
 
-import { TKmsDALFactory } from "./kms-dal";
+import { TOrgDALFactory } from "../org/org-dal";
+import { TProjectDALFactory } from "../project/project-dal";
+import { TInternalKmsDALFactory } from "./internal-kms-dal";
+import { TKmsKeyDALFactory } from "./kms-key-dal";
 import { TKmsRootConfigDALFactory } from "./kms-root-config-dal";
-import { TDecryptWithKmsDTO, TEncryptWithKmsDTO, TGenerateKMSDTO } from "./kms-types";
+import { EncryptionMode, TGenerateKMSDTO, TKmsServiceDecryptionDTO, TKmsServiceEncryptionDTO } from "./kms-types";
 
 type TKmsServiceFactoryDep = {
-  kmsDAL: TKmsDALFactory;
+  kmsDAL: TKmsKeyDALFactory;
+  projectDAL: Pick<TProjectDALFactory, "findById" | "updateById" | "transaction">;
+  orgDAL: Pick<TOrgDALFactory, "findById" | "updateById" | "transaction">;
   kmsRootConfigDAL: Pick<TKmsRootConfigDALFactory, "findById" | "create">;
   keyStore: Pick<TKeyStoreFactory, "acquireLock" | "waitTillReady" | "setItemWithExpiry">;
+  internalKmsDAL: Pick<TInternalKmsDALFactory, "create">;
 };
 
 export type TKmsServiceFactory = ReturnType<typeof kmsServiceFactory>;
@@ -25,36 +35,71 @@ const KMS_ROOT_CREATION_WAIT_TIME = 10;
 // akhilmhdh: Don't edit this value. This is measured for blob concatination in kms
 const KMS_VERSION = "v01";
 const KMS_VERSION_BLOB_LENGTH = 3;
-export const kmsServiceFactory = ({ kmsDAL, kmsRootConfigDAL, keyStore }: TKmsServiceFactoryDep) => {
+export const kmsServiceFactory = ({
+  kmsDAL,
+  kmsRootConfigDAL,
+  keyStore,
+  internalKmsDAL,
+  orgDAL,
+  projectDAL
+}: TKmsServiceFactoryDep) => {
   let ROOT_ENCRYPTION_KEY = Buffer.alloc(0);
 
   // this is used symmetric encryption
-  const generateKmsKey = async ({ scopeId, scopeType, isReserved = true, tx }: TGenerateKMSDTO) => {
+  const generateKmsKey = async ({ orgId, isReserved = true, tx, slug }: TGenerateKMSDTO) => {
     const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
     const kmsKeyMaterial = randomSecureBytes(32);
     const encryptedKeyMaterial = cipher.encrypt(kmsKeyMaterial, ROOT_ENCRYPTION_KEY);
+    const sanitizedSlug = slug ? slugify(slug) : slugify(alphaNumericNanoId(32));
+    const dbQuery = async (db: Knex) => {
+      const kmsDoc = await kmsDAL.create({
+        slug: sanitizedSlug,
+        orgId,
+        isReserved
+      });
 
-    const { encryptedKey, ...doc } = await kmsDAL.create(
+      const { encryptedKey, ...doc } = await internalKmsDAL.create(
-      {
+        {
-        version: 1,
+          version: 1,
-        encryptedKey: encryptedKeyMaterial,
+          encryptedKey: encryptedKeyMaterial,
-        encryptionAlgorithm: SymmetricEncryption.AES_GCM_256,
+          encryptionAlgorithm: SymmetricEncryption.AES_GCM_256,
-        isReserved,
+          kmsKeyId: kmsDoc.id
-        orgId: scopeType === "org" ? scopeId : undefined,
+        },
-        projectId: scopeType === "project" ? scopeId : undefined
+        db
-      },
+      );
-      tx
+      return doc;
-    );
+    };
+    if (tx) return dbQuery(tx);
+    const doc = await kmsDAL.transaction(async (tx2) => dbQuery(tx2));
     return doc;
   };
 
-  const encrypt = async ({ kmsId, plainText }: TEncryptWithKmsDTO) => {
+  /*
-    const kmsDoc = await kmsDAL.findById(kmsId);
+   * KMS encryption service
-    if (!kmsDoc) throw new BadRequestError({ message: "KMS ID not found" });
+   * Function to handle various kinds of encryption like
+   * Normal encryption
+   * Encrypt with KMS key - internal or external
+   */
+  const encrypt = async (encryptionDetails: TKmsServiceEncryptionDTO) => {
     // akhilmhdh: as more encryption are added do a check here on kmsDoc.encryptionAlgorithm
     const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+    // instead of using kms key encrypt with the provided key
+    if (encryptionDetails.type === EncryptionMode.EncryptionKey) {
+      const { plainText, encryptionKey } = encryptionDetails;
 
-    const kmsKey = cipher.decrypt(kmsDoc.encryptedKey, ROOT_ENCRYPTION_KEY);
+      const encryptedPlainTextBlob = cipher.encrypt(plainText, encryptionKey);
+      // Buffer#1 encrypted text + Buffer#2 version number
+      const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3
+      const cipherTextBlob = Buffer.concat([encryptedPlainTextBlob, versionBlob]);
+      return { cipherTextBlob };
+    }
+
+    // this mean use kms to encrypt it
+    const { plainText, kmsId } = encryptionDetails;
+    const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId);
+    if (!kmsDoc) throw new BadRequestError({ message: "KMS ID not found" });
+
+    const kmsKey = cipher.decrypt(kmsDoc.internalKms?.encryptedKey as Buffer, ROOT_ENCRYPTION_KEY);
     const encryptedPlainTextBlob = cipher.encrypt(plainText, kmsKey);
 
     // Buffer#1 encrypted text + Buffer#2 version number
@@ -63,18 +108,96 @@ export const kmsServiceFactory = ({ kmsDAL, kmsRootConfigDAL, keyStore }: TKmsSe
     return { cipherTextBlob };
   };
 
-  const decrypt = async ({ cipherTextBlob: versionedCipherTextBlob, kmsId }: TDecryptWithKmsDTO) => {
+  /*
-    const kmsDoc = await kmsDAL.findById(kmsId);
+   * KMS decryption service
-    if (!kmsDoc) throw new BadRequestError({ message: "KMS ID not found" });
+   * Function to handle various kinds of decryptionlike
+   * Normal decryption with a key
+   * Encrypt with KMS key - internal or external
+   */
+  const decrypt = async (encryptionDetails: TKmsServiceDecryptionDTO) => {
     // akhilmhdh: as more encryption are added do a check here on kmsDoc.encryptionAlgorithm
     const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
-    const kmsKey = cipher.decrypt(kmsDoc.encryptedKey, ROOT_ENCRYPTION_KEY);
+    if (encryptionDetails.type === EncryptionMode.EncryptionKey) {
+      const { cipherTextBlob: versionedCipherTextBlob, encryptionKey } = encryptionDetails;
+      const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH);
+      const decryptedBlob = cipher.decrypt(cipherTextBlob, encryptionKey);
+      return decryptedBlob;
+    }
+
+    const { cipherTextBlob: versionedCipherTextBlob, kmsId } = encryptionDetails;
+    const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId);
+    if (!kmsDoc) throw new BadRequestError({ message: "KMS ID not found" });
+    const kmsKey = cipher.decrypt(kmsDoc.internalKms?.encryptedKey as Buffer, ROOT_ENCRYPTION_KEY);
 
     const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH);
     const decryptedBlob = cipher.decrypt(cipherTextBlob, kmsKey);
     return decryptedBlob;
   };
 
+  const getOrgKmsKeyId = async (orgId: string) => {
+    const keyId = await orgDAL.transaction(async (tx) => {
+      const org = await orgDAL.findById(orgId, tx);
+      if (!org) {
+        throw new BadRequestError({ message: "Org not found" });
+      }
+
+      if (!org.kmsDefaultKeyId) {
+        // create default kms key for certificate service
+        const key = await generateKmsKey({
+          isReserved: true,
+          orgId: org.id,
+          tx
+        });
+
+        await orgDAL.updateById(
+          org.id,
+          {
+            kmsDefaultKeyId: key.id
+          },
+          tx
+        );
+
+        return key.id;
+      }
+
+      return org.kmsDefaultKeyId;
+    });
+
+    return keyId;
+  };
+
+  const getProjectSecretManagerKmsKeyId = async (projectId: string) => {
+    const keyId = await projectDAL.transaction(async (tx) => {
+      const project = await projectDAL.findById(projectId, tx);
+      if (!project) {
+        throw new BadRequestError({ message: "Project not found" });
+      }
+
+      if (!project.kmsSecretManagerKeyId) {
+        // create default kms key for certificate service
+        const key = await generateKmsKey({
+          isReserved: true,
+          orgId: project.orgId,
+          tx
+        });
+
+        await projectDAL.updateById(
+          projectId,
+          {
+            kmsSecretManagerKeyId: key.id
+          },
+          tx
+        );
+
+        return key.id;
+      }
+
+      return project.kmsSecretManagerKeyId;
+    });
+
+    return keyId;
+  };
+
   const startService = async () => {
     const appCfg = getConfig();
     // This will switch to a seal process and HMS flow in future
@@ -124,6 +247,8 @@ export const kmsServiceFactory = ({ kmsDAL, kmsRootConfigDAL, keyStore }: TKmsSe
     startService,
     generateKmsKey,
     encrypt,
-    decrypt
+    decrypt,
+    getOrgKmsKeyId,
+    getProjectSecretManagerKmsKeyId
   };
 };

backend/src/services/kms/kms-types.ts

@@ -1,18 +1,41 @@
 import { Knex } from "knex";
 
 export type TGenerateKMSDTO = {
-  scopeType: "project" | "org";
+  orgId: string;
-  scopeId: string;
   isReserved?: boolean;
+  slug?: string;
   tx?: Knex;
 };
 
+export enum EncryptionMode {
+  KMS = "kms",
+  EncryptionKey = "encryption-key"
+}
+
 export type TEncryptWithKmsDTO = {
+  type?: EncryptionMode.KMS;
   kmsId: string;
   plainText: Buffer;
 };
 
+export type TEncryptionWithKeyDTO = {
+  type: EncryptionMode.EncryptionKey;
+  encryptionKey: Buffer;
+  plainText: Buffer;
+};
+
+export type TKmsServiceEncryptionDTO = TEncryptWithKmsDTO | TEncryptionWithKeyDTO;
+
 export type TDecryptWithKmsDTO = {
+  type?: EncryptionMode.KMS;
   kmsId: string;
   cipherTextBlob: Buffer;
 };
+
+export type TDecryptWithEncryptionKeyDTO = {
+  type: EncryptionMode.EncryptionKey;
+  encryptionKey: Buffer;
+  cipherTextBlob: Buffer;
+};
+
+export type TKmsServiceDecryptionDTO = TDecryptWithKmsDTO | TDecryptWithEncryptionKeyDTO;

backend/src/services/org/org-dal.ts

@@ -207,9 +207,9 @@ export const orgDALFactory = (db: TDbClient) => {
     }
   };
 
-  const updateById = async (orgId: string, data: Partial<TOrganizations>) => {
+  const updateById = async (orgId: string, data: Partial<TOrganizations>, tx?: Knex) => {
     try {
-      const [org] = await db(TableName.Organization)
+      const [org] = await (tx || db)(TableName.Organization)
         .where({ id: orgId })
         .update({ ...data })
         .returning("*");

backend/src/services/project/project-fns.ts

@@ -71,9 +71,8 @@ export const getProjectKmsCertificateKeyId = async ({
     if (!project.kmsCertificateKeyId) {
       // create default kms key for certificate service
       const key = await kmsService.generateKmsKey({
-        scopeId: projectId,
-        scopeType: "project",
         isReserved: true,
+        orgId: project.orgId,
         tx
       });