feat: kms db schema changes to support external and internal kms uniformly

backend/src/db/migrations/20240708100026_external-kms.ts

@@ -0,0 +1,188 @@
+import slugify from "@sindresorhus/slugify";
+import { Knex } from "knex";
+
+import { alphaNumericNanoId } from "@app/lib/nanoid";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  // rename old kms key table to internal kms table
+  // the kms key table would be a container to hold external and internal respectively
+  const doesOldKmsKeyTableExist = await knex.schema.hasTable(TableName.KmsKey);
+  const doesOldKmsKeyVersionTableExist = await knex.schema.hasTable(TableName.KmsKeyVersion);
+  const doesInternalKmsTableExist = await knex.schema.hasTable(TableName.InternalKms);
+
+  if (doesOldKmsKeyTableExist && !doesInternalKmsTableExist) {
+    await knex.schema.createTable(TableName.InternalKms, (tb) => {
+      tb.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      tb.binary("encryptedKey").notNullable();
+      tb.string("encryptionAlgorithm").notNullable();
+      tb.integer("version").defaultTo(1).notNullable();
+      tb.uuid("kmsKeyId").unique().notNullable();
+      tb.foreign("kmsKeyId").references("id").inTable(TableName.KmsKey).onDelete("CASCADE");
+    });
+    // copy the old kms and build the data
+    const oldKmsKey = await knex(TableName.KmsKey).select("version", "encryptedKey", "encryptionAlgorithm", "id");
+    if (oldKmsKey.length) {
+      await knex(TableName.InternalKms).insert(
+        oldKmsKey.map((el) => ({
+          encryptionAlgorithm: el.encryptionAlgorithm,
+          encryptedKey: el.encryptedKey,
+          kmsKeyId: el.id,
+          version: el.version
+        }))
+      );
+    }
+
+    if (doesOldKmsKeyVersionTableExist) {
+      // because we haven't started using versioning for kms thus no data exist
+      await knex.schema.renameTable(TableName.KmsKeyVersion, TableName.InternalKmsKeyVersion);
+      await knex.schema.alterTable(TableName.InternalKmsKeyVersion, (tb) => {
+        tb.dropColumn("kmsKeyId");
+        tb.uuid("internalKmsId").notNullable();
+        tb.foreign("internalKmsId").references("id").inTable(TableName.InternalKms).onDelete("CASCADE");
+      });
+    }
+
+    await knex.schema.alterTable(TableName.KmsKey, (tb) => {
+      tb.string("slug", 32);
+      tb.dropColumn("encryptedKey");
+      tb.dropColumn("encryptionAlgorithm");
+      tb.dropColumn("version");
+    });
+    // backfill all org id in kms key
+    await knex(TableName.KmsKey)
+      .whereNull("orgId")
+      .update({
+        // eslint-disable-next-line
+        // @ts-ignore because generate schema happens after this
+        orgId: knex(TableName.Project)
+          .select("orgId")
+          .where("id", knex.raw("??", [`${TableName.KmsKey}.projectId`]))
+      });
+    // backfill slugs in kms
+    const missingSlugs = await knex(TableName.KmsKey).whereNull("slug").select("id");
+    if (missingSlugs.length) {
+      await knex(TableName.KmsKey)
+        // eslint-disable-next-line
+        // @ts-ignore because generate schema happens after this
+        .insert(missingSlugs.map(({ id }) => ({ id, slug: slugify(alphaNumericNanoId(32)) })))
+        .onConflict("id")
+        .merge();
+    }
+
+    await knex.schema.alterTable(TableName.KmsKey, (tb) => {
+      tb.uuid("orgId").notNullable().alter();
+      tb.string("slug", 32).notNullable().alter();
+      tb.dropColumn("projectId");
+    });
+  }
+
+  const doesExternalKmsServiceExist = await knex.schema.hasTable(TableName.ExternalKms);
+  if (!doesExternalKmsServiceExist) {
+    await knex.schema.createTable(TableName.ExternalKms, (tb) => {
+      tb.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      tb.string("provider").notNullable();
+      tb.binary("encryptedProviderInputs").notNullable();
+      tb.string("status");
+      tb.string("statusDetails");
+      tb.uuid("kmsKeyId").unique().notNullable();
+      tb.foreign("kmsKeyId").references("id").inTable(TableName.KmsKey).onDelete("CASCADE");
+    });
+  }
+
+  const doesOrgKmsKeyExist = await knex.schema.hasColumn(TableName.Organization, "kmsDefaultKeyId");
+  if (!doesOrgKmsKeyExist) {
+    await knex.schema.alterTable(TableName.Organization, (tb) => {
+      tb.uuid("kmsDefaultKeyId").nullable();
+      tb.foreign("kmsDefaultKeyId").references("id").inTable(TableName.KmsKey);
+    });
+  }
+
+  const doesProjectKmsSecretManagerKeyExist = await knex.schema.hasColumn(TableName.Project, "kmsSecretManagerKeyId");
+  if (!doesProjectKmsSecretManagerKeyExist) {
+    await knex.schema.alterTable(TableName.Project, (tb) => {
+      tb.uuid("kmsSecretManagerKeyId").nullable();
+      tb.foreign("kmsSecretManagerKeyId").references("id").inTable(TableName.KmsKey);
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesOrgKmsKeyExist = await knex.schema.hasColumn(TableName.Organization, "kmsDefaultKeyId");
+  if (doesOrgKmsKeyExist) {
+    await knex.schema.alterTable(TableName.Organization, (tb) => {
+      tb.dropColumn("kmsDefaultKeyId");
+    });
+  }
+
+  const doesProjectKmsSecretManagerKeyExist = await knex.schema.hasColumn(TableName.Project, "kmsSecretManagerKeyId");
+  if (doesProjectKmsSecretManagerKeyExist) {
+    await knex.schema.alterTable(TableName.Project, (tb) => {
+      tb.dropColumn("kmsSecretManagerKeyId");
+    });
+  }
+
+  const doesInternalKmsKeyVersionTableExist = await knex.schema.hasTable(TableName.InternalKmsKeyVersion);
+  const doesInternalKmsTableExist = await knex.schema.hasTable(TableName.InternalKms);
+  if (doesInternalKmsKeyVersionTableExist) {
+    // because we haven't started using versioning for kms thus no data exist
+    await knex.schema.renameTable(TableName.InternalKmsKeyVersion, TableName.KmsKeyVersion);
+    await knex.schema.alterTable(TableName.KmsKeyVersion, (tb) => {
+      tb.dropColumn("internalKmsId");
+      tb.uuid("kmsKeyId").notNullable();
+      tb.foreign("kmsKeyId").references("id").inTable(TableName.KmsKey).onDelete("CASCADE");
+    });
+  }
+
+  const doesOldKmsKeyTableExist = await knex.schema.hasTable(TableName.KmsKey);
+  const doesKmsSlugExist = await knex.schema.hasColumn(TableName.KmsKey, "slug");
+  if (doesInternalKmsTableExist && doesOldKmsKeyTableExist) {
+    // converting kms key to old one
+    // backfill so not setting it as not nullable
+    await knex.schema.alterTable(TableName.KmsKey, (tb) => {
+      tb.binary("encryptedKey");
+      tb.string("encryptionAlgorithm");
+      tb.integer("version").defaultTo(1);
+      tb.string("projectId");
+      tb.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
+      if (doesKmsSlugExist) {
+        tb.dropColumn("slug");
+      }
+    });
+    // backfill kms key with internal kms data
+    await knex(TableName.KmsKey).update({
+      // eslint-disable-next-line
+      // @ts-ignore because generate schema happens after this
+      encryptedKey: knex(TableName.InternalKms)
+        .select("encryptedKey")
+        .where("kmsKeyId", knex.raw("??", [`${TableName.KmsKey}.id`])),
+      // eslint-disable-next-line
+      // @ts-ignore because generate schema happens after this
+      encryptionAlgorithm: knex(TableName.InternalKms)
+        .select("encryptionAlgorithm")
+        .where("kmsKeyId", knex.raw("??", [`${TableName.KmsKey}.id`])),
+      // eslint-disable-next-line
+      // @ts-ignore because generate schema happens after this
+      projectId: knex(TableName.Project)
+        .select("id")
+        .where("kmsCertificateKeyId", knex.raw("??", [`${TableName.KmsKey}.id`]))
+    });
+    await knex.schema.alterTable(TableName.KmsKey, (tb) => {
+      tb.binary("encryptedKey").notNullable().alter();
+      tb.string("encryptionAlgorithm").notNullable().alter();
+    });
+    await knex.schema.alterTable(TableName.InternalKms, (tb) => {
+      tb.dropForeign("kmsKeyId");
+    });
+    await knex.schema.dropTable(TableName.InternalKms);
+  }
+
+  const doesExternalKmsServiceExist = await knex.schema.hasTable(TableName.ExternalKms);
+  if (doesExternalKmsServiceExist) {
+    await knex.schema.alterTable(TableName.ExternalKms, (tb) => {
+      tb.dropForeign("kmsKeyId");
+    });
+    await knex.schema.dropTable(TableName.ExternalKms);
+  }
+}

backend/src/db/schemas/external-kms.ts

@@ -0,0 +1,23 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { zodBuffer } from "@app/lib/zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const ExternalKmsSchema = z.object({
+  id: z.string().uuid(),
+  provider: z.string(),
+  encryptedProviderInputs: zodBuffer,
+  status: z.string().nullable().optional(),
+  statusDetails: z.string().nullable().optional(),
+  kmsKeyId: z.string().uuid()
+});
+
+export type TExternalKms = z.infer<typeof ExternalKmsSchema>;
+export type TExternalKmsInsert = Omit<z.input<typeof ExternalKmsSchema>, TImmutableDBKeys>;
+export type TExternalKmsUpdate = Partial<Omit<z.input<typeof ExternalKmsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/index.ts

@@ -17,6 +17,7 @@ export * from "./certificate-secrets";
 export * from "./certificates";
 export * from "./dynamic-secret-leases";
 export * from "./dynamic-secrets";
+export * from "./external-kms";
 export * from "./git-app-install-sessions";
 export * from "./git-app-org";
 export * from "./group-project-membership-roles";
@@ -38,6 +39,7 @@ export * from "./identity-universal-auths";
 export * from "./incident-contacts";
 export * from "./integration-auths";
 export * from "./integrations";
+export * from "./internal-kms";
 export * from "./kms-key-versions";
 export * from "./kms-keys";
 export * from "./kms-root-config";

backend/src/db/schemas/internal-kms-key-version.ts

@@ -0,0 +1,21 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { zodBuffer } from "@app/lib/zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const InternalKmsKeyVersionSchema = z.object({
+  id: z.string().uuid(),
+  encryptedKey: zodBuffer,
+  version: z.number(),
+  internalKmsId: z.string().uuid()
+});
+
+export type TInternalKmsKeyVersion = z.infer<typeof InternalKmsKeyVersionSchema>;
+export type TInternalKmsKeyVersionInsert = Omit<z.input<typeof InternalKmsKeyVersionSchema>, TImmutableDBKeys>;
+export type TInternalKmsKeyVersionUpdate = Partial<Omit<z.input<typeof InternalKmsKeyVersionSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/internal-kms.ts

@@ -0,0 +1,22 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { zodBuffer } from "@app/lib/zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const InternalKmsSchema = z.object({
+  id: z.string().uuid(),
+  encryptedKey: zodBuffer,
+  encryptionAlgorithm: z.string(),
+  version: z.number().default(1),
+  kmsKeyId: z.string().uuid()
+});
+
+export type TInternalKms = z.infer<typeof InternalKmsSchema>;
+export type TInternalKmsInsert = Omit<z.input<typeof InternalKmsSchema>, TImmutableDBKeys>;
+export type TInternalKmsUpdate = Partial<Omit<z.input<typeof InternalKmsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/kms-keys.ts

@@ -5,20 +5,15 @@
 
 import { z } from "zod";
 
-import { zodBuffer } from "@app/lib/zod";
-
 import { TImmutableDBKeys } from "./models";
 
 export const KmsKeysSchema = z.object({
   id: z.string().uuid(),
-  encryptedKey: zodBuffer,
-  encryptionAlgorithm: z.string(),
-  version: z.number().default(1),
   description: z.string().nullable().optional(),
   isDisabled: z.boolean().default(false).nullable().optional(),
   isReserved: z.boolean().default(true).nullable().optional(),
-  projectId: z.string().nullable().optional(),
+  orgId: z.string().uuid(),
-  orgId: z.string().uuid().nullable().optional()
+  slug: z.string()
 });
 
 export type TKmsKeys = z.infer<typeof KmsKeysSchema>;

backend/src/db/schemas/models.ts

@@ -95,7 +95,10 @@ export enum TableName {
   // KMS Service
   KmsServerRootConfig = "kms_root_config",
   KmsKey = "kms_keys",
-  KmsKeyVersion = "kms_key_versions"
+  KmsKeyVersion = "kms_key_versions",
+  ExternalKms = "external_kms",
+  InternalKms = "internal_kms",
+  InternalKmsKeyVersion = "internal_kms_key_version"
 }
 
 export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt";

backend/src/db/schemas/organizations.ts

@@ -15,7 +15,8 @@ export const OrganizationsSchema = z.object({
   createdAt: z.date(),
   updatedAt: z.date(),
   authEnforced: z.boolean().default(false).nullable().optional(),
-  scimEnabled: z.boolean().default(false).nullable().optional()
+  scimEnabled: z.boolean().default(false).nullable().optional(),
+  kmsDefaultKeyId: z.string().uuid().nullable().optional()
 });
 
 export type TOrganizations = z.infer<typeof OrganizationsSchema>;

backend/src/db/schemas/projects.ts

@@ -19,7 +19,8 @@ export const ProjectsSchema = z.object({
   upgradeStatus: z.string().nullable().optional(),
   pitVersionLimit: z.number().default(10),
   kmsCertificateKeyId: z.string().uuid().nullable().optional(),
-  auditLogsRetentionDays: z.number().nullable().optional()
+  auditLogsRetentionDays: z.number().nullable().optional(),
+  kmsSecretManagerKeyId: z.string().uuid().nullable().optional()
 });
 
 export type TProjects = z.infer<typeof ProjectsSchema>;