Merge pull request #2095 from akhilmhdh/feat/aws-kms-sm

aws kms support base setup

backend/src/db/migrations/20240708100026_external-kms.ts

@@ -0,0 +1,256 @@
+import slugify from "@sindresorhus/slugify";
+import { Knex } from "knex";
+
+import { alphaNumericNanoId } from "@app/lib/nanoid";
+
+import { TableName } from "../schemas";
+
+const createInternalKmsTableAndBackfillData = async (knex: Knex) => {
+  const doesOldKmsKeyTableExist = await knex.schema.hasTable(TableName.KmsKey);
+  const doesInternalKmsTableExist = await knex.schema.hasTable(TableName.InternalKms);
+
+  // building the internal kms table by filling from old kms table
+  if (doesOldKmsKeyTableExist && !doesInternalKmsTableExist) {
+    await knex.schema.createTable(TableName.InternalKms, (tb) => {
+      tb.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      tb.binary("encryptedKey").notNullable();
+      tb.string("encryptionAlgorithm").notNullable();
+      tb.integer("version").defaultTo(1).notNullable();
+      tb.uuid("kmsKeyId").unique().notNullable();
+      tb.foreign("kmsKeyId").references("id").inTable(TableName.KmsKey).onDelete("CASCADE");
+    });
+
+    // copy the old kms and backfill
+    const oldKmsKey = await knex(TableName.KmsKey).select("version", "encryptedKey", "encryptionAlgorithm", "id");
+    if (oldKmsKey.length) {
+      await knex(TableName.InternalKms).insert(
+        oldKmsKey.map((el) => ({
+          encryptionAlgorithm: el.encryptionAlgorithm,
+          encryptedKey: el.encryptedKey,
+          kmsKeyId: el.id,
+          version: el.version
+        }))
+      );
+    }
+  }
+};
+
+const renameKmsKeyVersionTableAsInternalKmsKeyVersion = async (knex: Knex) => {
+  const doesOldKmsKeyVersionTableExist = await knex.schema.hasTable(TableName.KmsKeyVersion);
+  const doesNewKmsKeyVersionTableExist = await knex.schema.hasTable(TableName.InternalKmsKeyVersion);
+
+  if (doesOldKmsKeyVersionTableExist && !doesNewKmsKeyVersionTableExist) {
+    // because we haven't started using versioning for kms thus no data exist
+    await knex.schema.renameTable(TableName.KmsKeyVersion, TableName.InternalKmsKeyVersion);
+    const hasKmsKeyIdColumn = await knex.schema.hasColumn(TableName.InternalKmsKeyVersion, "kmsKeyId");
+    const hasInternalKmsIdColumn = await knex.schema.hasColumn(TableName.InternalKmsKeyVersion, "internalKmsId");
+
+    await knex.schema.alterTable(TableName.InternalKmsKeyVersion, (tb) => {
+      if (hasKmsKeyIdColumn) tb.dropColumn("kmsKeyId");
+      if (!hasInternalKmsIdColumn) {
+        tb.uuid("internalKmsId").notNullable();
+        tb.foreign("internalKmsId").references("id").inTable(TableName.InternalKms).onDelete("CASCADE");
+      }
+    });
+  }
+};
+
+const createExternalKmsKeyTable = async (knex: Knex) => {
+  const doesExternalKmsServiceExist = await knex.schema.hasTable(TableName.ExternalKms);
+  if (!doesExternalKmsServiceExist) {
+    await knex.schema.createTable(TableName.ExternalKms, (tb) => {
+      tb.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      tb.string("provider").notNullable();
+      tb.binary("encryptedProviderInputs").notNullable();
+      tb.string("status");
+      tb.string("statusDetails");
+      tb.uuid("kmsKeyId").unique().notNullable();
+      tb.foreign("kmsKeyId").references("id").inTable(TableName.KmsKey).onDelete("CASCADE");
+    });
+  }
+};
+
+const removeNonRequiredFieldsFromKmsKeyTableAndBackfillRequiredData = async (knex: Knex) => {
+  const doesOldKmsKeyTableExist = await knex.schema.hasTable(TableName.KmsKey);
+
+  // building the internal kms table by filling from old kms table
+  if (doesOldKmsKeyTableExist) {
+    const hasSlugColumn = await knex.schema.hasColumn(TableName.KmsKey, "slug");
+    const hasEncryptedKeyColumn = await knex.schema.hasColumn(TableName.KmsKey, "encryptedKey");
+    const hasEncryptionAlgorithmColumn = await knex.schema.hasColumn(TableName.KmsKey, "encryptionAlgorithm");
+    const hasVersionColumn = await knex.schema.hasColumn(TableName.KmsKey, "version");
+    const hasTimestamps = await knex.schema.hasColumn(TableName.KmsKey, "createdAt");
+    const hasProjectId = await knex.schema.hasColumn(TableName.KmsKey, "projectId");
+    const hasOrgId = await knex.schema.hasColumn(TableName.KmsKey, "orgId");
+
+    await knex.schema.alterTable(TableName.KmsKey, (tb) => {
+      if (!hasSlugColumn) tb.string("slug", 32);
+      if (hasEncryptedKeyColumn) tb.dropColumn("encryptedKey");
+      if (hasEncryptionAlgorithmColumn) tb.dropColumn("encryptionAlgorithm");
+      if (hasVersionColumn) tb.dropColumn("version");
+      if (!hasTimestamps) tb.timestamps(true, true, true);
+    });
+
+    // backfill all org id in kms key because its gonna be changed to non nullable
+    if (hasProjectId && hasOrgId) {
+      await knex(TableName.KmsKey)
+        .whereNull("orgId")
+        .update({
+          // eslint-disable-next-line
+          // @ts-ignore because generate schema happens after this
+          orgId: knex(TableName.Project)
+            .select("orgId")
+            .where("id", knex.raw("??", [`${TableName.KmsKey}.projectId`]))
+        });
+    }
+
+    // backfill slugs in kms
+    const missingSlugs = await knex(TableName.KmsKey).whereNull("slug").select("id");
+    if (missingSlugs.length) {
+      await knex(TableName.KmsKey)
+        // eslint-disable-next-line
+        // @ts-ignore because generate schema happens after this
+        .insert(missingSlugs.map(({ id }) => ({ id, slug: slugify(alphaNumericNanoId(8).toLowerCase()) })))
+        .onConflict("id")
+        .merge();
+    }
+
+    await knex.schema.alterTable(TableName.KmsKey, (tb) => {
+      if (hasOrgId) tb.uuid("orgId").notNullable().alter();
+      tb.string("slug", 32).notNullable().alter();
+      if (hasProjectId) tb.dropColumn("projectId");
+      if (hasOrgId) tb.unique(["orgId", "slug"]);
+    });
+  }
+};
+
+/*
+ * The goal for this migration is split the existing kms key into three table
+ * the kms-key table would be a container table that contains
+ * the internal kms key table and external kms table
+ */
+export async function up(knex: Knex): Promise<void> {
+  await createInternalKmsTableAndBackfillData(knex);
+  await renameKmsKeyVersionTableAsInternalKmsKeyVersion(knex);
+  await removeNonRequiredFieldsFromKmsKeyTableAndBackfillRequiredData(knex);
+  await createExternalKmsKeyTable(knex);
+
+  const doesOrgKmsKeyExist = await knex.schema.hasColumn(TableName.Organization, "kmsDefaultKeyId");
+  if (!doesOrgKmsKeyExist) {
+    await knex.schema.alterTable(TableName.Organization, (tb) => {
+      tb.uuid("kmsDefaultKeyId").nullable();
+      tb.foreign("kmsDefaultKeyId").references("id").inTable(TableName.KmsKey);
+    });
+  }
+
+  const doesProjectKmsSecretManagerKeyExist = await knex.schema.hasColumn(TableName.Project, "kmsSecretManagerKeyId");
+  if (!doesProjectKmsSecretManagerKeyExist) {
+    await knex.schema.alterTable(TableName.Project, (tb) => {
+      tb.uuid("kmsSecretManagerKeyId").nullable();
+      tb.foreign("kmsSecretManagerKeyId").references("id").inTable(TableName.KmsKey);
+    });
+  }
+}
+
+const renameInternalKmsKeyVersionBackToKmsKeyVersion = async (knex: Knex) => {
+  const doesInternalKmsKeyVersionTableExist = await knex.schema.hasTable(TableName.InternalKmsKeyVersion);
+  const doesKmsKeyVersionTableExist = await knex.schema.hasTable(TableName.KmsKeyVersion);
+  if (doesInternalKmsKeyVersionTableExist && !doesKmsKeyVersionTableExist) {
+    // because we haven't started using versioning for kms thus no data exist
+    await knex.schema.renameTable(TableName.InternalKmsKeyVersion, TableName.KmsKeyVersion);
+    const hasInternalKmsIdColumn = await knex.schema.hasColumn(TableName.KmsKeyVersion, "internalKmsId");
+    const hasKmsKeyIdColumn = await knex.schema.hasColumn(TableName.KmsKeyVersion, "kmsKeyId");
+
+    await knex.schema.alterTable(TableName.KmsKeyVersion, (tb) => {
+      if (hasInternalKmsIdColumn) tb.dropColumn("internalKmsId");
+      if (!hasKmsKeyIdColumn) {
+        tb.uuid("kmsKeyId").notNullable();
+        tb.foreign("kmsKeyId").references("id").inTable(TableName.KmsKey).onDelete("CASCADE");
+      }
+    });
+  }
+};
+
+const bringBackKmsKeyFields = async (knex: Knex) => {
+  const doesOldKmsKeyTableExist = await knex.schema.hasTable(TableName.KmsKey);
+  const doesInternalKmsTableExist = await knex.schema.hasTable(TableName.InternalKms);
+  if (doesOldKmsKeyTableExist && doesInternalKmsTableExist) {
+    const hasSlug = await knex.schema.hasColumn(TableName.KmsKey, "slug");
+    const hasEncryptedKeyColumn = await knex.schema.hasColumn(TableName.KmsKey, "encryptedKey");
+    const hasEncryptionAlgorithmColumn = await knex.schema.hasColumn(TableName.KmsKey, "encryptionAlgorithm");
+    const hasVersionColumn = await knex.schema.hasColumn(TableName.KmsKey, "version");
+    const hasNullableOrgId = await knex.schema.hasColumn(TableName.KmsKey, "orgId");
+    const hasProjectIdColumn = await knex.schema.hasColumn(TableName.KmsKey, "projectId");
+
+    await knex.schema.alterTable(TableName.KmsKey, (tb) => {
+      if (!hasEncryptedKeyColumn) tb.binary("encryptedKey");
+      if (!hasEncryptionAlgorithmColumn) tb.string("encryptionAlgorithm");
+      if (!hasVersionColumn) tb.integer("version").defaultTo(1);
+      if (hasNullableOrgId) tb.uuid("orgId").nullable().alter();
+      if (!hasProjectIdColumn) {
+        tb.string("projectId");
+        tb.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
+      }
+      if (hasSlug) tb.dropColumn("slug");
+    });
+  }
+};
+
+const backfillKmsKeyFromInternalKmsTable = async (knex: Knex) => {
+  const doesOldKmsKeyTableExist = await knex.schema.hasTable(TableName.KmsKey);
+  const doesInternalKmsTableExist = await knex.schema.hasTable(TableName.InternalKms);
+  if (doesInternalKmsTableExist && doesOldKmsKeyTableExist) {
+    // backfill kms key with internal kms data
+    await knex(TableName.KmsKey).update({
+      // eslint-disable-next-line
+      // @ts-ignore because generate schema happens after this
+      encryptedKey: knex(TableName.InternalKms)
+        .select("encryptedKey")
+        .where("kmsKeyId", knex.raw("??", [`${TableName.KmsKey}.id`])),
+      // eslint-disable-next-line
+      // @ts-ignore because generate schema happens after this
+      encryptionAlgorithm: knex(TableName.InternalKms)
+        .select("encryptionAlgorithm")
+        .where("kmsKeyId", knex.raw("??", [`${TableName.KmsKey}.id`])),
+      // eslint-disable-next-line
+      // @ts-ignore because generate schema happens after this
+      projectId: knex(TableName.Project)
+        .select("id")
+        .where("kmsCertificateKeyId", knex.raw("??", [`${TableName.KmsKey}.id`]))
+    });
+  }
+};
+
+export async function down(knex: Knex): Promise<void> {
+  const doesOrgKmsKeyExist = await knex.schema.hasColumn(TableName.Organization, "kmsDefaultKeyId");
+  if (doesOrgKmsKeyExist) {
+    await knex.schema.alterTable(TableName.Organization, (tb) => {
+      tb.dropColumn("kmsDefaultKeyId");
+    });
+  }
+
+  const doesProjectKmsSecretManagerKeyExist = await knex.schema.hasColumn(TableName.Project, "kmsSecretManagerKeyId");
+  if (doesProjectKmsSecretManagerKeyExist) {
+    await knex.schema.alterTable(TableName.Project, (tb) => {
+      tb.dropColumn("kmsSecretManagerKeyId");
+    });
+  }
+
+  await renameInternalKmsKeyVersionBackToKmsKeyVersion(knex);
+  await bringBackKmsKeyFields(knex);
+  await backfillKmsKeyFromInternalKmsTable(knex);
+
+  const doesOldKmsKeyTableExist = await knex.schema.hasTable(TableName.KmsKey);
+  if (doesOldKmsKeyTableExist) {
+    await knex.schema.alterTable(TableName.KmsKey, (tb) => {
+      tb.binary("encryptedKey").notNullable().alter();
+      tb.string("encryptionAlgorithm").notNullable().alter();
+    });
+  }
+
+  const doesInternalKmsTableExist = await knex.schema.hasTable(TableName.InternalKms);
+  if (doesInternalKmsTableExist) await knex.schema.dropTable(TableName.InternalKms);
+
+  const doesExternalKmsServiceExist = await knex.schema.hasTable(TableName.ExternalKms);
+  if (doesExternalKmsServiceExist) await knex.schema.dropTable(TableName.ExternalKms);
+}

backend/src/db/schemas/external-kms.ts

@@ -0,0 +1,23 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { zodBuffer } from "@app/lib/zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const ExternalKmsSchema = z.object({
+  id: z.string().uuid(),
+  provider: z.string(),
+  encryptedProviderInputs: zodBuffer,
+  status: z.string().nullable().optional(),
+  statusDetails: z.string().nullable().optional(),
+  kmsKeyId: z.string().uuid()
+});
+
+export type TExternalKms = z.infer<typeof ExternalKmsSchema>;
+export type TExternalKmsInsert = Omit<z.input<typeof ExternalKmsSchema>, TImmutableDBKeys>;
+export type TExternalKmsUpdate = Partial<Omit<z.input<typeof ExternalKmsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/index.ts

@@ -17,6 +17,7 @@ export * from "./certificate-secrets";
 export * from "./certificates";
 export * from "./dynamic-secret-leases";
 export * from "./dynamic-secrets";
+export * from "./external-kms";
 export * from "./git-app-install-sessions";
 export * from "./git-app-org";
 export * from "./group-project-membership-roles";
@@ -39,6 +40,7 @@ export * from "./identity-universal-auths";
 export * from "./incident-contacts";
 export * from "./integration-auths";
 export * from "./integrations";
+export * from "./internal-kms";
 export * from "./kms-key-versions";
 export * from "./kms-keys";
 export * from "./kms-root-config";

backend/src/db/schemas/internal-kms-key-version.ts

@@ -0,0 +1,21 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { zodBuffer } from "@app/lib/zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const InternalKmsKeyVersionSchema = z.object({
+  id: z.string().uuid(),
+  encryptedKey: zodBuffer,
+  version: z.number(),
+  internalKmsId: z.string().uuid()
+});
+
+export type TInternalKmsKeyVersion = z.infer<typeof InternalKmsKeyVersionSchema>;
+export type TInternalKmsKeyVersionInsert = Omit<z.input<typeof InternalKmsKeyVersionSchema>, TImmutableDBKeys>;
+export type TInternalKmsKeyVersionUpdate = Partial<Omit<z.input<typeof InternalKmsKeyVersionSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/internal-kms.ts

@@ -0,0 +1,22 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { zodBuffer } from "@app/lib/zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const InternalKmsSchema = z.object({
+  id: z.string().uuid(),
+  encryptedKey: zodBuffer,
+  encryptionAlgorithm: z.string(),
+  version: z.number().default(1),
+  kmsKeyId: z.string().uuid()
+});
+
+export type TInternalKms = z.infer<typeof InternalKmsSchema>;
+export type TInternalKmsInsert = Omit<z.input<typeof InternalKmsSchema>, TImmutableDBKeys>;
+export type TInternalKmsUpdate = Partial<Omit<z.input<typeof InternalKmsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/kms-keys.ts

@@ -5,20 +5,17 @@
 
 import { z } from "zod";
 
-import { zodBuffer } from "@app/lib/zod";
-
 import { TImmutableDBKeys } from "./models";
 
 export const KmsKeysSchema = z.object({
   id: z.string().uuid(),
-  encryptedKey: zodBuffer,
-  encryptionAlgorithm: z.string(),
-  version: z.number().default(1),
   description: z.string().nullable().optional(),
   isDisabled: z.boolean().default(false).nullable().optional(),
   isReserved: z.boolean().default(true).nullable().optional(),
-  projectId: z.string().nullable().optional(),
-  orgId: z.string().uuid().nullable().optional()
+  orgId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  slug: z.string()
 });
 
 export type TKmsKeys = z.infer<typeof KmsKeysSchema>;

backend/src/db/schemas/models.ts

@@ -96,6 +96,10 @@ export enum TableName {
   // KMS Service
   KmsServerRootConfig = "kms_root_config",
   KmsKey = "kms_keys",
+  ExternalKms = "external_kms",
+  InternalKms = "internal_kms",
+  InternalKmsKeyVersion = "internal_kms_key_version",
+  // @depreciated
   KmsKeyVersion = "kms_key_versions"
 }
 

backend/src/db/schemas/organizations.ts

@@ -15,7 +15,8 @@ export const OrganizationsSchema = z.object({
   createdAt: z.date(),
   updatedAt: z.date(),
   authEnforced: z.boolean().default(false).nullable().optional(),
-  scimEnabled: z.boolean().default(false).nullable().optional()
+  scimEnabled: z.boolean().default(false).nullable().optional(),
+  kmsDefaultKeyId: z.string().uuid().nullable().optional()
 });
 
 export type TOrganizations = z.infer<typeof OrganizationsSchema>;

backend/src/db/schemas/projects.ts

@@ -19,7 +19,8 @@ export const ProjectsSchema = z.object({
   upgradeStatus: z.string().nullable().optional(),
   pitVersionLimit: z.number().default(10),
   kmsCertificateKeyId: z.string().uuid().nullable().optional(),
-  auditLogsRetentionDays: z.number().nullable().optional()
+  auditLogsRetentionDays: z.number().nullable().optional(),
+  kmsSecretManagerKeyId: z.string().uuid().nullable().optional()
 });
 
 export type TProjects = z.infer<typeof ProjectsSchema>;