From 71c49c8b9008a9ae01cd4db7787a3860f33fca15 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 10 Jul 2024 12:10:03 +0530 Subject: [PATCH] feat: kms db schema changes to support external and internal kms uniformly --- .../migrations/20240708100026_external-kms.ts | 188 ++++++++++++++++++ backend/src/db/schemas/external-kms.ts | 23 +++ backend/src/db/schemas/index.ts | 2 + .../db/schemas/internal-kms-key-version.ts | 21 ++ backend/src/db/schemas/internal-kms.ts | 22 ++ backend/src/db/schemas/kms-keys.ts | 9 +- backend/src/db/schemas/models.ts | 5 +- backend/src/db/schemas/organizations.ts | 3 +- backend/src/db/schemas/projects.ts | 3 +- 9 files changed, 266 insertions(+), 10 deletions(-) create mode 100644 backend/src/db/migrations/20240708100026_external-kms.ts create mode 100644 backend/src/db/schemas/external-kms.ts create mode 100644 backend/src/db/schemas/internal-kms-key-version.ts create mode 100644 backend/src/db/schemas/internal-kms.ts diff --git a/backend/src/db/migrations/20240708100026_external-kms.ts b/backend/src/db/migrations/20240708100026_external-kms.ts new file mode 100644 index 000000000..3c4bceb93 --- /dev/null +++ b/backend/src/db/migrations/20240708100026_external-kms.ts @@ -0,0 +1,188 @@ +import slugify from "@sindresorhus/slugify"; +import { Knex } from "knex"; + +import { alphaNumericNanoId } from "@app/lib/nanoid"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + // rename old kms key table to internal kms table + // the kms key table would be a container to hold external and internal respectively + const doesOldKmsKeyTableExist = await knex.schema.hasTable(TableName.KmsKey); + const doesOldKmsKeyVersionTableExist = await knex.schema.hasTable(TableName.KmsKeyVersion); + const doesInternalKmsTableExist = await knex.schema.hasTable(TableName.InternalKms); + + if (doesOldKmsKeyTableExist && !doesInternalKmsTableExist) { + await knex.schema.createTable(TableName.InternalKms, (tb) => { + tb.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + tb.binary("encryptedKey").notNullable(); + tb.string("encryptionAlgorithm").notNullable(); + tb.integer("version").defaultTo(1).notNullable(); + tb.uuid("kmsKeyId").unique().notNullable(); + tb.foreign("kmsKeyId").references("id").inTable(TableName.KmsKey).onDelete("CASCADE"); + }); + // copy the old kms and build the data + const oldKmsKey = await knex(TableName.KmsKey).select("version", "encryptedKey", "encryptionAlgorithm", "id"); + if (oldKmsKey.length) { + await knex(TableName.InternalKms).insert( + oldKmsKey.map((el) => ({ + encryptionAlgorithm: el.encryptionAlgorithm, + encryptedKey: el.encryptedKey, + kmsKeyId: el.id, + version: el.version + })) + ); + } + + if (doesOldKmsKeyVersionTableExist) { + // because we haven't started using versioning for kms thus no data exist + await knex.schema.renameTable(TableName.KmsKeyVersion, TableName.InternalKmsKeyVersion); + await knex.schema.alterTable(TableName.InternalKmsKeyVersion, (tb) => { + tb.dropColumn("kmsKeyId"); + tb.uuid("internalKmsId").notNullable(); + tb.foreign("internalKmsId").references("id").inTable(TableName.InternalKms).onDelete("CASCADE"); + }); + } + + await knex.schema.alterTable(TableName.KmsKey, (tb) => { + tb.string("slug", 32); + tb.dropColumn("encryptedKey"); + tb.dropColumn("encryptionAlgorithm"); + tb.dropColumn("version"); + }); + // backfill all org id in kms key + await knex(TableName.KmsKey) + .whereNull("orgId") + .update({ + // eslint-disable-next-line + // @ts-ignore because generate schema happens after this + orgId: knex(TableName.Project) + .select("orgId") + .where("id", knex.raw("??", [`${TableName.KmsKey}.projectId`])) + }); + // backfill slugs in kms + const missingSlugs = await knex(TableName.KmsKey).whereNull("slug").select("id"); + if (missingSlugs.length) { + await knex(TableName.KmsKey) + // eslint-disable-next-line + // @ts-ignore because generate schema happens after this + .insert(missingSlugs.map(({ id }) => ({ id, slug: slugify(alphaNumericNanoId(32)) }))) + .onConflict("id") + .merge(); + } + + await knex.schema.alterTable(TableName.KmsKey, (tb) => { + tb.uuid("orgId").notNullable().alter(); + tb.string("slug", 32).notNullable().alter(); + tb.dropColumn("projectId"); + }); + } + + const doesExternalKmsServiceExist = await knex.schema.hasTable(TableName.ExternalKms); + if (!doesExternalKmsServiceExist) { + await knex.schema.createTable(TableName.ExternalKms, (tb) => { + tb.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + tb.string("provider").notNullable(); + tb.binary("encryptedProviderInputs").notNullable(); + tb.string("status"); + tb.string("statusDetails"); + tb.uuid("kmsKeyId").unique().notNullable(); + tb.foreign("kmsKeyId").references("id").inTable(TableName.KmsKey).onDelete("CASCADE"); + }); + } + + const doesOrgKmsKeyExist = await knex.schema.hasColumn(TableName.Organization, "kmsDefaultKeyId"); + if (!doesOrgKmsKeyExist) { + await knex.schema.alterTable(TableName.Organization, (tb) => { + tb.uuid("kmsDefaultKeyId").nullable(); + tb.foreign("kmsDefaultKeyId").references("id").inTable(TableName.KmsKey); + }); + } + + const doesProjectKmsSecretManagerKeyExist = await knex.schema.hasColumn(TableName.Project, "kmsSecretManagerKeyId"); + if (!doesProjectKmsSecretManagerKeyExist) { + await knex.schema.alterTable(TableName.Project, (tb) => { + tb.uuid("kmsSecretManagerKeyId").nullable(); + tb.foreign("kmsSecretManagerKeyId").references("id").inTable(TableName.KmsKey); + }); + } +} + +export async function down(knex: Knex): Promise { + const doesOrgKmsKeyExist = await knex.schema.hasColumn(TableName.Organization, "kmsDefaultKeyId"); + if (doesOrgKmsKeyExist) { + await knex.schema.alterTable(TableName.Organization, (tb) => { + tb.dropColumn("kmsDefaultKeyId"); + }); + } + + const doesProjectKmsSecretManagerKeyExist = await knex.schema.hasColumn(TableName.Project, "kmsSecretManagerKeyId"); + if (doesProjectKmsSecretManagerKeyExist) { + await knex.schema.alterTable(TableName.Project, (tb) => { + tb.dropColumn("kmsSecretManagerKeyId"); + }); + } + + const doesInternalKmsKeyVersionTableExist = await knex.schema.hasTable(TableName.InternalKmsKeyVersion); + const doesInternalKmsTableExist = await knex.schema.hasTable(TableName.InternalKms); + if (doesInternalKmsKeyVersionTableExist) { + // because we haven't started using versioning for kms thus no data exist + await knex.schema.renameTable(TableName.InternalKmsKeyVersion, TableName.KmsKeyVersion); + await knex.schema.alterTable(TableName.KmsKeyVersion, (tb) => { + tb.dropColumn("internalKmsId"); + tb.uuid("kmsKeyId").notNullable(); + tb.foreign("kmsKeyId").references("id").inTable(TableName.KmsKey).onDelete("CASCADE"); + }); + } + + const doesOldKmsKeyTableExist = await knex.schema.hasTable(TableName.KmsKey); + const doesKmsSlugExist = await knex.schema.hasColumn(TableName.KmsKey, "slug"); + if (doesInternalKmsTableExist && doesOldKmsKeyTableExist) { + // converting kms key to old one + // backfill so not setting it as not nullable + await knex.schema.alterTable(TableName.KmsKey, (tb) => { + tb.binary("encryptedKey"); + tb.string("encryptionAlgorithm"); + tb.integer("version").defaultTo(1); + tb.string("projectId"); + tb.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); + if (doesKmsSlugExist) { + tb.dropColumn("slug"); + } + }); + // backfill kms key with internal kms data + await knex(TableName.KmsKey).update({ + // eslint-disable-next-line + // @ts-ignore because generate schema happens after this + encryptedKey: knex(TableName.InternalKms) + .select("encryptedKey") + .where("kmsKeyId", knex.raw("??", [`${TableName.KmsKey}.id`])), + // eslint-disable-next-line + // @ts-ignore because generate schema happens after this + encryptionAlgorithm: knex(TableName.InternalKms) + .select("encryptionAlgorithm") + .where("kmsKeyId", knex.raw("??", [`${TableName.KmsKey}.id`])), + // eslint-disable-next-line + // @ts-ignore because generate schema happens after this + projectId: knex(TableName.Project) + .select("id") + .where("kmsCertificateKeyId", knex.raw("??", [`${TableName.KmsKey}.id`])) + }); + await knex.schema.alterTable(TableName.KmsKey, (tb) => { + tb.binary("encryptedKey").notNullable().alter(); + tb.string("encryptionAlgorithm").notNullable().alter(); + }); + await knex.schema.alterTable(TableName.InternalKms, (tb) => { + tb.dropForeign("kmsKeyId"); + }); + await knex.schema.dropTable(TableName.InternalKms); + } + + const doesExternalKmsServiceExist = await knex.schema.hasTable(TableName.ExternalKms); + if (doesExternalKmsServiceExist) { + await knex.schema.alterTable(TableName.ExternalKms, (tb) => { + tb.dropForeign("kmsKeyId"); + }); + await knex.schema.dropTable(TableName.ExternalKms); + } +} diff --git a/backend/src/db/schemas/external-kms.ts b/backend/src/db/schemas/external-kms.ts new file mode 100644 index 000000000..810c3f70f --- /dev/null +++ b/backend/src/db/schemas/external-kms.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { zodBuffer } from "@app/lib/zod"; + +import { TImmutableDBKeys } from "./models"; + +export const ExternalKmsSchema = z.object({ + id: z.string().uuid(), + provider: z.string(), + encryptedProviderInputs: zodBuffer, + status: z.string().nullable().optional(), + statusDetails: z.string().nullable().optional(), + kmsKeyId: z.string().uuid() +}); + +export type TExternalKms = z.infer; +export type TExternalKmsInsert = Omit, TImmutableDBKeys>; +export type TExternalKmsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index bce99dfea..ff8d44de2 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -17,6 +17,7 @@ export * from "./certificate-secrets"; export * from "./certificates"; export * from "./dynamic-secret-leases"; export * from "./dynamic-secrets"; +export * from "./external-kms"; export * from "./git-app-install-sessions"; export * from "./git-app-org"; export * from "./group-project-membership-roles"; @@ -38,6 +39,7 @@ export * from "./identity-universal-auths"; export * from "./incident-contacts"; export * from "./integration-auths"; export * from "./integrations"; +export * from "./internal-kms"; export * from "./kms-key-versions"; export * from "./kms-keys"; export * from "./kms-root-config"; diff --git a/backend/src/db/schemas/internal-kms-key-version.ts b/backend/src/db/schemas/internal-kms-key-version.ts new file mode 100644 index 000000000..fc1e3c3db --- /dev/null +++ b/backend/src/db/schemas/internal-kms-key-version.ts @@ -0,0 +1,21 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { zodBuffer } from "@app/lib/zod"; + +import { TImmutableDBKeys } from "./models"; + +export const InternalKmsKeyVersionSchema = z.object({ + id: z.string().uuid(), + encryptedKey: zodBuffer, + version: z.number(), + internalKmsId: z.string().uuid() +}); + +export type TInternalKmsKeyVersion = z.infer; +export type TInternalKmsKeyVersionInsert = Omit, TImmutableDBKeys>; +export type TInternalKmsKeyVersionUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/internal-kms.ts b/backend/src/db/schemas/internal-kms.ts new file mode 100644 index 000000000..38e64dc5b --- /dev/null +++ b/backend/src/db/schemas/internal-kms.ts @@ -0,0 +1,22 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { zodBuffer } from "@app/lib/zod"; + +import { TImmutableDBKeys } from "./models"; + +export const InternalKmsSchema = z.object({ + id: z.string().uuid(), + encryptedKey: zodBuffer, + encryptionAlgorithm: z.string(), + version: z.number().default(1), + kmsKeyId: z.string().uuid() +}); + +export type TInternalKms = z.infer; +export type TInternalKmsInsert = Omit, TImmutableDBKeys>; +export type TInternalKmsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/kms-keys.ts b/backend/src/db/schemas/kms-keys.ts index 503c270d9..5e8dcf166 100644 --- a/backend/src/db/schemas/kms-keys.ts +++ b/backend/src/db/schemas/kms-keys.ts @@ -5,20 +5,15 @@ import { z } from "zod"; -import { zodBuffer } from "@app/lib/zod"; - import { TImmutableDBKeys } from "./models"; export const KmsKeysSchema = z.object({ id: z.string().uuid(), - encryptedKey: zodBuffer, - encryptionAlgorithm: z.string(), - version: z.number().default(1), description: z.string().nullable().optional(), isDisabled: z.boolean().default(false).nullable().optional(), isReserved: z.boolean().default(true).nullable().optional(), - projectId: z.string().nullable().optional(), - orgId: z.string().uuid().nullable().optional() + orgId: z.string().uuid(), + slug: z.string() }); export type TKmsKeys = z.infer; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 1dba71209..646e0455e 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -95,7 +95,10 @@ export enum TableName { // KMS Service KmsServerRootConfig = "kms_root_config", KmsKey = "kms_keys", - KmsKeyVersion = "kms_key_versions" + KmsKeyVersion = "kms_key_versions", + ExternalKms = "external_kms", + InternalKms = "internal_kms", + InternalKmsKeyVersion = "internal_kms_key_version" } export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt"; diff --git a/backend/src/db/schemas/organizations.ts b/backend/src/db/schemas/organizations.ts index f2933af86..7b7a004fc 100644 --- a/backend/src/db/schemas/organizations.ts +++ b/backend/src/db/schemas/organizations.ts @@ -15,7 +15,8 @@ export const OrganizationsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), authEnforced: z.boolean().default(false).nullable().optional(), - scimEnabled: z.boolean().default(false).nullable().optional() + scimEnabled: z.boolean().default(false).nullable().optional(), + kmsDefaultKeyId: z.string().uuid().nullable().optional() }); export type TOrganizations = z.infer; diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index f776e864c..19597c9df 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -19,7 +19,8 @@ export const ProjectsSchema = z.object({ upgradeStatus: z.string().nullable().optional(), pitVersionLimit: z.number().default(10), kmsCertificateKeyId: z.string().uuid().nullable().optional(), - auditLogsRetentionDays: z.number().nullable().optional() + auditLogsRetentionDays: z.number().nullable().optional(), + kmsSecretManagerKeyId: z.string().uuid().nullable().optional() }); export type TProjects = z.infer;