feat: changed webhook and dynamic secret change to migration mode, resolved snapshot deletion issue in update

2026-10-14 23:52:09 +00:00 · 2024-07-30 23:19:33 +05:30
parent c821bc0e14
commit 9063787772
20 changed files with 398 additions and 414 deletions
@@ -1,7 +1,178 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment */
 import { Knex } from "knex";

-import { SecretEncryptionAlgo, SecretKeyEncoding, SecretType, TableName } from "../schemas";
+import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption";
+import { selectAllTableCols } from "@app/lib/knex/select";
+
+import { SecretKeyEncoding, SecretType, TableName } from "../schemas";
 import { createJunctionTable, createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+import { getSecretManagerDataKey } from "./utils/kms";
+
+const backfillWebhooks = async (knex: Knex) => {
+  const hasEncryptedSecretKeyWithKms = await knex.schema.hasColumn(TableName.Webhook, "encryptedSecretKeyWithKms");
+  const hasEncryptedWebhookUrl = await knex.schema.hasColumn(TableName.Webhook, "encryptedUrl");
+  const hasUrlCipherText = await knex.schema.hasColumn(TableName.Webhook, "urlCipherText");
+  const hasUrlIV = await knex.schema.hasColumn(TableName.Webhook, "urlIV");
+  const hasUrlTag = await knex.schema.hasColumn(TableName.Webhook, "urlTag");
+  const hasEncryptedSecretKey = await knex.schema.hasColumn(TableName.Webhook, "encryptedSecretKey");
+  const hasIV = await knex.schema.hasColumn(TableName.Webhook, "iv");
+  const hasTag = await knex.schema.hasColumn(TableName.Webhook, "tag");
+  const hasKeyEncoding = await knex.schema.hasColumn(TableName.Webhook, "keyEncoding");
+  const hasAlgorithm = await knex.schema.hasColumn(TableName.Webhook, "algorithm");
+  const hasUrl = await knex.schema.hasColumn(TableName.Webhook, "url");
+
+  await knex.schema.alterTable(TableName.Webhook, (t) => {
+    if (!hasEncryptedSecretKeyWithKms) t.binary("encryptedSecretKeyWithKms");
+    if (!hasEncryptedWebhookUrl) t.binary("encryptedUrl");
+    if (hasUrl) t.string("url").nullable().alter();
+  });
+
+  const kmsEncryptorGroupByProjectId: Record<string, Awaited<ReturnType<typeof getSecretManagerDataKey>>["encryptor"]> =
+    {};
+  if (hasUrlCipherText && hasUrlIV && hasUrlTag && hasEncryptedSecretKey && hasIV && hasTag) {
+    // eslint-disable-next-line
+    const webhooksToFill = await knex(TableName.Webhook)
+      .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.Webhook}.envId`)
+      .whereNull("encryptedUrl")
+      // eslint-disable-next-line
+      // @ts-ignore knex migration fails
+      .select(selectAllTableCols(TableName.Webhook))
+      .select("projectId");
+
+    const updatedWebhooks = [];
+    for (const webhook of webhooksToFill) {
+      if (!kmsEncryptorGroupByProjectId[webhook.projectId]) {
+        // eslint-disable-next-line
+        const { encryptor } = await getSecretManagerDataKey(knex, webhook.projectId);
+        kmsEncryptorGroupByProjectId[webhook.projectId] = encryptor;
+      }
+
+      const kmsEncryptor = kmsEncryptorGroupByProjectId[webhook.projectId];
+
+      // @ts-ignore post migration fails
+      let webhookUrl = webhook.url;
+      let webhookSecretKey;
+
+      // @ts-ignore post migration fails
+      if (webhook.urlTag && webhook.urlCipherText && webhook.urlIV) {
+        webhookUrl = infisicalSymmetricDecrypt({
+          // @ts-ignore post migration fails
+          keyEncoding: webhook.keyEncoding as SecretKeyEncoding,
+          // @ts-ignore post migration fails
+          ciphertext: webhook.urlCipherText,
+          // @ts-ignore post migration fails
+          iv: webhook.urlIV,
+          // @ts-ignore post migration fails
+          tag: webhook.urlTag
+        });
+      }
+      // @ts-ignore post migration fails
+      if (webhook.encryptedSecretKey && webhook.iv && webhook.tag) {
+        webhookSecretKey = infisicalSymmetricDecrypt({
+          // @ts-ignore post migration fails
+          keyEncoding: webhook.keyEncoding as SecretKeyEncoding,
+          // @ts-ignore post migration fails
+          ciphertext: webhook.encryptedSecretKey,
+          // @ts-ignore post migration fails
+          iv: webhook.iv,
+          // @ts-ignore post migration fails
+          tag: webhook.tag
+        });
+      }
+      const { projectId, ...el } = webhook;
+      updatedWebhooks.push({
+        ...el,
+        encryptedSecretKeyWithKms: webhookSecretKey
+          ? kmsEncryptor({ plainText: Buffer.from(webhookSecretKey) }).cipherTextBlob
+          : null,
+        encryptedUrl: kmsEncryptor({ plainText: Buffer.from(webhookUrl) }).cipherTextBlob
+      });
+    }
+    if (updatedWebhooks.length) {
+      // eslint-disable-next-line
+      await knex(TableName.Webhook).insert(updatedWebhooks).onConflict("id").merge();
+    }
+  }
+  await knex.schema.alterTable(TableName.Webhook, (t) => {
+    t.binary("encryptedUrl").notNullable().alter();
+
+    if (hasUrlIV) t.dropColumn("urlIV");
+    if (hasUrlCipherText) t.dropColumn("urlCipherText");
+    if (hasUrlTag) t.dropColumn("urlTag");
+    if (hasIV) t.dropColumn("iv");
+    if (hasTag) t.dropColumn("tag");
+    if (hasEncryptedSecretKey) t.dropColumn("encryptedSecretKey");
+    if (hasKeyEncoding) t.dropColumn("keyEncoding");
+    if (hasAlgorithm) t.dropColumn("algorithm");
+    if (hasUrl) t.dropColumn("url");
+  });
+};
+
+const backfillDynamicSecretConfigs = async (knex: Knex) => {
+  const hasEncryptedConfig = await knex.schema.hasColumn(TableName.DynamicSecret, "encryptedConfig");
+
+  const hasInputCipherText = await knex.schema.hasColumn(TableName.DynamicSecret, "inputCiphertext");
+  const hasInputIV = await knex.schema.hasColumn(TableName.DynamicSecret, "inputIV");
+  const hasInputTag = await knex.schema.hasColumn(TableName.DynamicSecret, "inputTag");
+  const hasKeyEncoding = await knex.schema.hasColumn(TableName.DynamicSecret, "keyEncoding");
+  const hasAlgorithm = await knex.schema.hasColumn(TableName.DynamicSecret, "algorithm");
+
+  await knex.schema.alterTable(TableName.DynamicSecret, (t) => {
+    if (!hasEncryptedConfig) t.binary("encryptedConfig");
+  });
+  const kmsEncryptorGroupByProjectId: Record<string, Awaited<ReturnType<typeof getSecretManagerDataKey>>["encryptor"]> =
+    {};
+  if (hasInputCipherText && hasInputIV && hasInputTag) {
+    // eslint-disable-next-line
+    const dynamicSecretConfigs = await knex(TableName.DynamicSecret)
+      .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.DynamicSecret}.folderId`)
+      .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
+      .whereNull("encryptedConfig")
+      // @ts-ignore post migration fails
+      .select(selectAllTableCols(TableName.DynamicSecret))
+      .select("projectId");
+
+    const updatedConfigs = [];
+    for (const dynamicSecretConfig of dynamicSecretConfigs) {
+      if (!kmsEncryptorGroupByProjectId[dynamicSecretConfig.projectId]) {
+        // eslint-disable-next-line
+        const { encryptor } = await getSecretManagerDataKey(knex, dynamicSecretConfig.projectId);
+        kmsEncryptorGroupByProjectId[dynamicSecretConfig.projectId] = encryptor;
+      }
+
+      const kmsEncryptor = kmsEncryptorGroupByProjectId[dynamicSecretConfig.projectId];
+      const inputConfig = infisicalSymmetricDecrypt({
+        // @ts-ignore post migration fails
+        keyEncoding: dynamicSecretConfig.keyEncoding as SecretKeyEncoding,
+        // @ts-ignore post migration fails
+        ciphertext: dynamicSecretConfig.inputCiphertext as string,
+        // @ts-ignore post migration fails
+        iv: dynamicSecretConfig.inputIV as string,
+        // @ts-ignore post migration fails
+        tag: dynamicSecretConfig.inputTag as string
+      });
+
+      const { projectId, ...el } = dynamicSecretConfig;
+      updatedConfigs.push({
+        ...el,
+        encryptedConfig: kmsEncryptor({ plainText: Buffer.from(inputConfig) }).cipherTextBlob
+      });
+    }
+    if (updatedConfigs.length) {
+      // eslint-disable-next-line
+      await knex(TableName.DynamicSecret).insert(updatedConfigs).onConflict("id").merge();
+    }
+  }
+  await knex.schema.alterTable(TableName.DynamicSecret, (t) => {
+    t.binary("encryptedConfig").notNullable().alter();
+
+    if (hasInputTag) t.dropColumn("inputTag");
+    if (hasInputIV) t.dropColumn("inputIV");
+    if (hasInputCipherText) t.dropColumn("inputCiphertext");
+    if (hasKeyEncoding) t.dropColumn("keyEncoding");
+    if (hasAlgorithm) t.dropColumn("algorithm");
+  });
+};

 export async function up(knex: Knex): Promise<void> {
  const doesSecretV2TableExist = await knex.schema.hasTable(TableName.SecretV2);
@@ -145,29 +316,11 @@ export async function up(knex: Knex): Promise<void> {
  }

  if (await knex.schema.hasTable(TableName.Webhook)) {
-    const hasEncryptedWebhookSecretKey = await knex.schema.hasColumn(TableName.Webhook, "encryptedSecretKeyWithKms");
-    const hasEncryptedWebhookUrl = await knex.schema.hasColumn(TableName.Webhook, "encryptedUrl");
-    await knex.schema.alterTable(TableName.Webhook, (t) => {
-      if (!hasEncryptedWebhookSecretKey) t.binary("encryptedSecretKeyWithKms");
-      if (!hasEncryptedWebhookUrl) t.binary("encryptedUrl");
-    });
+    await backfillWebhooks(knex);
  }

  if (await knex.schema.hasTable(TableName.DynamicSecret)) {
-    const hasInputIV = await knex.schema.hasColumn(TableName.DynamicSecret, "inputIV");
-    const hasInputCipherText = await knex.schema.hasColumn(TableName.DynamicSecret, "inputCiphertext");
-    const hasInputTag = await knex.schema.hasColumn(TableName.DynamicSecret, "inputTag");
-    const hasAlgorithm = await knex.schema.hasColumn(TableName.DynamicSecret, "algorithm");
-    const hasKeyEncoding = await knex.schema.hasColumn(TableName.DynamicSecret, "keyEncoding");
-    const hasEncryptedConfig = await knex.schema.hasColumn(TableName.DynamicSecret, "encryptedConfig");
-    await knex.schema.alterTable(TableName.DynamicSecret, (t) => {
-      if (hasInputIV) t.string("inputIV").alter();
-      if (hasInputCipherText) t.text("inputCiphertext").alter();
-      if (hasInputTag) t.string("inputTag").alter();
-      if (hasAlgorithm) t.string("algorithm").defaultTo(SecretEncryptionAlgo.AES_256_GCM).alter();
-      if (hasKeyEncoding) t.string("keyEncoding").defaultTo(SecretKeyEncoding.UTF8).alter();
-      if (!hasEncryptedConfig) t.binary("encryptedConfig");
-    });
+    await backfillDynamicSecretConfigs(knex);
  }
 }

@@ -206,16 +359,46 @@ export async function down(knex: Knex): Promise<void> {
  if (await knex.schema.hasTable(TableName.Webhook)) {
    const hasEncryptedWebhookSecretKey = await knex.schema.hasColumn(TableName.Webhook, "encryptedSecretKeyWithKms");
    const hasEncryptedWebhookUrl = await knex.schema.hasColumn(TableName.Webhook, "encryptedUrl");
+    const hasUrlCipherText = await knex.schema.hasColumn(TableName.Webhook, "urlCipherText");
+    const hasUrlIV = await knex.schema.hasColumn(TableName.Webhook, "urlIV");
+    const hasUrlTag = await knex.schema.hasColumn(TableName.Webhook, "urlTag");
+    const hasEncryptedSecretKey = await knex.schema.hasColumn(TableName.Webhook, "encryptedSecretKey");
+    const hasIV = await knex.schema.hasColumn(TableName.Webhook, "iv");
+    const hasTag = await knex.schema.hasColumn(TableName.Webhook, "tag");
+    const hasKeyEncoding = await knex.schema.hasColumn(TableName.Webhook, "keyEncoding");
+    const hasAlgorithm = await knex.schema.hasColumn(TableName.Webhook, "algorithm");
+    const hasUrl = await knex.schema.hasColumn(TableName.Webhook, "url");
+
    await knex.schema.alterTable(TableName.Webhook, (t) => {
      if (hasEncryptedWebhookSecretKey) t.dropColumn("encryptedSecretKeyWithKms");
      if (hasEncryptedWebhookUrl) t.dropColumn("encryptedUrl");
+      if (!hasUrl) t.string("url");
+      if (!hasEncryptedSecretKey) t.string("encryptedSecretKey");
+      if (!hasIV) t.string("iv");
+      if (!hasTag) t.string("tag");
+      if (!hasAlgorithm) t.string("algorithm");
+      if (!hasKeyEncoding) t.string("keyEncoding");
+      if (!hasUrlCipherText) t.string("urlCipherText");
+      if (!hasUrlIV) t.string("urlIV");
+      if (!hasUrlTag) t.string("urlTag");
    });
  }

  if (await knex.schema.hasTable(TableName.DynamicSecret)) {
    const hasEncryptedConfig = await knex.schema.hasColumn(TableName.DynamicSecret, "encryptedConfig");
+
+    const hasInputIV = await knex.schema.hasColumn(TableName.DynamicSecret, "inputIV");
+    const hasInputCipherText = await knex.schema.hasColumn(TableName.DynamicSecret, "inputCiphertext");
+    const hasInputTag = await knex.schema.hasColumn(TableName.DynamicSecret, "inputTag");
+    const hasAlgorithm = await knex.schema.hasColumn(TableName.DynamicSecret, "algorithm");
+    const hasKeyEncoding = await knex.schema.hasColumn(TableName.DynamicSecret, "keyEncoding");
    await knex.schema.alterTable(TableName.DynamicSecret, (t) => {
      if (hasEncryptedConfig) t.dropColumn("encryptedConfig");
+      if (!hasInputIV) t.string("inputIV");
+      if (!hasInputCipherText) t.text("inputCiphertext");
+      if (!hasInputTag) t.string("inputTag");
+      if (!hasAlgorithm) t.string("algorithm");
+      if (!hasKeyEncoding) t.string("keyEncoding");
    });
  }
 }
@@ -0,0 +1,105 @@
+import slugify from "@sindresorhus/slugify";
+import { Knex } from "knex";
+
+import { TableName } from "@app/db/schemas";
+import { randomSecureBytes } from "@app/lib/crypto";
+import { symmetricCipherService, SymmetricEncryption } from "@app/lib/crypto/cipher";
+import { alphaNumericNanoId } from "@app/lib/nanoid";
+
+const getInstanceRootKey = async (knex: Knex) => {
+  const encryptionKey = process.env.ENCRYPTION_KEY || process.env.ROOT_ENCRYPTION_KEY;
+  // if root key its base64 encoded
+  const isBase64 = !process.env.ENCRYPTION_KEY;
+  if (!encryptionKey) throw new Error("ENCRYPTION_KEY variable needed for migration");
+  const encryptionKeyBuffer = Buffer.from(encryptionKey, isBase64 ? "base64" : "utf8");
+
+  const KMS_ROOT_CONFIG_UUID = "00000000-0000-0000-0000-000000000000";
+  const kmsRootConfig = await knex(TableName.KmsServerRootConfig).where({ id: KMS_ROOT_CONFIG_UUID }).first();
+  const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+  if (kmsRootConfig) {
+    const decryptedRootKey = cipher.decrypt(kmsRootConfig.encryptedRootKey, encryptionKeyBuffer);
+    // set the flag so that other instancen nodes can start
+    return decryptedRootKey;
+  }
+
+  const newRootKey = randomSecureBytes(32);
+  const encryptedRootKey = cipher.encrypt(newRootKey, encryptionKeyBuffer);
+  await knex(TableName.KmsServerRootConfig).insert({
+    encryptedRootKey,
+    // eslint-disable-next-line
+    // @ts-ignore id is kept as fixed for idempotence and to avoid race condition
+    id: KMS_ROOT_CONFIG_UUID
+  });
+  return encryptedRootKey;
+};
+
+export const getSecretManagerDataKey = async (knex: Knex, projectId: string) => {
+  const KMS_VERSION = "v01";
+  const KMS_VERSION_BLOB_LENGTH = 3;
+  const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+  const project = await knex(TableName.Project).where({ id: projectId }).first();
+  if (!project) throw new Error("Missing project id");
+
+  const ROOT_ENCRYPTION_KEY = await getInstanceRootKey(knex);
+
+  let secretManagerKmsKey;
+  const projectSecretManagerKmsId = project?.kmsSecretManagerKeyId;
+  if (projectSecretManagerKmsId) {
+    const kmsDoc = await knex(TableName.KmsKey)
+      .leftJoin(TableName.InternalKms, `${TableName.KmsKey}.id`, `${TableName.InternalKms}.kmsKeyId`)
+      .where({ [`${TableName.KmsKey}.id` as "id"]: projectSecretManagerKmsId })
+      .first();
+    if (!kmsDoc) throw new Error("missing kms");
+    secretManagerKmsKey = cipher.decrypt(kmsDoc.encryptedKey, ROOT_ENCRYPTION_KEY);
+  } else {
+    const [kmsDoc] = await knex(TableName.KmsKey)
+      .insert({
+        slug: slugify(alphaNumericNanoId(8).toLowerCase()),
+        orgId: project.orgId,
+        isReserved: false
+      })
+      .returning("*");
+
+    secretManagerKmsKey = randomSecureBytes(32);
+    const encryptedKeyMaterial = cipher.encrypt(secretManagerKmsKey, ROOT_ENCRYPTION_KEY);
+    await knex(TableName.InternalKms).insert({
+      version: 1,
+      encryptedKey: encryptedKeyMaterial,
+      encryptionAlgorithm: SymmetricEncryption.AES_GCM_256,
+      kmsKeyId: kmsDoc.id
+    });
+  }
+
+  const encryptedSecretManagerDataKey = project?.kmsSecretManagerEncryptedDataKey;
+  let dataKey: Buffer;
+  if (!encryptedSecretManagerDataKey) {
+    dataKey = randomSecureBytes();
+    // the below versioning we do it automatically in kms service
+    const unversionedDataKey = cipher.encrypt(dataKey, secretManagerKmsKey);
+    const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3
+    await knex(TableName.Project)
+      .where({ id: projectId })
+      .update({
+        kmsSecretManagerEncryptedDataKey: Buffer.concat([unversionedDataKey, versionBlob])
+      });
+  } else {
+    const cipherTextBlob = encryptedSecretManagerDataKey.subarray(0, -KMS_VERSION_BLOB_LENGTH);
+    dataKey = cipher.decrypt(cipherTextBlob, secretManagerKmsKey);
+  }
+
+  return {
+    encryptor: ({ plainText }: { plainText: Buffer }) => {
+      const encryptedPlainTextBlob = cipher.encrypt(plainText, dataKey);
+
+      // Buffer#1 encrypted text + Buffer#2 version number
+      const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3
+      const cipherTextBlob = Buffer.concat([encryptedPlainTextBlob, versionBlob]);
+      return { cipherTextBlob };
+    },
+    decryptor: ({ cipherTextBlob: versionedCipherTextBlob }: { cipherTextBlob: Buffer }) => {
+      const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH);
+      const decryptedBlob = cipher.decrypt(cipherTextBlob, dataKey);
+      return decryptedBlob;
+    }
+  };
+};
@@ -16,17 +16,12 @@ export const DynamicSecretsSchema = z.object({
  type: z.string(),
  defaultTTL: z.string(),
  maxTTL: z.string().nullable().optional(),
-  inputIV: z.string().nullable().optional(),
-  inputCiphertext: z.string().nullable().optional(),
-  inputTag: z.string().nullable().optional(),
-  algorithm: z.string().default("aes-256-gcm").nullable().optional(),
-  keyEncoding: z.string().default("utf8").nullable().optional(),
  folderId: z.string().uuid(),
  status: z.string().nullable().optional(),
  statusDetails: z.string().nullable().optional(),
  createdAt: z.date(),
  updatedAt: z.date(),
-  encryptedConfig: zodBuffer.nullable().optional()
+  encryptedConfig: zodBuffer
 });

 export type TDynamicSecrets = z.infer<typeof DynamicSecretsSchema>;
@@ -12,24 +12,15 @@ import { TImmutableDBKeys } from "./models";
 export const WebhooksSchema = z.object({
  id: z.string().uuid(),
  secretPath: z.string().default("/"),
-  url: z.string(),
  lastStatus: z.string().nullable().optional(),
  lastRunErrorMessage: z.string().nullable().optional(),
  isDisabled: z.boolean().default(false),
-  encryptedSecretKey: z.string().nullable().optional(),
-  iv: z.string().nullable().optional(),
-  tag: z.string().nullable().optional(),
-  algorithm: z.string().nullable().optional(),
-  keyEncoding: z.string().nullable().optional(),
  createdAt: z.date(),
  updatedAt: z.date(),
  envId: z.string().uuid(),
-  urlCipherText: z.string().nullable().optional(),
-  urlIV: z.string().nullable().optional(),
-  urlTag: z.string().nullable().optional(),
  type: z.string().default("general").nullable().optional(),
  encryptedSecretKeyWithKms: zodBuffer.nullable().optional(),
-  encryptedUrl: zodBuffer.nullable().optional()
+  encryptedUrl: zodBuffer
 });

 export type TWebhooks = z.infer<typeof WebhooksSchema>;