fix: migrate project gateway

backend/src/@types/knex.d.ts

@@ -218,6 +218,9 @@ import {
   TProjectEnvironments,
   TProjectEnvironmentsInsert,
   TProjectEnvironmentsUpdate,
+  TProjectGateways,
+  TProjectGatewaysInsert,
+  TProjectGatewaysUpdate,
   TProjectKeys,
   TProjectKeysInsert,
   TProjectKeysUpdate,
@@ -1023,6 +1026,11 @@ declare module "knex/types/tables" {
       TKmipClientCertificatesUpdate
     >;
     [TableName.Gateway]: KnexOriginal.CompositeTableType<TGateways, TGatewaysInsert, TGatewaysUpdate>;
+    [TableName.ProjectGateway]: KnexOriginal.CompositeTableType<
+      TProjectGateways,
+      TProjectGatewaysInsert,
+      TProjectGatewaysUpdate
+    >;
     [TableName.OrgGatewayConfig]: KnexOriginal.CompositeTableType<
       TOrgGatewayConfig,
       TOrgGatewayConfigInsert,

backend/src/db/migrations/20250513081738_remove-gateway-project-link.ts

@@ -1,22 +1,110 @@
 import { Knex } from "knex";
 
+import { inMemoryKeyStore } from "@app/keystore/memory";
+import { selectAllTableCols } from "@app/lib/knex";
+import { initLogger } from "@app/lib/logger";
+import { KmsDataKey } from "@app/services/kms/kms-types";
+
 import { TableName } from "../schemas";
+import { getMigrationEnvConfig } from "./utils/env-config";
+import { getMigrationEncryptionServices } from "./utils/services";
 
 // Note(daniel): We aren't dropping tables or columns in this migrations so we can easily rollback if needed.
 // In the future we need to drop the projectGatewayId on the dynamic secrets table, and drop the project_gateways table entirely.
 
-export async function up(knex: Knex): Promise<void> {
-  await knex.schema.alterTable(TableName.DynamicSecret, (table) => {
-    table.uuid("gatewayId").nullable();
-    table.foreign("gatewayId").references("id").inTable(TableName.Gateway).onDelete("SET NULL");
+const BATCH_SIZE = 500;
 
-    table.index("gatewayId");
-  });
+export async function up(knex: Knex): Promise<void> {
+  // eslint-disable-next-line no-param-reassign
+  knex.replicaNode = () => {
+    return knex;
+  };
+
+  if (!(await knex.schema.hasColumn(TableName.DynamicSecret, "gatewayId"))) {
+    await knex.schema.alterTable(TableName.DynamicSecret, (table) => {
+      table.uuid("gatewayId").nullable();
+      table.foreign("gatewayId").references("id").inTable(TableName.Gateway).onDelete("SET NULL");
+
+      table.index("gatewayId");
+    });
+
+    const existingDynamicSecretsWithProjectGatewayId = await knex(TableName.DynamicSecret)
+      .select(selectAllTableCols(TableName.DynamicSecret))
+      .whereNotNull(`${TableName.DynamicSecret}.projectGatewayId`)
+      .join(TableName.ProjectGateway, `${TableName.ProjectGateway}.id`, `${TableName.DynamicSecret}.projectGatewayId`)
+      .whereNotNull(`${TableName.ProjectGateway}.gatewayId`)
+      .select(
+        knex.ref("projectId").withSchema(TableName.ProjectGateway).as("projectId"),
+        knex.ref("gatewayId").withSchema(TableName.ProjectGateway).as("projectGatewayGatewayId")
+      );
+
+    initLogger();
+    const envConfig = getMigrationEnvConfig();
+    const keyStore = inMemoryKeyStore();
+    const { kmsService } = await getMigrationEncryptionServices({ envConfig, keyStore, db: knex });
+
+    const updatedDynamicSecrets = await Promise.all(
+      existingDynamicSecretsWithProjectGatewayId.map(async (existingDynamicSecret) => {
+        if (!existingDynamicSecret.projectGatewayGatewayId) {
+          const result = {
+            ...existingDynamicSecret,
+            gatewayId: null
+          };
+
+          const { projectId, projectGatewayGatewayId, ...rest } = result;
+          return rest;
+        }
+
+        const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
+          type: KmsDataKey.SecretManager,
+          projectId: existingDynamicSecret.projectId
+        });
+        const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({
+          type: KmsDataKey.SecretManager,
+          projectId: existingDynamicSecret.projectId
+        });
+
+        let decryptedStoredInput = JSON.parse(
+          secretManagerDecryptor({ cipherTextBlob: Buffer.from(existingDynamicSecret.encryptedInput) }).toString()
+        ) as object;
+
+        // We're not removing the existing projectGatewayId from the input so we can easily rollback without having to re-encrypt the input
+        decryptedStoredInput = {
+          ...decryptedStoredInput,
+          gatewayId: existingDynamicSecret.projectGatewayGatewayId
+        };
+
+        const encryptedInput = secretManagerEncryptor({
+          plainText: Buffer.from(JSON.stringify(decryptedStoredInput))
+        }).cipherTextBlob;
+
+        const result = {
+          ...existingDynamicSecret,
+          encryptedInput,
+          gatewayId: existingDynamicSecret.projectGatewayGatewayId
+        };
+
+        const { projectId, projectGatewayGatewayId, ...rest } = result;
+        return rest;
+      })
+    );
+
+    for (let i = 0; i < updatedDynamicSecrets.length; i += BATCH_SIZE) {
+      // eslint-disable-next-line no-await-in-loop
+      await knex(TableName.DynamicSecret)
+        .insert(updatedDynamicSecrets.slice(i, i + BATCH_SIZE))
+        .onConflict("id")
+        .merge();
+    }
+  }
 }
 
 export async function down(knex: Knex): Promise<void> {
-  await knex.schema.alterTable(TableName.DynamicSecret, (table) => {
-    table.dropForeign("gatewayId");
-    table.dropColumn("gatewayId");
-  });
+  // no re-encryption needed as we keep the old projectGatewayId in the input
+  if (await knex.schema.hasColumn(TableName.DynamicSecret, "gatewayId")) {
+    await knex.schema.alterTable(TableName.DynamicSecret, (table) => {
+      table.dropForeign("gatewayId");
+      table.dropColumn("gatewayId");
+    });
+  }
 }

backend/src/db/schemas/index.ts

@@ -72,6 +72,7 @@ export * from "./pki-collections";
 export * from "./pki-subscribers";
 export * from "./project-bots";
 export * from "./project-environments";
+export * from "./project-gateways";
 export * from "./project-keys";
 export * from "./project-memberships";
 export * from "./project-roles";

backend/src/db/schemas/models.ts

@@ -124,6 +124,7 @@ export enum TableName {
   // Gateway
   OrgGatewayConfig = "org_gateway_config",
   Gateway = "gateways",
+  ProjectGateway = "project_gateways",
   // junction tables with tags
   SecretV2JnTag = "secret_v2_tag_junction",
   JnSecretTag = "secret_tag_junction",