Merge branch 'main' into feat/point-in-time-revamp

backend/src/db/migrations/20250429232917_store-cert-secret-key-and-chain.ts

@@ -0,0 +1,33 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasColumn(TableName.CertificateBody, "encryptedCertificateChain"))) {
+    await knex.schema.alterTable(TableName.CertificateBody, (t) => {
+      t.binary("encryptedCertificateChain").nullable();
+    });
+  }
+
+  if (!(await knex.schema.hasTable(TableName.CertificateSecret))) {
+    await knex.schema.createTable(TableName.CertificateSecret, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.timestamps(true, true, true);
+      t.uuid("certId").notNullable().unique();
+      t.foreign("certId").references("id").inTable(TableName.Certificate).onDelete("CASCADE");
+      t.binary("encryptedPrivateKey").notNullable();
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasTable(TableName.CertificateSecret)) {
+    await knex.schema.dropTable(TableName.CertificateSecret);
+  }
+
+  if (await knex.schema.hasColumn(TableName.CertificateBody, "encryptedCertificateChain")) {
+    await knex.schema.alterTable(TableName.CertificateBody, (t) => {
+      t.dropColumn("encryptedCertificateChain");
+    });
+  }
+}

backend/src/db/migrations/20250501164905_add-groups-to-ssh-host-login-user-mappings.ts

@@ -0,0 +1,22 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasColumn(TableName.SshHostLoginUserMapping, "groupId"))) {
+    await knex.schema.alterTable(TableName.SshHostLoginUserMapping, (t) => {
+      t.uuid("groupId").nullable();
+      t.foreign("groupId").references("id").inTable(TableName.Groups).onDelete("CASCADE");
+      t.unique(["sshHostLoginUserId", "groupId"]);
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.SshHostLoginUserMapping, "groupId")) {
+    await knex.schema.alterTable(TableName.SshHostLoginUserMapping, (t) => {
+      t.dropUnique(["sshHostLoginUserId", "groupId"]);
+      t.dropColumn("groupId");
+    });
+  }
+}

backend/src/db/migrations/20250505203703_project-templates-type-col.ts

@@ -0,0 +1,22 @@
+import { Knex } from "knex";
+
+import { ProjectType, TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasColumn(TableName.ProjectTemplates, "type"))) {
+    await knex.schema.alterTable(TableName.ProjectTemplates, (t) => {
+      // defaulting to sm for migration to set existing, new ones will always be specified on creation
+      t.string("type").defaultTo(ProjectType.SecretManager).notNullable();
+      t.jsonb("environments").nullable().alter();
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.ProjectTemplates, "type")) {
+    await knex.schema.alterTable(TableName.ProjectTemplates, (t) => {
+      t.dropColumn("type");
+      // not reverting nullable environments
+    });
+  }
+}

backend/src/db/migrations/20250507003056_identity-ldap-auth.ts

@@ -0,0 +1,39 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.IdentityLdapAuth))) {
+    await knex.schema.createTable(TableName.IdentityLdapAuth, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+
+      t.bigInteger("accessTokenTTL").defaultTo(7200).notNullable();
+      t.bigInteger("accessTokenMaxTTL").defaultTo(7200).notNullable();
+      t.bigInteger("accessTokenNumUsesLimit").defaultTo(0).notNullable();
+      t.jsonb("accessTokenTrustedIps").notNullable();
+
+      t.uuid("identityId").notNullable().unique();
+      t.foreign("identityId").references("id").inTable(TableName.Identity).onDelete("CASCADE");
+
+      t.binary("encryptedBindDN").notNullable();
+      t.binary("encryptedBindPass").notNullable();
+      t.binary("encryptedLdapCaCertificate").nullable();
+
+      t.string("url").notNullable();
+      t.string("searchBase").notNullable();
+      t.string("searchFilter").notNullable();
+
+      t.jsonb("allowedFields").nullable();
+
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.IdentityLdapAuth);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.IdentityLdapAuth);
+  await dropOnUpdateTrigger(knex, TableName.IdentityLdapAuth);
+}

backend/src/db/migrations/20250508160957_pki-subscriber.ts

@@ -0,0 +1,46 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.PkiSubscriber))) {
+    await knex.schema.createTable(TableName.PkiSubscriber, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.timestamps(true, true, true);
+      t.string("projectId").notNullable();
+      t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
+      t.uuid("caId").nullable();
+      t.foreign("caId").references("id").inTable(TableName.CertificateAuthority).onDelete("SET NULL");
+      t.string("name").notNullable();
+      t.string("commonName").notNullable();
+      t.specificType("subjectAlternativeNames", "text[]").notNullable();
+      t.string("ttl").notNullable();
+      t.specificType("keyUsages", "text[]").notNullable();
+      t.specificType("extendedKeyUsages", "text[]").notNullable();
+      t.string("status").notNullable(); // active / disabled
+      t.unique(["projectId", "name"]);
+    });
+    await createOnUpdateTrigger(knex, TableName.PkiSubscriber);
+  }
+
+  const hasSubscriberCol = await knex.schema.hasColumn(TableName.Certificate, "pkiSubscriberId");
+  if (!hasSubscriberCol) {
+    await knex.schema.alterTable(TableName.Certificate, (t) => {
+      t.uuid("pkiSubscriberId").nullable();
+      t.foreign("pkiSubscriberId").references("id").inTable(TableName.PkiSubscriber).onDelete("SET NULL");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasSubscriberCol = await knex.schema.hasColumn(TableName.Certificate, "pkiSubscriberId");
+  if (hasSubscriberCol) {
+    await knex.schema.alterTable(TableName.Certificate, (t) => {
+      t.dropColumn("pkiSubscriberId");
+    });
+  }
+
+  await knex.schema.dropTableIfExists(TableName.PkiSubscriber);
+  await dropOnUpdateTrigger(knex, TableName.PkiSubscriber);
+}

backend/src/db/migrations/20250508210717_identity-oci-auth.ts

@@ -0,0 +1,30 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.IdentityOciAuth))) {
+    await knex.schema.createTable(TableName.IdentityOciAuth, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.bigInteger("accessTokenTTL").defaultTo(7200).notNullable();
+      t.bigInteger("accessTokenMaxTTL").defaultTo(7200).notNullable();
+      t.bigInteger("accessTokenNumUsesLimit").defaultTo(0).notNullable();
+      t.jsonb("accessTokenTrustedIps").notNullable();
+      t.timestamps(true, true, true);
+      t.uuid("identityId").notNullable().unique();
+      t.foreign("identityId").references("id").inTable(TableName.Identity).onDelete("CASCADE");
+      t.string("type").notNullable();
+
+      t.string("tenancyOcid").notNullable();
+      t.string("allowedUsernames").nullable();
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.IdentityOciAuth);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.IdentityOciAuth);
+  await dropOnUpdateTrigger(knex, TableName.IdentityOciAuth);
+}

backend/src/db/migrations/20250512103022_identity-kubernetes-auth-gateway.ts

@@ -0,0 +1,25 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const hasGatewayIdColumn = await knex.schema.hasColumn(TableName.IdentityKubernetesAuth, "gatewayId");
+
+  if (!hasGatewayIdColumn) {
+    await knex.schema.alterTable(TableName.IdentityKubernetesAuth, (table) => {
+      table.uuid("gatewayId").nullable();
+      table.foreign("gatewayId").references("id").inTable(TableName.Gateway).onDelete("SET NULL");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasGatewayIdColumn = await knex.schema.hasColumn(TableName.IdentityKubernetesAuth, "gatewayId");
+
+  if (hasGatewayIdColumn) {
+    await knex.schema.alterTable(TableName.IdentityKubernetesAuth, (table) => {
+      table.dropForeign("gatewayId");
+      table.dropColumn("gatewayId");
+    });
+  }
+}

backend/src/db/migrations/20250513081738_remove-gateway-project-link.ts

@@ -0,0 +1,110 @@
+import { Knex } from "knex";
+
+import { inMemoryKeyStore } from "@app/keystore/memory";
+import { selectAllTableCols } from "@app/lib/knex";
+import { initLogger } from "@app/lib/logger";
+import { KmsDataKey } from "@app/services/kms/kms-types";
+
+import { TableName } from "../schemas";
+import { getMigrationEnvConfig } from "./utils/env-config";
+import { getMigrationEncryptionServices } from "./utils/services";
+
+// Note(daniel): We aren't dropping tables or columns in this migrations so we can easily rollback if needed.
+// In the future we need to drop the projectGatewayId on the dynamic secrets table, and drop the project_gateways table entirely.
+
+const BATCH_SIZE = 500;
+
+export async function up(knex: Knex): Promise<void> {
+  // eslint-disable-next-line no-param-reassign
+  knex.replicaNode = () => {
+    return knex;
+  };
+
+  if (!(await knex.schema.hasColumn(TableName.DynamicSecret, "gatewayId"))) {
+    await knex.schema.alterTable(TableName.DynamicSecret, (table) => {
+      table.uuid("gatewayId").nullable();
+      table.foreign("gatewayId").references("id").inTable(TableName.Gateway).onDelete("SET NULL");
+
+      table.index("gatewayId");
+    });
+
+    const existingDynamicSecretsWithProjectGatewayId = await knex(TableName.DynamicSecret)
+      .select(selectAllTableCols(TableName.DynamicSecret))
+      .whereNotNull(`${TableName.DynamicSecret}.projectGatewayId`)
+      .join(TableName.ProjectGateway, `${TableName.ProjectGateway}.id`, `${TableName.DynamicSecret}.projectGatewayId`)
+      .whereNotNull(`${TableName.ProjectGateway}.gatewayId`)
+      .select(
+        knex.ref("projectId").withSchema(TableName.ProjectGateway).as("projectId"),
+        knex.ref("gatewayId").withSchema(TableName.ProjectGateway).as("projectGatewayGatewayId")
+      );
+
+    initLogger();
+    const envConfig = getMigrationEnvConfig();
+    const keyStore = inMemoryKeyStore();
+    const { kmsService } = await getMigrationEncryptionServices({ envConfig, keyStore, db: knex });
+
+    const updatedDynamicSecrets = await Promise.all(
+      existingDynamicSecretsWithProjectGatewayId.map(async (existingDynamicSecret) => {
+        if (!existingDynamicSecret.projectGatewayGatewayId) {
+          const result = {
+            ...existingDynamicSecret,
+            gatewayId: null
+          };
+
+          const { projectId, projectGatewayGatewayId, ...rest } = result;
+          return rest;
+        }
+
+        const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
+          type: KmsDataKey.SecretManager,
+          projectId: existingDynamicSecret.projectId
+        });
+        const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({
+          type: KmsDataKey.SecretManager,
+          projectId: existingDynamicSecret.projectId
+        });
+
+        let decryptedStoredInput = JSON.parse(
+          secretManagerDecryptor({ cipherTextBlob: Buffer.from(existingDynamicSecret.encryptedInput) }).toString()
+        ) as object;
+
+        // We're not removing the existing projectGatewayId from the input so we can easily rollback without having to re-encrypt the input
+        decryptedStoredInput = {
+          ...decryptedStoredInput,
+          gatewayId: existingDynamicSecret.projectGatewayGatewayId
+        };
+
+        const encryptedInput = secretManagerEncryptor({
+          plainText: Buffer.from(JSON.stringify(decryptedStoredInput))
+        }).cipherTextBlob;
+
+        const result = {
+          ...existingDynamicSecret,
+          encryptedInput,
+          gatewayId: existingDynamicSecret.projectGatewayGatewayId
+        };
+
+        const { projectId, projectGatewayGatewayId, ...rest } = result;
+        return rest;
+      })
+    );
+
+    for (let i = 0; i < updatedDynamicSecrets.length; i += BATCH_SIZE) {
+      // eslint-disable-next-line no-await-in-loop
+      await knex(TableName.DynamicSecret)
+        .insert(updatedDynamicSecrets.slice(i, i + BATCH_SIZE))
+        .onConflict("id")
+        .merge();
+    }
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  // no re-encryption needed as we keep the old projectGatewayId in the input
+  if (await knex.schema.hasColumn(TableName.DynamicSecret, "gatewayId")) {
+    await knex.schema.alterTable(TableName.DynamicSecret, (table) => {
+      table.dropForeign("gatewayId");
+      table.dropColumn("gatewayId");
+    });
+  }
+}

backend/src/db/migrations/20250515164622_select-org-products.ts

@@ -0,0 +1,53 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const columns = await knex.table(TableName.Organization).columnInfo();
+
+  await knex.schema.alterTable(TableName.Organization, (t) => {
+    if (!columns.secretsProductEnabled) {
+      t.boolean("secretsProductEnabled").defaultTo(true);
+    }
+    if (!columns.pkiProductEnabled) {
+      t.boolean("pkiProductEnabled").defaultTo(true);
+    }
+    if (!columns.kmsProductEnabled) {
+      t.boolean("kmsProductEnabled").defaultTo(true);
+    }
+    if (!columns.sshProductEnabled) {
+      t.boolean("sshProductEnabled").defaultTo(true);
+    }
+    if (!columns.scannerProductEnabled) {
+      t.boolean("scannerProductEnabled").defaultTo(true);
+    }
+    if (!columns.shareSecretsProductEnabled) {
+      t.boolean("shareSecretsProductEnabled").defaultTo(true);
+    }
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const columns = await knex.table(TableName.Organization).columnInfo();
+
+  await knex.schema.alterTable(TableName.Organization, (t) => {
+    if (columns.secretsProductEnabled) {
+      t.dropColumn("secretsProductEnabled");
+    }
+    if (columns.pkiProductEnabled) {
+      t.dropColumn("pkiProductEnabled");
+    }
+    if (columns.kmsProductEnabled) {
+      t.dropColumn("kmsProductEnabled");
+    }
+    if (columns.sshProductEnabled) {
+      t.dropColumn("sshProductEnabled");
+    }
+    if (columns.scannerProductEnabled) {
+      t.dropColumn("scannerProductEnabled");
+    }
+    if (columns.shareSecretsProductEnabled) {
+      t.dropColumn("shareSecretsProductEnabled");
+    }
+  });
+}

backend/src/db/migrations/20250516021501_toggle-secret-sharing-on-project.ts

@@ -0,0 +1,21 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const hasSecretSharingColumn = await knex.schema.hasColumn(TableName.Project, "secretSharing");
+  if (!hasSecretSharingColumn) {
+    await knex.schema.table(TableName.Project, (table) => {
+      table.boolean("secretSharing").notNullable().defaultTo(true);
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasSecretSharingColumn = await knex.schema.hasColumn(TableName.Project, "secretSharing");
+  if (hasSecretSharingColumn) {
+    await knex.schema.table(TableName.Project, (table) => {
+      table.dropColumn("secretSharing");
+    });
+  }
+}

backend/src/db/migrations/20250516192508_secret-sharing-limits-for-org.ts

@@ -0,0 +1,35 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const hasLifetimeColumn = await knex.schema.hasColumn(TableName.Organization, "maxSharedSecretLifetime");
+  const hasViewLimitColumn = await knex.schema.hasColumn(TableName.Organization, "maxSharedSecretViewLimit");
+
+  if (!hasLifetimeColumn || !hasViewLimitColumn) {
+    await knex.schema.alterTable(TableName.Organization, (t) => {
+      if (!hasLifetimeColumn) {
+        t.integer("maxSharedSecretLifetime").nullable().defaultTo(2592000); // 30 days in seconds
+      }
+      if (!hasViewLimitColumn) {
+        t.integer("maxSharedSecretViewLimit").nullable();
+      }
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasLifetimeColumn = await knex.schema.hasColumn(TableName.Organization, "maxSharedSecretLifetime");
+  const hasViewLimitColumn = await knex.schema.hasColumn(TableName.Organization, "maxSharedSecretViewLimit");
+
+  if (hasLifetimeColumn || hasViewLimitColumn) {
+    await knex.schema.alterTable(TableName.Organization, (t) => {
+      if (hasLifetimeColumn) {
+        t.dropColumn("maxSharedSecretLifetime");
+      }
+      if (hasViewLimitColumn) {
+        t.dropColumn("maxSharedSecretViewLimit");
+      }
+    });
+  }
+}

backend/src/db/migrations/20250517002223_secret-share-to-specific-emails.ts

@@ -0,0 +1,43 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (await knex.schema.hasTable(TableName.SecretSharing)) {
+    const hasEncryptedSalt = await knex.schema.hasColumn(TableName.SecretSharing, "encryptedSalt");
+    const hasAuthorizedEmails = await knex.schema.hasColumn(TableName.SecretSharing, "authorizedEmails");
+
+    if (!hasEncryptedSalt || !hasAuthorizedEmails) {
+      await knex.schema.alterTable(TableName.SecretSharing, (t) => {
+        // These two columns are only needed when secrets are shared with a specific list of emails
+
+        if (!hasEncryptedSalt) {
+          t.binary("encryptedSalt").nullable();
+        }
+
+        if (!hasAuthorizedEmails) {
+          t.json("authorizedEmails").nullable();
+        }
+      });
+    }
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasTable(TableName.SecretSharing)) {
+    const hasEncryptedSalt = await knex.schema.hasColumn(TableName.SecretSharing, "encryptedSalt");
+    const hasAuthorizedEmails = await knex.schema.hasColumn(TableName.SecretSharing, "authorizedEmails");
+
+    if (hasEncryptedSalt || hasAuthorizedEmails) {
+      await knex.schema.alterTable(TableName.SecretSharing, (t) => {
+        if (hasEncryptedSalt) {
+          t.dropColumn("encryptedSalt");
+        }
+
+        if (hasAuthorizedEmails) {
+          t.dropColumn("authorizedEmails");
+        }
+      });
+    }
+  }
+}