Merge branch 'main' into feature/slack-secret-sync-error-notification

backend/src/db/migrations/20251018061215_sub-org.ts

@@ -2,7 +2,7 @@ import { Knex } from "knex";
 
 import { dropConstraintIfExists } from "@app/db/migrations/utils/dropConstraintIfExists";
 
-import { AccessScope, TableName } from "../schemas";
+import { TableName } from "../schemas";
 
 export async function up(knex: Knex): Promise<void> {
   const hasParentOrgId = await knex.schema.hasColumn(TableName.Organization, "parentOrgId");
@@ -18,8 +18,6 @@ export async function up(knex: Knex): Promise<void> {
       await dropConstraintIfExists(TableName.Organization, "organizations_slug_unique", knex);
       t.unique(["rootOrgId", "parentOrgId", "slug"]);
     });
-
-    // had to switch to raw for null not distinct
   }
 
   const hasIdentityOrgCol = await knex.schema.hasColumn(TableName.Identity, "orgId");
@@ -28,24 +26,6 @@ export async function up(knex: Knex): Promise<void> {
       t.uuid("orgId");
       t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
     });
-
-    await knex.raw(
-      `
-  UPDATE ?? AS identity
-  SET "orgId" = membership."scopeOrgId"
-  FROM ?? AS membership
-  WHERE 
-    membership."actorIdentityId" = identity."id"
-    AND membership."scope" = ?
-`,
-      [TableName.Identity, TableName.Membership, AccessScope.Organization]
-    );
-
-    await knex.raw(`DELETE FROM ?? WHERE "orgId" IS NULL`, [TableName.Identity]);
-
-    await knex.schema.alterTable(TableName.Identity, (t) => {
-      t.uuid("orgId").notNullable().alter();
-    });
   }
 }
 

backend/src/db/migrations/20251019061215_sub-org-identity-backfill.ts

@@ -0,0 +1,48 @@
+import { Knex } from "knex";
+
+import { chunkArray } from "@app/lib/fn";
+
+import { AccessScope, TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.transaction(async (tx) => {
+    const hasIdentityOrgCol = await tx.schema.hasColumn(TableName.Identity, "orgId");
+    if (hasIdentityOrgCol) {
+      const identityMemberships = await tx(TableName.Membership)
+        .where({
+          scope: AccessScope.Organization
+        })
+        .whereNotNull("actorIdentityId")
+        .select("actorIdentityId", "scopeOrgId");
+
+      const identityToOrgMapping: Record<string, string> = {};
+      identityMemberships.forEach((el) => {
+        if (el.actorIdentityId) {
+          identityToOrgMapping[el.actorIdentityId] = el.scopeOrgId;
+        }
+      });
+
+      const batchMemberships = chunkArray(identityMemberships, 500);
+      for await (const membership of batchMemberships) {
+        const identityIds = membership.map((el) => el.actorIdentityId).filter(Boolean) as string[];
+        if (identityIds.length) {
+          const identities = await tx(TableName.Identity).whereIn("id", identityIds).select("*");
+          await tx(TableName.Identity)
+            .insert(
+              identities.map((el) => ({
+                ...el,
+                orgId: identityToOrgMapping[el.id]
+              }))
+            )
+            .onConflict("id")
+            .merge();
+        }
+      }
+    }
+  });
+}
+
+export async function down(): Promise<void> {}
+
+const config = { transaction: false };
+export { config };

backend/src/db/migrations/20251021112356_add-certificate-auto-renewal.ts

@@ -0,0 +1,51 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.PkiApiEnrollmentConfig, "autoRenewDays")) {
+    await knex.schema.alterTable(TableName.PkiApiEnrollmentConfig, (t) => {
+      t.renameColumn("autoRenewDays", "renewBeforeDays");
+    });
+  }
+
+  if (!(await knex.schema.hasColumn(TableName.Certificate, "renewBeforeDays"))) {
+    await knex.schema.alterTable(TableName.Certificate, (t) => {
+      t.integer("renewBeforeDays").nullable();
+      t.uuid("renewedFromCertificateId").nullable();
+      t.uuid("renewedByCertificateId").nullable();
+      t.text("renewalError").nullable();
+      t.string("keyAlgorithm").nullable();
+      t.string("signatureAlgorithm").nullable();
+      t.foreign("renewedFromCertificateId").references("id").inTable(TableName.Certificate).onDelete("SET NULL");
+      t.foreign("renewedByCertificateId").references("id").inTable(TableName.Certificate).onDelete("SET NULL");
+      t.index("renewedFromCertificateId");
+      t.index("renewedByCertificateId");
+      t.index("renewBeforeDays");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.Certificate, "renewBeforeDays")) {
+    await knex.schema.alterTable(TableName.Certificate, (t) => {
+      t.dropForeign(["renewedFromCertificateId"]);
+      t.dropForeign(["renewedByCertificateId"]);
+      t.dropIndex("renewedFromCertificateId");
+      t.dropIndex("renewedByCertificateId");
+      t.dropIndex("renewBeforeDays");
+      t.dropColumn("renewBeforeDays");
+      t.dropColumn("renewedFromCertificateId");
+      t.dropColumn("renewedByCertificateId");
+      t.dropColumn("renewalError");
+      t.dropColumn("keyAlgorithm");
+      t.dropColumn("signatureAlgorithm");
+    });
+  }
+
+  if (await knex.schema.hasColumn(TableName.PkiApiEnrollmentConfig, "renewBeforeDays")) {
+    await knex.schema.alterTable(TableName.PkiApiEnrollmentConfig, (t) => {
+      t.renameColumn("renewBeforeDays", "autoRenewDays");
+    });
+  }
+}

backend/src/db/migrations/20251021124744_fix-project-deletion-approval-policy-constraint.ts

@@ -0,0 +1,68 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+// Fix for 20250722152841_add-policies-environments-table.ts migration.
+// 20250722152841_add-policies-environments-table.ts introduced a bug where you can no longer delete a project if it has any approval policy environments.
+
+export async function up(knex: Knex): Promise<void> {
+  // Fix SecretApprovalPolicyEnvironment to cascade delete when environment is deleted
+  // note: this won't actually happen, as we prevent deletion of environments with active approval policies
+
+  // in the old migration it was ON DELETE SET NULL, which doesn't work because envId is not a nullable col
+  await knex.schema.alterTable(TableName.SecretApprovalPolicyEnvironment, (t) => {
+    t.dropForeign(["envId"]);
+    t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE");
+  });
+
+  // Fix AccessApprovalPolicyEnvironment to cascade delete when environment is deleted
+  // note: this won't actually happen, as we prevent deletion of environments with active approval policies
+
+  // in the old migration it was ON DELETE SET NULL, which doesn't work because envId is not a nullable col
+  await knex.schema.alterTable(TableName.AccessApprovalPolicyEnvironment, (t) => {
+    t.dropForeign(["envId"]);
+    t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE");
+  });
+
+  // Fix SecretApprovalPolicy to CASCADE instead of SET NULL
+
+  // in the old migration it was ON DELETE SET NULL, which doesn't work because envId is not a nullable col
+  await knex.schema.alterTable(TableName.SecretApprovalPolicy, (t) => {
+    t.dropForeign(["envId"]);
+    t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE");
+  });
+
+  // Fix AccessApprovalPolicy to CASCADE instead of SET NULL
+
+  // in the old migration it was ON DELETE SET NULL, which doesn't work because envId is not a nullable col
+  await knex.schema.alterTable(TableName.AccessApprovalPolicy, (t) => {
+    t.dropForeign(["envId"]);
+    t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE");
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  // Revert SecretApprovalPolicyEnvironment
+  await knex.schema.alterTable(TableName.SecretApprovalPolicyEnvironment, (t) => {
+    t.dropForeign(["envId"]);
+    t.foreign("envId").references("id").inTable(TableName.Environment);
+  });
+
+  // Revert AccessApprovalPolicyEnvironment
+  await knex.schema.alterTable(TableName.AccessApprovalPolicyEnvironment, (t) => {
+    t.dropForeign(["envId"]);
+    t.foreign("envId").references("id").inTable(TableName.Environment);
+  });
+
+  // Revert SecretApprovalPolicy back to SET NULL
+  await knex.schema.alterTable(TableName.SecretApprovalPolicy, (t) => {
+    t.dropForeign(["envId"]);
+    t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("SET NULL");
+  });
+
+  // Revert AccessApprovalPolicy back to SET NULL
+  await knex.schema.alterTable(TableName.AccessApprovalPolicy, (t) => {
+    t.dropForeign(["envId"]);
+    t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("SET NULL");
+  });
+}

backend/src/db/migrations/20251023123213_block-duplicate-sync-destinations-setting.ts

@@ -0,0 +1,27 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const hasOrgBlockDuplicateColumn = await knex.schema.hasColumn(
+    TableName.Organization,
+    "blockDuplicateSecretSyncDestinations"
+  );
+  if (!hasOrgBlockDuplicateColumn) {
+    await knex.schema.table(TableName.Organization, (table) => {
+      table.boolean("blockDuplicateSecretSyncDestinations").notNullable().defaultTo(false);
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasOrgBlockDuplicateColumn = await knex.schema.hasColumn(
+    TableName.Organization,
+    "blockDuplicateSecretSyncDestinations"
+  );
+  if (hasOrgBlockDuplicateColumn) {
+    await knex.schema.table(TableName.Organization, (table) => {
+      table.dropColumn("blockDuplicateSecretSyncDestinations");
+    });
+  }
+}

backend/src/db/migrations/20251028064623_pam-account-rotation-status.ts

@@ -0,0 +1,29 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasColumn(TableName.PamAccount, "rotationStatus"))) {
+    await knex.schema.alterTable(TableName.PamAccount, (t) => {
+      t.string("rotationStatus").nullable();
+    });
+  }
+  if (!(await knex.schema.hasColumn(TableName.PamAccount, "encryptedLastRotationMessage"))) {
+    await knex.schema.alterTable(TableName.PamAccount, (t) => {
+      t.binary("encryptedLastRotationMessage").nullable();
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.PamAccount, "rotationStatus")) {
+    await knex.schema.alterTable(TableName.PamAccount, (t) => {
+      t.dropColumn("rotationStatus");
+    });
+  }
+  if (await knex.schema.hasColumn(TableName.PamAccount, "encryptedLastRotationMessage")) {
+    await knex.schema.alterTable(TableName.PamAccount, (t) => {
+      t.dropColumn("encryptedLastRotationMessage");
+    });
+  }
+}

backend/src/db/migrations/20251028120000_add-certificate-sync-table.ts

@@ -0,0 +1,35 @@
+import { Knex } from "knex";
+
+import { TableName } from "@app/db/schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "@app/db/utils";
+import { CertificateSyncStatus } from "@app/services/certificate-sync/certificate-sync-enums";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.CertificateSync))) {
+    await knex.schema.createTable(TableName.CertificateSync, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.uuid("pkiSyncId").notNullable();
+      t.foreign("pkiSyncId").references("id").inTable(TableName.PkiSync).onDelete("CASCADE");
+      t.uuid("certificateId").notNullable();
+      t.foreign("certificateId").references("id").inTable(TableName.Certificate).onDelete("CASCADE");
+      t.string("syncStatus").defaultTo(CertificateSyncStatus.Pending);
+      t.text("lastSyncMessage");
+      t.datetime("lastSyncedAt");
+      t.timestamps(true, true, true);
+
+      // Ensure unique combination of pki sync and certificate
+      t.unique(["pkiSyncId", "certificateId"]);
+
+      t.index("pkiSyncId");
+      t.index("certificateId");
+      t.index("syncStatus");
+    });
+
+    await createOnUpdateTrigger(knex, TableName.CertificateSync);
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.CertificateSync);
+  await dropOnUpdateTrigger(knex, TableName.CertificateSync);
+}

backend/src/db/migrations/20251028155708_identity-access-token-remove-fk-for-identity-id.ts

@@ -0,0 +1,22 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.transaction(async (tx) => {
+    await tx.schema.alterTable(TableName.IdentityAccessToken, (table) => {
+      table.dropForeign("identityId");
+    });
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.transaction(async (tx) => {
+    await tx.schema.alterTable(TableName.IdentityAccessToken, (table) => {
+      table.foreign("identityId").references("id").inTable(TableName.Identity);
+    });
+  });
+}
+
+const config = { transaction: false };
+export { config };

backend/src/db/migrations/20251028160921_delete-no-org-identities.ts

@@ -0,0 +1,30 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+const MIGRATION_TIMEOUT = 30 * 60 * 1000; // 30 minutes
+
+export async function up(knex: Knex): Promise<void> {
+  const result = await knex.raw("SHOW statement_timeout");
+  const originalTimeout = result.rows[0].statement_timeout;
+
+  await knex.transaction(async (tx) => {
+    try {
+      await tx.raw(`SET statement_timeout = ${MIGRATION_TIMEOUT}`);
+      const hasIdentityOrgCol = await tx.schema.hasColumn(TableName.Identity, "orgId");
+      if (hasIdentityOrgCol) {
+        await tx(TableName.Identity).whereNull("orgId").delete();
+        await tx.schema.alterTable(TableName.Identity, (t) => {
+          t.uuid("orgId").notNullable().alter();
+        });
+      }
+    } finally {
+      await tx.raw(`SET statement_timeout = '${originalTimeout}'`);
+    }
+  });
+}
+
+export async function down(): Promise<void> {}
+
+const config = { transaction: false };
+export { config };

backend/src/db/migrations/20251031044512_add-certificate-sync-external-identifier.ts

@@ -0,0 +1,21 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasColumn(TableName.CertificateSync, "externalIdentifier"))) {
+    await knex.schema.alterTable(TableName.CertificateSync, (t) => {
+      t.text("externalIdentifier").nullable();
+      t.index("externalIdentifier");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.CertificateSync, "externalIdentifier")) {
+    await knex.schema.alterTable(TableName.CertificateSync, (t) => {
+      t.dropIndex("externalIdentifier");
+      t.dropColumn("externalIdentifier");
+    });
+  }
+}

backend/src/db/schemas/certificate-syncs.ts

@@ -0,0 +1,24 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const CertificateSyncsSchema = z.object({
+  id: z.string().uuid(),
+  pkiSyncId: z.string().uuid(),
+  certificateId: z.string().uuid(),
+  syncStatus: z.string().default("pending").nullable().optional(),
+  lastSyncMessage: z.string().nullable().optional(),
+  lastSyncedAt: z.date().nullable().optional(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  externalIdentifier: z.string().nullable().optional()
+});
+
+export type TCertificateSyncs = z.infer<typeof CertificateSyncsSchema>;
+export type TCertificateSyncsInsert = Omit<z.input<typeof CertificateSyncsSchema>, TImmutableDBKeys>;
+export type TCertificateSyncsUpdate = Partial<Omit<z.input<typeof CertificateSyncsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/certificates.ts

@@ -27,7 +27,13 @@ export const CertificatesSchema = z.object({
   extendedKeyUsages: z.string().array().nullable().optional(),
   projectId: z.string(),
   pkiSubscriberId: z.string().uuid().nullable().optional(),
-  profileId: z.string().uuid().nullable().optional()
+  profileId: z.string().uuid().nullable().optional(),
+  renewBeforeDays: z.number().nullable().optional(),
+  renewedFromCertificateId: z.string().uuid().nullable().optional(),
+  renewedByCertificateId: z.string().uuid().nullable().optional(),
+  renewalError: z.string().nullable().optional(),
+  keyAlgorithm: z.string().nullable().optional(),
+  signatureAlgorithm: z.string().nullable().optional()
 });
 
 export type TCertificates = z.infer<typeof CertificatesSchema>;

backend/src/db/schemas/index.ts

@@ -17,6 +17,7 @@ export * from "./certificate-authority-crl";
 export * from "./certificate-authority-secret";
 export * from "./certificate-bodies";
 export * from "./certificate-secrets";
+export * from "./certificate-syncs";
 export * from "./certificate-template-est-configs";
 export * from "./certificate-templates";
 export * from "./certificates";

backend/src/db/schemas/models.ts

@@ -161,6 +161,7 @@ export enum TableName {
   AppConnection = "app_connections",
   SecretSync = "secret_syncs",
   PkiSync = "pki_syncs",
+  CertificateSync = "certificate_syncs",
   KmipClient = "kmip_clients",
   KmipOrgConfig = "kmip_org_configs",
   KmipOrgServerCertificates = "kmip_org_server_certificates",

backend/src/db/schemas/organizations.ts

@@ -40,7 +40,8 @@ export const OrganizationsSchema = z.object({
   googleSsoAuthEnforced: z.boolean().default(false),
   googleSsoAuthLastUsed: z.date().nullable().optional(),
   parentOrgId: z.string().uuid().nullable().optional(),
-  rootOrgId: z.string().uuid().nullable().optional()
+  rootOrgId: z.string().uuid().nullable().optional(),
+  blockDuplicateSecretSyncDestinations: z.boolean().default(false)
 });
 
 export type TOrganizations = z.infer<typeof OrganizationsSchema>;

backend/src/db/schemas/pam-accounts.ts

@@ -21,7 +21,9 @@ export const PamAccountsSchema = z.object({
   updatedAt: z.date(),
   rotationEnabled: z.boolean().default(false),
   rotationIntervalSeconds: z.number().nullable().optional(),
-  lastRotatedAt: z.date().nullable().optional()
+  lastRotatedAt: z.date().nullable().optional(),
+  rotationStatus: z.string().nullable().optional(),
+  encryptedLastRotationMessage: zodBuffer.nullable().optional()
 });
 
 export type TPamAccounts = z.infer<typeof PamAccountsSchema>;

backend/src/db/schemas/pki-api-enrollment-configs.ts

@@ -10,7 +10,7 @@ import { TImmutableDBKeys } from "./models";
 export const PkiApiEnrollmentConfigsSchema = z.object({
   id: z.string().uuid(),
   autoRenew: z.boolean().default(false).nullable().optional(),
-  autoRenewDays: z.number().nullable().optional(),
+  renewBeforeDays: z.number().nullable().optional(),
   createdAt: z.date(),
   updatedAt: z.date()
 });