From 8dbdb798333709dac0592719e58a05e980bef899 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 4 Oct 2024 21:43:33 +0800 Subject: [PATCH] misc: finalized partition migration script --- backend/src/db/auditlog-knexfile.ts | 2 + .../20241003075413_partition-audit-logs.ts | 90 +++++++++++-------- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/backend/src/db/auditlog-knexfile.ts b/backend/src/db/auditlog-knexfile.ts index f6b84d453..3ceef65a0 100644 --- a/backend/src/db/auditlog-knexfile.ts +++ b/backend/src/db/auditlog-knexfile.ts @@ -18,6 +18,8 @@ if (!process.env.AUDIT_LOGS_DB_CONNECTION_URI && !process.env.AUDIT_LOGS_DB_HOST process.exit(0); } +console.info("Executing migration on audit log database..."); + export default { development: { client: "postgres", diff --git a/backend/src/db/migrations/20241003075413_partition-audit-logs.ts b/backend/src/db/migrations/20241003075413_partition-audit-logs.ts index 3fd9bf200..d0c923103 100644 --- a/backend/src/db/migrations/20241003075413_partition-audit-logs.ts +++ b/backend/src/db/migrations/20241003075413_partition-audit-logs.ts @@ -24,9 +24,11 @@ const createAuditLogPartition = async (knex: Knex, startDate: Date, endDate: Dat ); }; +const isUsingDedicatedAuditLogDb = Boolean(process.env.AUDIT_LOGS_DB_CONNECTION_URI); + export async function up(knex: Knex): Promise { - // prepare the existing audit log table for it to become a partition - if (await knex.schema.hasTable(TableName.AuditLog)) { + if (!isUsingDedicatedAuditLogDb && (await knex.schema.hasTable(TableName.AuditLog))) { + // prepare the existing audit log table for it to become a partition const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); @@ -84,11 +86,11 @@ export async function up(knex: Knex): Promise { }); await knex.raw( - `CREATE INDEX "audit_logs_actorMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("actorMetadata" jsonb_path_ops)` + `CREATE INDEX IF NOT EXISTS "audit_logs_actorMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("actorMetadata" jsonb_path_ops)` ); await knex.raw( - `CREATE INDEX "audit_logs_eventMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("eventMetadata" jsonb_path_ops)` + `CREATE INDEX IF NOT EXISTS "audit_logs_eventMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("eventMetadata" jsonb_path_ops)` ); // create default partition @@ -100,14 +102,16 @@ export async function up(knex: Knex): Promise { nextDate.setDate(nextDate.getDate() + 1); const nextDateStr = formatPartitionDate(nextDate); - // attach existing audit log table as a partition - await knex.schema.raw(` - ALTER TABLE ${TableName.AuditLog} ADD CONSTRAINT audit_log_old - CHECK ( "createdAt" < DATE '${nextDateStr}' ); - - ALTER TABLE ${TableName.PartitionedAuditLog} ATTACH PARTITION ${TableName.AuditLog} - FOR VALUES FROM (MINVALUE) TO ('${nextDateStr}' ); - `); + // attach existing audit log table as a partition ONLY if using the same DB + if (!isUsingDedicatedAuditLogDb) { + await knex.schema.raw(` + ALTER TABLE ${TableName.AuditLog} ADD CONSTRAINT audit_log_old + CHECK ( "createdAt" < DATE '${nextDateStr}' ); + + ALTER TABLE ${TableName.PartitionedAuditLog} ATTACH PARTITION ${TableName.AuditLog} + FOR VALUES FROM (MINVALUE) TO ('${nextDateStr}' ); + `); + } // create partition from now until end of month await createAuditLogPartition(knex, nextDate, new Date(nextDate.getFullYear(), nextDate.getMonth() + 1)); @@ -130,40 +134,50 @@ export async function up(knex: Knex): Promise { } export async function down(knex: Knex): Promise { - // detach audit log from partition - await knex.schema.raw(` + const result = await knex.raw(` + SELECT inhrelid::regclass::text + FROM pg_inherits + WHERE inhparent::regclass::text = '${TableName.PartitionedAuditLog}' + AND inhrelid::regclass::text = '${TableName.AuditLog}' + `); + + const isAuditLogAPartition = result.rows.length > 0; + if (isAuditLogAPartition) { + // detach audit log from partition + await knex.schema.raw(` ALTER TABLE ${TableName.PartitionedAuditLog} DETACH PARTITION ${TableName.AuditLog}; ALTER TABLE ${TableName.AuditLog} DROP CONSTRAINT audit_log_old; `); - // revert audit log modifications - const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); - const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); - const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); + // revert audit log modifications + const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); + const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); + const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); - if (await knex.schema.hasTable(TableName.AuditLog)) { - await knex.schema.alterTable(TableName.AuditLog, (t) => { - // we drop this first because adding to the partition results in a new primary key - t.dropPrimary(); + if (await knex.schema.hasTable(TableName.AuditLog)) { + await knex.schema.alterTable(TableName.AuditLog, (t) => { + // we drop this first because adding to the partition results in a new primary key + t.dropPrimary(); - // add back the original keys of the audit logs table - t.primary(["id"], { - constraintName: "audit_logs_pkey" + // add back the original keys of the audit logs table + t.primary(["id"], { + constraintName: "audit_logs_pkey" + }); + + if (doesOrgIdExist) { + t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); + } + if (doesProjectIdExist) { + t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); + } + + // remove normalized fields + if (doesProjectNameExist) { + t.dropColumn("projectName"); + } }); - - if (doesOrgIdExist) { - t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); - } - if (doesProjectIdExist) { - t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); - } - - // remove normalized fields - if (doesProjectNameExist) { - t.dropColumn("projectName"); - } - }); + } } await knex.schema.dropTableIfExists(TableName.PartitionedAuditLog);