Merge pull request #2523 from Infisical/misc/move-audit-logs-to-dedicated

misc: audit log migration + special handing

backend/src/db/auditlog-knexfile.ts

@@ -0,0 +1,75 @@
+// eslint-disable-next-line
+import "ts-node/register";
+
+import dotenv from "dotenv";
+import type { Knex } from "knex";
+import path from "path";
+
+// Update with your config settings. .
+dotenv.config({
+  path: path.join(__dirname, "../../../.env.migration")
+});
+dotenv.config({
+  path: path.join(__dirname, "../../../.env")
+});
+
+if (!process.env.AUDIT_LOGS_DB_CONNECTION_URI && !process.env.AUDIT_LOGS_DB_HOST) {
+  console.info("Dedicated audit log database not found. No further migrations necessary");
+  process.exit(0);
+}
+
+console.info("Executing migration on audit log database...");
+
+export default {
+  development: {
+    client: "postgres",
+    connection: {
+      connectionString: process.env.AUDIT_LOGS_DB_CONNECTION_URI,
+      host: process.env.AUDIT_LOGS_DB_HOST,
+      port: process.env.AUDIT_LOGS_DB_PORT,
+      user: process.env.AUDIT_LOGS_DB_USER,
+      database: process.env.AUDIT_LOGS_DB_NAME,
+      password: process.env.AUDIT_LOGS_DB_PASSWORD,
+      ssl: process.env.AUDIT_LOGS_DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.AUDIT_LOGS_DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
+    pool: {
+      min: 2,
+      max: 10
+    },
+    seeds: {
+      directory: "./seeds"
+    },
+    migrations: {
+      tableName: "infisical_migrations"
+    }
+  },
+  production: {
+    client: "postgres",
+    connection: {
+      connectionString: process.env.AUDIT_LOGS_DB_CONNECTION_URI,
+      host: process.env.AUDIT_LOGS_DB_HOST,
+      port: process.env.AUDIT_LOGS_DB_PORT,
+      user: process.env.AUDIT_LOGS_DB_USER,
+      database: process.env.AUDIT_LOGS_DB_NAME,
+      password: process.env.AUDIT_LOGS_DB_PASSWORD,
+      ssl: process.env.AUDIT_LOGS_DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.AUDIT_LOGS_DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
+    pool: {
+      min: 2,
+      max: 10
+    },
+    migrations: {
+      tableName: "infisical_migrations"
+    }
+  }
+} as Knex.Config;

backend/src/db/index.ts

@@ -1,2 +1,2 @@
 export type { TDbClient } from "./instance";
-export { initDbConnection } from "./instance";
+export { initAuditLogDbConnection, initDbConnection } from "./instance";

backend/src/db/instance.ts

@@ -70,3 +70,45 @@ export const initDbConnection = ({
 
   return db;
 };
+
+export const initAuditLogDbConnection = ({
+  dbConnectionUri,
+  dbRootCert
+}: {
+  dbConnectionUri: string;
+  dbRootCert?: string;
+}) => {
+  // akhilmhdh: the default Knex is knex.Knex<any, any[]>. but when assigned with knex({<config>}) the value is knex.Knex<any, unknown[]>
+  // this was causing issue with files like `snapshot-dal` `findRecursivelySnapshots` this i am explicitly putting the any and unknown[]
+  // eslint-disable-next-line
+  const db: Knex<any, unknown[]> = knex({
+    client: "pg",
+    connection: {
+      connectionString: dbConnectionUri,
+      host: process.env.AUDIT_LOGS_DB_HOST,
+      // @ts-expect-error I have no clue why only for the port there is a type error
+      // eslint-disable-next-line
+      port: process.env.AUDIT_LOGS_DB_PORT,
+      user: process.env.AUDIT_LOGS_DB_USER,
+      database: process.env.AUDIT_LOGS_DB_NAME,
+      password: process.env.AUDIT_LOGS_DB_PASSWORD,
+      ssl: dbRootCert
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(dbRootCert, "base64").toString("ascii")
+          }
+        : false
+    }
+  });
+
+  // we add these overrides so that auditLogDb and the primary DB are interchangeable
+  db.primaryNode = () => {
+    return db;
+  };
+
+  db.replicaNode = () => {
+    return db;
+  };
+
+  return db;
+};

backend/src/db/migrations/20241007052025_make-audit-log-independent.ts

@@ -0,0 +1,48 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (await knex.schema.hasTable(TableName.AuditLog)) {
+    const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId");
+    const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId");
+    const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName");
+
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      if (doesOrgIdExist) {
+        t.dropForeign("orgId");
+      }
+
+      if (doesProjectIdExist) {
+        t.dropForeign("projectId");
+      }
+
+      // add normalized field
+      if (!doesProjectNameExist) {
+        t.string("projectName");
+      }
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId");
+  const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId");
+  const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName");
+
+  if (await knex.schema.hasTable(TableName.AuditLog)) {
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      if (doesOrgIdExist) {
+        t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      }
+      if (doesProjectIdExist) {
+        t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
+      }
+
+      // remove normalized field
+      if (doesProjectNameExist) {
+        t.dropColumn("projectName");
+      }
+    });
+  }
+}

backend/src/db/migrations/20241007052449_partition-audit-logs.ts

@@ -0,0 +1,164 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+const formatPartitionDate = (date: Date) => {
+  const year = date.getFullYear();
+  const month = String(date.getMonth() + 1).padStart(2, "0");
+  const day = String(date.getDate()).padStart(2, "0");
+
+  return `${year}-${month}-${day}`;
+};
+
+const createAuditLogPartition = async (knex: Knex, startDate: Date, endDate: Date) => {
+  const startDateStr = formatPartitionDate(startDate);
+  const endDateStr = formatPartitionDate(endDate);
+
+  const partitionName = `${TableName.PartitionedAuditLog}_${startDateStr.replace(/-/g, "")}_${endDateStr.replace(
+    /-/g,
+    ""
+  )}`;
+
+  await knex.schema.raw(
+    `CREATE TABLE ${partitionName} PARTITION OF ${TableName.PartitionedAuditLog} FOR VALUES FROM ('${startDateStr}') TO ('${endDateStr}')`
+  );
+};
+
+const isUsingDedicatedAuditLogDb = Boolean(process.env.AUDIT_LOGS_DB_CONNECTION_URI);
+
+export async function up(knex: Knex): Promise<void> {
+  if (!isUsingDedicatedAuditLogDb && (await knex.schema.hasTable(TableName.AuditLog))) {
+    console.info("Dropping primary key of Audit Log table...");
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      // remove existing keys
+      t.dropPrimary();
+    });
+  }
+
+  // create a new partitioned table for audit logs
+  if (!(await knex.schema.hasTable(TableName.PartitionedAuditLog))) {
+    const createTableSql = knex.schema
+      .createTable(TableName.PartitionedAuditLog, (t) => {
+        t.uuid("id").defaultTo(knex.fn.uuid());
+        t.string("actor").notNullable();
+        t.jsonb("actorMetadata").notNullable();
+        t.string("ipAddress");
+        t.string("eventType").notNullable();
+        t.jsonb("eventMetadata");
+        t.string("userAgent");
+        t.string("userAgentType");
+        t.datetime("expiresAt");
+        t.timestamps(true, true, true);
+        t.uuid("orgId");
+        t.string("projectId");
+        t.string("projectName");
+        t.primary(["id", "createdAt"]);
+      })
+      .toString();
+
+    console.info("Creating partition table...");
+    await knex.schema.raw(`
+        ${createTableSql} PARTITION BY RANGE ("createdAt");
+    `);
+
+    console.log("Adding indices...");
+    await knex.schema.alterTable(TableName.PartitionedAuditLog, (t) => {
+      t.index(["projectId", "createdAt"]);
+      t.index(["orgId", "createdAt"]);
+      t.index("expiresAt");
+      t.index("orgId");
+      t.index("projectId");
+    });
+
+    console.log("Adding GIN indices...");
+
+    await knex.raw(
+      `CREATE INDEX IF NOT EXISTS "audit_logs_actorMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("actorMetadata" jsonb_path_ops)`
+    );
+    console.log("GIN index for actorMetadata done");
+
+    await knex.raw(
+      `CREATE INDEX IF NOT EXISTS "audit_logs_eventMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("eventMetadata" jsonb_path_ops)`
+    );
+    console.log("GIN index for eventMetadata done");
+
+    // create default partition
+    console.log("Creating default partition...");
+    await knex.schema.raw(
+      `CREATE TABLE ${TableName.PartitionedAuditLog}_default PARTITION OF ${TableName.PartitionedAuditLog} DEFAULT`
+    );
+
+    const nextDate = new Date();
+    nextDate.setDate(nextDate.getDate() + 1);
+    const nextDateStr = formatPartitionDate(nextDate);
+
+    // attach existing audit log table as a partition ONLY if using the same DB
+    if (!isUsingDedicatedAuditLogDb) {
+      console.log("Attaching existing audit log table as a partition...");
+      await knex.schema.raw(`
+      ALTER TABLE ${TableName.AuditLog} ADD CONSTRAINT audit_log_old
+      CHECK ( "createdAt" < DATE '${nextDateStr}' );
+  
+      ALTER TABLE ${TableName.PartitionedAuditLog} ATTACH PARTITION ${TableName.AuditLog}
+      FOR VALUES FROM (MINVALUE) TO ('${nextDateStr}' );
+      `);
+    }
+
+    // create partition from now until end of month
+    console.log("Creating audit log partitions ahead of time... next date:", nextDateStr);
+    await createAuditLogPartition(knex, nextDate, new Date(nextDate.getFullYear(), nextDate.getMonth() + 1));
+
+    // create partitions 4 years ahead
+    const partitionMonths = 4 * 12;
+    const partitionPromises: Promise<void>[] = [];
+    for (let x = 1; x <= partitionMonths; x += 1) {
+      partitionPromises.push(
+        createAuditLogPartition(
+          knex,
+          new Date(nextDate.getFullYear(), nextDate.getMonth() + x, 1),
+          new Date(nextDate.getFullYear(), nextDate.getMonth() + (x + 1), 1)
+        )
+      );
+    }
+
+    await Promise.all(partitionPromises);
+    console.log("Partition migration complete");
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const partitionSearchResult = await knex.raw(`
+      SELECT inhrelid::regclass::text
+      FROM pg_inherits
+      WHERE inhparent::regclass::text = '${TableName.PartitionedAuditLog}'
+      AND inhrelid::regclass::text = '${TableName.AuditLog}'
+  `);
+
+  const isAuditLogAPartition = partitionSearchResult.rows.length > 0;
+  if (isAuditLogAPartition) {
+    // detach audit log from partition
+    console.log("Detaching original audit log table from new partition table...");
+    await knex.schema.raw(`
+    ALTER TABLE ${TableName.PartitionedAuditLog} DETACH PARTITION ${TableName.AuditLog};
+
+    ALTER TABLE ${TableName.AuditLog} DROP CONSTRAINT audit_log_old;
+  `);
+
+    // revert audit log modifications
+    console.log("Reverting changes made to the audit log table...");
+    if (await knex.schema.hasTable(TableName.AuditLog)) {
+      await knex.schema.alterTable(TableName.AuditLog, (t) => {
+        // we drop this first because adding to the partition results in a new primary key
+        t.dropPrimary();
+
+        // add back the original keys of the audit logs table
+        t.primary(["id"], {
+          constraintName: "audit_logs_pkey"
+        });
+      });
+    }
+  }
+
+  await knex.schema.dropTableIfExists(TableName.PartitionedAuditLog);
+  console.log("Partition rollback complete");
+}

backend/src/db/schemas/index.ts

@@ -55,6 +55,7 @@ export * from "./org-bots";
 export * from "./org-memberships";
 export * from "./org-roles";
 export * from "./organizations";
+export * from "./partitioned-audit-logs";
 export * from "./pki-alerts";
 export * from "./pki-collection-items";
 export * from "./pki-collections";

backend/src/db/schemas/models.ts

@@ -90,6 +90,7 @@ export enum TableName {
   OidcConfig = "oidc_configs",
   LdapGroupMap = "ldap_group_maps",
   AuditLog = "audit_logs",
+  PartitionedAuditLog = "partitioned_audit_logs",
   AuditLogStream = "audit_log_streams",
   GitAppInstallSession = "git_app_install_sessions",
   GitAppOrg = "git_app_org",

backend/src/db/schemas/partitioned-audit-logs.ts

@@ -0,0 +1,29 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const PartitionedAuditLogsSchema = z.object({
+  id: z.string().uuid(),
+  actor: z.string(),
+  actorMetadata: z.unknown(),
+  ipAddress: z.string().nullable().optional(),
+  eventType: z.string(),
+  eventMetadata: z.unknown().nullable().optional(),
+  userAgent: z.string().nullable().optional(),
+  userAgentType: z.string().nullable().optional(),
+  expiresAt: z.date().nullable().optional(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  orgId: z.string().uuid().nullable().optional(),
+  projectId: z.string().nullable().optional(),
+  projectName: z.string().nullable().optional()
+});
+
+export type TPartitionedAuditLogs = z.infer<typeof PartitionedAuditLogsSchema>;
+export type TPartitionedAuditLogsInsert = Omit<z.input<typeof PartitionedAuditLogsSchema>, TImmutableDBKeys>;
+export type TPartitionedAuditLogsUpdate = Partial<Omit<z.input<typeof PartitionedAuditLogsSchema>, TImmutableDBKeys>>;