From 98ed063ce6d230c4c2e636274717d16d56876358 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 8 Oct 2024 12:52:43 +0800 Subject: [PATCH 1/6] misc: enabled audit log exploration --- backend/src/server/routes/v1/organization-router.ts | 7 ------- frontend/src/views/Org/AuditLogsPage/AuditLogsPage.tsx | 9 +-------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/backend/src/server/routes/v1/organization-router.ts b/backend/src/server/routes/v1/organization-router.ts index 68e186468..e9421af7f 100644 --- a/backend/src/server/routes/v1/organization-router.ts +++ b/backend/src/server/routes/v1/organization-router.ts @@ -11,8 +11,6 @@ import { } from "@app/db/schemas"; import { EventType, UserAgentType } from "@app/ee/services/audit-log/audit-log-types"; import { AUDIT_LOGS, ORGANIZATIONS } from "@app/lib/api-docs"; -import { getConfig } from "@app/lib/config/env"; -import { BadRequestError } from "@app/lib/errors"; import { getLastMidnightDateISO } from "@app/lib/fn"; import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; @@ -141,11 +139,6 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { - const appCfg = getConfig(); - if (appCfg.isCloud) { - throw new BadRequestError({ message: "Infisical cloud audit log is in maintenance mode." }); - } - const auditLogs = await server.services.auditLog.listAuditLogs({ filter: { ...req.query, diff --git a/frontend/src/views/Org/AuditLogsPage/AuditLogsPage.tsx b/frontend/src/views/Org/AuditLogsPage/AuditLogsPage.tsx index 3fef39340..2b6ec6744 100644 --- a/frontend/src/views/Org/AuditLogsPage/AuditLogsPage.tsx +++ b/frontend/src/views/Org/AuditLogsPage/AuditLogsPage.tsx @@ -1,4 +1,3 @@ -import { NoticeBanner } from "@app/components/v2"; import { OrgPermissionActions, OrgPermissionSubjects } from "@app/context"; import { withPermission } from "@app/hoc"; @@ -11,15 +10,9 @@ export const AuditLogsPage = withPermission(

Audit Logs

- {(window.location.origin.includes("https://app.infisical.com") || - window.location.origin.includes("https://gamma.infisical.com")) && ( - - We are currently working on improving the performance of audit log queries. During this time, querying logs is temporarily disabled. However, audit logs are still being generated as usual, so there is no disruption to log collection. - - )}
- {!window.location.origin.includes("https://app.infisical.com") && } +
); From 22b417b50b1aa8dcc92a769b2da7d3cb47049a8d Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 8 Oct 2024 17:53:53 +0800 Subject: [PATCH 2/6] misc: made partition opt-in --- backend/scripts/generate-schema-types.ts | 7 +- backend/src/@types/knex.d.ts | 8 - .../manual-migrations/partition-audit-logs.ts | 145 ++++++++++++++++ .../20241007052449_partition-audit-logs.ts | 164 ------------------ backend/src/db/schemas/audit-logs.ts | 3 +- backend/src/db/schemas/index.ts | 1 - backend/src/db/schemas/models.ts | 1 - .../src/db/schemas/partitioned-audit-logs.ts | 29 ---- backend/src/ee/routes/v1/project-router.ts | 4 +- .../ee/services/audit-log/audit-log-dal.ts | 20 +-- .../server/routes/v1/organization-router.ts | 4 +- frontend/src/hooks/api/auditLogs/types.tsx | 1 + .../AuditLogsPage/components/LogsTableRow.tsx | 2 +- 13 files changed, 169 insertions(+), 220 deletions(-) create mode 100644 backend/src/db/manual-migrations/partition-audit-logs.ts delete mode 100644 backend/src/db/migrations/20241007052449_partition-audit-logs.ts delete mode 100644 backend/src/db/schemas/partitioned-audit-logs.ts diff --git a/backend/scripts/generate-schema-types.ts b/backend/scripts/generate-schema-types.ts index c7b1fcc7a..fc398c2ac 100644 --- a/backend/scripts/generate-schema-types.ts +++ b/backend/scripts/generate-schema-types.ts @@ -90,7 +90,12 @@ const main = async () => { .whereRaw("table_schema = current_schema()") .select<{ tableName: string }[]>("table_name as tableName") .orderBy("table_name") - ).filter((el) => !el.tableName.includes("_migrations") && !el.tableName.includes("partitioned_audit_logs_")); + ).filter( + (el) => + !el.tableName.includes("_migrations") && + !el.tableName.includes("audit_logs_") && + el.tableName !== "intermediate_audit_logs" + ); for (let i = 0; i < tables.length; i += 1) { const { tableName } = tables[i]; diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index d40a4f148..624915276 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -170,9 +170,6 @@ import { TOrgRoles, TOrgRolesInsert, TOrgRolesUpdate, - TPartitionedAuditLogs, - TPartitionedAuditLogsInsert, - TPartitionedAuditLogsUpdate, TPkiAlerts, TPkiAlertsInsert, TPkiAlertsUpdate, @@ -718,11 +715,6 @@ declare module "knex/types/tables" { TAuditLogStreamsInsert, TAuditLogStreamsUpdate >; - [TableName.PartitionedAuditLog]: KnexOriginal.CompositeTableType< - TPartitionedAuditLogs, - TPartitionedAuditLogsInsert, - TPartitionedAuditLogsUpdate - >; [TableName.GitAppInstallSession]: KnexOriginal.CompositeTableType< TGitAppInstallSessions, TGitAppInstallSessionsInsert, diff --git a/backend/src/db/manual-migrations/partition-audit-logs.ts b/backend/src/db/manual-migrations/partition-audit-logs.ts new file mode 100644 index 000000000..6b18aafa4 --- /dev/null +++ b/backend/src/db/manual-migrations/partition-audit-logs.ts @@ -0,0 +1,145 @@ +import kx, { Knex } from "knex"; + +import { TableName } from "../schemas"; + +const INTERMEDIATE_AUDIT_LOG_TABLE = "intermediate_audit_logs"; + +const formatPartitionDate = (date: Date) => { + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, "0"); + const day = String(date.getDate()).padStart(2, "0"); + + return `${year}-${month}-${day}`; +}; + +const createAuditLogPartition = async (knex: Knex, startDate: Date, endDate: Date) => { + const startDateStr = formatPartitionDate(startDate); + const endDateStr = formatPartitionDate(endDate); + + const partitionName = `${TableName.AuditLog}_${startDateStr.replace(/-/g, "")}_${endDateStr.replace(/-/g, "")}`; + + await knex.schema.raw( + `CREATE TABLE ${partitionName} PARTITION OF ${TableName.AuditLog} FOR VALUES FROM ('${startDateStr}') TO ('${endDateStr}')` + ); +}; + +const up = async (knex: Knex): Promise => { + console.info("Dropping primary key of audit log table..."); + await knex.schema.alterTable(TableName.AuditLog, (t) => { + // remove existing keys + t.dropPrimary(); + }); + + // renaming audit log to intermediate table + console.log("Renaming audit log table to the intermediate name"); + await knex.schema.renameTable(TableName.AuditLog, INTERMEDIATE_AUDIT_LOG_TABLE); + + if (!(await knex.schema.hasTable(TableName.AuditLog))) { + const createTableSql = knex.schema + .createTable(TableName.AuditLog, (t) => { + t.uuid("id").defaultTo(knex.fn.uuid()); + t.string("actor").notNullable(); + t.jsonb("actorMetadata").notNullable(); + t.string("ipAddress"); + t.string("eventType").notNullable(); + t.jsonb("eventMetadata"); + t.string("userAgent"); + t.string("userAgentType"); + t.datetime("expiresAt"); + t.timestamps(true, true, true); + t.uuid("orgId"); + t.string("projectId"); + t.string("projectName"); + t.primary(["id", "createdAt"]); + }) + .toString(); + + console.info("Creating partition table..."); + await knex.schema.raw(` + ${createTableSql} PARTITION BY RANGE ("createdAt"); + `); + + console.log("Adding indices..."); + await knex.schema.alterTable(TableName.AuditLog, (t) => { + t.index(["projectId", "createdAt"]); + t.index(["orgId", "createdAt"]); + t.index("expiresAt"); + t.index("orgId"); + t.index("projectId"); + }); + + console.log("Adding GIN indices..."); + + await knex.raw( + `CREATE INDEX IF NOT EXISTS "audit_logs_actorMetadata_idx" ON ${TableName.AuditLog} USING gin("actorMetadata" jsonb_path_ops)` + ); + console.log("GIN index for actorMetadata done"); + + await knex.raw( + `CREATE INDEX IF NOT EXISTS "audit_logs_eventMetadata_idx" ON ${TableName.AuditLog} USING gin("eventMetadata" jsonb_path_ops)` + ); + console.log("GIN index for eventMetadata done"); + + // create default partition + console.log("Creating default partition..."); + await knex.schema.raw(`CREATE TABLE ${TableName.AuditLog}_default PARTITION OF ${TableName.AuditLog} DEFAULT`); + + const nextDate = new Date(); + nextDate.setDate(nextDate.getDate() + 1); + const nextDateStr = formatPartitionDate(nextDate); + + console.log("Attaching existing audit log table as a partition..."); + await knex.schema.raw(` + ALTER TABLE ${INTERMEDIATE_AUDIT_LOG_TABLE} ADD CONSTRAINT audit_log_old + CHECK ( "createdAt" < DATE '${nextDateStr}' ); + + ALTER TABLE ${TableName.AuditLog} ATTACH PARTITION ${INTERMEDIATE_AUDIT_LOG_TABLE} + FOR VALUES FROM (MINVALUE) TO ('${nextDateStr}' ); + `); + + // create partition from now until end of month + console.log("Creating audit log partitions ahead of time... next date:", nextDateStr); + await createAuditLogPartition(knex, nextDate, new Date(nextDate.getFullYear(), nextDate.getMonth() + 1)); + + // create partitions 4 years ahead + const partitionMonths = 4 * 12; + const partitionPromises: Promise[] = []; + for (let x = 1; x <= partitionMonths; x += 1) { + partitionPromises.push( + createAuditLogPartition( + knex, + new Date(nextDate.getFullYear(), nextDate.getMonth() + x, 1), + new Date(nextDate.getFullYear(), nextDate.getMonth() + (x + 1), 1) + ) + ); + } + + await Promise.all(partitionPromises); + console.log("Partition migration complete"); + process.exit(0); + } +}; + +export const executeMigration = async (url: string) => { + console.log("Executing migration to:", url); + const knex = kx({ + client: "pg", + connection: url + }); + + await knex.transaction(async (tx) => { + await up(tx); + }); +}; + +const args = process.argv.slice(2); +const dbUrl = args[0]; + +if (!dbUrl) { + console.error("Please provide a DB connection URL as the first argument."); + process.exit(1); +} + +void executeMigration(dbUrl).then(() => { + console.log("Migration: partition-audit-logs DONE"); +}); diff --git a/backend/src/db/migrations/20241007052449_partition-audit-logs.ts b/backend/src/db/migrations/20241007052449_partition-audit-logs.ts deleted file mode 100644 index e132bb5ef..000000000 --- a/backend/src/db/migrations/20241007052449_partition-audit-logs.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { Knex } from "knex"; - -import { TableName } from "../schemas"; - -const formatPartitionDate = (date: Date) => { - const year = date.getFullYear(); - const month = String(date.getMonth() + 1).padStart(2, "0"); - const day = String(date.getDate()).padStart(2, "0"); - - return `${year}-${month}-${day}`; -}; - -const createAuditLogPartition = async (knex: Knex, startDate: Date, endDate: Date) => { - const startDateStr = formatPartitionDate(startDate); - const endDateStr = formatPartitionDate(endDate); - - const partitionName = `${TableName.PartitionedAuditLog}_${startDateStr.replace(/-/g, "")}_${endDateStr.replace( - /-/g, - "" - )}`; - - await knex.schema.raw( - `CREATE TABLE ${partitionName} PARTITION OF ${TableName.PartitionedAuditLog} FOR VALUES FROM ('${startDateStr}') TO ('${endDateStr}')` - ); -}; - -const isUsingDedicatedAuditLogDb = Boolean(process.env.AUDIT_LOGS_DB_CONNECTION_URI); - -export async function up(knex: Knex): Promise { - if (!isUsingDedicatedAuditLogDb && (await knex.schema.hasTable(TableName.AuditLog))) { - console.info("Dropping primary key of Audit Log table..."); - await knex.schema.alterTable(TableName.AuditLog, (t) => { - // remove existing keys - t.dropPrimary(); - }); - } - - // create a new partitioned table for audit logs - if (!(await knex.schema.hasTable(TableName.PartitionedAuditLog))) { - const createTableSql = knex.schema - .createTable(TableName.PartitionedAuditLog, (t) => { - t.uuid("id").defaultTo(knex.fn.uuid()); - t.string("actor").notNullable(); - t.jsonb("actorMetadata").notNullable(); - t.string("ipAddress"); - t.string("eventType").notNullable(); - t.jsonb("eventMetadata"); - t.string("userAgent"); - t.string("userAgentType"); - t.datetime("expiresAt"); - t.timestamps(true, true, true); - t.uuid("orgId"); - t.string("projectId"); - t.string("projectName"); - t.primary(["id", "createdAt"]); - }) - .toString(); - - console.info("Creating partition table..."); - await knex.schema.raw(` - ${createTableSql} PARTITION BY RANGE ("createdAt"); - `); - - console.log("Adding indices..."); - await knex.schema.alterTable(TableName.PartitionedAuditLog, (t) => { - t.index(["projectId", "createdAt"]); - t.index(["orgId", "createdAt"]); - t.index("expiresAt"); - t.index("orgId"); - t.index("projectId"); - }); - - console.log("Adding GIN indices..."); - - await knex.raw( - `CREATE INDEX IF NOT EXISTS "audit_logs_actorMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("actorMetadata" jsonb_path_ops)` - ); - console.log("GIN index for actorMetadata done"); - - await knex.raw( - `CREATE INDEX IF NOT EXISTS "audit_logs_eventMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("eventMetadata" jsonb_path_ops)` - ); - console.log("GIN index for eventMetadata done"); - - // create default partition - console.log("Creating default partition..."); - await knex.schema.raw( - `CREATE TABLE ${TableName.PartitionedAuditLog}_default PARTITION OF ${TableName.PartitionedAuditLog} DEFAULT` - ); - - const nextDate = new Date(); - nextDate.setDate(nextDate.getDate() + 1); - const nextDateStr = formatPartitionDate(nextDate); - - // attach existing audit log table as a partition ONLY if using the same DB - if (!isUsingDedicatedAuditLogDb) { - console.log("Attaching existing audit log table as a partition..."); - await knex.schema.raw(` - ALTER TABLE ${TableName.AuditLog} ADD CONSTRAINT audit_log_old - CHECK ( "createdAt" < DATE '${nextDateStr}' ); - - ALTER TABLE ${TableName.PartitionedAuditLog} ATTACH PARTITION ${TableName.AuditLog} - FOR VALUES FROM (MINVALUE) TO ('${nextDateStr}' ); - `); - } - - // create partition from now until end of month - console.log("Creating audit log partitions ahead of time... next date:", nextDateStr); - await createAuditLogPartition(knex, nextDate, new Date(nextDate.getFullYear(), nextDate.getMonth() + 1)); - - // create partitions 4 years ahead - const partitionMonths = 4 * 12; - const partitionPromises: Promise[] = []; - for (let x = 1; x <= partitionMonths; x += 1) { - partitionPromises.push( - createAuditLogPartition( - knex, - new Date(nextDate.getFullYear(), nextDate.getMonth() + x, 1), - new Date(nextDate.getFullYear(), nextDate.getMonth() + (x + 1), 1) - ) - ); - } - - await Promise.all(partitionPromises); - console.log("Partition migration complete"); - } -} - -export async function down(knex: Knex): Promise { - const partitionSearchResult = await knex.raw(` - SELECT inhrelid::regclass::text - FROM pg_inherits - WHERE inhparent::regclass::text = '${TableName.PartitionedAuditLog}' - AND inhrelid::regclass::text = '${TableName.AuditLog}' - `); - - const isAuditLogAPartition = partitionSearchResult.rows.length > 0; - if (isAuditLogAPartition) { - // detach audit log from partition - console.log("Detaching original audit log table from new partition table..."); - await knex.schema.raw(` - ALTER TABLE ${TableName.PartitionedAuditLog} DETACH PARTITION ${TableName.AuditLog}; - - ALTER TABLE ${TableName.AuditLog} DROP CONSTRAINT audit_log_old; - `); - - // revert audit log modifications - console.log("Reverting changes made to the audit log table..."); - if (await knex.schema.hasTable(TableName.AuditLog)) { - await knex.schema.alterTable(TableName.AuditLog, (t) => { - // we drop this first because adding to the partition results in a new primary key - t.dropPrimary(); - - // add back the original keys of the audit logs table - t.primary(["id"], { - constraintName: "audit_logs_pkey" - }); - }); - } - } - - await knex.schema.dropTableIfExists(TableName.PartitionedAuditLog); - console.log("Partition rollback complete"); -} diff --git a/backend/src/db/schemas/audit-logs.ts b/backend/src/db/schemas/audit-logs.ts index b8906698b..d1c239724 100644 --- a/backend/src/db/schemas/audit-logs.ts +++ b/backend/src/db/schemas/audit-logs.ts @@ -20,7 +20,8 @@ export const AuditLogsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), orgId: z.string().uuid().nullable().optional(), - projectId: z.string().nullable().optional() + projectId: z.string().nullable().optional(), + projectName: z.string().nullable().optional() }); export type TAuditLogs = z.infer; diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index 86ff5e612..4fcf26c1a 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -55,7 +55,6 @@ export * from "./org-bots"; export * from "./org-memberships"; export * from "./org-roles"; export * from "./organizations"; -export * from "./partitioned-audit-logs"; export * from "./pki-alerts"; export * from "./pki-collection-items"; export * from "./pki-collections"; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 4e241439e..08f3e79ce 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -90,7 +90,6 @@ export enum TableName { OidcConfig = "oidc_configs", LdapGroupMap = "ldap_group_maps", AuditLog = "audit_logs", - PartitionedAuditLog = "partitioned_audit_logs", AuditLogStream = "audit_log_streams", GitAppInstallSession = "git_app_install_sessions", GitAppOrg = "git_app_org", diff --git a/backend/src/db/schemas/partitioned-audit-logs.ts b/backend/src/db/schemas/partitioned-audit-logs.ts deleted file mode 100644 index dd9500e7a..000000000 --- a/backend/src/db/schemas/partitioned-audit-logs.ts +++ /dev/null @@ -1,29 +0,0 @@ -// Code generated by automation script, DO NOT EDIT. -// Automated by pulling database and generating zod schema -// To update. Just run npm run generate:schema -// Written by akhilmhdh. - -import { z } from "zod"; - -import { TImmutableDBKeys } from "./models"; - -export const PartitionedAuditLogsSchema = z.object({ - id: z.string().uuid(), - actor: z.string(), - actorMetadata: z.unknown(), - ipAddress: z.string().nullable().optional(), - eventType: z.string(), - eventMetadata: z.unknown().nullable().optional(), - userAgent: z.string().nullable().optional(), - userAgentType: z.string().nullable().optional(), - expiresAt: z.date().nullable().optional(), - createdAt: z.date(), - updatedAt: z.date(), - orgId: z.string().uuid().nullable().optional(), - projectId: z.string().nullable().optional(), - projectName: z.string().nullable().optional() -}); - -export type TPartitionedAuditLogs = z.infer; -export type TPartitionedAuditLogsInsert = Omit, TImmutableDBKeys>; -export type TPartitionedAuditLogsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/ee/routes/v1/project-router.ts b/backend/src/ee/routes/v1/project-router.ts index aefca8a7b..e3956731e 100644 --- a/backend/src/ee/routes/v1/project-router.ts +++ b/backend/src/ee/routes/v1/project-router.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -import { PartitionedAuditLogsSchema, SecretSnapshotsSchema } from "@app/db/schemas"; +import { AuditLogsSchema, SecretSnapshotsSchema } from "@app/db/schemas"; import { EventType, UserAgentType } from "@app/ee/services/audit-log/audit-log-types"; import { AUDIT_LOGS, PROJECTS } from "@app/lib/api-docs"; import { getLastMidnightDateISO, removeTrailingSlash } from "@app/lib/fn"; @@ -120,7 +120,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }), response: { 200: z.object({ - auditLogs: PartitionedAuditLogsSchema.omit({ + auditLogs: AuditLogsSchema.omit({ eventMetadata: true, eventType: true, actor: true, diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index 214fc7b4f..b2c80aa0b 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -26,7 +26,7 @@ type TFindQuery = { }; export const auditLogDALFactory = (db: TDbClient) => { - const auditLogOrm = ormify(db, TableName.PartitionedAuditLog); + const auditLogOrm = ormify(db, TableName.AuditLog); const find = async ( { @@ -55,13 +55,13 @@ export const auditLogDALFactory = (db: TDbClient) => { try { // Find statements - const sqlQuery = (tx || db.replicaNode())(TableName.PartitionedAuditLog) + const sqlQuery = (tx || db.replicaNode())(TableName.AuditLog) // eslint-disable-next-line func-names .where(function () { if (orgId) { - void this.where(`${TableName.PartitionedAuditLog}.orgId`, orgId); + void this.where(`${TableName.AuditLog}.orgId`, orgId); } else if (projectId) { - void this.where(`${TableName.PartitionedAuditLog}.projectId`, projectId); + void this.where(`${TableName.AuditLog}.projectId`, projectId); } }); @@ -71,10 +71,10 @@ export const auditLogDALFactory = (db: TDbClient) => { // Select statements void sqlQuery - .select(selectAllTableCols(TableName.PartitionedAuditLog)) + .select(selectAllTableCols(TableName.AuditLog)) .limit(limit) .offset(offset) - .orderBy(`${TableName.PartitionedAuditLog}.createdAt`, "desc"); + .orderBy(`${TableName.AuditLog}.createdAt`, "desc"); // Special case: Filter by actor ID if (actorId) { @@ -100,10 +100,10 @@ export const auditLogDALFactory = (db: TDbClient) => { // Filter by date range if (startDate) { - void sqlQuery.where(`${TableName.PartitionedAuditLog}.createdAt`, ">=", startDate); + void sqlQuery.where(`${TableName.AuditLog}.createdAt`, ">=", startDate); } if (endDate) { - void sqlQuery.where(`${TableName.PartitionedAuditLog}.createdAt`, "<=", endDate); + void sqlQuery.where(`${TableName.AuditLog}.createdAt`, "<=", endDate); } // we timeout long running queries to prevent DB resource issues (2 minutes) @@ -135,13 +135,13 @@ export const auditLogDALFactory = (db: TDbClient) => { logger.info(`${QueueName.DailyResourceCleanUp}: audit log started`); do { try { - const findExpiredLogSubQuery = (tx || db)(TableName.PartitionedAuditLog) + const findExpiredLogSubQuery = (tx || db)(TableName.AuditLog) .where("expiresAt", "<", today) .select("id") .limit(AUDIT_LOG_PRUNE_BATCH_SIZE); // eslint-disable-next-line no-await-in-loop - deletedAuditLogIds = await (tx || db)(TableName.PartitionedAuditLog) + deletedAuditLogIds = await (tx || db)(TableName.AuditLog) .whereIn("id", findExpiredLogSubQuery) .del() .returning("id"); diff --git a/backend/src/server/routes/v1/organization-router.ts b/backend/src/server/routes/v1/organization-router.ts index e9421af7f..d19317c4b 100644 --- a/backend/src/server/routes/v1/organization-router.ts +++ b/backend/src/server/routes/v1/organization-router.ts @@ -1,12 +1,12 @@ import { z } from "zod"; import { + AuditLogsSchema, GroupsSchema, IncidentContactsSchema, OrganizationsSchema, OrgMembershipsSchema, OrgRolesSchema, - PartitionedAuditLogsSchema, UsersSchema } from "@app/db/schemas"; import { EventType, UserAgentType } from "@app/ee/services/audit-log/audit-log-types"; @@ -115,7 +115,7 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { response: { 200: z.object({ - auditLogs: PartitionedAuditLogsSchema.omit({ + auditLogs: AuditLogsSchema.omit({ eventMetadata: true, eventType: true, actor: true, diff --git a/frontend/src/hooks/api/auditLogs/types.tsx b/frontend/src/hooks/api/auditLogs/types.tsx index 76eb58517..567e1d0b2 100644 --- a/frontend/src/hooks/api/auditLogs/types.tsx +++ b/frontend/src/hooks/api/auditLogs/types.tsx @@ -887,4 +887,5 @@ export type AuditLog = { createdAt: string; updatedAt: string; projectName?: string; + projectId?: string; }; diff --git a/frontend/src/views/Org/AuditLogsPage/components/LogsTableRow.tsx b/frontend/src/views/Org/AuditLogsPage/components/LogsTableRow.tsx index e4620a477..2e8e01871 100644 --- a/frontend/src/views/Org/AuditLogsPage/components/LogsTableRow.tsx +++ b/frontend/src/views/Org/AuditLogsPage/components/LogsTableRow.tsx @@ -573,7 +573,7 @@ export const LogsTableRow = ({ auditLog, isOrgAuditLogs, showActorColumn }: Prop {formatDate(auditLog.createdAt)} {`${eventToNameMap[auditLog.event.type]}`} - {isOrgAuditLogs && {auditLog?.projectName ?? "N/A"}} + {isOrgAuditLogs && {auditLog?.projectName ?? auditLog?.projectId ?? "N/A"}} {showActorColumn && renderActor(auditLog.actor)} {renderSource()} {renderMetadata(auditLog.event)} From a823347c9949d4c7d46b3c0c1a824496734f573d Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 8 Oct 2024 21:21:32 +0800 Subject: [PATCH 3/6] misc: added proper deletion of indices --- .../manual-migrations/partition-audit-logs.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/backend/src/db/manual-migrations/partition-audit-logs.ts b/backend/src/db/manual-migrations/partition-audit-logs.ts index 6b18aafa4..1bdbb82f7 100644 --- a/backend/src/db/manual-migrations/partition-audit-logs.ts +++ b/backend/src/db/manual-migrations/partition-audit-logs.ts @@ -30,6 +30,24 @@ const up = async (knex: Knex): Promise => { t.dropPrimary(); }); + // Get all indices of the audit log table and drop them + const indexNames: { rows: { indexname: string }[] } = await knex.raw( + ` + SELECT indexname + FROM pg_indexes + WHERE tablename = '${TableName.AuditLog}' + ` + ); + + console.log( + "Deleting existing audit log indices:", + indexNames.rows.map((e) => e.indexname) + ); + + for await (const row of indexNames.rows) { + await knex.raw(`DROP INDEX IF EXISTS ${row.indexname}`); + } + // renaming audit log to intermediate table console.log("Renaming audit log table to the intermediate name"); await knex.schema.renameTable(TableName.AuditLog, INTERMEDIATE_AUDIT_LOG_TABLE); @@ -116,7 +134,6 @@ const up = async (knex: Knex): Promise => { await Promise.all(partitionPromises); console.log("Partition migration complete"); - process.exit(0); } }; @@ -142,4 +159,5 @@ if (!dbUrl) { void executeMigration(dbUrl).then(() => { console.log("Migration: partition-audit-logs DONE"); + process.exit(0); }); From 68c38f228dfad9894289493dcd7e4380b3589ee4 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 8 Oct 2024 21:29:36 +0800 Subject: [PATCH 4/6] misc: moved to using env --- backend/src/db/manual-migrations/partition-audit-logs.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/src/db/manual-migrations/partition-audit-logs.ts b/backend/src/db/manual-migrations/partition-audit-logs.ts index 1bdbb82f7..5c3913341 100644 --- a/backend/src/db/manual-migrations/partition-audit-logs.ts +++ b/backend/src/db/manual-migrations/partition-audit-logs.ts @@ -138,7 +138,7 @@ const up = async (knex: Knex): Promise => { }; export const executeMigration = async (url: string) => { - console.log("Executing migration to:", url); + console.log("Executing migration..."); const knex = kx({ client: "pg", connection: url @@ -149,9 +149,7 @@ export const executeMigration = async (url: string) => { }); }; -const args = process.argv.slice(2); -const dbUrl = args[0]; - +const dbUrl = process.env.AUDIT_LOGS_DB_URL; if (!dbUrl) { console.error("Please provide a DB connection URL as the first argument."); process.exit(1); From e191a72ca079d4cf908dcce95c5ec0eef9c489a8 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 8 Oct 2024 21:38:38 +0800 Subject: [PATCH 5/6] misc: finalized env name --- backend/src/db/manual-migrations/partition-audit-logs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/db/manual-migrations/partition-audit-logs.ts b/backend/src/db/manual-migrations/partition-audit-logs.ts index 5c3913341..4d2d837ff 100644 --- a/backend/src/db/manual-migrations/partition-audit-logs.ts +++ b/backend/src/db/manual-migrations/partition-audit-logs.ts @@ -149,7 +149,7 @@ export const executeMigration = async (url: string) => { }); }; -const dbUrl = process.env.AUDIT_LOGS_DB_URL; +const dbUrl = process.env.AUDIT_LOGS_DB_CONNECTION_URI; if (!dbUrl) { console.error("Please provide a DB connection URL as the first argument."); process.exit(1); From 391c9abbb03bd6311136a87b51076aab3d7d02b4 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 8 Oct 2024 22:49:11 +0800 Subject: [PATCH 6/6] misc: updated error description --- backend/src/db/manual-migrations/partition-audit-logs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/db/manual-migrations/partition-audit-logs.ts b/backend/src/db/manual-migrations/partition-audit-logs.ts index 4d2d837ff..382ef0dbf 100644 --- a/backend/src/db/manual-migrations/partition-audit-logs.ts +++ b/backend/src/db/manual-migrations/partition-audit-logs.ts @@ -151,7 +151,7 @@ export const executeMigration = async (url: string) => { const dbUrl = process.env.AUDIT_LOGS_DB_CONNECTION_URI; if (!dbUrl) { - console.error("Please provide a DB connection URL as the first argument."); + console.error("Please provide a DB connection URL to the AUDIT_LOGS_DB_CONNECTION_URI env"); process.exit(1); }