From 147c21ab9f257c3d92d99bd35ea3be113688a724 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 2 Jul 2025 19:35:51 +0530 Subject: [PATCH] feat: updated backend logic to use parition and speed up audit log queries --- backend/src/auto-start-migrations.ts | 56 +++++++++---------- backend/src/db/instance.ts | 3 +- backend/src/ee/routes/v1/project-router.ts | 2 +- .../ee/services/audit-log/audit-log-dal.ts | 26 +++------ .../services/audit-log/audit-log-service.ts | 3 +- .../ee/services/audit-log/audit-log-types.ts | 4 +- .../server/routes/v1/organization-router.ts | 4 +- .../resource-cleanup-queue.ts | 2 +- 8 files changed, 46 insertions(+), 54 deletions(-) diff --git a/backend/src/auto-start-migrations.ts b/backend/src/auto-start-migrations.ts index 88f6dea69..4df6aa1cf 100644 --- a/backend/src/auto-start-migrations.ts +++ b/backend/src/auto-start-migrations.ts @@ -44,17 +44,17 @@ export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs }); } } - if (auditLogDb) { - const hasMigrationTableInAuditLog = await auditLogDb.schema.hasTable(migrationTable); - if (hasMigrationTableInAuditLog) { - const firstFile = (await auditLogDb(migrationTable).where({}).first()) as { name: string }; - if (firstFile?.name?.includes(".ts")) { - await auditLogDb(migrationTable).update({ - name: auditLogDb.raw("REPLACE(name, '.ts', '.mjs')") - }); - } - } - } + // if (auditLogDb) { + // const hasMigrationTableInAuditLog = await auditLogDb.schema.hasTable(migrationTable); + // if (hasMigrationTableInAuditLog) { + // const firstFile = (await auditLogDb(migrationTable).where({}).first()) as { name: string }; + // if (firstFile?.name?.includes(".ts")) { + // await auditLogDb(migrationTable).update({ + // name: auditLogDb.raw("REPLACE(name, '.ts', '.mjs')") + // }); + // } + // } + // } } const shouldRunMigration = Boolean( @@ -65,23 +65,23 @@ export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs return; } - if (auditLogDb) { - await auditLogDb.transaction(async (tx) => { - await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]); - logger.info("Running audit log migrations."); - - const didPreviousInstanceRunMigration = !(await auditLogDb.migrate - .status(migrationConfig) - .catch(migrationStatusCheckErrorHandler)); - if (didPreviousInstanceRunMigration) { - logger.info("No audit log migrations pending: Applied by previous instance. Skipping migration process."); - return; - } - - await auditLogDb.migrate.latest(migrationConfig); - logger.info("Finished audit log migrations."); - }); - } + // if (auditLogDb) { + // await auditLogDb.transaction(async (tx) => { + // await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]); + // logger.info("Running audit log migrations."); + // + // const didPreviousInstanceRunMigration = !(await auditLogDb.migrate + // .status(migrationConfig) + // .catch(migrationStatusCheckErrorHandler)); + // if (didPreviousInstanceRunMigration) { + // logger.info("No audit log migrations pending: Applied by previous instance. Skipping migration process."); + // return; + // } + // + // await auditLogDb.migrate.latest(migrationConfig); + // logger.info("Finished audit log migrations."); + // }); + // } await applicationDb.transaction(async (tx) => { await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]); diff --git a/backend/src/db/instance.ts b/backend/src/db/instance.ts index 3ce8148aa..b1ecf7d65 100644 --- a/backend/src/db/instance.ts +++ b/backend/src/db/instance.ts @@ -110,7 +110,8 @@ export const initAuditLogDbConnection = ({ }, migrations: { tableName: "infisical_migrations" - } + }, + pool: { min: 0, max: 10 } }); // we add these overrides so that auditLogDb and the primary DB are interchangeable diff --git a/backend/src/ee/routes/v1/project-router.ts b/backend/src/ee/routes/v1/project-router.ts index ab9d1be6c..f8b0770ee 100644 --- a/backend/src/ee/routes/v1/project-router.ts +++ b/backend/src/ee/routes/v1/project-router.ts @@ -115,7 +115,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { eventType: z.nativeEnum(EventType).optional().describe(AUDIT_LOGS.EXPORT.eventType), userAgentType: z.nativeEnum(UserAgentType).optional().describe(AUDIT_LOGS.EXPORT.userAgentType), startDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.startDate), - endDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.endDate), + endDate: z.string().datetime().default(new Date().toISOString()).describe(AUDIT_LOGS.EXPORT.endDate), offset: z.coerce.number().default(0).describe(AUDIT_LOGS.EXPORT.offset), limit: z.coerce.number().default(20).describe(AUDIT_LOGS.EXPORT.limit), actor: z.string().optional().describe(AUDIT_LOGS.EXPORT.actor) diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index 874460b36..5e0a2ef05 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -30,10 +30,10 @@ type TFindQuery = { actor?: string; projectId?: string; environment?: string; - orgId?: string; + orgId: string; eventType?: string; - startDate?: string; - endDate?: string; + startDate: string; + endDate: string; userAgentType?: string; limit?: number; offset?: number; @@ -61,18 +61,16 @@ export const auditLogDALFactory = (db: TDbClient) => { }, tx ) => { - if (!orgId && !projectId) { - throw new Error("Either orgId or projectId must be provided"); - } try { // Find statements const sqlQuery = (tx || db.replicaNode())(TableName.AuditLog) + .where(`${TableName.AuditLog}.orgId`, orgId) + .whereRaw(`"${TableName.AuditLog}"."createdAt" >= ?::timestamptz`, [startDate]) + .andWhereRaw(`"${TableName.AuditLog}"."createdAt" < ?::timestamptz`, [endDate]) // eslint-disable-next-line func-names .where(function () { - if (orgId) { - void this.where(`${TableName.AuditLog}.orgId`, orgId); - } else if (projectId) { + if (projectId) { void this.where(`${TableName.AuditLog}.projectId`, projectId); } }); @@ -135,14 +133,6 @@ export const auditLogDALFactory = (db: TDbClient) => { void sqlQuery.whereIn("eventType", eventType); } - // Filter by date range - if (startDate) { - void sqlQuery.whereRaw(`"${TableName.AuditLog}"."createdAt" >= ?::timestamptz`, [startDate]); - } - if (endDate) { - void sqlQuery.whereRaw(`"${TableName.AuditLog}"."createdAt" <= ?::timestamptz`, [endDate]); - } - // we timeout long running queries to prevent DB resource issues (2 minutes) const docs = await sqlQuery.timeout(1000 * 120); @@ -174,6 +164,8 @@ export const auditLogDALFactory = (db: TDbClient) => { try { const findExpiredLogSubQuery = (tx || db)(TableName.AuditLog) .where("expiresAt", "<", today) + .where("createdAt", "<", today) // to use audit log partition + .orderBy(`${TableName.AuditLog}.createdAt`, "desc") .select("id") .limit(AUDIT_LOG_PRUNE_BATCH_SIZE); diff --git a/backend/src/ee/services/audit-log/audit-log-service.ts b/backend/src/ee/services/audit-log/audit-log-service.ts index bf00a499a..333847734 100644 --- a/backend/src/ee/services/audit-log/audit-log-service.ts +++ b/backend/src/ee/services/audit-log/audit-log-service.ts @@ -67,7 +67,8 @@ export const auditLogServiceFactory = ({ secretPath: filter.secretPath, secretKey: filter.secretKey, environment: filter.environment, - ...(filter.projectId ? { projectId: filter.projectId } : { orgId: actorOrgId }) + orgId: actorOrgId, + ...(filter.projectId ? { projectId: filter.projectId } : {}) }); return auditLogs.map(({ eventType: logEventType, actor: eActor, actorMetadata, eventMetadata, ...el }) => ({ diff --git a/backend/src/ee/services/audit-log/audit-log-types.ts b/backend/src/ee/services/audit-log/audit-log-types.ts index a2acb62c4..625dd6556 100644 --- a/backend/src/ee/services/audit-log/audit-log-types.ts +++ b/backend/src/ee/services/audit-log/audit-log-types.ts @@ -56,8 +56,8 @@ export type TListProjectAuditLogDTO = { eventType?: EventType[]; offset?: number; limit: number; - endDate?: string; - startDate?: string; + endDate: string; + startDate: string; projectId?: string; environment?: string; auditLogActorId?: string; diff --git a/backend/src/server/routes/v1/organization-router.ts b/backend/src/server/routes/v1/organization-router.ts index b3fceb201..3d900b403 100644 --- a/backend/src/server/routes/v1/organization-router.ts +++ b/backend/src/server/routes/v1/organization-router.ts @@ -153,12 +153,11 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { }) .describe(AUDIT_LOGS.EXPORT.eventMetadata), startDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.startDate), - endDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.endDate), + endDate: z.string().datetime().default(new Date().toISOString()).describe(AUDIT_LOGS.EXPORT.endDate), offset: z.coerce.number().default(0).describe(AUDIT_LOGS.EXPORT.offset), limit: z.coerce.number().default(20).describe(AUDIT_LOGS.EXPORT.limit), actor: z.string().optional().describe(AUDIT_LOGS.EXPORT.actor) }), - response: { 200: z.object({ auditLogs: AuditLogsSchema.omit({ @@ -195,7 +194,6 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { actorType: req.query.actorType, eventType: req.query.eventType as EventType[] | undefined }, - actorId: req.permission.id, actorOrgId: req.permission.orgId, actorAuthMethod: req.permission.authMethod, diff --git a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts index deb4d0cb2..39926488f 100644 --- a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts +++ b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts @@ -47,7 +47,6 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ queueService.start(QueueName.DailyResourceCleanUp, async () => { logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`); await secretDAL.pruneSecretReminders(queueService); - await auditLogDAL.pruneAuditLog(); await identityAccessTokenDAL.removeExpiredTokens(); await identityUniversalAuthClientSecretDAL.removeExpiredClientSecrets(); await secretSharingDAL.pruneExpiredSharedSecrets(); @@ -58,6 +57,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ await secretFolderVersionDAL.pruneExcessVersions(); await serviceTokenService.notifyExpiringTokens(); await orgService.notifyInvitedUsers(); + await auditLogDAL.pruneAuditLog(); logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`); });