awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-07 15:27:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: updated backend logic to use parition and speed up audit log queries

Browse Source
This commit is contained in:
=
2025-07-02 23:04:36 +05:30
parent f62eb9f8a2
commit 147c21ab9f
8 changed files with 46 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/auto-start-migrations.ts
+28 -28
View File
@@ -44,17 +44,17 @@ export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs
}); });
} }
} }
if (auditLogDb) { // if (auditLogDb) {
const hasMigrationTableInAuditLog = await auditLogDb.schema.hasTable(migrationTable); // const hasMigrationTableInAuditLog = await auditLogDb.schema.hasTable(migrationTable);
if (hasMigrationTableInAuditLog) { // if (hasMigrationTableInAuditLog) {
const firstFile = (await auditLogDb(migrationTable).where({}).first()) as { name: string }; // const firstFile = (await auditLogDb(migrationTable).where({}).first()) as { name: string };
if (firstFile?.name?.includes(".ts")) { // if (firstFile?.name?.includes(".ts")) {
await auditLogDb(migrationTable).update({ // await auditLogDb(migrationTable).update({
name: auditLogDb.raw("REPLACE(name, '.ts', '.mjs')") // name: auditLogDb.raw("REPLACE(name, '.ts', '.mjs')")
}); // });
} // }
} // }
} // }
} }
const shouldRunMigration = Boolean( const shouldRunMigration = Boolean(
@@ -65,23 +65,23 @@ export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs
return; return;
} }
if (auditLogDb) { // if (auditLogDb) {
await auditLogDb.transaction(async (tx) => { // await auditLogDb.transaction(async (tx) => {
await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]); // await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]);
logger.info("Running audit log migrations."); // logger.info("Running audit log migrations.");
//
const didPreviousInstanceRunMigration = !(await auditLogDb.migrate // const didPreviousInstanceRunMigration = !(await auditLogDb.migrate
.status(migrationConfig) // .status(migrationConfig)
.catch(migrationStatusCheckErrorHandler)); // .catch(migrationStatusCheckErrorHandler));
if (didPreviousInstanceRunMigration) { // if (didPreviousInstanceRunMigration) {
logger.info("No audit log migrations pending: Applied by previous instance. Skipping migration process."); // logger.info("No audit log migrations pending: Applied by previous instance. Skipping migration process.");
return; // return;
} // }
//
await auditLogDb.migrate.latest(migrationConfig); // await auditLogDb.migrate.latest(migrationConfig);
logger.info("Finished audit log migrations."); // logger.info("Finished audit log migrations.");
}); // });
} // }
await applicationDb.transaction(async (tx) => { await applicationDb.transaction(async (tx) => {
await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]); await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]);
backend/src/db/instance.ts
+2 -1
View File
@@ -110,7 +110,8 @@ export const initAuditLogDbConnection = ({
}, },
migrations: { migrations: {
tableName: "infisical_migrations" tableName: "infisical_migrations"
} },
pool: { min: 0, max: 10 }
}); });
// we add these overrides so that auditLogDb and the primary DB are interchangeable // we add these overrides so that auditLogDb and the primary DB are interchangeable
backend/src/ee/routes/v1/project-router.ts
+1 -1
View File
@@ -115,7 +115,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
eventType: z.nativeEnum(EventType).optional().describe(AUDIT_LOGS.EXPORT.eventType), eventType: z.nativeEnum(EventType).optional().describe(AUDIT_LOGS.EXPORT.eventType),
userAgentType: z.nativeEnum(UserAgentType).optional().describe(AUDIT_LOGS.EXPORT.userAgentType), userAgentType: z.nativeEnum(UserAgentType).optional().describe(AUDIT_LOGS.EXPORT.userAgentType),
startDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.startDate), startDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.startDate),
endDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.endDate), endDate: z.string().datetime().default(new Date().toISOString()).describe(AUDIT_LOGS.EXPORT.endDate),
offset: z.coerce.number().default(0).describe(AUDIT_LOGS.EXPORT.offset), offset: z.coerce.number().default(0).describe(AUDIT_LOGS.EXPORT.offset),
limit: z.coerce.number().default(20).describe(AUDIT_LOGS.EXPORT.limit), limit: z.coerce.number().default(20).describe(AUDIT_LOGS.EXPORT.limit),
actor: z.string().optional().describe(AUDIT_LOGS.EXPORT.actor) actor: z.string().optional().describe(AUDIT_LOGS.EXPORT.actor)
backend/src/ee/services/audit-log/audit-log-dal.ts
+9 -17
View File
@@ -30,10 +30,10 @@ type TFindQuery = {
actor?: string; actor?: string;
projectId?: string; projectId?: string;
environment?: string; environment?: string;
orgId?: string; orgId: string;
eventType?: string; eventType?: string;
startDate?: string; startDate: string;
endDate?: string; endDate: string;
userAgentType?: string; userAgentType?: string;
limit?: number; limit?: number;
offset?: number; offset?: number;
@@ -61,18 +61,16 @@ export const auditLogDALFactory = (db: TDbClient) => {
}, },
tx tx
) => { ) => {
if (!orgId && !projectId) {
throw new Error("Either orgId or projectId must be provided");
}
try { try {
// Find statements // Find statements
const sqlQuery = (tx || db.replicaNode())(TableName.AuditLog) const sqlQuery = (tx || db.replicaNode())(TableName.AuditLog)
.where(`${TableName.AuditLog}.orgId`, orgId)
.whereRaw(`"${TableName.AuditLog}"."createdAt" >= ?::timestamptz`, [startDate])
.andWhereRaw(`"${TableName.AuditLog}"."createdAt" < ?::timestamptz`, [endDate])
// eslint-disable-next-line func-names // eslint-disable-next-line func-names
.where(function () { .where(function () {
if (orgId) { if (projectId) {
void this.where(`${TableName.AuditLog}.orgId`, orgId);
} else if (projectId) {
void this.where(`${TableName.AuditLog}.projectId`, projectId); void this.where(`${TableName.AuditLog}.projectId`, projectId);
} }
}); });
@@ -135,14 +133,6 @@ export const auditLogDALFactory = (db: TDbClient) => {
void sqlQuery.whereIn("eventType", eventType); void sqlQuery.whereIn("eventType", eventType);
} }
// Filter by date range
if (startDate) {
void sqlQuery.whereRaw(`"${TableName.AuditLog}"."createdAt" >= ?::timestamptz`, [startDate]);
}
if (endDate) {
void sqlQuery.whereRaw(`"${TableName.AuditLog}"."createdAt" <= ?::timestamptz`, [endDate]);
}
// we timeout long running queries to prevent DB resource issues (2 minutes) // we timeout long running queries to prevent DB resource issues (2 minutes)
const docs = await sqlQuery.timeout(1000 * 120); const docs = await sqlQuery.timeout(1000 * 120);
@@ -174,6 +164,8 @@ export const auditLogDALFactory = (db: TDbClient) => {
try { try {
const findExpiredLogSubQuery = (tx || db)(TableName.AuditLog) const findExpiredLogSubQuery = (tx || db)(TableName.AuditLog)
.where("expiresAt", "<", today) .where("expiresAt", "<", today)
.where("createdAt", "<", today) // to use audit log partition
.orderBy(`${TableName.AuditLog}.createdAt`, "desc")
.select("id") .select("id")
.limit(AUDIT_LOG_PRUNE_BATCH_SIZE); .limit(AUDIT_LOG_PRUNE_BATCH_SIZE);
backend/src/ee/services/audit-log/audit-log-service.ts
+2 -1
View File
@@ -67,7 +67,8 @@ export const auditLogServiceFactory = ({
secretPath: filter.secretPath, secretPath: filter.secretPath,
secretKey: filter.secretKey, secretKey: filter.secretKey,
environment: filter.environment, environment: filter.environment,
...(filter.projectId ? { projectId: filter.projectId } : { orgId: actorOrgId }) orgId: actorOrgId,
...(filter.projectId ? { projectId: filter.projectId } : {})
}); });
return auditLogs.map(({ eventType: logEventType, actor: eActor, actorMetadata, eventMetadata, ...el }) => ({ return auditLogs.map(({ eventType: logEventType, actor: eActor, actorMetadata, eventMetadata, ...el }) => ({
backend/src/ee/services/audit-log/audit-log-types.ts
+2 -2
View File
@@ -56,8 +56,8 @@ export type TListProjectAuditLogDTO = {
eventType?: EventType[]; eventType?: EventType[];
offset?: number; offset?: number;
limit: number; limit: number;
endDate?: string; endDate: string;
startDate?: string; startDate: string;
projectId?: string; projectId?: string;
environment?: string; environment?: string;
auditLogActorId?: string; auditLogActorId?: string;
backend/src/server/routes/v1/organization-router.ts
+1 -3
View File
@@ -153,12 +153,11 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
}) })
.describe(AUDIT_LOGS.EXPORT.eventMetadata), .describe(AUDIT_LOGS.EXPORT.eventMetadata),
startDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.startDate), startDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.startDate),
endDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.endDate), endDate: z.string().datetime().default(new Date().toISOString()).describe(AUDIT_LOGS.EXPORT.endDate),
offset: z.coerce.number().default(0).describe(AUDIT_LOGS.EXPORT.offset), offset: z.coerce.number().default(0).describe(AUDIT_LOGS.EXPORT.offset),
limit: z.coerce.number().default(20).describe(AUDIT_LOGS.EXPORT.limit), limit: z.coerce.number().default(20).describe(AUDIT_LOGS.EXPORT.limit),
actor: z.string().optional().describe(AUDIT_LOGS.EXPORT.actor) actor: z.string().optional().describe(AUDIT_LOGS.EXPORT.actor)
}), }),
response: { response: {
200: z.object({ 200: z.object({
auditLogs: AuditLogsSchema.omit({ auditLogs: AuditLogsSchema.omit({
@@ -195,7 +194,6 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
actorType: req.query.actorType, actorType: req.query.actorType,
eventType: req.query.eventType as EventType[] | undefined eventType: req.query.eventType as EventType[] | undefined
}, },
actorId: req.permission.id, actorId: req.permission.id,
actorOrgId: req.permission.orgId, actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod, actorAuthMethod: req.permission.authMethod,
backend/src/services/resource-cleanup/resource-cleanup-queue.ts
+1 -1
View File
@@ -47,7 +47,6 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
queueService.start(QueueName.DailyResourceCleanUp, async () => { queueService.start(QueueName.DailyResourceCleanUp, async () => {
logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`); logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`);
await secretDAL.pruneSecretReminders(queueService); await secretDAL.pruneSecretReminders(queueService);
await auditLogDAL.pruneAuditLog();
await identityAccessTokenDAL.removeExpiredTokens(); await identityAccessTokenDAL.removeExpiredTokens();
await identityUniversalAuthClientSecretDAL.removeExpiredClientSecrets(); await identityUniversalAuthClientSecretDAL.removeExpiredClientSecrets();
await secretSharingDAL.pruneExpiredSharedSecrets(); await secretSharingDAL.pruneExpiredSharedSecrets();
@@ -58,6 +57,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
await secretFolderVersionDAL.pruneExcessVersions(); await secretFolderVersionDAL.pruneExcessVersions();
await serviceTokenService.notifyExpiringTokens(); await serviceTokenService.notifyExpiringTokens();
await orgService.notifyInvitedUsers(); await orgService.notifyInvitedUsers();
await auditLogDAL.pruneAuditLog();
logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`); logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`);
}); });