awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: updated backend logic to use parition and speed up audit log queries

Browse Source
This commit is contained in:
=
2025-07-02 19:35:51 +05:30
parent f62eb9f8a2
commit 147c21ab9f
8 changed files with 46 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
backend/src/auto-start-migrations.ts
View File

@@ -44,17 +44,17 @@ export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs
});
}
}
if (auditLogDb) {
const hasMigrationTableInAuditLog = await auditLogDb.schema.hasTable(migrationTable);
if (hasMigrationTableInAuditLog) {
const firstFile = (await auditLogDb(migrationTable).where({}).first()) as { name: string };
if (firstFile?.name?.includes(".ts")) {
await auditLogDb(migrationTable).update({
name: auditLogDb.raw("REPLACE(name, '.ts', '.mjs')")
});
}
}
}
// if (auditLogDb) {
// const hasMigrationTableInAuditLog = await auditLogDb.schema.hasTable(migrationTable);
// if (hasMigrationTableInAuditLog) {
// const firstFile = (await auditLogDb(migrationTable).where({}).first()) as { name: string };
// if (firstFile?.name?.includes(".ts")) {
// await auditLogDb(migrationTable).update({
// name: auditLogDb.raw("REPLACE(name, '.ts', '.mjs')")
// });
// }
// }
// }
}
const shouldRunMigration = Boolean(
@@ -65,23 +65,23 @@ export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs
return;
}
if (auditLogDb) {
await auditLogDb.transaction(async (tx) => {
await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]);
logger.info("Running audit log migrations.");
const didPreviousInstanceRunMigration = !(await auditLogDb.migrate
.status(migrationConfig)
.catch(migrationStatusCheckErrorHandler));
if (didPreviousInstanceRunMigration) {
logger.info("No audit log migrations pending: Applied by previous instance. Skipping migration process.");
return;
}
await auditLogDb.migrate.latest(migrationConfig);
logger.info("Finished audit log migrations.");
});
}
// if (auditLogDb) {
// await auditLogDb.transaction(async (tx) => {
// await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]);
// logger.info("Running audit log migrations.");
//
// const didPreviousInstanceRunMigration = !(await auditLogDb.migrate
// .status(migrationConfig)
// .catch(migrationStatusCheckErrorHandler));
// if (didPreviousInstanceRunMigration) {
// logger.info("No audit log migrations pending: Applied by previous instance. Skipping migration process.");
// return;
// }
//
// await auditLogDb.migrate.latest(migrationConfig);
// logger.info("Finished audit log migrations.");
// });
// }
await applicationDb.transaction(async (tx) => {
await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]);

3
backend/src/db/instance.ts
View File

@@ -110,7 +110,8 @@ export const initAuditLogDbConnection = ({
},
migrations: {
tableName: "infisical_migrations"
}
},
pool: { min: 0, max: 10 }
});
// we add these overrides so that auditLogDb and the primary DB are interchangeable

2
backend/src/ee/routes/v1/project-router.ts
View File

@@ -115,7 +115,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
eventType: z.nativeEnum(EventType).optional().describe(AUDIT_LOGS.EXPORT.eventType),
userAgentType: z.nativeEnum(UserAgentType).optional().describe(AUDIT_LOGS.EXPORT.userAgentType),
startDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.startDate),
endDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.endDate),
endDate: z.string().datetime().default(new Date().toISOString()).describe(AUDIT_LOGS.EXPORT.endDate),
offset: z.coerce.number().default(0).describe(AUDIT_LOGS.EXPORT.offset),
limit: z.coerce.number().default(20).describe(AUDIT_LOGS.EXPORT.limit),
actor: z.string().optional().describe(AUDIT_LOGS.EXPORT.actor)

26
backend/src/ee/services/audit-log/audit-log-dal.ts
View File

@@ -30,10 +30,10 @@ type TFindQuery = {
actor?: string;
projectId?: string;
environment?: string;
orgId?: string;
orgId: string;
eventType?: string;
startDate?: string;
endDate?: string;
startDate: string;
endDate: string;
userAgentType?: string;
limit?: number;
offset?: number;
@@ -61,18 +61,16 @@ export const auditLogDALFactory = (db: TDbClient) => {
},
tx
) => {
if (!orgId && !projectId) {
throw new Error("Either orgId or projectId must be provided");
}
try {
// Find statements
const sqlQuery = (tx || db.replicaNode())(TableName.AuditLog)
.where(`${TableName.AuditLog}.orgId`, orgId)
.whereRaw(`"${TableName.AuditLog}"."createdAt" >= ?::timestamptz`, [startDate])
.andWhereRaw(`"${TableName.AuditLog}"."createdAt" < ?::timestamptz`, [endDate])
// eslint-disable-next-line func-names
.where(function () {
if (orgId) {
void this.where(`${TableName.AuditLog}.orgId`, orgId);
} else if (projectId) {
if (projectId) {
void this.where(`${TableName.AuditLog}.projectId`, projectId);
}
});
@@ -135,14 +133,6 @@ export const auditLogDALFactory = (db: TDbClient) => {
void sqlQuery.whereIn("eventType", eventType);
}
// Filter by date range
if (startDate) {
void sqlQuery.whereRaw(`"${TableName.AuditLog}"."createdAt" >= ?::timestamptz`, [startDate]);
}
if (endDate) {
void sqlQuery.whereRaw(`"${TableName.AuditLog}"."createdAt" <= ?::timestamptz`, [endDate]);
}
// we timeout long running queries to prevent DB resource issues (2 minutes)
const docs = await sqlQuery.timeout(1000 * 120);
@@ -174,6 +164,8 @@ export const auditLogDALFactory = (db: TDbClient) => {
try {
const findExpiredLogSubQuery = (tx || db)(TableName.AuditLog)
.where("expiresAt", "<", today)
.where("createdAt", "<", today) // to use audit log partition
.orderBy(`${TableName.AuditLog}.createdAt`, "desc")
.select("id")
.limit(AUDIT_LOG_PRUNE_BATCH_SIZE);

3
backend/src/ee/services/audit-log/audit-log-service.ts
View File

@@ -67,7 +67,8 @@ export const auditLogServiceFactory = ({
secretPath: filter.secretPath,
secretKey: filter.secretKey,
environment: filter.environment,
...(filter.projectId ? { projectId: filter.projectId } : { orgId: actorOrgId })
orgId: actorOrgId,
...(filter.projectId ? { projectId: filter.projectId } : {})
});
return auditLogs.map(({ eventType: logEventType, actor: eActor, actorMetadata, eventMetadata, ...el }) => ({

4
backend/src/ee/services/audit-log/audit-log-types.ts
View File

@@ -56,8 +56,8 @@ export type TListProjectAuditLogDTO = {
eventType?: EventType[];
offset?: number;
limit: number;
endDate?: string;
startDate?: string;
endDate: string;
startDate: string;
projectId?: string;
environment?: string;
auditLogActorId?: string;

4
backend/src/server/routes/v1/organization-router.ts
View File

@@ -153,12 +153,11 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
})
.describe(AUDIT_LOGS.EXPORT.eventMetadata),
startDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.startDate),
endDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.endDate),
endDate: z.string().datetime().default(new Date().toISOString()).describe(AUDIT_LOGS.EXPORT.endDate),
offset: z.coerce.number().default(0).describe(AUDIT_LOGS.EXPORT.offset),
limit: z.coerce.number().default(20).describe(AUDIT_LOGS.EXPORT.limit),
actor: z.string().optional().describe(AUDIT_LOGS.EXPORT.actor)
}),
response: {
200: z.object({
auditLogs: AuditLogsSchema.omit({
@@ -195,7 +194,6 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
actorType: req.query.actorType,
eventType: req.query.eventType as EventType[] | undefined
},
actorId: req.permission.id,
actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod,

2
backend/src/services/resource-cleanup/resource-cleanup-queue.ts
View File

@@ -47,7 +47,6 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
queueService.start(QueueName.DailyResourceCleanUp, async () => {
logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`);
await secretDAL.pruneSecretReminders(queueService);
await auditLogDAL.pruneAuditLog();
await identityAccessTokenDAL.removeExpiredTokens();
await identityUniversalAuthClientSecretDAL.removeExpiredClientSecrets();
await secretSharingDAL.pruneExpiredSharedSecrets();
@@ -58,6 +57,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
await secretFolderVersionDAL.pruneExcessVersions();
await serviceTokenService.notifyExpiringTokens();
await orgService.notifyInvitedUsers();
await auditLogDAL.pruneAuditLog();
logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`);
});