awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-18 13:15:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into ENG-3506

Browse Source
This commit is contained in:
x032205
2025-08-27 04:10:12 -04:00
parent 8d5b6a17b1 bceaac844f
commit f52dbaa2f2
181 changed files with 7268 additions and 1288 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/ee/services/audit-log/audit-log-dal.ts
+35 -40
View File
@@ -14,7 +14,7 @@ import { ActorType } from "@app/services/auth/auth-type";
import { EventType, filterableSecretEvents } from "./audit-log-types";
export interface TAuditLogDALFactory extends Omit<TOrmify<TableName.AuditLog>, "find"> {
pruneAuditLog: (tx?: knex.Knex) => Promise<void>;
pruneAuditLog: () => Promise<void>;
find: (
arg: Omit<TFindQuery, "actor" | "eventType"> & {
actorId?: string | undefined;
@@ -41,6 +41,10 @@ type TFindQuery = {
offset?: number;
};
const QUERY_TIMEOUT_MS = 10 * 60 * 1000; // 10 minutes
const AUDIT_LOG_PRUNE_BATCH_SIZE = 10000;
const MAX_RETRY_ON_FAILURE = 3;
export const auditLogDALFactory = (db: TDbClient) => {
const auditLogOrm = ormify(db, TableName.AuditLog);
@@ -151,20 +155,20 @@ export const auditLogDALFactory = (db: TDbClient) => {
};
// delete all audit log that have expired
const pruneAuditLog: TAuditLogDALFactory["pruneAuditLog"] = async (tx) => {
const runPrune = async (dbClient: knex.Knex) => {
const AUDIT_LOG_PRUNE_BATCH_SIZE = 10000;
const MAX_RETRY_ON_FAILURE = 3;
const pruneAuditLog: TAuditLogDALFactory["pruneAuditLog"] = async () => {
const today = new Date();
let deletedAuditLogIds: { id: string }[] = [];
let numberOfRetryOnFailure = 0;
let isRetrying = false;
const today = new Date();
let deletedAuditLogIds: { id: string }[] = [];
let numberOfRetryOnFailure = 0;
let isRetrying = false;
logger.info(`${QueueName.DailyResourceCleanUp}: audit log started`);
do {
try {
// eslint-disable-next-line no-await-in-loop
deletedAuditLogIds = await db.transaction(async (trx) => {
await trx.raw(`SET statement_timeout = ${QUERY_TIMEOUT_MS}`);
logger.info(`${QueueName.DailyResourceCleanUp}: audit log started`);
do {
try {
const findExpiredLogSubQuery = dbClient(TableName.AuditLog)
const findExpiredLogSubQuery = trx(TableName.AuditLog)
.where("expiresAt", "<", today)
.where("createdAt", "<", today) // to use audit log partition
.orderBy(`${TableName.AuditLog}.createdAt`, "desc")
@@ -172,34 +176,25 @@ export const auditLogDALFactory = (db: TDbClient) => {
.limit(AUDIT_LOG_PRUNE_BATCH_SIZE);
// eslint-disable-next-line no-await-in-loop
deletedAuditLogIds = await dbClient(TableName.AuditLog)
.whereIn("id", findExpiredLogSubQuery)
.del()
.returning("id");
numberOfRetryOnFailure = 0; // reset
} catch (error) {
numberOfRetryOnFailure += 1;
logger.error(error, "Failed to delete audit log on pruning");
} finally {
// eslint-disable-next-line no-await-in-loop
await new Promise((resolve) => {
setTimeout(resolve, 10); // time to breathe for db
});
}
isRetrying = numberOfRetryOnFailure > 0;
} while (deletedAuditLogIds.length > 0 || (isRetrying && numberOfRetryOnFailure < MAX_RETRY_ON_FAILURE));
logger.info(`${QueueName.DailyResourceCleanUp}: audit log completed`);
};
const results = await trx(TableName.AuditLog).whereIn("id", findExpiredLogSubQuery).del().returning("id");
if (tx) {
await runPrune(tx);
} else {
const QUERY_TIMEOUT_MS = 10 * 60 * 1000; // 10 minutes
await db.transaction(async (trx) => {
await trx.raw(`SET statement_timeout = ${QUERY_TIMEOUT_MS}`);
await runPrune(trx);
});
}
return results;
});
numberOfRetryOnFailure = 0; // reset
} catch (error) {
numberOfRetryOnFailure += 1;
deletedAuditLogIds = [];
logger.error(error, "Failed to delete audit log on pruning");
} finally {
// eslint-disable-next-line no-await-in-loop
await new Promise((resolve) => {
setTimeout(resolve, 10); // time to breathe for db
});
}
isRetrying = numberOfRetryOnFailure > 0;
} while (deletedAuditLogIds.length > 0 || (isRetrying && numberOfRetryOnFailure < MAX_RETRY_ON_FAILURE));
logger.info(`${QueueName.DailyResourceCleanUp}: audit log completed`);
};
const create: TAuditLogDALFactory["create"] = async (tx) => {
backend/src/ee/services/audit-log/audit-log-queue.ts
+2 -16
View File
@@ -1,8 +1,6 @@
import { AxiosError, RawAxiosRequestHeaders } from "axios";
import { ProjectType, SecretKeyEncoding } from "@app/db/schemas";
import { TEventBusService } from "@app/ee/services/event/event-bus-service";
import { TopicName, toPublishableEvent } from "@app/ee/services/event/types";
import { SecretKeyEncoding } from "@app/db/schemas";
import { request } from "@app/lib/config/request";
import { crypto } from "@app/lib/crypto/cryptography";
import { logger } from "@app/lib/logger";
@@ -22,7 +20,6 @@ type TAuditLogQueueServiceFactoryDep = {
queueService: TQueueServiceFactory;
projectDAL: Pick<TProjectDALFactory, "findById">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
eventBusService: TEventBusService;
};
export type TAuditLogQueueServiceFactory = {
@@ -38,8 +35,7 @@ export const auditLogQueueServiceFactory = async ({
queueService,
projectDAL,
licenseService,
auditLogStreamDAL,
eventBusService
auditLogStreamDAL
}: TAuditLogQueueServiceFactoryDep): Promise<TAuditLogQueueServiceFactory> => {
const pushToLog = async (data: TCreateAuditLogDTO) => {
await queueService.queue<QueueName.AuditLog>(QueueName.AuditLog, QueueJobs.AuditLog, data, {
@@ -145,16 +141,6 @@ export const auditLogQueueServiceFactory = async ({
)
);
}
const publishable = toPublishableEvent(event);
if (publishable) {
await eventBusService.publish(TopicName.CoreServers, {
type: ProjectType.SecretManager,
source: "infiscal",
data: publishable.data
});
}
});
return {