awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-07 07:27:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2842 from Infisical/misc/add-pg-queue-init-flag

Browse Source 
misc: added pg queue init flag
This commit is contained in:
Maidul Islam
2024-12-05 11:12:29 -05:00
committed by GitHub
parent ef82c664a6 0a1242db75
commit 2098bd3be2
3 changed files with 95 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/ee/services/audit-log/audit-log-queue.ts
+84 -82
View File
@@ -37,7 +37,7 @@ export const auditLogQueueServiceFactory = async ({
const appCfg = getConfig(); const appCfg = getConfig();
const pushToLog = async (data: TCreateAuditLogDTO) => { const pushToLog = async (data: TCreateAuditLogDTO) => {
if (appCfg.USE_PG_QUEUE) { if (appCfg.USE_PG_QUEUE && appCfg.SHOULD_INIT_PG_QUEUE) {
await queueService.queuePg<QueueName.AuditLog>(QueueJobs.AuditLog, data, { await queueService.queuePg<QueueName.AuditLog>(QueueJobs.AuditLog, data, {
retryLimit: 10, retryLimit: 10,
retryBackoff: true retryBackoff: true
@@ -52,96 +52,98 @@ export const auditLogQueueServiceFactory = async ({
} }
}; };
await queueService.startPg<QueueName.AuditLog>( if (appCfg.SHOULD_INIT_PG_QUEUE) {
QueueJobs.AuditLog, await queueService.startPg<QueueName.AuditLog>(
async ([job]) => { QueueJobs.AuditLog,
const { actor, event, ipAddress, projectId, userAgent, userAgentType } = job.data; async ([job]) => {
let { orgId } = job.data; const { actor, event, ipAddress, projectId, userAgent, userAgentType } = job.data;
const MS_IN_DAY = 24 * 60 * 60 * 1000; let { orgId } = job.data;
let project; const MS_IN_DAY = 24 * 60 * 60 * 1000;
let project;
if (!orgId) { if (!orgId) {
// it will never be undefined for both org and project id // it will never be undefined for both org and project id
// TODO(akhilmhdh): use caching here in dal to avoid db calls // TODO(akhilmhdh): use caching here in dal to avoid db calls
project = await projectDAL.findById(projectId as string); project = await projectDAL.findById(projectId as string);
orgId = project.orgId; orgId = project.orgId;
} }
const plan = await licenseService.getPlan(orgId); const plan = await licenseService.getPlan(orgId);
if (plan.auditLogsRetentionDays === 0) { if (plan.auditLogsRetentionDays === 0) {
// skip inserting if audit log retention is 0 meaning its not supported // skip inserting if audit log retention is 0 meaning its not supported
return; return;
} }
// For project actions, set TTL to project-level audit log retention config // For project actions, set TTL to project-level audit log retention config
// This condition ensures that the plan's audit log retention days cannot be bypassed // This condition ensures that the plan's audit log retention days cannot be bypassed
const ttlInDays = const ttlInDays =
project?.auditLogsRetentionDays && project.auditLogsRetentionDays < plan.auditLogsRetentionDays project?.auditLogsRetentionDays && project.auditLogsRetentionDays < plan.auditLogsRetentionDays
? project.auditLogsRetentionDays ? project.auditLogsRetentionDays
: plan.auditLogsRetentionDays; : plan.auditLogsRetentionDays;
const ttl = ttlInDays * MS_IN_DAY; const ttl = ttlInDays * MS_IN_DAY;
const auditLog = await auditLogDAL.create({ const auditLog = await auditLogDAL.create({
actor: actor.type, actor: actor.type,
actorMetadata: actor.metadata, actorMetadata: actor.metadata,
userAgent, userAgent,
projectId, projectId,
projectName: project?.name, projectName: project?.name,
ipAddress, ipAddress,
orgId, orgId,
eventType: event.type, eventType: event.type,
expiresAt: new Date(Date.now() + ttl), expiresAt: new Date(Date.now() + ttl),
eventMetadata: event.metadata, eventMetadata: event.metadata,
userAgentType userAgentType
}); });
const logStreams = orgId ? await auditLogStreamDAL.find({ orgId }) : []; const logStreams = orgId ? await auditLogStreamDAL.find({ orgId }) : [];
await Promise.allSettled( await Promise.allSettled(
logStreams.map( logStreams.map(
async ({ async ({
url, url,
encryptedHeadersTag, encryptedHeadersTag,
encryptedHeadersIV, encryptedHeadersIV,
encryptedHeadersKeyEncoding, encryptedHeadersKeyEncoding,
encryptedHeadersCiphertext encryptedHeadersCiphertext
}) => { }) => {
const streamHeaders = const streamHeaders =
encryptedHeadersIV && encryptedHeadersCiphertext && encryptedHeadersTag encryptedHeadersIV && encryptedHeadersCiphertext && encryptedHeadersTag
? (JSON.parse( ? (JSON.parse(
infisicalSymmetricDecrypt({ infisicalSymmetricDecrypt({
keyEncoding: encryptedHeadersKeyEncoding as SecretKeyEncoding, keyEncoding: encryptedHeadersKeyEncoding as SecretKeyEncoding,
iv: encryptedHeadersIV, iv: encryptedHeadersIV,
tag: encryptedHeadersTag, tag: encryptedHeadersTag,
ciphertext: encryptedHeadersCiphertext ciphertext: encryptedHeadersCiphertext
}) })
) as LogStreamHeaders[]) ) as LogStreamHeaders[])
: []; : [];
const headers: RawAxiosRequestHeaders = { "Content-Type": "application/json" }; const headers: RawAxiosRequestHeaders = { "Content-Type": "application/json" };
if (streamHeaders.length) if (streamHeaders.length)
streamHeaders.forEach(({ key, value }) => { streamHeaders.forEach(({ key, value }) => {
headers[key] = value; headers[key] = value;
});
return request.post(url, auditLog, {
headers,
// request timeout
timeout: AUDIT_LOG_STREAM_TIMEOUT,
// connection timeout
signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT)
}); });
}
return request.post(url, auditLog, { )
headers, );
// request timeout },
timeout: AUDIT_LOG_STREAM_TIMEOUT, {
// connection timeout batchSize: 1,
signal: AbortSignal.timeout(AUDIT_LOG_STREAM_TIMEOUT) workerCount: 30,
}); pollingIntervalSeconds: 0.5
} }
) );
); }
},
{
batchSize: 1,
workerCount: 30,
pollingIntervalSeconds: 0.5
}
);
queueService.start(QueueName.AuditLog, async (job) => { queueService.start(QueueName.AuditLog, async (job) => {
const { actor, event, ipAddress, projectId, userAgent, userAgentType } = job.data; const { actor, event, ipAddress, projectId, userAgent, userAgentType } = job.data;
backend/src/lib/config/env.ts
+2 -1
View File
@@ -180,7 +180,8 @@ const envSchema = z
HSM_KEY_LABEL: zpStr(z.string().optional()), HSM_KEY_LABEL: zpStr(z.string().optional()),
HSM_SLOT: z.coerce.number().optional().default(0), HSM_SLOT: z.coerce.number().optional().default(0),
USE_PG_QUEUE: zodStrBool.default("false") USE_PG_QUEUE: zodStrBool.default("false"),
SHOULD_INIT_PG_QUEUE: zodStrBool.default("false")
}) })
// To ensure that basic encryption is always possible. // To ensure that basic encryption is always possible.
.refine( .refine(
backend/src/queue/queue-service.ts
+9 -4
View File
@@ -8,6 +8,7 @@ import {
TScanFullRepoEventPayload, TScanFullRepoEventPayload,
TScanPushEventPayload TScanPushEventPayload
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types"; } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import { getConfig } from "@app/lib/config/env";
import { logger } from "@app/lib/logger"; import { logger } from "@app/lib/logger";
import { import {
TFailedIntegrationSyncEmailsPayload, TFailedIntegrationSyncEmailsPayload,
@@ -208,11 +209,15 @@ export const queueServiceFactory = (redisUrl: string, dbConnectionUrl: string) =
>; >;
const initialize = async () => { const initialize = async () => {
await pgBoss.start(); const appCfg = getConfig();
if (appCfg.SHOULD_INIT_PG_QUEUE) {
logger.info("Initializing pg-queue...");
await pgBoss.start();
pgBoss.on("error", (error) => { pgBoss.on("error", (error) => {
logger.error(error, "pg-queue error"); logger.error(error, "pg-queue error");
}); });
}
}; };
const start = <T extends QueueName>( const start = <T extends QueueName>(