From b9c824559c03e66900d4410469afa23537376120 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 28 Oct 2025 04:57:25 +0800 Subject: [PATCH 1/7] misc: added health and ready probe endpoints --- backend/src/main.ts | 30 +++++++++++----- backend/src/server/app.ts | 76 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 9 deletions(-) diff --git a/backend/src/main.ts b/backend/src/main.ts index 400804804..3936d1c47 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -16,7 +16,7 @@ import { buildRedisFromConfig } from "./lib/config/redis"; import { removeTemporaryBaseDirectory } from "./lib/files"; import { initLogger } from "./lib/logger"; import { queueServiceFactory } from "./queue"; -import { main } from "./server/app"; +import { main, markServerReady } from "./server/app"; import { bootstrapCheck } from "./server/boot-strap-check"; import { kmsRootConfigDALFactory } from "./services/kms/kms-root-config-dal"; import { smtpServiceFactory } from "./services/smtp/smtp-service"; @@ -59,8 +59,6 @@ const run = async () => { }) : undefined; - await runMigrations({ applicationDb: db, auditLogDb, logger }); - const smtp = smtpServiceFactory(formatSmtpConfig()); const queue = queueServiceFactory(envConfig, { @@ -87,8 +85,8 @@ const run = async () => { redis, envConfig }); - const bootstrap = await bootstrapCheck({ db }); + // Setup signal handlers // eslint-disable-next-line process.on("SIGINT", async () => { await server.close(); @@ -119,14 +117,28 @@ const run = async () => { }); } + // Start listening BEFORE migrations + // At this point: /api/health returns 200, /api/ready returns 503 await server.listen({ port: envConfig.PORT, - host: envConfig.HOST, - listenTextResolver: (address) => { - void bootstrap(); - return address; - } + host: envConfig.HOST }); + + logger.info(`Server listening on ${envConfig.HOST}:${envConfig.PORT}`); + logger.info("Running migrations - health check available, other endpoints blocked..."); + + // Run migrations while server is up + await runMigrations({ applicationDb: db, auditLogDb, logger }); + + logger.info("Migrations complete. Marking server as ready..."); + + // Mark server as ready - now all endpoints work + markServerReady(); + + logger.info("Server is ready to accept traffic"); + + const bootstrap = await bootstrapCheck({ db }); + void bootstrap(); }; void run(); diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index f1176b932..eadda27cd 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -1,5 +1,6 @@ /* eslint-disable import/extensions */ import path from "node:path"; +import { monitorEventLoopDelay } from "perf_hooks"; import type { FastifyCookieOptions } from "@fastify/cookie"; import cookie from "@fastify/cookie"; @@ -24,6 +25,7 @@ import { TQueueServiceFactory } from "@app/queue"; import { TKmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal"; import { TSmtpService } from "@app/services/smtp/smtp-service"; import { TSuperAdminDALFactory } from "@app/services/super-admin/super-admin-dal"; +import { getServerCfg } from "@app/services/super-admin/super-admin-service"; import { globalRateLimiterCfg } from "./config/rateLimiter"; import { addErrorsToResponseSchemas } from "./plugins/add-errors-to-response-schemas"; @@ -36,6 +38,15 @@ import { registerServeUI } from "./plugins/serve-ui"; import { fastifySwagger } from "./plugins/swagger"; import { registerRoutes } from "./routes"; +// Monitor event loop for readiness checks +const histogram = monitorEventLoopDelay({ resolution: 20 }); +histogram.enable(); + +// Readiness state +const readinessState = { + isReady: false +}; + type TMain = { auditLogDb?: Knex; db: Knex; @@ -145,6 +156,66 @@ export const main = async ({ }) }); + // Health check - always returns 200 when server is running + server.get("/api/health", async () => { + return { status: "ok", message: "Server is alive" }; + }); + + // Global preHandler to block requests during migrations + // Excludes /api/health and /api/ready endpoints + server.addHook("preHandler", async (request, reply) => { + if (request.url === "/api/health" || request.url === "/api/ready") { + return; + } + if (!readinessState.isReady) { + return reply.code(503).send({ + status: "unavailable", + message: "Server is starting up, migrations in progress. Please try again in a moment." + }); + } + }); + + // Readiness check - returns 503 until migrations are complete + server.get("/api/ready", async (request, reply) => { + const cfg = getConfig(); + + // Calculate event loop statistics + const meanLagMs = histogram.mean / 1e6; + const maxLagMs = histogram.max / 1e6; + const p99LagMs = histogram.percentile(99) / 1e6; + + request.log.info( + `Event loop stats - Mean: ${meanLagMs.toFixed(2)}ms, Max: ${maxLagMs.toFixed(2)}ms, p99: ${p99LagMs.toFixed(2)}ms` + ); + + request.log.info(`Raw event loop stats: ${JSON.stringify(histogram, null, 2)}`); + + if (!readinessState.isReady) { + return reply.code(503).send({ + date: new Date(), + message: "Server is starting up, migrations in progress", + emailConfigured: cfg.isSmtpConfigured, + redisConfigured: cfg.isRedisConfigured, + secretScanningConfigured: cfg.isSecretScanningConfigured, + samlDefaultOrgSlug: cfg.samlDefaultOrgSlug, + auditLogStorageDisabled: Boolean(cfg.DISABLE_AUDIT_LOG_STORAGE) + }); + } + + const serverCfg = await getServerCfg(); + + return { + date: new Date(), + message: "Ok", + emailConfigured: cfg.isSmtpConfigured, + inviteOnlySignup: Boolean(serverCfg.allowSignUp), + redisConfigured: cfg.isRedisConfigured, + secretScanningConfigured: cfg.isSecretScanningConfigured, + samlDefaultOrgSlug: cfg.samlDefaultOrgSlug, + auditLogStorageDisabled: Boolean(cfg.DISABLE_AUDIT_LOG_STORAGE) + }; + }); + await server.register(registerRoutes, { smtp, queue, @@ -171,3 +242,8 @@ export const main = async ({ process.exit(1); } }; + +// Function to mark server as ready after migrations +export const markServerReady = () => { + readinessState.isReady = true; +}; From 8214e07abd531af01c4335005398c0b52d6acbea Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 28 Oct 2025 05:52:58 +0800 Subject: [PATCH 2/7] misc: properly handled multi-container setups --- backend/e2e-test/vitest-environment-knex.ts | 4 ++- backend/src/auto-start-migrations.ts | 9 +++++- backend/src/main.ts | 25 ++++++++++----- backend/src/server/app.ts | 34 +++++++++++++++------ 4 files changed, 54 insertions(+), 18 deletions(-) diff --git a/backend/e2e-test/vitest-environment-knex.ts b/backend/e2e-test/vitest-environment-knex.ts index 0f84dbee2..f33d32f6a 100644 --- a/backend/e2e-test/vitest-environment-knex.ts +++ b/backend/e2e-test/vitest-environment-knex.ts @@ -8,7 +8,7 @@ import path from "path"; import { seedData1 } from "@app/db/seed-data"; import { getDatabaseCredentials, getHsmConfig, initEnvConfig } from "@app/lib/config/env"; import { initLogger } from "@app/lib/logger"; -import { main } from "@app/server/app"; +import { main, markServerReady } from "@app/server/app"; import { AuthMethod, AuthTokenType } from "@app/services/auth/auth-type"; import { mockSmtpServer } from "./mocks/smtp"; @@ -96,6 +96,8 @@ export default { envConfig: envCfg }); + markServerReady(); + await bootstrapCheck({ db }); // @ts-expect-error type diff --git a/backend/src/auto-start-migrations.ts b/backend/src/auto-start-migrations.ts index 88f6dea69..38c30366d 100644 --- a/backend/src/auto-start-migrations.ts +++ b/backend/src/auto-start-migrations.ts @@ -12,6 +12,7 @@ type TArgs = { auditLogDb?: Knex; applicationDb: Knex; logger: Logger; + onMigrationLockAcquired?: () => void; }; const isProduction = process.env.NODE_ENV === "production"; @@ -30,7 +31,7 @@ const migrationStatusCheckErrorHandler = (err: Error) => { throw err; }; -export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs) => { +export const runMigrations = async ({ applicationDb, auditLogDb, logger, onMigrationLockAcquired }: TArgs) => { try { // akhilmhdh(Feb 10 2025): 2 years from now remove this if (isProduction) { @@ -85,6 +86,12 @@ export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs await applicationDb.transaction(async (tx) => { await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]); + + // Signal that this container is running migrations + if (onMigrationLockAcquired) { + onMigrationLockAcquired(); + } + logger.info("Running application migrations."); const didPreviousInstanceRunMigration = !(await applicationDb.migrate diff --git a/backend/src/main.ts b/backend/src/main.ts index 3936d1c47..d680fc006 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -16,7 +16,7 @@ import { buildRedisFromConfig } from "./lib/config/redis"; import { removeTemporaryBaseDirectory } from "./lib/files"; import { initLogger } from "./lib/logger"; import { queueServiceFactory } from "./queue"; -import { main, markServerReady } from "./server/app"; +import { main, markRunningMigrations, markServerReady } from "./server/app"; import { bootstrapCheck } from "./server/boot-strap-check"; import { kmsRootConfigDALFactory } from "./services/kms/kms-root-config-dal"; import { smtpServiceFactory } from "./services/smtp/smtp-service"; @@ -118,25 +118,36 @@ const run = async () => { } // Start listening BEFORE migrations - // At this point: /api/health returns 200, /api/ready returns 503 await server.listen({ port: envConfig.PORT, host: envConfig.HOST }); logger.info(`Server listening on ${envConfig.HOST}:${envConfig.PORT}`); - logger.info("Running migrations - health check available, other endpoints blocked..."); + logger.info("Running migrations..."); // Run migrations while server is up - await runMigrations({ applicationDb: db, auditLogDb, logger }); + // All containers start as NOT HEALTHY (waiting for migrations) + // Container that acquires lock: becomes HEALTHY (running migrations) + NOT READY (no traffic) + // Other containers waiting: stay NOT HEALTHY (waiting) + NOT READY (no traffic) + await runMigrations({ + applicationDb: db, + auditLogDb, + logger, + onMigrationLockAcquired: () => { + // Called after successfully acquiring the lock + // This container is now the migration runner + markRunningMigrations(); + logger.info("Migration lock acquired! This container is running migrations."); + } + }); - logger.info("Migrations complete. Marking server as ready..."); + logger.info("Migrations complete. Marking server as READY..."); - // Mark server as ready - now all endpoints work + // Now mark server as ready - it can accept traffic markServerReady(); logger.info("Server is ready to accept traffic"); - const bootstrap = await bootstrapCheck({ db }); void bootstrap(); }; diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index eadda27cd..2de20ab46 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -42,9 +42,11 @@ import { registerRoutes } from "./routes"; const histogram = monitorEventLoopDelay({ resolution: 20 }); histogram.enable(); -// Readiness state -const readinessState = { - isReady: false +// Server state tracking +const serverState = { + isReady: false, + isRunningMigrations: false, + isWaitingForMigrations: true // Start as true - containers are unhealthy until they acquire migration lock or complete }; type TMain = { @@ -156,8 +158,15 @@ export const main = async ({ }) }); - // Health check - always returns 200 when server is running - server.get("/api/health", async () => { + // Health check - returns 200 only if doing useful work (running migrations or ready) + // Returns 503 if waiting for another container to finish migrations + server.get("/api/health", async (_, reply) => { + if (serverState.isWaitingForMigrations) { + return reply.code(503).send({ + status: "waiting", + message: "Waiting for migrations to complete in another container" + }); + } return { status: "ok", message: "Server is alive" }; }); @@ -167,7 +176,7 @@ export const main = async ({ if (request.url === "/api/health" || request.url === "/api/ready") { return; } - if (!readinessState.isReady) { + if (!serverState.isReady) { return reply.code(503).send({ status: "unavailable", message: "Server is starting up, migrations in progress. Please try again in a moment." @@ -190,7 +199,7 @@ export const main = async ({ request.log.info(`Raw event loop stats: ${JSON.stringify(histogram, null, 2)}`); - if (!readinessState.isReady) { + if (!serverState.isReady) { return reply.code(503).send({ date: new Date(), message: "Server is starting up, migrations in progress", @@ -243,7 +252,14 @@ export const main = async ({ } }; -// Function to mark server as ready after migrations +// Functions to manage server state export const markServerReady = () => { - readinessState.isReady = true; + serverState.isReady = true; + serverState.isRunningMigrations = false; + serverState.isWaitingForMigrations = false; +}; + +export const markRunningMigrations = () => { + serverState.isRunningMigrations = true; + serverState.isWaitingForMigrations = false; }; From d310fef792a65eca995da9958bc5097f7396d7e5 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 28 Oct 2025 06:06:45 +0800 Subject: [PATCH 3/7] misc: improved commnts --- backend/src/auto-start-migrations.ts | 3 ++- backend/src/main.ts | 3 --- backend/src/server/app.ts | 5 ----- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/backend/src/auto-start-migrations.ts b/backend/src/auto-start-migrations.ts index 38c30366d..28f66e20e 100644 --- a/backend/src/auto-start-migrations.ts +++ b/backend/src/auto-start-migrations.ts @@ -87,7 +87,8 @@ export const runMigrations = async ({ applicationDb, auditLogDb, logger, onMigra await applicationDb.transaction(async (tx) => { await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]); - // Signal that this container is running migrations + // Signal that this container is running migrations so that it can be marked as healthy/alive + // This is to prevent the container from being killed by the orchestrator if (onMigrationLockAcquired) { onMigrationLockAcquired(); } diff --git a/backend/src/main.ts b/backend/src/main.ts index d680fc006..9d7812c96 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -86,7 +86,6 @@ const run = async () => { envConfig }); - // Setup signal handlers // eslint-disable-next-line process.on("SIGINT", async () => { await server.close(); @@ -117,7 +116,6 @@ const run = async () => { }); } - // Start listening BEFORE migrations await server.listen({ port: envConfig.PORT, host: envConfig.HOST @@ -144,7 +142,6 @@ const run = async () => { logger.info("Migrations complete. Marking server as READY..."); - // Now mark server as ready - it can accept traffic markServerReady(); logger.info("Server is ready to accept traffic"); diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index 2de20ab46..daf49ed10 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -38,11 +38,9 @@ import { registerServeUI } from "./plugins/serve-ui"; import { fastifySwagger } from "./plugins/swagger"; import { registerRoutes } from "./routes"; -// Monitor event loop for readiness checks const histogram = monitorEventLoopDelay({ resolution: 20 }); histogram.enable(); -// Server state tracking const serverState = { isReady: false, isRunningMigrations: false, @@ -171,7 +169,6 @@ export const main = async ({ }); // Global preHandler to block requests during migrations - // Excludes /api/health and /api/ready endpoints server.addHook("preHandler", async (request, reply) => { if (request.url === "/api/health" || request.url === "/api/ready") { return; @@ -188,7 +185,6 @@ export const main = async ({ server.get("/api/ready", async (request, reply) => { const cfg = getConfig(); - // Calculate event loop statistics const meanLagMs = histogram.mean / 1e6; const maxLagMs = histogram.max / 1e6; const p99LagMs = histogram.percentile(99) / 1e6; @@ -252,7 +248,6 @@ export const main = async ({ } }; -// Functions to manage server state export const markServerReady = () => { serverState.isReady = true; serverState.isRunningMigrations = false; From 1cff5cd7d7cd33c3df42a738c20731f5b5aa9e5f Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 28 Oct 2025 21:50:51 +0800 Subject: [PATCH 4/7] misc: added health and ready probe to docs --- .../guides/production-hardening.mdx | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/docs/self-hosting/guides/production-hardening.mdx b/docs/self-hosting/guides/production-hardening.mdx index dfd7b575f..e8f05730c 100644 --- a/docs/self-hosting/guides/production-hardening.mdx +++ b/docs/self-hosting/guides/production-hardening.mdx @@ -137,6 +137,33 @@ Configure database read replicas for high availability PostgreSQL setups: DB_READ_REPLICAS='[{"DB_CONNECTION_URI":"postgresql://user:pass@replica:5432/db?sslmode=require"}]' ``` +### Health Check Endpoints + +Infisical provides two health check endpoints for proper container orchestration and load balancer integration: + +#### `/api/health` - Container Health Check + +Determines whether the application container should be kept alive or terminated. + +- Returns `200` if the application is running and operational +- Returns `200` even during startup tasks +- Returns `503` only if the application has crashed or is unable to start + +**Use for**: Docker health checks, Kubernetes liveness probes, ECS task health checks. + +#### `/api/ready` - Traffic Readiness Check + +Determines whether the application instance is ready to receive production traffic. + +- Returns `200` when the application is fully ready to serve requests +- Returns `503` during startup tasks (e.g., database migrations, initialization) + +**Use for**: Load balancer health checks, Kubernetes readiness probes, ALB target health checks. + +#### Why Two Endpoints? + +Using both endpoints together enables zero-downtime deployments: containers stay alive during startup tasks (`/api/health` returns `200`) while load balancers avoid sending traffic to instances that aren't ready (`/api/ready` returns `503`). This ensures existing instances continue serving traffic until new instances complete their initialization. + ### Operational Security #### User Access Management @@ -207,14 +234,17 @@ docker run --memory=1g --cpus=0.5 infisical/infisical:latest #### Health Monitoring -**Configure health checks**. Set up Docker health checks: +**Configure health checks**. Set up Docker health checks using the appropriate endpoint: ```dockerfile # In Dockerfile or docker-compose.yml +# Use /api/health for container health (keeps container alive during startup) HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ - CMD curl -f http://localhost:8080/api/status || exit 1 + CMD curl -f http://localhost:8080/api/health || exit 1 ``` +**Note**: Use `/api/health` for container health checks and `/api/ready` for load balancer readiness checks. See [Health Check Endpoints](#health-check-endpoints) for detailed information. + #### Network Security **Host firewall configuration**. Configure host-level firewall for Docker deployments: @@ -433,26 +463,32 @@ stringData: #### Health Monitoring -**Set up health checks**. Configure readiness and liveness probes: +**Set up health checks**. Configure readiness and liveness probes using the appropriate endpoints: ```yaml # Health check configuration containers: - name: infisical + # Use /api/ready for readiness (traffic routing) readinessProbe: httpGet: - path: /api/status + path: /api/ready port: 8080 initialDelaySeconds: 10 periodSeconds: 5 + failureThreshold: 3 + # Use /api/health for liveness (container restart) livenessProbe: httpGet: - path: /api/status + path: /api/health port: 8080 initialDelaySeconds: 30 periodSeconds: 10 + failureThreshold: 3 ``` +**Important**: The `readinessProbe` uses `/api/ready` to ensure traffic is only sent to pods that are fully initialized. The `livenessProbe` uses `/api/health` to keep the container alive during startup. See [Health Check Endpoints](#health-check-endpoints) for detailed information. + #### Infrastructure Considerations **Use managed databases (if possible)**. For production deployments, consider using managed PostgreSQL and Redis services instead of in-cluster instances when feasible, as they typically provide better security, backup, and maintenance capabilities. From 9d61b06eec16daa4be2377eac29ce38abe804dae Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 28 Oct 2025 23:08:39 +0800 Subject: [PATCH 5/7] misc: updated processes to run after db migration --- .../secret-rotation-v2-queue.ts | 300 +++---- .../secret-scanning-v2-queue.ts | 825 +++++++++--------- backend/src/main.ts | 15 +- backend/src/server/app.ts | 4 +- backend/src/server/routes/index.ts | 148 ++-- .../notification/notification-queue.ts | 30 +- 6 files changed, 675 insertions(+), 647 deletions(-) diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts index f653802b6..829c17e7b 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts @@ -42,168 +42,172 @@ export const secretRotationV2QueueServiceFactory = async ({ smtpService, notificationService }: TSecretRotationV2QueueServiceFactoryDep) => { - const appCfg = getConfig(); + const init = async () => { + const appCfg = getConfig(); - if (appCfg.isRotationDevelopmentMode) { - logger.warn("Secret Rotation V2 is in development mode."); - } + if (appCfg.isRotationDevelopmentMode) { + logger.warn("Secret Rotation V2 is in development mode."); + } - await queueService.startPg( - QueueJobs.SecretRotationV2QueueRotations, - async () => { - try { - const rotateBy = getNextUtcRotationInterval(); + await queueService.startPg( + QueueJobs.SecretRotationV2QueueRotations, + async () => { + try { + const rotateBy = getNextUtcRotationInterval(); - const currentTime = new Date(); + const currentTime = new Date(); - const secretRotations = await secretRotationV2DAL.findSecretRotationsToQueue(rotateBy); + const secretRotations = await secretRotationV2DAL.findSecretRotationsToQueue(rotateBy); - logger.info( - `secretRotationV2Queue: Queue Rotations [currentTime=${currentTime.toISOString()}] [rotateBy=${rotateBy.toISOString()}] [count=${ - secretRotations.length - }]` - ); - - for await (const rotation of secretRotations) { logger.info( - `secretRotationV2Queue: Queue Rotation [rotationId=${rotation.id}] [lastRotatedAt=${new Date( - rotation.lastRotatedAt - ).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]` + `secretRotationV2Queue: Queue Rotations [currentTime=${currentTime.toISOString()}] [rotateBy=${rotateBy.toISOString()}] [count=${ + secretRotations.length + }]` ); - const data = { - rotationId: rotation.id, - queuedAt: currentTime - } as TSecretRotationRotateSecretsJobPayload; - - if (appCfg.isTestMode) { - logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode"); - await rotateSecretsFns({ - job: { - id: uuidv4(), - data, - retryCount: 0, - retryLimit: 0 - }, - secretRotationV2DAL, - secretRotationV2Service - }); - } else { - await queueService.queuePg( - QueueJobs.SecretRotationV2RotateSecrets, - { - rotationId: rotation.id, - queuedAt: currentTime - }, - getSecretRotationRotateSecretJobOptions(rotation) + for await (const rotation of secretRotations) { + logger.info( + `secretRotationV2Queue: Queue Rotation [rotationId=${rotation.id}] [lastRotatedAt=${new Date( + rotation.lastRotatedAt + ).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]` ); + + const data = { + rotationId: rotation.id, + queuedAt: currentTime + } as TSecretRotationRotateSecretsJobPayload; + + if (appCfg.isTestMode) { + logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode"); + await rotateSecretsFns({ + job: { + id: uuidv4(), + data, + retryCount: 0, + retryLimit: 0 + }, + secretRotationV2DAL, + secretRotationV2Service + }); + } else { + await queueService.queuePg( + QueueJobs.SecretRotationV2RotateSecrets, + { + rotationId: rotation.id, + queuedAt: currentTime + }, + getSecretRotationRotateSecretJobOptions(rotation) + ); + } } + } catch (error) { + logger.error(error, "secretRotationV2Queue: Queue Rotations Error:"); + throw error; } - } catch (error) { - logger.error(error, "secretRotationV2Queue: Queue Rotations Error:"); - throw error; + }, + { + batchSize: 1, + workerCount: 1, + pollingIntervalSeconds: appCfg.isRotationDevelopmentMode ? 0.5 : 30 } - }, - { - batchSize: 1, - workerCount: 1, - pollingIntervalSeconds: appCfg.isRotationDevelopmentMode ? 0.5 : 30 - } - ); + ); - await queueService.startPg( - QueueJobs.SecretRotationV2RotateSecrets, - async ([job]) => { - await rotateSecretsFns({ - job: { - ...job, - data: job.data as TSecretRotationRotateSecretsJobPayload - }, - secretRotationV2DAL, - secretRotationV2Service - }); - }, - { - batchSize: 1, - workerCount: 2, - pollingIntervalSeconds: 0.5 - } - ); - - await queueService.startPg( - QueueJobs.SecretRotationV2SendNotification, - async ([job]) => { - const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload; - try { - const { - name: rotationName, - type, - projectId, - lastRotationAttemptedAt, - folder, - environment, - id: rotationId - } = secretRotation; - - logger.info(`secretRotationV2Queue: Sending Status Notification [rotationId=${rotationId}]`); - - const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId); - const project = await projectDAL.findById(projectId); - - const projectAdmins = projectMembers.filter((member) => - member.roles.some((role) => role.role === ProjectMembershipRole.Admin) - ); - - const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation]; - - const rotationPath = `/projects/secret-management/${projectId}/secrets/${environment.slug}`; - - await notificationService.createUserNotifications( - projectAdmins.map((admin) => ({ - userId: admin.userId, - orgId: project.orgId, - type: NotificationType.SECRET_ROTATION_FAILED, - title: "Secret Rotation Failed", - body: `Your **${rotationType}** rotation **${rotationName}** failed to rotate.`, - link: rotationPath - })) - ); - - await smtpService.sendMail({ - recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean), - template: SmtpTemplates.SecretRotationFailed, - subjectLine: `Secret Rotation Failed`, - substitutions: { - rotationName, - rotationType, - content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date( - lastRotationAttemptedAt - ).toISOString()}. Please check the rotation status in Infisical for more details.`, - secretPath: folder.path, - environment: environment.name, - projectName: project.name, - rotationUrl: encodeURI(`${appCfg.SITE_URL}${rotationPath}`) - } + await queueService.startPg( + QueueJobs.SecretRotationV2RotateSecrets, + async ([job]) => { + await rotateSecretsFns({ + job: { + ...job, + data: job.data as TSecretRotationRotateSecretsJobPayload + }, + secretRotationV2DAL, + secretRotationV2Service }); - } catch (error) { - logger.error( - error, - `secretRotationV2Queue: Failed to Send Status Notification [rotationId=${secretRotation.id}]` - ); - throw error; + }, + { + batchSize: 1, + workerCount: 2, + pollingIntervalSeconds: 0.5 } - }, - { - batchSize: 1, - workerCount: 2, - pollingIntervalSeconds: 1 - } - ); + ); - await queueService.schedulePg( - QueueJobs.SecretRotationV2QueueRotations, - appCfg.isRotationDevelopmentMode ? "* * * * *" : "0 0 * * *", - undefined, - { tz: "UTC" } - ); + await queueService.startPg( + QueueJobs.SecretRotationV2SendNotification, + async ([job]) => { + const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload; + try { + const { + name: rotationName, + type, + projectId, + lastRotationAttemptedAt, + folder, + environment, + id: rotationId + } = secretRotation; + + logger.info(`secretRotationV2Queue: Sending Status Notification [rotationId=${rotationId}]`); + + const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId); + const project = await projectDAL.findById(projectId); + + const projectAdmins = projectMembers.filter((member) => + member.roles.some((role) => role.role === ProjectMembershipRole.Admin) + ); + + const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation]; + + const rotationPath = `/projects/secret-management/${projectId}/secrets/${environment.slug}`; + + await notificationService.createUserNotifications( + projectAdmins.map((admin) => ({ + userId: admin.userId, + orgId: project.orgId, + type: NotificationType.SECRET_ROTATION_FAILED, + title: "Secret Rotation Failed", + body: `Your **${rotationType}** rotation **${rotationName}** failed to rotate.`, + link: rotationPath + })) + ); + + await smtpService.sendMail({ + recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean), + template: SmtpTemplates.SecretRotationFailed, + subjectLine: `Secret Rotation Failed`, + substitutions: { + rotationName, + rotationType, + content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date( + lastRotationAttemptedAt + ).toISOString()}. Please check the rotation status in Infisical for more details.`, + secretPath: folder.path, + environment: environment.name, + projectName: project.name, + rotationUrl: encodeURI(`${appCfg.SITE_URL}${rotationPath}`) + } + }); + } catch (error) { + logger.error( + error, + `secretRotationV2Queue: Failed to Send Status Notification [rotationId=${secretRotation.id}]` + ); + throw error; + } + }, + { + batchSize: 1, + workerCount: 2, + pollingIntervalSeconds: 1 + } + ); + + await queueService.schedulePg( + QueueJobs.SecretRotationV2QueueRotations, + appCfg.isRotationDevelopmentMode ? "* * * * *" : "0 0 * * *", + undefined, + { tz: "UTC" } + ); + }; + + return { init }; }; diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts index 406c25e03..4830196f6 100644 --- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts +++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts @@ -141,202 +141,6 @@ export const secretScanningV2QueueServiceFactory = async ({ } }; - await queueService.startPg( - QueueJobs.SecretScanningV2FullScan, - async ([job]) => { - const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan; - const { retryCount, retryLimit } = job; - - const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`; - - const tempFolder = await createTempFolder(); - - const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); - - if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`); - - const resource = await secretScanningV2DAL.resources.findById(resourceId); - - if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`); - - let lock: Awaited> | undefined; - - try { - try { - lock = await keyStore.acquireLock( - [KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)], - 60 * 1000 * 5 - ); - } catch (e) { - throw new Error("Failed to acquire scanning lock."); - } - - await secretScanningV2DAL.scans.update( - { id: scanId }, - { - status: SecretScanningScanStatus.Scanning - } - ); - - let connection: TAppConnection | null = null; - if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService); - - const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({ - kmsService, - appConnectionDAL - }); - - const findingsPath = join(tempFolder, "findings.json"); - - const scanPath = await factory.getFullScanPath({ - dataSource: { - ...dataSource, - connection - } as TSecretScanningDataSourceWithConnection, - resourceName: resource.name, - tempFolder - }); - - const config = await secretScanningV2DAL.configs.findOne({ - projectId: dataSource.projectId - }); - - let configPath: string | undefined; - - if (config && config.content) { - configPath = join(tempFolder, "infisical-scan.toml"); - await writeTextToFile(configPath, config.content); - } - - let findingsPayload: TFindingsPayload; - switch (resource.type) { - case SecretScanningResource.Repository: - case SecretScanningResource.Project: - findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath); - break; - default: - throw new Error("Unhandled resource type"); - } - - const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => { - let findings: TSecretScanningFindings[] = []; - if (findingsPayload.length) { - findings = await secretScanningV2DAL.findings.upsert( - findingsPayload.map((finding) => ({ - ...finding, - projectId: dataSource.projectId, - dataSourceName: dataSource.name, - dataSourceType: dataSource.type, - resourceName: resource.name, - resourceType: resource.type, - scanId - })), - ["projectId", "fingerprint"], - tx, - ["resourceName", "dataSourceName"] - ); - } - - await secretScanningV2DAL.scans.update( - { id: scanId }, - { - status: SecretScanningScanStatus.Completed, - statusMessage: null - } - ); - - return findings; - }); - - const newFindings = allFindings.filter((finding) => finding.scanId === scanId); - - if (newFindings.length) { - await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, { - status: SecretScanningScanStatus.Completed, - resourceName: resource.name, - isDiffScan: false, - dataSource, - numberOfSecrets: newFindings.length, - scanId - }); - } - - await auditLogService.createAuditLog({ - projectId: dataSource.projectId, - actor: { - type: ActorType.PLATFORM, - metadata: {} - }, - event: { - type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN, - metadata: { - dataSourceId: dataSource.id, - dataSourceType: dataSource.type, - resourceId: resource.id, - resourceType: resource.type, - scanId, - scanStatus: SecretScanningScanStatus.Completed, - scanType: SecretScanningScanType.FullScan, - numberOfSecretsDetected: findingsPayload.length - } - } - }); - - logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`); - } catch (error) { - if (retryCount === retryLimit) { - const errorMessage = parseScanErrorMessage(error); - - await secretScanningV2DAL.scans.update( - { id: scanId }, - { - status: SecretScanningScanStatus.Failed, - statusMessage: errorMessage - } - ); - - await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, { - status: SecretScanningScanStatus.Failed, - resourceName: resource.name, - dataSource, - errorMessage - }); - - await auditLogService.createAuditLog({ - projectId: dataSource.projectId, - actor: { - type: ActorType.PLATFORM, - metadata: {} - }, - event: { - type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN, - metadata: { - dataSourceId: dataSource.id, - dataSourceType: dataSource.type, - resourceId: resource.id, - resourceType: resource.type, - scanId, - scanStatus: SecretScanningScanStatus.Failed, - scanType: SecretScanningScanType.FullScan - } - } - }); - } - - logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`); - throw error; - } finally { - await deleteTempFolder(tempFolder); - await lock?.release(); - } - }, - { - batchSize: 1, - workerCount: 2, - pollingIntervalSeconds: 1 - } - ); - const queueResourceDiffScan = async ({ payload, dataSourceId, @@ -391,148 +195,127 @@ export const secretScanningV2QueueServiceFactory = async ({ } }; - await queueService.startPg( - QueueJobs.SecretScanningV2DiffScan, - async ([job]) => { - const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan; - const { retryCount, retryLimit } = job; + const init = async () => { + await queueService.startPg( + QueueJobs.SecretScanningV2FullScan, + async ([job]) => { + const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan; + const { retryCount, retryLimit } = job; - const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`; + const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`; - const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); + const tempFolder = await createTempFolder(); - if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`); + const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); - const resource = await secretScanningV2DAL.resources.findById(resourceId); + if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`); - if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`); + const resource = await secretScanningV2DAL.resources.findById(resourceId); - const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({ - kmsService, - appConnectionDAL - }); + if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`); - const tempFolder = await createTempFolder(); + let lock: Awaited> | undefined; - try { - await secretScanningV2DAL.scans.update( - { id: scanId }, - { - status: SecretScanningScanStatus.Scanning - } - ); - - let connection: TAppConnection | null = null; - if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService); - - const config = await secretScanningV2DAL.configs.findOne({ - projectId: dataSource.projectId - }); - - let configPath: string | undefined; - - if (config && config.content) { - configPath = join(tempFolder, "infisical-scan.toml"); - await writeTextToFile(configPath, config.content); - } - - const findingsPayload = await factory.getDiffScanFindingsPayload({ - dataSource: { - ...dataSource, - connection - } as TSecretScanningDataSourceWithConnection, - resourceName: resource.name, - payload, - configPath - }); - - const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => { - let findings: TSecretScanningFindings[] = []; - - if (findingsPayload.length) { - findings = await secretScanningV2DAL.findings.upsert( - findingsPayload.map((finding) => ({ - ...finding, - projectId: dataSource.projectId, - dataSourceName: dataSource.name, - dataSourceType: dataSource.type, - resourceName: resource.name, - resourceType: resource.type, - scanId - })), - ["projectId", "fingerprint"], - tx, - ["resourceName", "dataSourceName"] + try { + try { + lock = await keyStore.acquireLock( + [KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)], + 60 * 1000 * 5 ); + } catch (e) { + throw new Error("Failed to acquire scanning lock."); } await secretScanningV2DAL.scans.update( { id: scanId }, { - status: SecretScanningScanStatus.Completed + status: SecretScanningScanStatus.Scanning } ); - return findings; - }); + let connection: TAppConnection | null = null; + if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService); - const newFindings = allFindings.filter((finding) => finding.scanId === scanId); - - if (newFindings.length) { - const finding = newFindings[0] as TSecretScanningFinding; - await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, { - status: SecretScanningScanStatus.Completed, - resourceName: resource.name, - isDiffScan: true, - dataSource, - numberOfSecrets: newFindings.length, - scanId, - authorName: finding?.details?.author, - authorEmail: finding?.details?.email + const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({ + kmsService, + appConnectionDAL }); - } - await auditLogService.createAuditLog({ - projectId: dataSource.projectId, - actor: { - type: ActorType.PLATFORM, - metadata: {} - }, - event: { - type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN, - metadata: { - dataSourceId: dataSource.id, - dataSourceType: dataSource.type, - resourceId, - resourceType: resource.type, - scanId, - scanStatus: SecretScanningScanStatus.Completed, - scanType: SecretScanningScanType.DiffScan, - numberOfSecretsDetected: findingsPayload.length - } + const findingsPath = join(tempFolder, "findings.json"); + + const scanPath = await factory.getFullScanPath({ + dataSource: { + ...dataSource, + connection + } as TSecretScanningDataSourceWithConnection, + resourceName: resource.name, + tempFolder + }); + + const config = await secretScanningV2DAL.configs.findOne({ + projectId: dataSource.projectId + }); + + let configPath: string | undefined; + + if (config && config.content) { + configPath = join(tempFolder, "infisical-scan.toml"); + await writeTextToFile(configPath, config.content); } - }); - logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`); - } catch (error) { - if (retryCount === retryLimit) { - const errorMessage = parseScanErrorMessage(error); + let findingsPayload: TFindingsPayload; + switch (resource.type) { + case SecretScanningResource.Repository: + case SecretScanningResource.Project: + findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath); + break; + default: + throw new Error("Unhandled resource type"); + } - await secretScanningV2DAL.scans.update( - { id: scanId }, - { - status: SecretScanningScanStatus.Failed, - statusMessage: errorMessage + const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => { + let findings: TSecretScanningFindings[] = []; + if (findingsPayload.length) { + findings = await secretScanningV2DAL.findings.upsert( + findingsPayload.map((finding) => ({ + ...finding, + projectId: dataSource.projectId, + dataSourceName: dataSource.name, + dataSourceType: dataSource.type, + resourceName: resource.name, + resourceType: resource.type, + scanId + })), + ["projectId", "fingerprint"], + tx, + ["resourceName", "dataSourceName"] + ); } - ); - await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, { - status: SecretScanningScanStatus.Failed, - resourceName: resource.name, - dataSource, - errorMessage + await secretScanningV2DAL.scans.update( + { id: scanId }, + { + status: SecretScanningScanStatus.Completed, + statusMessage: null + } + ); + + return findings; }); + const newFindings = allFindings.filter((finding) => finding.scanId === scanId); + + if (newFindings.length) { + await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, { + status: SecretScanningScanStatus.Completed, + resourceName: resource.name, + isDiffScan: false, + dataSource, + numberOfSecrets: newFindings.length, + scanId + }); + } + await auditLogService.createAuditLog({ projectId: dataSource.projectId, actor: { @@ -547,128 +330,348 @@ export const secretScanningV2QueueServiceFactory = async ({ resourceId: resource.id, resourceType: resource.type, scanId, - scanStatus: SecretScanningScanStatus.Failed, - scanType: SecretScanningScanType.DiffScan + scanStatus: SecretScanningScanStatus.Completed, + scanType: SecretScanningScanType.FullScan, + numberOfSecretsDetected: findingsPayload.length } } }); + + logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`); + } catch (error) { + if (retryCount === retryLimit) { + const errorMessage = parseScanErrorMessage(error); + + await secretScanningV2DAL.scans.update( + { id: scanId }, + { + status: SecretScanningScanStatus.Failed, + statusMessage: errorMessage + } + ); + + await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, { + status: SecretScanningScanStatus.Failed, + resourceName: resource.name, + dataSource, + errorMessage + }); + + await auditLogService.createAuditLog({ + projectId: dataSource.projectId, + actor: { + type: ActorType.PLATFORM, + metadata: {} + }, + event: { + type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN, + metadata: { + dataSourceId: dataSource.id, + dataSourceType: dataSource.type, + resourceId: resource.id, + resourceType: resource.type, + scanId, + scanStatus: SecretScanningScanStatus.Failed, + scanType: SecretScanningScanType.FullScan + } + } + }); + } + + logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`); + throw error; + } finally { + await deleteTempFolder(tempFolder); + await lock?.release(); } - - logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`); - throw error; - } finally { - await deleteTempFolder(tempFolder); + }, + { + batchSize: 1, + workerCount: 2, + pollingIntervalSeconds: 1 } - }, - { - batchSize: 1, - workerCount: 2, - pollingIntervalSeconds: 1 - } - ); + ); - await queueService.startPg( - QueueJobs.SecretScanningV2SendNotification, - async ([job]) => { - const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification; + await queueService.startPg( + QueueJobs.SecretScanningV2DiffScan, + async ([job]) => { + const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan; + const { retryCount, retryLimit } = job; - const appCfg = getConfig(); + const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`; - if (!appCfg.isSmtpConfigured) return; + const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); - try { - const { projectId } = dataSource; + if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`); - logger.info( - `secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]` - ); + const resource = await secretScanningV2DAL.resources.findById(resourceId); - const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId); - const project = await projectDAL.findById(projectId); + if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`); - const recipients = projectMembers.filter((member) => { - const isAdmin = member.roles.some((role) => role.role === ProjectMembershipRole.Admin); - const isCompleted = payload.status === SecretScanningScanStatus.Completed; - // We assume that the committer is one of the project members - const isCommitter = isCompleted && payload.authorEmail === member.user.email; - return isAdmin || isCommitter; + const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({ + kmsService, + appConnectionDAL }); - const timestamp = new Date().toISOString(); + const tempFolder = await createTempFolder(); - const subjectLine = - payload.status === SecretScanningScanStatus.Completed - ? "Incident Alert: Secret(s) Leaked" - : `Secret Scanning Failed`; + try { + await secretScanningV2DAL.scans.update( + { id: scanId }, + { + status: SecretScanningScanStatus.Scanning + } + ); - await notificationService.createUserNotifications( - recipients.map((member) => ({ - userId: member.userId, - orgId: project.orgId, - type: - payload.status === SecretScanningScanStatus.Completed - ? NotificationType.SECRET_SCANNING_SECRETS_DETECTED - : NotificationType.SECRET_SCANNING_SCAN_FAILED, - title: subjectLine, - body: - payload.status === SecretScanningScanStatus.Completed - ? `Uncovered **${payload.numberOfSecrets}** secret(s) ${payload.isDiffScan ? " from a recent commit to" : " in"} **${resourceName}**.` - : `Encountered an error while attempting to scan the resource **${resourceName}**: ${payload.errorMessage}`, - link: - payload.status === SecretScanningScanStatus.Completed - ? `/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}` - : `/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}` - })) - ); + let connection: TAppConnection | null = null; + if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService); - await smtpService.sendMail({ - recipients: recipients.map((member) => member.user.email!).filter(Boolean), - template: - payload.status === SecretScanningScanStatus.Completed - ? SmtpTemplates.SecretScanningV2SecretsDetected - : SmtpTemplates.SecretScanningV2ScanFailed, - subjectLine, - substitutions: - payload.status === SecretScanningScanStatus.Completed - ? { - authorName: payload.authorName, - authorEmail: payload.authorEmail, - resourceName, - numberOfSecrets: payload.numberOfSecrets, - isDiffScan: payload.isDiffScan, - url: encodeURI( - `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}` - ), - timestamp - } - : { + const config = await secretScanningV2DAL.configs.findOne({ + projectId: dataSource.projectId + }); + + let configPath: string | undefined; + + if (config && config.content) { + configPath = join(tempFolder, "infisical-scan.toml"); + await writeTextToFile(configPath, config.content); + } + + const findingsPayload = await factory.getDiffScanFindingsPayload({ + dataSource: { + ...dataSource, + connection + } as TSecretScanningDataSourceWithConnection, + resourceName: resource.name, + payload, + configPath + }); + + const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => { + let findings: TSecretScanningFindings[] = []; + + if (findingsPayload.length) { + findings = await secretScanningV2DAL.findings.upsert( + findingsPayload.map((finding) => ({ + ...finding, + projectId: dataSource.projectId, dataSourceName: dataSource.name, - resourceName, - projectName: project.name, - timestamp, - errorMessage: payload.errorMessage, - url: encodeURI( - `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}` - ) + dataSourceType: dataSource.type, + resourceName: resource.name, + resourceType: resource.type, + scanId + })), + ["projectId", "fingerprint"], + tx, + ["resourceName", "dataSourceName"] + ); + } + + await secretScanningV2DAL.scans.update( + { id: scanId }, + { + status: SecretScanningScanStatus.Completed + } + ); + + return findings; + }); + + const newFindings = allFindings.filter((finding) => finding.scanId === scanId); + + if (newFindings.length) { + const finding = newFindings[0] as TSecretScanningFinding; + await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, { + status: SecretScanningScanStatus.Completed, + resourceName: resource.name, + isDiffScan: true, + dataSource, + numberOfSecrets: newFindings.length, + scanId, + authorName: finding?.details?.author, + authorEmail: finding?.details?.email + }); + } + + await auditLogService.createAuditLog({ + projectId: dataSource.projectId, + actor: { + type: ActorType.PLATFORM, + metadata: {} + }, + event: { + type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN, + metadata: { + dataSourceId: dataSource.id, + dataSourceType: dataSource.type, + resourceId, + resourceType: resource.type, + scanId, + scanStatus: SecretScanningScanStatus.Completed, + scanType: SecretScanningScanType.DiffScan, + numberOfSecretsDetected: findingsPayload.length + } + } + }); + + logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`); + } catch (error) { + if (retryCount === retryLimit) { + const errorMessage = parseScanErrorMessage(error); + + await secretScanningV2DAL.scans.update( + { id: scanId }, + { + status: SecretScanningScanStatus.Failed, + statusMessage: errorMessage + } + ); + + await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, { + status: SecretScanningScanStatus.Failed, + resourceName: resource.name, + dataSource, + errorMessage + }); + + await auditLogService.createAuditLog({ + projectId: dataSource.projectId, + actor: { + type: ActorType.PLATFORM, + metadata: {} + }, + event: { + type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN, + metadata: { + dataSourceId: dataSource.id, + dataSourceType: dataSource.type, + resourceId: resource.id, + resourceType: resource.type, + scanId, + scanStatus: SecretScanningScanStatus.Failed, + scanType: SecretScanningScanType.DiffScan } - }); - } catch (error) { - logger.error( - error, - `secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]` - ); - throw error; + } + }); + } + + logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`); + throw error; + } finally { + await deleteTempFolder(tempFolder); + } + }, + { + batchSize: 1, + workerCount: 2, + pollingIntervalSeconds: 1 } - }, - { - batchSize: 1, - workerCount: 2, - pollingIntervalSeconds: 1 - } - ); + ); + + await queueService.startPg( + QueueJobs.SecretScanningV2SendNotification, + async ([job]) => { + const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification; + + const appCfg = getConfig(); + + if (!appCfg.isSmtpConfigured) return; + + try { + const { projectId } = dataSource; + + logger.info( + `secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]` + ); + + const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId); + const project = await projectDAL.findById(projectId); + + const recipients = projectMembers.filter((member) => { + const isAdmin = member.roles.some((role) => role.role === ProjectMembershipRole.Admin); + const isCompleted = payload.status === SecretScanningScanStatus.Completed; + // We assume that the committer is one of the project members + const isCommitter = isCompleted && payload.authorEmail === member.user.email; + return isAdmin || isCommitter; + }); + + const timestamp = new Date().toISOString(); + + const subjectLine = + payload.status === SecretScanningScanStatus.Completed + ? "Incident Alert: Secret(s) Leaked" + : `Secret Scanning Failed`; + + await notificationService.createUserNotifications( + recipients.map((member) => ({ + userId: member.userId, + orgId: project.orgId, + type: + payload.status === SecretScanningScanStatus.Completed + ? NotificationType.SECRET_SCANNING_SECRETS_DETECTED + : NotificationType.SECRET_SCANNING_SCAN_FAILED, + title: subjectLine, + body: + payload.status === SecretScanningScanStatus.Completed + ? `Uncovered **${payload.numberOfSecrets}** secret(s) ${payload.isDiffScan ? " from a recent commit to" : " in"} **${resourceName}**.` + : `Encountered an error while attempting to scan the resource **${resourceName}**: ${payload.errorMessage}`, + link: + payload.status === SecretScanningScanStatus.Completed + ? `/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}` + : `/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}` + })) + ); + + await smtpService.sendMail({ + recipients: recipients.map((member) => member.user.email!).filter(Boolean), + template: + payload.status === SecretScanningScanStatus.Completed + ? SmtpTemplates.SecretScanningV2SecretsDetected + : SmtpTemplates.SecretScanningV2ScanFailed, + subjectLine, + substitutions: + payload.status === SecretScanningScanStatus.Completed + ? { + authorName: payload.authorName, + authorEmail: payload.authorEmail, + resourceName, + numberOfSecrets: payload.numberOfSecrets, + isDiffScan: payload.isDiffScan, + url: encodeURI( + `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}` + ), + timestamp + } + : { + dataSourceName: dataSource.name, + resourceName, + projectName: project.name, + timestamp, + errorMessage: payload.errorMessage, + url: encodeURI( + `${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}` + ) + } + }); + } catch (error) { + logger.error( + error, + `secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]` + ); + throw error; + } + }, + { + batchSize: 1, + workerCount: 2, + pollingIntervalSeconds: 1 + } + ); + }; return { queueDataSourceFullScan, - queueResourceDiffScan + queueResourceDiffScan, + init }; }; diff --git a/backend/src/main.ts b/backend/src/main.ts index 9d7812c96..d36d706ba 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -72,7 +72,7 @@ const run = async () => { const keyStore = keyStoreFactory(envConfig, keyValueStoreDAL); const redis = buildRedisFromConfig(envConfig); - const server = await main({ + const { server, completeServerInitialization } = await main({ db, auditLogDb, superAdminDAL, @@ -140,7 +140,18 @@ const run = async () => { } }); - logger.info("Migrations complete. Marking server as READY..."); + logger.info("Migrations complete. Completing server initialization..."); + + try { + await completeServerInitialization(); + } catch (error) { + logger.error(error, "Failed to complete server initialization"); + await server.close(); + await queue.shutdown(); + process.exit(1); + } + + logger.info("Server initialization complete. Marking server as READY..."); markServerReady(); diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index daf49ed10..17a9feb2a 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -221,7 +221,7 @@ export const main = async ({ }; }); - await server.register(registerRoutes, { + const completeServerInitialization = await registerRoutes(server, { smtp, queue, db, @@ -240,7 +240,7 @@ export const main = async ({ await server.ready(); server.swagger(); - return server; + return { server, completeServerInitialization }; } catch (err) { server.log.error(err); await queue.shutdown(); diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index c135db83e..44a5c8503 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -2206,7 +2206,7 @@ export const registerRoutes = async ( internalCaFns }); - await secretRotationV2QueueServiceFactory({ + const secretRotationV2Queue = await secretRotationV2QueueServiceFactory({ secretRotationV2Service, secretRotationV2DAL, queueService, @@ -2300,50 +2300,87 @@ export const registerRoutes = async ( // setup the communication with license key server await licenseService.init(); - // If FIPS is enabled, we check to ensure that the users license includes FIPS mode. - crypto.verifyFipsLicense(licenseService); + const completeServerInitialization = async () => { + await superAdminService.initServerCfg(); - await superAdminService.initServerCfg(); + // If FIPS is enabled, we check to ensure that the users license includes FIPS mode. + crypto.verifyFipsLicense(licenseService); - // Start HSM service if it's configured/enabled. - await hsmService.startService(); + // Start HSM service if it's configured/enabled. + await hsmService.startService(); - const hsmStatus = await isHsmActiveAndEnabled({ - hsmService, - kmsRootConfigDAL, - licenseService - }); + const hsmStatus = await isHsmActiveAndEnabled({ + hsmService, + kmsRootConfigDAL, + licenseService + }); - // if the encryption strategy is software - user needs to provide an encryption key - // if the encryption strategy is null AND the hsm is not configured - user needs to provide an encryption key - const needsEncryptionKey = - hsmStatus.rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.Software || - (hsmStatus.rootKmsConfigEncryptionStrategy === null && !hsmStatus.isHsmConfigured); + // if the encryption strategy is software - user needs to provide an encryption key + // if the encryption strategy is null AND the hsm is not configured - user needs to provide an encryption key + const needsEncryptionKey = + hsmStatus.rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.Software || + (hsmStatus.rootKmsConfigEncryptionStrategy === null && !hsmStatus.isHsmConfigured); - if (needsEncryptionKey) { - if (!envConfig.ROOT_ENCRYPTION_KEY && !envConfig.ENCRYPTION_KEY) { - throw new BadRequestError({ - message: - "Root KMS encryption strategy is set to software. Please set the ENCRYPTION_KEY environment variable and restart your deployment.\nYou can enable HSM encryption in the Server Console." - }); + if (needsEncryptionKey) { + if (!envConfig.ROOT_ENCRYPTION_KEY && !envConfig.ENCRYPTION_KEY) { + throw new BadRequestError({ + message: + "Root KMS encryption strategy is set to software. Please set the ENCRYPTION_KEY environment variable and restart your deployment.\nYou can enable HSM encryption in the Server Console." + }); + } } - } - await telemetryQueue.startTelemetryCheck(); - await telemetryQueue.startAggregatedEventsJob(); - await dailyResourceCleanUp.init(); - await healthAlert.init(); - await pkiSyncCleanup.init(); - await pamAccountRotation.init(); - await dailyReminderQueueService.startDailyRemindersJob(); - await dailyReminderQueueService.startSecretReminderMigrationJob(); - await dailyExpiringPkiItemAlert.startSendingAlerts(); - await pkiSubscriberQueue.startDailyAutoRenewalJob(); - await certificateV3Queue.init(); - await kmsService.startService(hsmStatus); - await microsoftTeamsService.start(); - await dynamicSecretQueueService.init(); - await eventBusService.init(); + await telemetryQueue.startTelemetryCheck(); + await telemetryQueue.startAggregatedEventsJob(); + await dailyResourceCleanUp.init(); + await healthAlert.init(); + await pkiSyncCleanup.init(); + await pamAccountRotation.init(); + await dailyReminderQueueService.startDailyRemindersJob(); + await dailyReminderQueueService.startSecretReminderMigrationJob(); + await dailyExpiringPkiItemAlert.startSendingAlerts(); + await pkiSubscriberQueue.startDailyAutoRenewalJob(); + await certificateV3Queue.init(); + await kmsService.startService(hsmStatus); + await microsoftTeamsService.start(); + await dynamicSecretQueueService.init(); + await secretScanningV2Queue.init(); + await secretRotationV2Queue.init(); + await notificationQueue.init(); + await eventBusService.init(); + + const cronJobs: CronJob[] = []; + if (appCfg.isProductionMode) { + const rateLimitSyncJob = await rateLimitService.initializeBackgroundSync(); + if (rateLimitSyncJob) { + cronJobs.push(rateLimitSyncJob); + } + const licenseSyncJob = await licenseService.initializeBackgroundSync(); + if (licenseSyncJob) { + cronJobs.push(licenseSyncJob); + } + + const microsoftTeamsSyncJob = await microsoftTeamsService.initializeBackgroundSync(); + if (microsoftTeamsSyncJob) { + cronJobs.push(microsoftTeamsSyncJob); + } + + const adminIntegrationsSyncJob = await superAdminService.initializeAdminIntegrationConfigSync(); + if (adminIntegrationsSyncJob) { + cronJobs.push(adminIntegrationsSyncJob); + } + } + + const configSyncJob = await superAdminService.initializeEnvConfigSync(); + if (configSyncJob) { + cronJobs.push(configSyncJob); + } + + const oauthConfigSyncJob = await initializeOauthConfigSync(); + if (oauthConfigSyncJob) { + cronJobs.push(oauthConfigSyncJob); + } + }; // inject all services server.decorate("services", { @@ -2473,38 +2510,6 @@ export const registerRoutes = async ( convertor: convertorService }); - const cronJobs: CronJob[] = []; - if (appCfg.isProductionMode) { - const rateLimitSyncJob = await rateLimitService.initializeBackgroundSync(); - if (rateLimitSyncJob) { - cronJobs.push(rateLimitSyncJob); - } - const licenseSyncJob = await licenseService.initializeBackgroundSync(); - if (licenseSyncJob) { - cronJobs.push(licenseSyncJob); - } - - const microsoftTeamsSyncJob = await microsoftTeamsService.initializeBackgroundSync(); - if (microsoftTeamsSyncJob) { - cronJobs.push(microsoftTeamsSyncJob); - } - - const adminIntegrationsSyncJob = await superAdminService.initializeAdminIntegrationConfigSync(); - if (adminIntegrationsSyncJob) { - cronJobs.push(adminIntegrationsSyncJob); - } - } - - const configSyncJob = await superAdminService.initializeEnvConfigSync(); - if (configSyncJob) { - cronJobs.push(configSyncJob); - } - - const oauthConfigSyncJob = await initializeOauthConfigSync(); - if (oauthConfigSyncJob) { - cronJobs.push(oauthConfigSyncJob); - } - server.decorate("store", { user: userDAL, kmipClient: kmipClientDAL @@ -2593,9 +2598,10 @@ export const registerRoutes = async ( await server.register(registerV4Routes, { prefix: "/api/v4" }); server.addHook("onClose", async () => { - cronJobs.forEach((job) => job.stop()); await telemetryService.flushAll(); await eventBusService.close(); sseService.close(); }); + + return completeServerInitialization; }; diff --git a/backend/src/services/notification/notification-queue.ts b/backend/src/services/notification/notification-queue.ts index e5d89c83b..aaff8e23f 100644 --- a/backend/src/services/notification/notification-queue.ts +++ b/backend/src/services/notification/notification-queue.ts @@ -10,6 +10,7 @@ type TNotificationQueueServiceFactoryDep = { export type TNotificationQueueServiceFactory = { pushUserNotifications: (data: TCreateUserNotificationDTO[]) => Promise; + init: () => Promise; }; export const notificationQueueServiceFactory = async ({ @@ -20,20 +21,23 @@ export const notificationQueueServiceFactory = async ({ await queueService.queuePg(QueueJobs.UserNotification, { notifications: data }); }; - await queueService.startPg( - QueueJobs.UserNotification, - async ([job]) => { - const { notifications } = job.data as { notifications: TCreateUserNotificationDTO[] }; - await userNotificationDAL.batchInsert(notifications); - }, - { - batchSize: 1, - workerCount: 2, - pollingIntervalSeconds: 1 - } - ); + const init = async () => { + await queueService.startPg( + QueueJobs.UserNotification, + async ([job]) => { + const { notifications } = job.data as { notifications: TCreateUserNotificationDTO[] }; + await userNotificationDAL.batchInsert(notifications); + }, + { + batchSize: 1, + workerCount: 2, + pollingIntervalSeconds: 1 + } + ); + }; return { - pushUserNotifications + pushUserNotifications, + init }; }; From 8c9a68346857ed175ba7cea5deb48df4e8c884a6 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 28 Oct 2025 23:20:18 +0800 Subject: [PATCH 6/7] misc: addressed e2e failure --- backend/e2e-test/vitest-environment-knex.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/e2e-test/vitest-environment-knex.ts b/backend/e2e-test/vitest-environment-knex.ts index f33d32f6a..5ee84a7c7 100644 --- a/backend/e2e-test/vitest-environment-knex.ts +++ b/backend/e2e-test/vitest-environment-knex.ts @@ -83,7 +83,7 @@ export default { await queue.initialize(); - const server = await main({ + const { server, completeServerInitialization } = await main({ db, smtp, logger, @@ -96,6 +96,8 @@ export default { envConfig: envCfg }); + await completeServerInitialization(); + markServerReady(); await bootstrapCheck({ db }); From 8c5c88aefb2ab922d1b9b603595aae940dfd9df8 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 28 Oct 2025 23:41:25 +0800 Subject: [PATCH 7/7] misc: corrected hsm placement --- backend/src/ee/services/hsm/hsm-fns.ts | 2 +- backend/src/server/routes/index.ts | 54 +++++++++++++------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/backend/src/ee/services/hsm/hsm-fns.ts b/backend/src/ee/services/hsm/hsm-fns.ts index 400fa31e9..d37766f71 100644 --- a/backend/src/ee/services/hsm/hsm-fns.ts +++ b/backend/src/ee/services/hsm/hsm-fns.ts @@ -84,7 +84,7 @@ export const isHsmActiveAndEnabled = async ({ rootKmsConfigEncryptionStrategy = (rootKmsConfig?.encryptionStrategy || null) as RootKeyEncryptionStrategy | null; if ( - rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.HSM && + (rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.HSM || isHsmConfigured) && licenseService && !licenseService.onPremFeatures.hsm ) { diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 44a5c8503..d50f3d01e 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -2300,36 +2300,36 @@ export const registerRoutes = async ( // setup the communication with license key server await licenseService.init(); + // If FIPS is enabled, we check to ensure that the users license includes FIPS mode. + crypto.verifyFipsLicense(licenseService); + + // Start HSM service if it's configured/enabled. + await hsmService.startService(); + + const hsmStatus = await isHsmActiveAndEnabled({ + hsmService, + kmsRootConfigDAL, + licenseService + }); + + // if the encryption strategy is software - user needs to provide an encryption key + // if the encryption strategy is null AND the hsm is not configured - user needs to provide an encryption key + const needsEncryptionKey = + hsmStatus.rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.Software || + (hsmStatus.rootKmsConfigEncryptionStrategy === null && !hsmStatus.isHsmConfigured); + + if (needsEncryptionKey) { + if (!envConfig.ROOT_ENCRYPTION_KEY && !envConfig.ENCRYPTION_KEY) { + throw new BadRequestError({ + message: + "Root KMS encryption strategy is set to software. Please set the ENCRYPTION_KEY environment variable and restart your deployment.\nYou can enable HSM encryption in the Server Console." + }); + } + } + const completeServerInitialization = async () => { await superAdminService.initServerCfg(); - // If FIPS is enabled, we check to ensure that the users license includes FIPS mode. - crypto.verifyFipsLicense(licenseService); - - // Start HSM service if it's configured/enabled. - await hsmService.startService(); - - const hsmStatus = await isHsmActiveAndEnabled({ - hsmService, - kmsRootConfigDAL, - licenseService - }); - - // if the encryption strategy is software - user needs to provide an encryption key - // if the encryption strategy is null AND the hsm is not configured - user needs to provide an encryption key - const needsEncryptionKey = - hsmStatus.rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.Software || - (hsmStatus.rootKmsConfigEncryptionStrategy === null && !hsmStatus.isHsmConfigured); - - if (needsEncryptionKey) { - if (!envConfig.ROOT_ENCRYPTION_KEY && !envConfig.ENCRYPTION_KEY) { - throw new BadRequestError({ - message: - "Root KMS encryption strategy is set to software. Please set the ENCRYPTION_KEY environment variable and restart your deployment.\nYou can enable HSM encryption in the Server Console." - }); - } - } - await telemetryQueue.startTelemetryCheck(); await telemetryQueue.startAggregatedEventsJob(); await dailyResourceCleanUp.init();