From 17f19220418d809f5f90122756e7f2816c565177 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Mon, 1 Dec 2025 14:14:17 -0800 Subject: [PATCH 01/13] Add mark as ready --- .../pki-acme/pki-acme-challenge-service.ts | 161 ++++++++++-------- .../ee/services/pki-acme/pki-acme-types.ts | 3 + 2 files changed, 91 insertions(+), 73 deletions(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts index e084c8f0a..e0d58144f 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts @@ -1,5 +1,6 @@ import axios, { AxiosError } from "axios"; +import { TPkiAcmeChallenges } from "@app/db/schemas/pki-acme-challenges"; import { getConfig } from "@app/lib/config/env"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { isPrivateIp } from "@app/lib/ip/ipRange"; @@ -18,7 +19,11 @@ import { TPkiAcmeChallengeServiceFactory } from "./pki-acme-types"; type TPkiAcmeChallengeServiceFactoryDep = { acmeChallengeDAL: Pick< TPkiAcmeChallengeDALFactory, - "transaction" | "findByIdForChallengeValidation" | "markAsValidCascadeById" | "markAsInvalidCascadeById" + | "transaction" + | "findByIdForChallengeValidation" + | "markAsValidCascadeById" + | "markAsInvalidCascadeById" + | "updateById" >; }; @@ -26,9 +31,8 @@ export const pkiAcmeChallengeServiceFactory = ({ acmeChallengeDAL }: TPkiAcmeChallengeServiceFactoryDep): TPkiAcmeChallengeServiceFactory => { const appCfg = getConfig(); - - const validateChallengeResponse = async (challengeId: string): Promise => { - const error: Error | undefined = await acmeChallengeDAL.transaction(async (tx) => { + const markChallengeAsyReady = async (challengeId: string): Promise => { + return acmeChallengeDAL.transaction(async (tx) => { logger.info({ challengeId }, "Validating ACME challenge response"); const challenge = await acmeChallengeDAL.findByIdForChallengeValidation(challengeId, tx); if (!challenge) { @@ -52,81 +56,92 @@ export const pkiAcmeChallengeServiceFactory = ({ if (challenge.type !== AcmeChallengeType.HTTP_01) { throw new BadRequestError({ message: "Only HTTP-01 challenges are supported for now" }); } - let host = challenge.auth.identifierValue; + const host = challenge.auth.identifierValue; // check if host is a private ip address if (isPrivateIp(host)) { throw new BadRequestError({ message: "Private IP addresses are not allowed" }); } - if (appCfg.isAcmeDevelopmentMode && appCfg.ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES[host]) { - host = appCfg.ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES[host]; - logger.warn( - { srcHost: challenge.auth.identifierValue, dstHost: host }, - "Using ACME development HTTP-01 challenge host override" - ); - } - const challengeUrl = new URL(`/.well-known/acme-challenge/${challenge.auth.token}`, `http://${host}`); - logger.info({ challengeUrl }, "Performing ACME HTTP-01 challenge validation"); - try { - // TODO: read config from the profile to get the timeout instead - const timeoutMs = 10 * 1000; // 10 seconds - // Notice: well, we are in a transaction, ideally we should not hold transaction and perform - // a long running operation for long time. But assuming we are not performing a tons of - // challenge validation at the same time, it should be fine. - const challengeResponse = await axios.get(challengeUrl.toString(), { - // In case if we override the host in the development mode, still provide the original host in the header - // to help the upstream server to validate the request - headers: { Host: challenge.auth.identifierValue }, - timeout: timeoutMs, - responseType: "text", - validateStatus: () => true - }); - if (challengeResponse.status !== 200) { - throw new AcmeIncorrectResponseError({ - message: `ACME challenge response is not 200: ${challengeResponse.status}` - }); - } - const challengeResponseBody: string = challengeResponse.data; - const thumbprint = challenge.auth.account.publicKeyThumbprint; - const expectedChallengeResponseBody = `${challenge.auth.token}.${thumbprint}`; - if (challengeResponseBody.trimEnd() !== expectedChallengeResponseBody) { - throw new AcmeIncorrectResponseError({ message: "ACME challenge response is not correct" }); - } - await acmeChallengeDAL.markAsValidCascadeById(challengeId, tx); - } catch (exp) { - // TODO: we should retry the challenge validation a few times, but let's keep it simple for now - await acmeChallengeDAL.markAsInvalidCascadeById(challengeId, tx); - // Properly type and inspect the error - if (axios.isAxiosError(exp)) { - const axiosError = exp as AxiosError; - const errorCode = axiosError.code; - const errorMessage = axiosError.message; - - if (errorCode === "ECONNREFUSED" || errorMessage.includes("ECONNREFUSED")) { - return new AcmeConnectionError({ message: "Connection refused" }); - } - if (errorCode === "ENOTFOUND" || errorMessage.includes("ENOTFOUND")) { - return new AcmeDnsFailureError({ message: "Hostname could not be resolved (DNS failure)" }); - } - if (errorCode === "ECONNABORTED" || errorMessage.includes("timeout")) { - logger.error(exp, "Connection timed out while validating ACME challenge response"); - return new AcmeConnectionError({ message: "Connection timed out" }); - } - logger.error(exp, "Unknown error validating ACME challenge response"); - return new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" }); - } - if (exp instanceof Error) { - logger.error(exp, "Error validating ACME challenge response"); - } else { - logger.error(exp, "Unknown error validating ACME challenge response"); - return new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" }); - } - return exp; - } + return acmeChallengeDAL.updateById(challengeId, { status: AcmeChallengeStatus.Processing }, tx); }); - if (error) { - throw error; + }; + + const validateChallengeResponse = async (challengeId: string): Promise => { + logger.info({ challengeId }, "Validating ACME challenge response"); + const challenge = await acmeChallengeDAL.findByIdForChallengeValidation(challengeId); + if (!challenge) { + throw new NotFoundError({ message: "ACME challenge not found" }); + } + if (challenge.status !== AcmeChallengeStatus.Processing) { + throw new BadRequestError({ + message: `ACME challenge is ${challenge.status} instead of ${AcmeChallengeStatus.Processing}` + }); + } + let host = challenge.auth.identifierValue; + if (appCfg.isAcmeDevelopmentMode && appCfg.ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES[host]) { + host = appCfg.ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES[host]; + logger.warn( + { srcHost: challenge.auth.identifierValue, dstHost: host }, + "Using ACME development HTTP-01 challenge host override" + ); + } + const challengeUrl = new URL(`/.well-known/acme-challenge/${challenge.auth.token}`, `http://${host}`); + logger.info({ challengeUrl }, "Performing ACME HTTP-01 challenge validation"); + try { + // TODO: read config from the profile to get the timeout instead + const timeoutMs = 10 * 1000; // 10 seconds + // Notice: well, we are in a transaction, ideally we should not hold transaction and perform + // a long running operation for long time. But assuming we are not performing a tons of + // challenge validation at the same time, it should be fine. + const challengeResponse = await axios.get(challengeUrl.toString(), { + // In case if we override the host in the development mode, still provide the original host in the header + // to help the upstream server to validate the request + headers: { Host: challenge.auth.identifierValue }, + timeout: timeoutMs, + responseType: "text", + validateStatus: () => true + }); + if (challengeResponse.status !== 200) { + throw new AcmeIncorrectResponseError({ + message: `ACME challenge response is not 200: ${challengeResponse.status}` + }); + } + const challengeResponseBody: string = challengeResponse.data; + const thumbprint = challenge.auth.account.publicKeyThumbprint; + const expectedChallengeResponseBody = `${challenge.auth.token}.${thumbprint}`; + if (challengeResponseBody.trimEnd() !== expectedChallengeResponseBody) { + throw new AcmeIncorrectResponseError({ message: "ACME challenge response is not correct" }); + } + await acmeChallengeDAL.markAsValidCascadeById(challengeId); + } catch (exp) { + // TODO: we should retry the challenge validation a few times, but let's keep it simple for now + await acmeChallengeDAL.markAsInvalidCascadeById(challengeId); + // Properly type and inspect the error + if (axios.isAxiosError(exp)) { + const axiosError = exp as AxiosError; + const errorCode = axiosError.code; + const errorMessage = axiosError.message; + + if (errorCode === "ECONNREFUSED" || errorMessage.includes("ECONNREFUSED")) { + throw new AcmeConnectionError({ message: "Connection refused" }); + } + if (errorCode === "ENOTFOUND" || errorMessage.includes("ENOTFOUND")) { + throw new AcmeDnsFailureError({ message: "Hostname could not be resolved (DNS failure)" }); + } + if (errorCode === "ECONNABORTED" || errorMessage.includes("timeout")) { + logger.error(exp, "Connection timed out while validating ACME challenge response"); + throw new AcmeConnectionError({ message: "Connection timed out" }); + } + logger.error(exp, "Unknown error validating ACME challenge response"); + throw new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" }); + } + if (exp instanceof Error) { + logger.error(exp, "Error validating ACME challenge response"); + throw exp; + } + logger.error(exp, "Unknown error validating ACME challenge response"); + throw new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" }); } }; - return { validateChallengeResponse }; + return { markChallengeAsyReady, validateChallengeResponse }; }; diff --git a/backend/src/ee/services/pki-acme/pki-acme-types.ts b/backend/src/ee/services/pki-acme/pki-acme-types.ts index 3ddb424f1..fb76aec79 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-types.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-types.ts @@ -1,6 +1,8 @@ import { JWSHeaderParameters } from "jose"; import { z } from "zod"; +import { TPkiAcmeChallenges } from "@app/db/schemas/pki-acme-challenges"; + import { AcmeOrderResourceSchema, CreateAcmeAccountBodySchema, @@ -176,5 +178,6 @@ export type TPkiAcmeServiceFactory = { }; export type TPkiAcmeChallengeServiceFactory = { + markChallengeAsyReady: (challengeId: string) => Promise; validateChallengeResponse: (challengeId: string) => Promise; }; From da7cc1e406e5b66ab6cfc548275b0c4090bebfcc Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Mon, 1 Dec 2025 15:09:48 -0800 Subject: [PATCH 02/13] Add queue --- .../pki-acme/pki-acme-challenge-service.ts | 16 +++-- .../ee/services/pki-acme/pki-acme-queue.ts | 67 +++++++++++++++++++ .../ee/services/pki-acme/pki-acme-service.ts | 10 ++- .../ee/services/pki-acme/pki-acme-types.ts | 4 +- backend/src/queue/queue-service.ts | 10 ++- backend/src/server/routes/index.ts | 10 ++- 6 files changed, 102 insertions(+), 15 deletions(-) create mode 100644 backend/src/ee/services/pki-acme/pki-acme-queue.ts diff --git a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts index e0d58144f..572d52bf5 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts @@ -31,7 +31,7 @@ export const pkiAcmeChallengeServiceFactory = ({ acmeChallengeDAL }: TPkiAcmeChallengeServiceFactoryDep): TPkiAcmeChallengeServiceFactory => { const appCfg = getConfig(); - const markChallengeAsyReady = async (challengeId: string): Promise => { + const markChallengeAsReady = async (challengeId: string): Promise => { return acmeChallengeDAL.transaction(async (tx) => { logger.info({ challengeId }, "Validating ACME challenge response"); const challenge = await acmeChallengeDAL.findByIdForChallengeValidation(challengeId, tx); @@ -65,8 +65,8 @@ export const pkiAcmeChallengeServiceFactory = ({ }); }; - const validateChallengeResponse = async (challengeId: string): Promise => { - logger.info({ challengeId }, "Validating ACME challenge response"); + const validateChallengeResponse = async (challengeId: string, retryCount: number): Promise => { + logger.info({ challengeId, retryCount }, "Validating ACME challenge response"); const challenge = await acmeChallengeDAL.findByIdForChallengeValidation(challengeId); if (!challenge) { throw new NotFoundError({ message: "ACME challenge not found" }); @@ -93,7 +93,7 @@ export const pkiAcmeChallengeServiceFactory = ({ // a long running operation for long time. But assuming we are not performing a tons of // challenge validation at the same time, it should be fine. const challengeResponse = await axios.get(challengeUrl.toString(), { - // In case if we override the host in the development mode, still provide the original host in the header + // In case if we override the hos2 in the development mode, still provide the original host in the header // to help the upstream server to validate the request headers: { Host: challenge.auth.identifierValue }, timeout: timeoutMs, @@ -113,8 +113,10 @@ export const pkiAcmeChallengeServiceFactory = ({ } await acmeChallengeDAL.markAsValidCascadeById(challengeId); } catch (exp) { - // TODO: we should retry the challenge validation a few times, but let's keep it simple for now - await acmeChallengeDAL.markAsInvalidCascadeById(challengeId); + if (retryCount >= 2) { + // This is the last attempt to validate the challenge response, if it fails, we mark the challenge as invalid + await acmeChallengeDAL.markAsInvalidCascadeById(challengeId); + } // Properly type and inspect the error if (axios.isAxiosError(exp)) { const axiosError = exp as AxiosError; @@ -143,5 +145,5 @@ export const pkiAcmeChallengeServiceFactory = ({ } }; - return { markChallengeAsyReady, validateChallengeResponse }; + return { markChallengeAsReady, validateChallengeResponse }; }; diff --git a/backend/src/ee/services/pki-acme/pki-acme-queue.ts b/backend/src/ee/services/pki-acme/pki-acme-queue.ts new file mode 100644 index 000000000..2adef485e --- /dev/null +++ b/backend/src/ee/services/pki-acme/pki-acme-queue.ts @@ -0,0 +1,67 @@ +import { getConfig } from "@app/lib/config/env"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { TPkiAcmeChallengeServiceFactory } from "./pki-acme-types"; + +type TPkiAcmeQueueServiceFactoryDep = { + queueService: TQueueServiceFactory; + acmeChallengeService: TPkiAcmeChallengeServiceFactory; +}; + +export const pkiAcmeQueueServiceFactory = async ({ + queueService, + acmeChallengeService +}: TPkiAcmeQueueServiceFactoryDep) => { + const appCfg = getConfig(); + + // Initialize the worker to process challenge validation jobs + await queueService.startPg( + QueueJobs.PkiAcmeChallengeValidation, + async ([job]) => { + const { challengeId } = job.data; + const retryCount = job.retryCount || 0; + try { + logger.info({ challengeId, retryCount }, "Processing ACME challenge validation job"); + await acmeChallengeService.validateChallengeResponse(challengeId, retryCount); + logger.info({ challengeId, retryCount }, "ACME challenge validation completed successfully"); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.error( + error, + `Failed to validate ACME challenge ${challengeId} (retryCount ${retryCount}): ${errorMessage}` + ); + // Re-throw to let pg-boss handle retries with exponential backoff + throw error; + } + }, + { + batchSize: 1, + workerCount: 2, + pollingIntervalSeconds: 1 + } + ); + + const queueChallengeValidation = async (challengeId: string): Promise => { + if (appCfg.isSecondaryInstance) { + return; + } + + logger.info({ challengeId }, "Queueing ACME challenge validation"); + await queueService.queuePg( + QueueJobs.PkiAcmeChallengeValidation, + { challengeId }, + { + retryLimit: 3, + retryDelay: 30 * 1000, // Base delay of 30 seconds + retryBackoff: true // Exponential backoff: 30s, 60s, 120s + } + ); + }; + + return { + queueChallengeValidation + }; +}; + +export type TPkiAcmeQueueServiceFactory = Awaited>; diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index 89ba17dd5..167161475 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -62,6 +62,7 @@ import { import { buildUrl, extractAccountIdFromKid, validateDnsIdentifier } from "./pki-acme-fns"; import { TPkiAcmeOrderAuthDALFactory } from "./pki-acme-order-auth-dal"; import { TPkiAcmeOrderDALFactory } from "./pki-acme-order-dal"; +import { TPkiAcmeQueueServiceFactory } from "./pki-acme-queue"; import { AcmeAuthStatus, AcmeChallengeStatus, @@ -126,7 +127,8 @@ type TPkiAcmeServiceFactoryDep = { >; licenseService: Pick; certificateV3Service: Pick; - acmeChallengeService: TPkiAcmeChallengeServiceFactory; + acmeChallengeService: Pick; + pkiAcmeQueueService: Pick; }; export const pkiAcmeServiceFactory = ({ @@ -147,7 +149,8 @@ export const pkiAcmeServiceFactory = ({ kmsService, licenseService, certificateV3Service, - acmeChallengeService + acmeChallengeService, + pkiAcmeQueueService }: TPkiAcmeServiceFactoryDep): TPkiAcmeServiceFactory => { const validateAcmeProfile = async (profileId: string): Promise => { const profile = await certificateProfileDAL.findByIdWithConfigs(profileId); @@ -975,7 +978,8 @@ export const pkiAcmeServiceFactory = ({ if (!result) { throw new NotFoundError({ message: "ACME challenge not found" }); } - await acmeChallengeService.validateChallengeResponse(challengeId); + await acmeChallengeService.markChallengeAsyReady(challengeId); + await pkiAcmeQueueService.queueChallengeValidation(challengeId); const challenge = (await acmeChallengeDAL.findByIdForChallengeValidation(challengeId))!; return { status: 200, diff --git a/backend/src/ee/services/pki-acme/pki-acme-types.ts b/backend/src/ee/services/pki-acme/pki-acme-types.ts index fb76aec79..6607ce711 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-types.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-types.ts @@ -178,6 +178,6 @@ export type TPkiAcmeServiceFactory = { }; export type TPkiAcmeChallengeServiceFactory = { - markChallengeAsyReady: (challengeId: string) => Promise; - validateChallengeResponse: (challengeId: string) => Promise; + markChallengeAsReady: (challengeId: string) => Promise; + validateChallengeResponse: (challengeId: string, retryCount: number) => Promise; }; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 57409d173..5ea6dff8e 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -81,7 +81,8 @@ export enum QueueName { UserNotification = "user-notification", HealthAlert = "health-alert", CertificateV3AutoRenewal = "certificate-v3-auto-renewal", - PamAccountRotation = "pam-account-rotation" + PamAccountRotation = "pam-account-rotation", + PkiAcmeChallengeValidation = "pki-acme-challenge-validation" } export enum QueueJobs { @@ -134,7 +135,8 @@ export enum QueueJobs { UserNotification = "user-notification-job", HealthAlert = "health-alert", CertificateV3DailyAutoRenewal = "certificate-v3-daily-auto-renewal", - PamAccountRotation = "pam-account-rotation" + PamAccountRotation = "pam-account-rotation", + PkiAcmeChallengeValidation = "pki-acme-challenge-validation" } export type TQueueJobTypes = { @@ -385,6 +387,10 @@ export type TQueueJobTypes = { name: QueueJobs.PamAccountRotation; payload: undefined; }; + [QueueName.PkiAcmeChallengeValidation]: { + name: QueueJobs.PkiAcmeChallengeValidation; + payload: { challengeId: string }; + }; }; const SECRET_SCANNING_JOBS = [ diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index ec9e56880..9aea9d5c1 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -374,6 +374,7 @@ import { initializeOauthConfigSync } from "./v1/sso-router"; import { registerV2Routes } from "./v2"; import { registerV3Routes } from "./v3"; import { registerV4Routes } from "./v4"; +import { pkiAcmeQueueServiceFactory } from "@app/ee/services/pki-acme/pki-acme-queue"; const histogram = monitorEventLoopDelay({ resolution: 20 }); histogram.enable(); @@ -2254,6 +2255,12 @@ export const registerRoutes = async ( const acmeChallengeService = pkiAcmeChallengeServiceFactory({ acmeChallengeDAL }); + + const pkiAcmeQueueService = pkiAcmeQueueServiceFactory({ + queueService, + acmeChallengeService + }); + const pkiAcmeService = pkiAcmeServiceFactory({ projectDAL, appConnectionDAL, @@ -2272,7 +2279,8 @@ export const registerRoutes = async ( kmsService, licenseService, certificateV3Service, - acmeChallengeService + acmeChallengeService, + pkiAcmeQueueService }); const pkiSubscriberService = pkiSubscriberServiceFactory({ From 33dd0ea5dee37b21c79ac3ab62d41bb31fbde0f9 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Mon, 1 Dec 2025 15:10:42 -0800 Subject: [PATCH 03/13] Fix typo --- backend/src/ee/services/pki-acme/pki-acme-service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index 167161475..a28e2aa32 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -978,7 +978,7 @@ export const pkiAcmeServiceFactory = ({ if (!result) { throw new NotFoundError({ message: "ACME challenge not found" }); } - await acmeChallengeService.markChallengeAsyReady(challengeId); + await acmeChallengeService.markChallengeAsReady(challengeId); await pkiAcmeQueueService.queueChallengeValidation(challengeId); const challenge = (await acmeChallengeDAL.findByIdForChallengeValidation(challengeId))!; return { From 2fec9c3ee6f7795f4cb28b08f31b0fc86b8bee25 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Mon, 1 Dec 2025 17:25:52 -0800 Subject: [PATCH 04/13] Fix return type --- backend/src/ee/services/pki-acme/pki-acme-queue.ts | 4 ++-- backend/src/server/routes/index.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-queue.ts b/backend/src/ee/services/pki-acme/pki-acme-queue.ts index 2adef485e..b070674ce 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-queue.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-queue.ts @@ -9,6 +9,8 @@ type TPkiAcmeQueueServiceFactoryDep = { acmeChallengeService: TPkiAcmeChallengeServiceFactory; }; +export type TPkiAcmeQueueServiceFactory = Awaited>; + export const pkiAcmeQueueServiceFactory = async ({ queueService, acmeChallengeService @@ -63,5 +65,3 @@ export const pkiAcmeQueueServiceFactory = async ({ queueChallengeValidation }; }; - -export type TPkiAcmeQueueServiceFactory = Awaited>; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 9aea9d5c1..ccf1b9a6c 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -81,6 +81,7 @@ import { pkiAcmeChallengeDALFactory } from "@app/ee/services/pki-acme/pki-acme-c import { pkiAcmeChallengeServiceFactory } from "@app/ee/services/pki-acme/pki-acme-challenge-service"; import { pkiAcmeOrderAuthDALFactory } from "@app/ee/services/pki-acme/pki-acme-order-auth-dal"; import { pkiAcmeOrderDALFactory } from "@app/ee/services/pki-acme/pki-acme-order-dal"; +import { pkiAcmeQueueServiceFactory } from "@app/ee/services/pki-acme/pki-acme-queue"; import { pkiAcmeServiceFactory } from "@app/ee/services/pki-acme/pki-acme-service"; import { projectTemplateDALFactory } from "@app/ee/services/project-template/project-template-dal"; import { projectTemplateServiceFactory } from "@app/ee/services/project-template/project-template-service"; @@ -374,7 +375,6 @@ import { initializeOauthConfigSync } from "./v1/sso-router"; import { registerV2Routes } from "./v2"; import { registerV3Routes } from "./v3"; import { registerV4Routes } from "./v4"; -import { pkiAcmeQueueServiceFactory } from "@app/ee/services/pki-acme/pki-acme-queue"; const histogram = monitorEventLoopDelay({ resolution: 20 }); histogram.enable(); @@ -2256,7 +2256,7 @@ export const registerRoutes = async ( acmeChallengeDAL }); - const pkiAcmeQueueService = pkiAcmeQueueServiceFactory({ + const pkiAcmeQueueService = await pkiAcmeQueueServiceFactory({ queueService, acmeChallengeService }); From 3cb3bd3d21e58b75d0097756777d96d2d2286774 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Mon, 1 Dec 2025 17:34:34 -0800 Subject: [PATCH 05/13] conn reset --- backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts index 572d52bf5..f477d0977 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts @@ -129,6 +129,9 @@ export const pkiAcmeChallengeServiceFactory = ({ if (errorCode === "ENOTFOUND" || errorMessage.includes("ENOTFOUND")) { throw new AcmeDnsFailureError({ message: "Hostname could not be resolved (DNS failure)" }); } + if (errorCode === "ECONNRESET" || errorMessage.includes("ECONNRESET")) { + throw new AcmeConnectionError({ message: "Connection reset by peer" }); + } if (errorCode === "ECONNABORTED" || errorMessage.includes("timeout")) { logger.error(exp, "Connection timed out while validating ACME challenge response"); throw new AcmeConnectionError({ message: "Connection timed out" }); From 64633b0abca3ea57543b258774de5793c89700c7 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Mon, 1 Dec 2025 17:46:24 -0800 Subject: [PATCH 06/13] Fix wrong timeout uit --- backend/src/ee/services/pki-acme/pki-acme-queue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-queue.ts b/backend/src/ee/services/pki-acme/pki-acme-queue.ts index b070674ce..851159981 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-queue.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-queue.ts @@ -55,7 +55,7 @@ export const pkiAcmeQueueServiceFactory = async ({ { challengeId }, { retryLimit: 3, - retryDelay: 30 * 1000, // Base delay of 30 seconds + retryDelay: 30, // Base delay of 30 seconds retryBackoff: true // Exponential backoff: 30s, 60s, 120s } ); From a2774bd18e71da961a85af68d3f9277696375850 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Mon, 1 Dec 2025 18:02:39 -0800 Subject: [PATCH 07/13] Add log --- .../src/ee/services/pki-acme/pki-acme-challenge-service.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts index f477d0977..74434f881 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts @@ -114,6 +114,10 @@ export const pkiAcmeChallengeServiceFactory = ({ await acmeChallengeDAL.markAsValidCascadeById(challengeId); } catch (exp) { if (retryCount >= 2) { + logger.error( + exp, + `Last attempt to validate ACME challenge response failed, marking ${challengeId} challenge as invalid` + ); // This is the last attempt to validate the challenge response, if it fails, we mark the challenge as invalid await acmeChallengeDAL.markAsInvalidCascadeById(challengeId); } From 27d027519b33fd15c709df5c64d0936a76a66d01 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Mon, 1 Dec 2025 18:51:48 -0800 Subject: [PATCH 08/13] More efficient challenge waiting --- backend/bdd/features/steps/pki_acme.py | 30 ++++++++++++------- .../pki-acme/pki-acme-challenge-service.ts | 1 + 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index e895ed69e..53ad2d15b 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -797,21 +797,25 @@ def select_challenge( return challenges[0] -def serve_challenge( +def serve_challenges( context: Context, - challenge: messages.ChallengeBody, + challenges: list[messages.ChallengeBody], ): if hasattr(context, "web_server"): context.web_server.shutdown_and_server_close() - response, validation = challenge.response_and_validation( - context.acme_client.net.key - ) - resource = standalone.HTTP01RequestHandler.HTTP01Resource( - chall=challenge.chall, response=response, validation=validation - ) + resources = set() + for challenge in challenges: + response, validation = challenge.response_and_validation( + context.acme_client.net.key + ) + resources.add( + standalone.HTTP01RequestHandler.HTTP01Resource( + chall=challenge.chall, response=response, validation=validation + ) + ) # TODO: make port configurable - servers = standalone.HTTP01DualNetworkedServers(("0.0.0.0", 8087), {resource}) + servers = standalone.HTTP01DualNetworkedServers(("0.0.0.0", 8087), resources) servers.serve_forever() context.web_server = servers @@ -865,6 +869,7 @@ def step_impl( f"Expected OrderResource but got {type(order)!r} at {order_var_path!r}" ) + challenges = {} for domain in order.body.identifiers: logger.info( "Selecting challenge for domain %s with type %s ...", @@ -877,6 +882,7 @@ def step_impl( domain=domain.value, order_var_path=order_var_path, ) + print("@" * 20, domain, challenge.chall.path) logger.info( "Found challenge for domain %s with type %s, challenge=%s", domain.value, @@ -889,8 +895,10 @@ def step_impl( domain.value, challenge_type, ) - serve_challenge(context=context, challenge=challenge) + challenges[domain] = challenge + serve_challenges(context=context, challenges=list(challenges.values())) + for domain, challenge in challenges.items(): logger.info( "Notifying challenge for domain %s with type %s ...", domain, challenge_type ) @@ -900,7 +908,7 @@ def step_impl( @then("I serve challenge response for {var_path} at {hostname}") def step_impl(context: Context, var_path: str, hostname: str): challenge = eval_var(context, var_path, as_json=False) - serve_challenge(context=context, challenge=challenge) + serve_challenges(context=context, challenges=[challenge]) @then("I tell ACME server that {var_path} is ready to be verified") diff --git a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts index 74434f881..c841b496d 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts @@ -111,6 +111,7 @@ export const pkiAcmeChallengeServiceFactory = ({ if (challengeResponseBody.trimEnd() !== expectedChallengeResponseBody) { throw new AcmeIncorrectResponseError({ message: "ACME challenge response is not correct" }); } + logger.info({ challengeId }, "ACME challenge response is correct, marking challenge as valid"); await acmeChallengeDAL.markAsValidCascadeById(challengeId); } catch (exp) { if (retryCount >= 2) { From d28bd1e4ecc1ea8f12c35456cc4c9ea10a285b6f Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Mon, 1 Dec 2025 19:06:31 -0800 Subject: [PATCH 09/13] Add test for retry logic --- .../bdd/features/pki/acme/challenge.feature | 22 ++++++++++++++++ backend/bdd/features/steps/pki_acme.py | 26 +++++++++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/backend/bdd/features/pki/acme/challenge.feature b/backend/bdd/features/pki/acme/challenge.feature index 21c63329f..1d16f6baa 100644 --- a/backend/bdd/features/pki/acme/challenge.feature +++ b/backend/bdd/features/pki/acme/challenge.feature @@ -22,6 +22,28 @@ Feature: Challenge And I parse the full-chain certificate from order finalized_order as cert And the value cert with jq ".subject.common_name" should be equal to "localhost" + Scenario: Validate challenge with retry + Given I have an ACME cert profile as "acme_profile" + When I have an ACME client connecting to "{BASE_URL}/api/v1/cert-manager/acme/profiles/{acme_profile.id}/directory" + Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account + When I create certificate signing request as csr + Then I add names to certificate signing request csr + """ + { + "COMMON_NAME": "localhost" + } + """ + And I create a RSA private key pair as cert_key + And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format + And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order + And I select challenge with type http-01 for domain localhost from order in order as challenge + And I wait 45 seconds before serve challenge response for challenge at localhost + And I tell ACME server that challenge is ready to be verified + And I poll and finalize the ACME order order as finalized_order + And the value finalized_order.body with jq ".status" should be equal to "valid" + And I parse the full-chain certificate from order finalized_order as cert + And the value cert with jq ".subject.common_name" should be equal to "localhost" + Scenario: Validate challenges for multiple domains Given I have an ACME cert profile as "acme_profile" When I have an ACME client connecting to "{BASE_URL}/api/v1/cert-manager/acme/profiles/{acme_profile.id}/directory" diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index 53ad2d15b..2dd61c123 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -2,6 +2,8 @@ import json import logging import re import urllib.parse +import time +import threading import acme.client import jq @@ -800,6 +802,7 @@ def select_challenge( def serve_challenges( context: Context, challenges: list[messages.ChallengeBody], + wait_time: int | None = None, ): if hasattr(context, "web_server"): context.web_server.shutdown_and_server_close() @@ -816,7 +819,19 @@ def serve_challenges( ) # TODO: make port configurable servers = standalone.HTTP01DualNetworkedServers(("0.0.0.0", 8087), resources) - servers.serve_forever() + if wait_time is None: + servers.serve_forever() + else: + + def wait_and_start(): + logger.info("Waiting %s seconds before we start serving.", wait_time) + time.sleep(wait_time) + logger.info("Start server now") + servers.serve_forever() + + thread = threading.Thread(target=wait_and_start) + thread.daemon = True + thread.start() context.web_server = servers @@ -882,7 +897,6 @@ def step_impl( domain=domain.value, order_var_path=order_var_path, ) - print("@" * 20, domain, challenge.chall.path) logger.info( "Found challenge for domain %s with type %s, challenge=%s", domain.value, @@ -905,6 +919,14 @@ def step_impl( notify_challenge_ready(context=context, challenge=challenge) +@then( + "I wait {wait_time} seconds before serve challenge response for {var_path} at {hostname}" +) +def step_impl(context: Context, wait_time: str, var_path: str, hostname: str): + challenge = eval_var(context, var_path, as_json=False) + serve_challenges(context=context, challenges=[challenge], wait_time=int(wait_time)) + + @then("I serve challenge response for {var_path} at {hostname}") def step_impl(context: Context, var_path: str, hostname: str): challenge = eval_var(context, var_path, as_json=False) From 74cdfe97da538746a8c12ffeab354cf34e9a54d1 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Mon, 1 Dec 2025 19:10:16 -0800 Subject: [PATCH 10/13] Fix grammar --- backend/bdd/features/pki/acme/challenge.feature | 2 +- backend/bdd/features/steps/pki_acme.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/bdd/features/pki/acme/challenge.feature b/backend/bdd/features/pki/acme/challenge.feature index 1d16f6baa..9a2e72e87 100644 --- a/backend/bdd/features/pki/acme/challenge.feature +++ b/backend/bdd/features/pki/acme/challenge.feature @@ -37,7 +37,7 @@ Feature: Challenge And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order And I select challenge with type http-01 for domain localhost from order in order as challenge - And I wait 45 seconds before serve challenge response for challenge at localhost + And I wait 45 seconds and serve challenge response for challenge at localhost And I tell ACME server that challenge is ready to be verified And I poll and finalize the ACME order order as finalized_order And the value finalized_order.body with jq ".status" should be equal to "valid" diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index 2dd61c123..062fd7c2f 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -920,7 +920,7 @@ def step_impl( @then( - "I wait {wait_time} seconds before serve challenge response for {var_path} at {hostname}" + "I wait {wait_time} seconds and serve challenge response for {var_path} at {hostname}" ) def step_impl(context: Context, wait_time: str, var_path: str, hostname: str): challenge = eval_var(context, var_path, as_json=False) From a6883895990763c9d67a2736a70e1abb2217f484 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Mon, 1 Dec 2025 20:50:05 -0800 Subject: [PATCH 11/13] Fix broken tests --- backend/bdd/features/pki/acme/challenge.feature | 2 ++ backend/bdd/features/steps/pki_acme.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/backend/bdd/features/pki/acme/challenge.feature b/backend/bdd/features/pki/acme/challenge.feature index 9a2e72e87..52090fdb0 100644 --- a/backend/bdd/features/pki/acme/challenge.feature +++ b/backend/bdd/features/pki/acme/challenge.feature @@ -207,8 +207,10 @@ Feature: Challenge Then the value response.status_code should be equal to 201 And I memorize response with jq ".finalize" as finalize_url And I memorize response.headers with jq ".["replay-nonce"]" as nonce + And I memorize response.headers with jq ".["location"]" as order_uri And I memorize response as order And I pass all challenges with type http-01 for order in order + And I wait until the status of order order_uri becomes ready And I encode CSR csr_pem as JOSE Base-64 DER as base64_csr_der When I send a raw ACME request to "{finalize_url}" """ diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index 062fd7c2f..ff90cf64d 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -939,6 +939,23 @@ def step_impl(context: Context, var_path: str): notify_challenge_ready(context=context, challenge=challenge) +@then("I wait until the status of order {order_var} becomes {status}") +def step_impl(context: Context, order_var: str, status: str): + acme_client = context.acme_client + attempt_count = 6 + while attempt_count: + order = eval_var(context, order_var, as_json=False) + response = acme_client._post_as_get( + order.uri if isinstance(order, messages.OrderResource) else order + ) + order = messages.Order.from_json(response.json()) + if order.status.name == status: + return + acme_client -= 1 + time.sleep(10) + raise TimeoutError(f"The status of order doesn't become {status} before timeout") + + @then("I poll and finalize the ACME order {var_path} as {finalized_var}") def step_impl(context: Context, var_path: str, finalized_var: str): order = eval_var(context, var_path, as_json=False) From 91b916c9ad846807304e5f6c1c391d792c3aaa97 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Mon, 1 Dec 2025 21:38:05 -0800 Subject: [PATCH 12/13] Fix broken tests --- .../bdd/features/pki/acme/challenge.feature | 54 +++---------------- backend/bdd/features/steps/pki_acme.py | 35 +++++++++++- 2 files changed, 42 insertions(+), 47 deletions(-) diff --git a/backend/bdd/features/pki/acme/challenge.feature b/backend/bdd/features/pki/acme/challenge.feature index 52090fdb0..80f6fed6c 100644 --- a/backend/bdd/features/pki/acme/challenge.feature +++ b/backend/bdd/features/pki/acme/challenge.feature @@ -85,13 +85,12 @@ Feature: Challenge When I create certificate signing request as csr Then I add names to certificate signing request csr """ - { - "COMMON_NAME": "localhost" - } + {} """ And I add subject alternative name to certificate signing request csr """ [ + "localhost", "infisical.com" ] """ @@ -104,56 +103,19 @@ Feature: Challenge # the localhost auth should be valid And I memorize order with jq ".authorizations | map(select(.body.identifier.value == "localhost")) | first | .uri" as localhost_auth - And I peak and memorize the next nonce as nonce - When I send a raw ACME request to "{localhost_auth}" - """ - { - "protected": { - "alg": "RS256", - "nonce": "{nonce}", - "url": "{localhost_auth}", - "kid": "{acme_account.uri}" - } - } - """ - Then the value response.status_code should be equal to 200 - And the value response with jq ".status" should be equal to "valid" + And I wait until the status of authorization localhost_auth becomes valid # the infisical.com auth should still be pending And I memorize order with jq ".authorizations | map(select(.body.identifier.value == "infisical.com")) | first | .uri" as infisical_auth - And I memorize response.headers with jq ".["replay-nonce"]" as nonce - When I send a raw ACME request to "{infisical_auth}" - """ - { - "protected": { - "alg": "RS256", - "nonce": "{nonce}", - "url": "{infisical_auth}", - "kid": "{acme_account.uri}" - } - } - """ - Then the value response.status_code should be equal to 200 - And the value response with jq ".status" should be equal to "pending" + And I post-as-get {infisical_auth} as infisical_auth_resp + And the value infisical_auth_resp with jq ".status" should be equal to "pending" # the order should be pending as well - And I memorize response.headers with jq ".["replay-nonce"]" as nonce - When I send a raw ACME request to "{order.uri}" - """ - { - "protected": { - "alg": "RS256", - "nonce": "{nonce}", - "url": "{order.uri}", - "kid": "{acme_account.uri}" - } - } - """ - Then the value response.status_code should be equal to 200 - And the value response with jq ".status" should be equal to "pending" + And I post-as-get {order.uri} as order_resp + And the value order_resp with jq ".status" should be equal to "pending" # finalize should not be allowed when all auths are not valid yet - And I memorize response.headers with jq ".["replay-nonce"]" as nonce + And I get a new-nonce as nonce When I send a raw ACME request to "{order.body.finalize}" """ { diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index ff90cf64d..8f0661c71 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -726,6 +726,15 @@ def step_impl(context: Context, var_path: str, jq_query, var_name: str): context.vars[var_name] = value +@then("I get a new-nonce as {var_name}") +def step_impl(context: Context, var_name: str): + acme_client = context.acme_client + nonce = acme_client.net._get_nonce( + url=None, new_nonce_url=acme_client.directory.newNonce + ) + context.vars[var_name] = json_util.encode_b64jose(nonce) + + @then("I peak and memorize the next nonce as {var_name}") def step_impl(context: Context, var_name: str): acme_client = context.acme_client @@ -951,11 +960,35 @@ def step_impl(context: Context, order_var: str, status: str): order = messages.Order.from_json(response.json()) if order.status.name == status: return - acme_client -= 1 + attempt_count -= 1 time.sleep(10) raise TimeoutError(f"The status of order doesn't become {status} before timeout") +@then("I wait until the status of authorization {auth_var} becomes {status}") +def step_impl(context: Context, auth_var: str, status: str): + acme_client = context.acme_client + attempt_count = 6 + while attempt_count: + auth = eval_var(context, auth_var, as_json=False) + response = acme_client._post_as_get( + auth.uri if isinstance(auth, messages.Authorization) else auth + ) + auth = messages.Authorization.from_json(response.json()) + if auth.status.name == status: + return + attempt_count -= 1 + time.sleep(10) + raise TimeoutError(f"The status of auth doesn't become {status} before timeout") + + +@then("I post-as-get {uri} as {resp_var}") +def step_impl(context: Context, uri: str, resp_var: str): + acme_client = context.acme_client + response = acme_client._post_as_get(replace_vars(uri, vars=context.vars)) + context.vars[resp_var] = response.json() + + @then("I poll and finalize the ACME order {var_path} as {finalized_var}") def step_impl(context: Context, var_path: str, finalized_var: str): order = eval_var(context, var_path, as_json=False) From a804d0101b343f68178d853d41d8de531709865b Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 2 Dec 2025 09:57:58 -0800 Subject: [PATCH 13/13] Fix a typo --- backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts index c841b496d..7379d7ece 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts @@ -93,7 +93,7 @@ export const pkiAcmeChallengeServiceFactory = ({ // a long running operation for long time. But assuming we are not performing a tons of // challenge validation at the same time, it should be fine. const challengeResponse = await axios.get(challengeUrl.toString(), { - // In case if we override the hos2 in the development mode, still provide the original host in the header + // In case if we override the host in the development mode, still provide the original host in the header // to help the upstream server to validate the request headers: { Host: challenge.auth.identifierValue }, timeout: timeoutMs,