Merge pull request #4975 from Infisical/PKI-38-better-http-01-challenge

improvement(api): perform http01-challenge in the background with retries

backend/bdd/features/pki/acme/challenge.feature

@@ -22,6 +22,28 @@ Feature: Challenge
     And I parse the full-chain certificate from order finalized_order as cert
     And the value cert with jq ".subject.common_name" should be equal to "localhost"
 
+  Scenario: Validate challenge with retry
+    Given I have an ACME cert profile as "acme_profile"
+    When I have an ACME client connecting to "{BASE_URL}/api/v1/cert-manager/acme/profiles/{acme_profile.id}/directory"
+    Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account
+    When I create certificate signing request as csr
+    Then I add names to certificate signing request csr
+      """
+      {
+        "COMMON_NAME": "localhost"
+      }
+      """
+    And I create a RSA private key pair as cert_key
+    And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format
+    And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order
+    And I select challenge with type http-01 for domain localhost from order in order as challenge
+    And I wait 45 seconds and serve challenge response for challenge at localhost
+    And I tell ACME server that challenge is ready to be verified
+    And I poll and finalize the ACME order order as finalized_order
+    And the value finalized_order.body with jq ".status" should be equal to "valid"
+    And I parse the full-chain certificate from order finalized_order as cert
+    And the value cert with jq ".subject.common_name" should be equal to "localhost"
+
   Scenario: Validate challenges for multiple domains
     Given I have an ACME cert profile as "acme_profile"
     When I have an ACME client connecting to "{BASE_URL}/api/v1/cert-manager/acme/profiles/{acme_profile.id}/directory"
@@ -63,13 +85,12 @@ Feature: Challenge
     When I create certificate signing request as csr
     Then I add names to certificate signing request csr
       """
-      {
+      {}
-        "COMMON_NAME": "localhost"
-      }
       """
     And I add subject alternative name to certificate signing request csr
       """
       [
+        "localhost",
         "infisical.com"
       ]
       """
@@ -82,56 +103,19 @@ Feature: Challenge
 
     # the localhost auth should be valid
     And I memorize order with jq ".authorizations | map(select(.body.identifier.value == "localhost")) | first | .uri" as localhost_auth
-    And I peak and memorize the next nonce as nonce
+    And I wait until the status of authorization localhost_auth becomes valid
-    When I send a raw ACME request to "{localhost_auth}"
-      """
-      {
-        "protected": {
-          "alg": "RS256",
-          "nonce": "{nonce}",
-          "url": "{localhost_auth}",
-          "kid": "{acme_account.uri}"
-        }
-      }
-      """
-    Then the value response.status_code should be equal to 200
-    And the value response with jq ".status" should be equal to "valid"
 
     # the infisical.com auth should still be pending
     And I memorize order with jq ".authorizations | map(select(.body.identifier.value == "infisical.com")) | first | .uri" as infisical_auth
-    And I memorize response.headers with jq ".["replay-nonce"]" as nonce
+    And I post-as-get {infisical_auth} as infisical_auth_resp
-    When I send a raw ACME request to "{infisical_auth}"
+    And the value infisical_auth_resp with jq ".status" should be equal to "pending"
-      """
-      {
-        "protected": {
-          "alg": "RS256",
-          "nonce": "{nonce}",
-          "url": "{infisical_auth}",
-          "kid": "{acme_account.uri}"
-        }
-      }
-      """
-    Then the value response.status_code should be equal to 200
-    And the value response with jq ".status" should be equal to "pending"
 
     # the order should be pending as well
-    And I memorize response.headers with jq ".["replay-nonce"]" as nonce
+    And I post-as-get {order.uri} as order_resp
-    When I send a raw ACME request to "{order.uri}"
+    And the value order_resp with jq ".status" should be equal to "pending"
-      """
-      {
-        "protected": {
-          "alg": "RS256",
-          "nonce": "{nonce}",
-          "url": "{order.uri}",
-          "kid": "{acme_account.uri}"
-        }
-      }
-      """
-    Then the value response.status_code should be equal to 200
-    And the value response with jq ".status" should be equal to "pending"
 
     # finalize should not be allowed when all auths are not valid yet
-    And I memorize response.headers with jq ".["replay-nonce"]" as nonce
+    And I get a new-nonce as nonce
     When I send a raw ACME request to "{order.body.finalize}"
       """
       {
@@ -185,8 +169,10 @@ Feature: Challenge
     Then the value response.status_code should be equal to 201
     And I memorize response with jq ".finalize" as finalize_url
     And I memorize response.headers with jq ".["replay-nonce"]" as nonce
+    And I memorize response.headers with jq ".["location"]" as order_uri
     And I memorize response as order
     And I pass all challenges with type http-01 for order in order
+    And I wait until the status of order order_uri becomes ready
     And I encode CSR csr_pem as JOSE Base-64 DER as base64_csr_der
     When I send a raw ACME request to "{finalize_url}"
       """

backend/bdd/features/steps/pki_acme.py

@@ -2,6 +2,8 @@ import json
 import logging
 import re
 import urllib.parse
+import time
+import threading
 
 import acme.client
 import jq
@@ -724,6 +726,15 @@ def step_impl(context: Context, var_path: str, jq_query, var_name: str):
     context.vars[var_name] = value
 
 
+@then("I get a new-nonce as {var_name}")
+def step_impl(context: Context, var_name: str):
+    acme_client = context.acme_client
+    nonce = acme_client.net._get_nonce(
+        url=None, new_nonce_url=acme_client.directory.newNonce
+    )
+    context.vars[var_name] = json_util.encode_b64jose(nonce)
+
+
 @then("I peak and memorize the next nonce as {var_name}")
 def step_impl(context: Context, var_name: str):
     acme_client = context.acme_client
@@ -797,22 +808,39 @@ def select_challenge(
     return challenges[0]
 
 
-def serve_challenge(
+def serve_challenges(
     context: Context,
-    challenge: messages.ChallengeBody,
+    challenges: list[messages.ChallengeBody],
+    wait_time: int | None = None,
 ):
     if hasattr(context, "web_server"):
         context.web_server.shutdown_and_server_close()
 
-    response, validation = challenge.response_and_validation(
+    resources = set()
-        context.acme_client.net.key
+    for challenge in challenges:
-    )
+        response, validation = challenge.response_and_validation(
-    resource = standalone.HTTP01RequestHandler.HTTP01Resource(
+            context.acme_client.net.key
-        chall=challenge.chall, response=response, validation=validation
+        )
-    )
+        resources.add(
+            standalone.HTTP01RequestHandler.HTTP01Resource(
+                chall=challenge.chall, response=response, validation=validation
+            )
+        )
     # TODO: make port configurable
-    servers = standalone.HTTP01DualNetworkedServers(("0.0.0.0", 8087), {resource})
+    servers = standalone.HTTP01DualNetworkedServers(("0.0.0.0", 8087), resources)
-    servers.serve_forever()
+    if wait_time is None:
+        servers.serve_forever()
+    else:
+
+        def wait_and_start():
+            logger.info("Waiting %s seconds before we start serving.", wait_time)
+            time.sleep(wait_time)
+            logger.info("Start server now")
+            servers.serve_forever()
+
+        thread = threading.Thread(target=wait_and_start)
+        thread.daemon = True
+        thread.start()
     context.web_server = servers
 
 
@@ -865,6 +893,7 @@ def step_impl(
             f"Expected OrderResource but got {type(order)!r} at {order_var_path!r}"
         )
 
+    challenges = {}
     for domain in order.body.identifiers:
         logger.info(
             "Selecting challenge for domain %s with type %s ...",
@@ -889,18 +918,28 @@ def step_impl(
             domain.value,
             challenge_type,
         )
-        serve_challenge(context=context, challenge=challenge)
+        challenges[domain] = challenge
 
+    serve_challenges(context=context, challenges=list(challenges.values()))
+    for domain, challenge in challenges.items():
         logger.info(
             "Notifying challenge for domain %s with type %s ...", domain, challenge_type
         )
         notify_challenge_ready(context=context, challenge=challenge)
 
 
+@then(
+    "I wait {wait_time} seconds and serve challenge response for {var_path} at {hostname}"
+)
+def step_impl(context: Context, wait_time: str, var_path: str, hostname: str):
+    challenge = eval_var(context, var_path, as_json=False)
+    serve_challenges(context=context, challenges=[challenge], wait_time=int(wait_time))
+
+
 @then("I serve challenge response for {var_path} at {hostname}")
 def step_impl(context: Context, var_path: str, hostname: str):
     challenge = eval_var(context, var_path, as_json=False)
-    serve_challenge(context=context, challenge=challenge)
+    serve_challenges(context=context, challenges=[challenge])
 
 
 @then("I tell ACME server that {var_path} is ready to be verified")
@@ -909,6 +948,47 @@ def step_impl(context: Context, var_path: str):
     notify_challenge_ready(context=context, challenge=challenge)
 
 
+@then("I wait until the status of order {order_var} becomes {status}")
+def step_impl(context: Context, order_var: str, status: str):
+    acme_client = context.acme_client
+    attempt_count = 6
+    while attempt_count:
+        order = eval_var(context, order_var, as_json=False)
+        response = acme_client._post_as_get(
+            order.uri if isinstance(order, messages.OrderResource) else order
+        )
+        order = messages.Order.from_json(response.json())
+        if order.status.name == status:
+            return
+        attempt_count -= 1
+        time.sleep(10)
+    raise TimeoutError(f"The status of order doesn't become {status} before timeout")
+
+
+@then("I wait until the status of authorization {auth_var} becomes {status}")
+def step_impl(context: Context, auth_var: str, status: str):
+    acme_client = context.acme_client
+    attempt_count = 6
+    while attempt_count:
+        auth = eval_var(context, auth_var, as_json=False)
+        response = acme_client._post_as_get(
+            auth.uri if isinstance(auth, messages.Authorization) else auth
+        )
+        auth = messages.Authorization.from_json(response.json())
+        if auth.status.name == status:
+            return
+        attempt_count -= 1
+        time.sleep(10)
+    raise TimeoutError(f"The status of auth doesn't become {status} before timeout")
+
+
+@then("I post-as-get {uri} as {resp_var}")
+def step_impl(context: Context, uri: str, resp_var: str):
+    acme_client = context.acme_client
+    response = acme_client._post_as_get(replace_vars(uri, vars=context.vars))
+    context.vars[resp_var] = response.json()
+
+
 @then("I poll and finalize the ACME order {var_path} as {finalized_var}")
 def step_impl(context: Context, var_path: str, finalized_var: str):
     order = eval_var(context, var_path, as_json=False)

backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts

@@ -1,5 +1,6 @@
 import axios, { AxiosError } from "axios";
 
+import { TPkiAcmeChallenges } from "@app/db/schemas/pki-acme-challenges";
 import { getConfig } from "@app/lib/config/env";
 import { BadRequestError, NotFoundError } from "@app/lib/errors";
 import { isPrivateIp } from "@app/lib/ip/ipRange";
@@ -18,7 +19,11 @@ import { TPkiAcmeChallengeServiceFactory } from "./pki-acme-types";
 type TPkiAcmeChallengeServiceFactoryDep = {
   acmeChallengeDAL: Pick<
     TPkiAcmeChallengeDALFactory,
-    "transaction" | "findByIdForChallengeValidation" | "markAsValidCascadeById" | "markAsInvalidCascadeById"
+    | "transaction"
+    | "findByIdForChallengeValidation"
+    | "markAsValidCascadeById"
+    | "markAsInvalidCascadeById"
+    | "updateById"
   >;
 };
 
@@ -26,9 +31,8 @@ export const pkiAcmeChallengeServiceFactory = ({
   acmeChallengeDAL
 }: TPkiAcmeChallengeServiceFactoryDep): TPkiAcmeChallengeServiceFactory => {
   const appCfg = getConfig();
-
+  const markChallengeAsReady = async (challengeId: string): Promise<TPkiAcmeChallenges> => {
-  const validateChallengeResponse = async (challengeId: string): Promise<void> => {
+    return acmeChallengeDAL.transaction(async (tx) => {
-    const error: Error | undefined = await acmeChallengeDAL.transaction(async (tx) => {
       logger.info({ challengeId }, "Validating ACME challenge response");
       const challenge = await acmeChallengeDAL.findByIdForChallengeValidation(challengeId, tx);
       if (!challenge) {
@@ -52,81 +56,102 @@ export const pkiAcmeChallengeServiceFactory = ({
       if (challenge.type !== AcmeChallengeType.HTTP_01) {
         throw new BadRequestError({ message: "Only HTTP-01 challenges are supported for now" });
       }
-      let host = challenge.auth.identifierValue;
+      const host = challenge.auth.identifierValue;
       // check if host is a private ip address
       if (isPrivateIp(host)) {
         throw new BadRequestError({ message: "Private IP addresses are not allowed" });
       }
-      if (appCfg.isAcmeDevelopmentMode && appCfg.ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES[host]) {
+      return acmeChallengeDAL.updateById(challengeId, { status: AcmeChallengeStatus.Processing }, tx);
-        host = appCfg.ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES[host];
-        logger.warn(
-          { srcHost: challenge.auth.identifierValue, dstHost: host },
-          "Using ACME development HTTP-01 challenge host override"
-        );
-      }
-      const challengeUrl = new URL(`/.well-known/acme-challenge/${challenge.auth.token}`, `http://${host}`);
-      logger.info({ challengeUrl }, "Performing ACME HTTP-01 challenge validation");
-      try {
-        // TODO: read config from the profile to get the timeout instead
-        const timeoutMs = 10 * 1000; // 10 seconds
-        // Notice: well, we are in a transaction, ideally we should not hold transaction and perform
-        //         a long running operation for long time. But assuming we are not performing a tons of
-        //         challenge validation at the same time, it should be fine.
-        const challengeResponse = await axios.get<string>(challengeUrl.toString(), {
-          // In case if we override the host in the development mode, still provide the original host in the header
-          // to help the upstream server to validate the request
-          headers: { Host: challenge.auth.identifierValue },
-          timeout: timeoutMs,
-          responseType: "text",
-          validateStatus: () => true
-        });
-        if (challengeResponse.status !== 200) {
-          throw new AcmeIncorrectResponseError({
-            message: `ACME challenge response is not 200: ${challengeResponse.status}`
-          });
-        }
-        const challengeResponseBody: string = challengeResponse.data;
-        const thumbprint = challenge.auth.account.publicKeyThumbprint;
-        const expectedChallengeResponseBody = `${challenge.auth.token}.${thumbprint}`;
-        if (challengeResponseBody.trimEnd() !== expectedChallengeResponseBody) {
-          throw new AcmeIncorrectResponseError({ message: "ACME challenge response is not correct" });
-        }
-        await acmeChallengeDAL.markAsValidCascadeById(challengeId, tx);
-      } catch (exp) {
-        // TODO: we should retry the challenge validation a few times, but let's keep it simple for now
-        await acmeChallengeDAL.markAsInvalidCascadeById(challengeId, tx);
-        // Properly type and inspect the error
-        if (axios.isAxiosError(exp)) {
-          const axiosError = exp as AxiosError;
-          const errorCode = axiosError.code;
-          const errorMessage = axiosError.message;
-
-          if (errorCode === "ECONNREFUSED" || errorMessage.includes("ECONNREFUSED")) {
-            return new AcmeConnectionError({ message: "Connection refused" });
-          }
-          if (errorCode === "ENOTFOUND" || errorMessage.includes("ENOTFOUND")) {
-            return new AcmeDnsFailureError({ message: "Hostname could not be resolved (DNS failure)" });
-          }
-          if (errorCode === "ECONNABORTED" || errorMessage.includes("timeout")) {
-            logger.error(exp, "Connection timed out while validating ACME challenge response");
-            return new AcmeConnectionError({ message: "Connection timed out" });
-          }
-          logger.error(exp, "Unknown error validating ACME challenge response");
-          return new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" });
-        }
-        if (exp instanceof Error) {
-          logger.error(exp, "Error validating ACME challenge response");
-        } else {
-          logger.error(exp, "Unknown error validating ACME challenge response");
-          return new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" });
-        }
-        return exp;
-      }
     });
-    if (error) {
+  };
-      throw error;
+
+  const validateChallengeResponse = async (challengeId: string, retryCount: number): Promise<void> => {
+    logger.info({ challengeId, retryCount }, "Validating ACME challenge response");
+    const challenge = await acmeChallengeDAL.findByIdForChallengeValidation(challengeId);
+    if (!challenge) {
+      throw new NotFoundError({ message: "ACME challenge not found" });
+    }
+    if (challenge.status !== AcmeChallengeStatus.Processing) {
+      throw new BadRequestError({
+        message: `ACME challenge is ${challenge.status} instead of ${AcmeChallengeStatus.Processing}`
+      });
+    }
+    let host = challenge.auth.identifierValue;
+    if (appCfg.isAcmeDevelopmentMode && appCfg.ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES[host]) {
+      host = appCfg.ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES[host];
+      logger.warn(
+        { srcHost: challenge.auth.identifierValue, dstHost: host },
+        "Using ACME development HTTP-01 challenge host override"
+      );
+    }
+    const challengeUrl = new URL(`/.well-known/acme-challenge/${challenge.auth.token}`, `http://${host}`);
+    logger.info({ challengeUrl }, "Performing ACME HTTP-01 challenge validation");
+    try {
+      // TODO: read config from the profile to get the timeout instead
+      const timeoutMs = 10 * 1000; // 10 seconds
+      // Notice: well, we are in a transaction, ideally we should not hold transaction and perform
+      //         a long running operation for long time. But assuming we are not performing a tons of
+      //         challenge validation at the same time, it should be fine.
+      const challengeResponse = await axios.get<string>(challengeUrl.toString(), {
+        // In case if we override the host in the development mode, still provide the original host in the header
+        // to help the upstream server to validate the request
+        headers: { Host: challenge.auth.identifierValue },
+        timeout: timeoutMs,
+        responseType: "text",
+        validateStatus: () => true
+      });
+      if (challengeResponse.status !== 200) {
+        throw new AcmeIncorrectResponseError({
+          message: `ACME challenge response is not 200: ${challengeResponse.status}`
+        });
+      }
+      const challengeResponseBody: string = challengeResponse.data;
+      const thumbprint = challenge.auth.account.publicKeyThumbprint;
+      const expectedChallengeResponseBody = `${challenge.auth.token}.${thumbprint}`;
+      if (challengeResponseBody.trimEnd() !== expectedChallengeResponseBody) {
+        throw new AcmeIncorrectResponseError({ message: "ACME challenge response is not correct" });
+      }
+      logger.info({ challengeId }, "ACME challenge response is correct, marking challenge as valid");
+      await acmeChallengeDAL.markAsValidCascadeById(challengeId);
+    } catch (exp) {
+      if (retryCount >= 2) {
+        logger.error(
+          exp,
+          `Last attempt to validate ACME challenge response failed, marking ${challengeId} challenge as invalid`
+        );
+        // This is the last attempt to validate the challenge response, if it fails, we mark the challenge as invalid
+        await acmeChallengeDAL.markAsInvalidCascadeById(challengeId);
+      }
+      // Properly type and inspect the error
+      if (axios.isAxiosError(exp)) {
+        const axiosError = exp as AxiosError;
+        const errorCode = axiosError.code;
+        const errorMessage = axiosError.message;
+
+        if (errorCode === "ECONNREFUSED" || errorMessage.includes("ECONNREFUSED")) {
+          throw new AcmeConnectionError({ message: "Connection refused" });
+        }
+        if (errorCode === "ENOTFOUND" || errorMessage.includes("ENOTFOUND")) {
+          throw new AcmeDnsFailureError({ message: "Hostname could not be resolved (DNS failure)" });
+        }
+        if (errorCode === "ECONNRESET" || errorMessage.includes("ECONNRESET")) {
+          throw new AcmeConnectionError({ message: "Connection reset by peer" });
+        }
+        if (errorCode === "ECONNABORTED" || errorMessage.includes("timeout")) {
+          logger.error(exp, "Connection timed out while validating ACME challenge response");
+          throw new AcmeConnectionError({ message: "Connection timed out" });
+        }
+        logger.error(exp, "Unknown error validating ACME challenge response");
+        throw new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" });
+      }
+      if (exp instanceof Error) {
+        logger.error(exp, "Error validating ACME challenge response");
+        throw exp;
+      }
+      logger.error(exp, "Unknown error validating ACME challenge response");
+      throw new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" });
     }
   };
 
-  return { validateChallengeResponse };
+  return { markChallengeAsReady, validateChallengeResponse };
 };

backend/src/ee/services/pki-acme/pki-acme-queue.ts

@@ -0,0 +1,67 @@
+import { getConfig } from "@app/lib/config/env";
+import { logger } from "@app/lib/logger";
+import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
+
+import { TPkiAcmeChallengeServiceFactory } from "./pki-acme-types";
+
+type TPkiAcmeQueueServiceFactoryDep = {
+  queueService: TQueueServiceFactory;
+  acmeChallengeService: TPkiAcmeChallengeServiceFactory;
+};
+
+export type TPkiAcmeQueueServiceFactory = Awaited<ReturnType<typeof pkiAcmeQueueServiceFactory>>;
+
+export const pkiAcmeQueueServiceFactory = async ({
+  queueService,
+  acmeChallengeService
+}: TPkiAcmeQueueServiceFactoryDep) => {
+  const appCfg = getConfig();
+
+  // Initialize the worker to process challenge validation jobs
+  await queueService.startPg<QueueName.PkiAcmeChallengeValidation>(
+    QueueJobs.PkiAcmeChallengeValidation,
+    async ([job]) => {
+      const { challengeId } = job.data;
+      const retryCount = job.retryCount || 0;
+      try {
+        logger.info({ challengeId, retryCount }, "Processing ACME challenge validation job");
+        await acmeChallengeService.validateChallengeResponse(challengeId, retryCount);
+        logger.info({ challengeId, retryCount }, "ACME challenge validation completed successfully");
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        logger.error(
+          error,
+          `Failed to validate ACME challenge ${challengeId} (retryCount ${retryCount}): ${errorMessage}`
+        );
+        // Re-throw to let pg-boss handle retries with exponential backoff
+        throw error;
+      }
+    },
+    {
+      batchSize: 1,
+      workerCount: 2,
+      pollingIntervalSeconds: 1
+    }
+  );
+
+  const queueChallengeValidation = async (challengeId: string): Promise<void> => {
+    if (appCfg.isSecondaryInstance) {
+      return;
+    }
+
+    logger.info({ challengeId }, "Queueing ACME challenge validation");
+    await queueService.queuePg(
+      QueueJobs.PkiAcmeChallengeValidation,
+      { challengeId },
+      {
+        retryLimit: 3,
+        retryDelay: 30, // Base delay of 30 seconds
+        retryBackoff: true // Exponential backoff: 30s, 60s, 120s
+      }
+    );
+  };
+
+  return {
+    queueChallengeValidation
+  };
+};

backend/src/ee/services/pki-acme/pki-acme-service.ts

@@ -62,6 +62,7 @@ import {
 import { buildUrl, extractAccountIdFromKid, validateDnsIdentifier } from "./pki-acme-fns";
 import { TPkiAcmeOrderAuthDALFactory } from "./pki-acme-order-auth-dal";
 import { TPkiAcmeOrderDALFactory } from "./pki-acme-order-dal";
+import { TPkiAcmeQueueServiceFactory } from "./pki-acme-queue";
 import {
   AcmeAuthStatus,
   AcmeChallengeStatus,
@@ -126,7 +127,8 @@ type TPkiAcmeServiceFactoryDep = {
   >;
   licenseService: Pick<TLicenseServiceFactory, "getPlan">;
   certificateV3Service: Pick<TCertificateV3ServiceFactory, "signCertificateFromProfile">;
-  acmeChallengeService: TPkiAcmeChallengeServiceFactory;
+  acmeChallengeService: Pick<TPkiAcmeChallengeServiceFactory, "markChallengeAsReady">;
+  pkiAcmeQueueService: Pick<TPkiAcmeQueueServiceFactory, "queueChallengeValidation">;
 };
 
 export const pkiAcmeServiceFactory = ({
@@ -147,7 +149,8 @@ export const pkiAcmeServiceFactory = ({
   kmsService,
   licenseService,
   certificateV3Service,
-  acmeChallengeService
+  acmeChallengeService,
+  pkiAcmeQueueService
 }: TPkiAcmeServiceFactoryDep): TPkiAcmeServiceFactory => {
   const validateAcmeProfile = async (profileId: string): Promise<TCertificateProfileWithConfigs> => {
     const profile = await certificateProfileDAL.findByIdWithConfigs(profileId);
@@ -975,7 +978,8 @@ export const pkiAcmeServiceFactory = ({
     if (!result) {
       throw new NotFoundError({ message: "ACME challenge not found" });
     }
-    await acmeChallengeService.validateChallengeResponse(challengeId);
+    await acmeChallengeService.markChallengeAsReady(challengeId);
+    await pkiAcmeQueueService.queueChallengeValidation(challengeId);
     const challenge = (await acmeChallengeDAL.findByIdForChallengeValidation(challengeId))!;
     return {
       status: 200,

backend/src/ee/services/pki-acme/pki-acme-types.ts

@@ -1,6 +1,8 @@
 import { JWSHeaderParameters } from "jose";
 import { z } from "zod";
 
+import { TPkiAcmeChallenges } from "@app/db/schemas/pki-acme-challenges";
+
 import {
   AcmeOrderResourceSchema,
   CreateAcmeAccountBodySchema,
@@ -176,5 +178,6 @@ export type TPkiAcmeServiceFactory = {
 };
 
 export type TPkiAcmeChallengeServiceFactory = {
-  validateChallengeResponse: (challengeId: string) => Promise<void>;
+  markChallengeAsReady: (challengeId: string) => Promise<TPkiAcmeChallenges>;
+  validateChallengeResponse: (challengeId: string, retryCount: number) => Promise<void>;
 };

backend/src/queue/queue-service.ts

@@ -81,7 +81,8 @@ export enum QueueName {
   UserNotification = "user-notification",
   HealthAlert = "health-alert",
   CertificateV3AutoRenewal = "certificate-v3-auto-renewal",
-  PamAccountRotation = "pam-account-rotation"
+  PamAccountRotation = "pam-account-rotation",
+  PkiAcmeChallengeValidation = "pki-acme-challenge-validation"
 }
 
 export enum QueueJobs {
@@ -134,7 +135,8 @@ export enum QueueJobs {
   UserNotification = "user-notification-job",
   HealthAlert = "health-alert",
   CertificateV3DailyAutoRenewal = "certificate-v3-daily-auto-renewal",
-  PamAccountRotation = "pam-account-rotation"
+  PamAccountRotation = "pam-account-rotation",
+  PkiAcmeChallengeValidation = "pki-acme-challenge-validation"
 }
 
 export type TQueueJobTypes = {
@@ -385,6 +387,10 @@ export type TQueueJobTypes = {
     name: QueueJobs.PamAccountRotation;
     payload: undefined;
   };
+  [QueueName.PkiAcmeChallengeValidation]: {
+    name: QueueJobs.PkiAcmeChallengeValidation;
+    payload: { challengeId: string };
+  };
 };
 
 const SECRET_SCANNING_JOBS = [

backend/src/server/routes/index.ts

@@ -81,6 +81,7 @@ import { pkiAcmeChallengeDALFactory } from "@app/ee/services/pki-acme/pki-acme-c
 import { pkiAcmeChallengeServiceFactory } from "@app/ee/services/pki-acme/pki-acme-challenge-service";
 import { pkiAcmeOrderAuthDALFactory } from "@app/ee/services/pki-acme/pki-acme-order-auth-dal";
 import { pkiAcmeOrderDALFactory } from "@app/ee/services/pki-acme/pki-acme-order-dal";
+import { pkiAcmeQueueServiceFactory } from "@app/ee/services/pki-acme/pki-acme-queue";
 import { pkiAcmeServiceFactory } from "@app/ee/services/pki-acme/pki-acme-service";
 import { projectTemplateDALFactory } from "@app/ee/services/project-template/project-template-dal";
 import { projectTemplateServiceFactory } from "@app/ee/services/project-template/project-template-service";
@@ -2254,6 +2255,12 @@ export const registerRoutes = async (
   const acmeChallengeService = pkiAcmeChallengeServiceFactory({
     acmeChallengeDAL
   });
+
+  const pkiAcmeQueueService = await pkiAcmeQueueServiceFactory({
+    queueService,
+    acmeChallengeService
+  });
+
   const pkiAcmeService = pkiAcmeServiceFactory({
     projectDAL,
     appConnectionDAL,
@@ -2272,7 +2279,8 @@ export const registerRoutes = async (
     kmsService,
     licenseService,
     certificateV3Service,
-    acmeChallengeService
+    acmeChallengeService,
+    pkiAcmeQueueService
   });
 
   const pkiSubscriberService = pkiSubscriberServiceFactory({