mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-09-22 13:39:35 +00:00
Merge pull request #4975 from Infisical/PKI-38-better-http-01-challenge
improvement(api): perform http01-challenge in the background with retries
This commit is contained in:
@@ -22,6 +22,28 @@ Feature: Challenge
|
||||
And I parse the full-chain certificate from order finalized_order as cert
|
||||
And the value cert with jq ".subject.common_name" should be equal to "localhost"
|
||||
|
||||
Scenario: Validate challenge with retry
|
||||
Given I have an ACME cert profile as "acme_profile"
|
||||
When I have an ACME client connecting to "{BASE_URL}/api/v1/cert-manager/acme/profiles/{acme_profile.id}/directory"
|
||||
Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account
|
||||
When I create certificate signing request as csr
|
||||
Then I add names to certificate signing request csr
|
||||
"""
|
||||
{
|
||||
"COMMON_NAME": "localhost"
|
||||
}
|
||||
"""
|
||||
And I create a RSA private key pair as cert_key
|
||||
And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format
|
||||
And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order
|
||||
And I select challenge with type http-01 for domain localhost from order in order as challenge
|
||||
And I wait 45 seconds and serve challenge response for challenge at localhost
|
||||
And I tell ACME server that challenge is ready to be verified
|
||||
And I poll and finalize the ACME order order as finalized_order
|
||||
And the value finalized_order.body with jq ".status" should be equal to "valid"
|
||||
And I parse the full-chain certificate from order finalized_order as cert
|
||||
And the value cert with jq ".subject.common_name" should be equal to "localhost"
|
||||
|
||||
Scenario: Validate challenges for multiple domains
|
||||
Given I have an ACME cert profile as "acme_profile"
|
||||
When I have an ACME client connecting to "{BASE_URL}/api/v1/cert-manager/acme/profiles/{acme_profile.id}/directory"
|
||||
@@ -63,13 +85,12 @@ Feature: Challenge
|
||||
When I create certificate signing request as csr
|
||||
Then I add names to certificate signing request csr
|
||||
"""
|
||||
{
|
||||
"COMMON_NAME": "localhost"
|
||||
}
|
||||
{}
|
||||
"""
|
||||
And I add subject alternative name to certificate signing request csr
|
||||
"""
|
||||
[
|
||||
"localhost",
|
||||
"infisical.com"
|
||||
]
|
||||
"""
|
||||
@@ -82,56 +103,19 @@ Feature: Challenge
|
||||
|
||||
# the localhost auth should be valid
|
||||
And I memorize order with jq ".authorizations | map(select(.body.identifier.value == "localhost")) | first | .uri" as localhost_auth
|
||||
And I peak and memorize the next nonce as nonce
|
||||
When I send a raw ACME request to "{localhost_auth}"
|
||||
"""
|
||||
{
|
||||
"protected": {
|
||||
"alg": "RS256",
|
||||
"nonce": "{nonce}",
|
||||
"url": "{localhost_auth}",
|
||||
"kid": "{acme_account.uri}"
|
||||
}
|
||||
}
|
||||
"""
|
||||
Then the value response.status_code should be equal to 200
|
||||
And the value response with jq ".status" should be equal to "valid"
|
||||
And I wait until the status of authorization localhost_auth becomes valid
|
||||
|
||||
# the infisical.com auth should still be pending
|
||||
And I memorize order with jq ".authorizations | map(select(.body.identifier.value == "infisical.com")) | first | .uri" as infisical_auth
|
||||
And I memorize response.headers with jq ".["replay-nonce"]" as nonce
|
||||
When I send a raw ACME request to "{infisical_auth}"
|
||||
"""
|
||||
{
|
||||
"protected": {
|
||||
"alg": "RS256",
|
||||
"nonce": "{nonce}",
|
||||
"url": "{infisical_auth}",
|
||||
"kid": "{acme_account.uri}"
|
||||
}
|
||||
}
|
||||
"""
|
||||
Then the value response.status_code should be equal to 200
|
||||
And the value response with jq ".status" should be equal to "pending"
|
||||
And I post-as-get {infisical_auth} as infisical_auth_resp
|
||||
And the value infisical_auth_resp with jq ".status" should be equal to "pending"
|
||||
|
||||
# the order should be pending as well
|
||||
And I memorize response.headers with jq ".["replay-nonce"]" as nonce
|
||||
When I send a raw ACME request to "{order.uri}"
|
||||
"""
|
||||
{
|
||||
"protected": {
|
||||
"alg": "RS256",
|
||||
"nonce": "{nonce}",
|
||||
"url": "{order.uri}",
|
||||
"kid": "{acme_account.uri}"
|
||||
}
|
||||
}
|
||||
"""
|
||||
Then the value response.status_code should be equal to 200
|
||||
And the value response with jq ".status" should be equal to "pending"
|
||||
And I post-as-get {order.uri} as order_resp
|
||||
And the value order_resp with jq ".status" should be equal to "pending"
|
||||
|
||||
# finalize should not be allowed when all auths are not valid yet
|
||||
And I memorize response.headers with jq ".["replay-nonce"]" as nonce
|
||||
And I get a new-nonce as nonce
|
||||
When I send a raw ACME request to "{order.body.finalize}"
|
||||
"""
|
||||
{
|
||||
@@ -185,8 +169,10 @@ Feature: Challenge
|
||||
Then the value response.status_code should be equal to 201
|
||||
And I memorize response with jq ".finalize" as finalize_url
|
||||
And I memorize response.headers with jq ".["replay-nonce"]" as nonce
|
||||
And I memorize response.headers with jq ".["location"]" as order_uri
|
||||
And I memorize response as order
|
||||
And I pass all challenges with type http-01 for order in order
|
||||
And I wait until the status of order order_uri becomes ready
|
||||
And I encode CSR csr_pem as JOSE Base-64 DER as base64_csr_der
|
||||
When I send a raw ACME request to "{finalize_url}"
|
||||
"""
|
||||
|
||||
@@ -2,6 +2,8 @@ import json
|
||||
import logging
|
||||
import re
|
||||
import urllib.parse
|
||||
import time
|
||||
import threading
|
||||
|
||||
import acme.client
|
||||
import jq
|
||||
@@ -724,6 +726,15 @@ def step_impl(context: Context, var_path: str, jq_query, var_name: str):
|
||||
context.vars[var_name] = value
|
||||
|
||||
|
||||
@then("I get a new-nonce as {var_name}")
|
||||
def step_impl(context: Context, var_name: str):
|
||||
acme_client = context.acme_client
|
||||
nonce = acme_client.net._get_nonce(
|
||||
url=None, new_nonce_url=acme_client.directory.newNonce
|
||||
)
|
||||
context.vars[var_name] = json_util.encode_b64jose(nonce)
|
||||
|
||||
|
||||
@then("I peak and memorize the next nonce as {var_name}")
|
||||
def step_impl(context: Context, var_name: str):
|
||||
acme_client = context.acme_client
|
||||
@@ -797,22 +808,39 @@ def select_challenge(
|
||||
return challenges[0]
|
||||
|
||||
|
||||
def serve_challenge(
|
||||
def serve_challenges(
|
||||
context: Context,
|
||||
challenge: messages.ChallengeBody,
|
||||
challenges: list[messages.ChallengeBody],
|
||||
wait_time: int | None = None,
|
||||
):
|
||||
if hasattr(context, "web_server"):
|
||||
context.web_server.shutdown_and_server_close()
|
||||
|
||||
response, validation = challenge.response_and_validation(
|
||||
context.acme_client.net.key
|
||||
)
|
||||
resource = standalone.HTTP01RequestHandler.HTTP01Resource(
|
||||
chall=challenge.chall, response=response, validation=validation
|
||||
)
|
||||
resources = set()
|
||||
for challenge in challenges:
|
||||
response, validation = challenge.response_and_validation(
|
||||
context.acme_client.net.key
|
||||
)
|
||||
resources.add(
|
||||
standalone.HTTP01RequestHandler.HTTP01Resource(
|
||||
chall=challenge.chall, response=response, validation=validation
|
||||
)
|
||||
)
|
||||
# TODO: make port configurable
|
||||
servers = standalone.HTTP01DualNetworkedServers(("0.0.0.0", 8087), {resource})
|
||||
servers.serve_forever()
|
||||
servers = standalone.HTTP01DualNetworkedServers(("0.0.0.0", 8087), resources)
|
||||
if wait_time is None:
|
||||
servers.serve_forever()
|
||||
else:
|
||||
|
||||
def wait_and_start():
|
||||
logger.info("Waiting %s seconds before we start serving.", wait_time)
|
||||
time.sleep(wait_time)
|
||||
logger.info("Start server now")
|
||||
servers.serve_forever()
|
||||
|
||||
thread = threading.Thread(target=wait_and_start)
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
context.web_server = servers
|
||||
|
||||
|
||||
@@ -865,6 +893,7 @@ def step_impl(
|
||||
f"Expected OrderResource but got {type(order)!r} at {order_var_path!r}"
|
||||
)
|
||||
|
||||
challenges = {}
|
||||
for domain in order.body.identifiers:
|
||||
logger.info(
|
||||
"Selecting challenge for domain %s with type %s ...",
|
||||
@@ -889,18 +918,28 @@ def step_impl(
|
||||
domain.value,
|
||||
challenge_type,
|
||||
)
|
||||
serve_challenge(context=context, challenge=challenge)
|
||||
challenges[domain] = challenge
|
||||
|
||||
serve_challenges(context=context, challenges=list(challenges.values()))
|
||||
for domain, challenge in challenges.items():
|
||||
logger.info(
|
||||
"Notifying challenge for domain %s with type %s ...", domain, challenge_type
|
||||
)
|
||||
notify_challenge_ready(context=context, challenge=challenge)
|
||||
|
||||
|
||||
@then(
|
||||
"I wait {wait_time} seconds and serve challenge response for {var_path} at {hostname}"
|
||||
)
|
||||
def step_impl(context: Context, wait_time: str, var_path: str, hostname: str):
|
||||
challenge = eval_var(context, var_path, as_json=False)
|
||||
serve_challenges(context=context, challenges=[challenge], wait_time=int(wait_time))
|
||||
|
||||
|
||||
@then("I serve challenge response for {var_path} at {hostname}")
|
||||
def step_impl(context: Context, var_path: str, hostname: str):
|
||||
challenge = eval_var(context, var_path, as_json=False)
|
||||
serve_challenge(context=context, challenge=challenge)
|
||||
serve_challenges(context=context, challenges=[challenge])
|
||||
|
||||
|
||||
@then("I tell ACME server that {var_path} is ready to be verified")
|
||||
@@ -909,6 +948,47 @@ def step_impl(context: Context, var_path: str):
|
||||
notify_challenge_ready(context=context, challenge=challenge)
|
||||
|
||||
|
||||
@then("I wait until the status of order {order_var} becomes {status}")
|
||||
def step_impl(context: Context, order_var: str, status: str):
|
||||
acme_client = context.acme_client
|
||||
attempt_count = 6
|
||||
while attempt_count:
|
||||
order = eval_var(context, order_var, as_json=False)
|
||||
response = acme_client._post_as_get(
|
||||
order.uri if isinstance(order, messages.OrderResource) else order
|
||||
)
|
||||
order = messages.Order.from_json(response.json())
|
||||
if order.status.name == status:
|
||||
return
|
||||
attempt_count -= 1
|
||||
time.sleep(10)
|
||||
raise TimeoutError(f"The status of order doesn't become {status} before timeout")
|
||||
|
||||
|
||||
@then("I wait until the status of authorization {auth_var} becomes {status}")
|
||||
def step_impl(context: Context, auth_var: str, status: str):
|
||||
acme_client = context.acme_client
|
||||
attempt_count = 6
|
||||
while attempt_count:
|
||||
auth = eval_var(context, auth_var, as_json=False)
|
||||
response = acme_client._post_as_get(
|
||||
auth.uri if isinstance(auth, messages.Authorization) else auth
|
||||
)
|
||||
auth = messages.Authorization.from_json(response.json())
|
||||
if auth.status.name == status:
|
||||
return
|
||||
attempt_count -= 1
|
||||
time.sleep(10)
|
||||
raise TimeoutError(f"The status of auth doesn't become {status} before timeout")
|
||||
|
||||
|
||||
@then("I post-as-get {uri} as {resp_var}")
|
||||
def step_impl(context: Context, uri: str, resp_var: str):
|
||||
acme_client = context.acme_client
|
||||
response = acme_client._post_as_get(replace_vars(uri, vars=context.vars))
|
||||
context.vars[resp_var] = response.json()
|
||||
|
||||
|
||||
@then("I poll and finalize the ACME order {var_path} as {finalized_var}")
|
||||
def step_impl(context: Context, var_path: str, finalized_var: str):
|
||||
order = eval_var(context, var_path, as_json=False)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import axios, { AxiosError } from "axios";
|
||||
|
||||
import { TPkiAcmeChallenges } from "@app/db/schemas/pki-acme-challenges";
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { BadRequestError, NotFoundError } from "@app/lib/errors";
|
||||
import { isPrivateIp } from "@app/lib/ip/ipRange";
|
||||
@@ -18,7 +19,11 @@ import { TPkiAcmeChallengeServiceFactory } from "./pki-acme-types";
|
||||
type TPkiAcmeChallengeServiceFactoryDep = {
|
||||
acmeChallengeDAL: Pick<
|
||||
TPkiAcmeChallengeDALFactory,
|
||||
"transaction" | "findByIdForChallengeValidation" | "markAsValidCascadeById" | "markAsInvalidCascadeById"
|
||||
| "transaction"
|
||||
| "findByIdForChallengeValidation"
|
||||
| "markAsValidCascadeById"
|
||||
| "markAsInvalidCascadeById"
|
||||
| "updateById"
|
||||
>;
|
||||
};
|
||||
|
||||
@@ -26,9 +31,8 @@ export const pkiAcmeChallengeServiceFactory = ({
|
||||
acmeChallengeDAL
|
||||
}: TPkiAcmeChallengeServiceFactoryDep): TPkiAcmeChallengeServiceFactory => {
|
||||
const appCfg = getConfig();
|
||||
|
||||
const validateChallengeResponse = async (challengeId: string): Promise<void> => {
|
||||
const error: Error | undefined = await acmeChallengeDAL.transaction(async (tx) => {
|
||||
const markChallengeAsReady = async (challengeId: string): Promise<TPkiAcmeChallenges> => {
|
||||
return acmeChallengeDAL.transaction(async (tx) => {
|
||||
logger.info({ challengeId }, "Validating ACME challenge response");
|
||||
const challenge = await acmeChallengeDAL.findByIdForChallengeValidation(challengeId, tx);
|
||||
if (!challenge) {
|
||||
@@ -52,81 +56,102 @@ export const pkiAcmeChallengeServiceFactory = ({
|
||||
if (challenge.type !== AcmeChallengeType.HTTP_01) {
|
||||
throw new BadRequestError({ message: "Only HTTP-01 challenges are supported for now" });
|
||||
}
|
||||
let host = challenge.auth.identifierValue;
|
||||
const host = challenge.auth.identifierValue;
|
||||
// check if host is a private ip address
|
||||
if (isPrivateIp(host)) {
|
||||
throw new BadRequestError({ message: "Private IP addresses are not allowed" });
|
||||
}
|
||||
if (appCfg.isAcmeDevelopmentMode && appCfg.ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES[host]) {
|
||||
host = appCfg.ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES[host];
|
||||
logger.warn(
|
||||
{ srcHost: challenge.auth.identifierValue, dstHost: host },
|
||||
"Using ACME development HTTP-01 challenge host override"
|
||||
);
|
||||
}
|
||||
const challengeUrl = new URL(`/.well-known/acme-challenge/${challenge.auth.token}`, `http://${host}`);
|
||||
logger.info({ challengeUrl }, "Performing ACME HTTP-01 challenge validation");
|
||||
try {
|
||||
// TODO: read config from the profile to get the timeout instead
|
||||
const timeoutMs = 10 * 1000; // 10 seconds
|
||||
// Notice: well, we are in a transaction, ideally we should not hold transaction and perform
|
||||
// a long running operation for long time. But assuming we are not performing a tons of
|
||||
// challenge validation at the same time, it should be fine.
|
||||
const challengeResponse = await axios.get<string>(challengeUrl.toString(), {
|
||||
// In case if we override the host in the development mode, still provide the original host in the header
|
||||
// to help the upstream server to validate the request
|
||||
headers: { Host: challenge.auth.identifierValue },
|
||||
timeout: timeoutMs,
|
||||
responseType: "text",
|
||||
validateStatus: () => true
|
||||
});
|
||||
if (challengeResponse.status !== 200) {
|
||||
throw new AcmeIncorrectResponseError({
|
||||
message: `ACME challenge response is not 200: ${challengeResponse.status}`
|
||||
});
|
||||
}
|
||||
const challengeResponseBody: string = challengeResponse.data;
|
||||
const thumbprint = challenge.auth.account.publicKeyThumbprint;
|
||||
const expectedChallengeResponseBody = `${challenge.auth.token}.${thumbprint}`;
|
||||
if (challengeResponseBody.trimEnd() !== expectedChallengeResponseBody) {
|
||||
throw new AcmeIncorrectResponseError({ message: "ACME challenge response is not correct" });
|
||||
}
|
||||
await acmeChallengeDAL.markAsValidCascadeById(challengeId, tx);
|
||||
} catch (exp) {
|
||||
// TODO: we should retry the challenge validation a few times, but let's keep it simple for now
|
||||
await acmeChallengeDAL.markAsInvalidCascadeById(challengeId, tx);
|
||||
// Properly type and inspect the error
|
||||
if (axios.isAxiosError(exp)) {
|
||||
const axiosError = exp as AxiosError;
|
||||
const errorCode = axiosError.code;
|
||||
const errorMessage = axiosError.message;
|
||||
|
||||
if (errorCode === "ECONNREFUSED" || errorMessage.includes("ECONNREFUSED")) {
|
||||
return new AcmeConnectionError({ message: "Connection refused" });
|
||||
}
|
||||
if (errorCode === "ENOTFOUND" || errorMessage.includes("ENOTFOUND")) {
|
||||
return new AcmeDnsFailureError({ message: "Hostname could not be resolved (DNS failure)" });
|
||||
}
|
||||
if (errorCode === "ECONNABORTED" || errorMessage.includes("timeout")) {
|
||||
logger.error(exp, "Connection timed out while validating ACME challenge response");
|
||||
return new AcmeConnectionError({ message: "Connection timed out" });
|
||||
}
|
||||
logger.error(exp, "Unknown error validating ACME challenge response");
|
||||
return new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" });
|
||||
}
|
||||
if (exp instanceof Error) {
|
||||
logger.error(exp, "Error validating ACME challenge response");
|
||||
} else {
|
||||
logger.error(exp, "Unknown error validating ACME challenge response");
|
||||
return new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" });
|
||||
}
|
||||
return exp;
|
||||
}
|
||||
return acmeChallengeDAL.updateById(challengeId, { status: AcmeChallengeStatus.Processing }, tx);
|
||||
});
|
||||
if (error) {
|
||||
throw error;
|
||||
};
|
||||
|
||||
const validateChallengeResponse = async (challengeId: string, retryCount: number): Promise<void> => {
|
||||
logger.info({ challengeId, retryCount }, "Validating ACME challenge response");
|
||||
const challenge = await acmeChallengeDAL.findByIdForChallengeValidation(challengeId);
|
||||
if (!challenge) {
|
||||
throw new NotFoundError({ message: "ACME challenge not found" });
|
||||
}
|
||||
if (challenge.status !== AcmeChallengeStatus.Processing) {
|
||||
throw new BadRequestError({
|
||||
message: `ACME challenge is ${challenge.status} instead of ${AcmeChallengeStatus.Processing}`
|
||||
});
|
||||
}
|
||||
let host = challenge.auth.identifierValue;
|
||||
if (appCfg.isAcmeDevelopmentMode && appCfg.ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES[host]) {
|
||||
host = appCfg.ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES[host];
|
||||
logger.warn(
|
||||
{ srcHost: challenge.auth.identifierValue, dstHost: host },
|
||||
"Using ACME development HTTP-01 challenge host override"
|
||||
);
|
||||
}
|
||||
const challengeUrl = new URL(`/.well-known/acme-challenge/${challenge.auth.token}`, `http://${host}`);
|
||||
logger.info({ challengeUrl }, "Performing ACME HTTP-01 challenge validation");
|
||||
try {
|
||||
// TODO: read config from the profile to get the timeout instead
|
||||
const timeoutMs = 10 * 1000; // 10 seconds
|
||||
// Notice: well, we are in a transaction, ideally we should not hold transaction and perform
|
||||
// a long running operation for long time. But assuming we are not performing a tons of
|
||||
// challenge validation at the same time, it should be fine.
|
||||
const challengeResponse = await axios.get<string>(challengeUrl.toString(), {
|
||||
// In case if we override the host in the development mode, still provide the original host in the header
|
||||
// to help the upstream server to validate the request
|
||||
headers: { Host: challenge.auth.identifierValue },
|
||||
timeout: timeoutMs,
|
||||
responseType: "text",
|
||||
validateStatus: () => true
|
||||
});
|
||||
if (challengeResponse.status !== 200) {
|
||||
throw new AcmeIncorrectResponseError({
|
||||
message: `ACME challenge response is not 200: ${challengeResponse.status}`
|
||||
});
|
||||
}
|
||||
const challengeResponseBody: string = challengeResponse.data;
|
||||
const thumbprint = challenge.auth.account.publicKeyThumbprint;
|
||||
const expectedChallengeResponseBody = `${challenge.auth.token}.${thumbprint}`;
|
||||
if (challengeResponseBody.trimEnd() !== expectedChallengeResponseBody) {
|
||||
throw new AcmeIncorrectResponseError({ message: "ACME challenge response is not correct" });
|
||||
}
|
||||
logger.info({ challengeId }, "ACME challenge response is correct, marking challenge as valid");
|
||||
await acmeChallengeDAL.markAsValidCascadeById(challengeId);
|
||||
} catch (exp) {
|
||||
if (retryCount >= 2) {
|
||||
logger.error(
|
||||
exp,
|
||||
`Last attempt to validate ACME challenge response failed, marking ${challengeId} challenge as invalid`
|
||||
);
|
||||
// This is the last attempt to validate the challenge response, if it fails, we mark the challenge as invalid
|
||||
await acmeChallengeDAL.markAsInvalidCascadeById(challengeId);
|
||||
}
|
||||
// Properly type and inspect the error
|
||||
if (axios.isAxiosError(exp)) {
|
||||
const axiosError = exp as AxiosError;
|
||||
const errorCode = axiosError.code;
|
||||
const errorMessage = axiosError.message;
|
||||
|
||||
if (errorCode === "ECONNREFUSED" || errorMessage.includes("ECONNREFUSED")) {
|
||||
throw new AcmeConnectionError({ message: "Connection refused" });
|
||||
}
|
||||
if (errorCode === "ENOTFOUND" || errorMessage.includes("ENOTFOUND")) {
|
||||
throw new AcmeDnsFailureError({ message: "Hostname could not be resolved (DNS failure)" });
|
||||
}
|
||||
if (errorCode === "ECONNRESET" || errorMessage.includes("ECONNRESET")) {
|
||||
throw new AcmeConnectionError({ message: "Connection reset by peer" });
|
||||
}
|
||||
if (errorCode === "ECONNABORTED" || errorMessage.includes("timeout")) {
|
||||
logger.error(exp, "Connection timed out while validating ACME challenge response");
|
||||
throw new AcmeConnectionError({ message: "Connection timed out" });
|
||||
}
|
||||
logger.error(exp, "Unknown error validating ACME challenge response");
|
||||
throw new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" });
|
||||
}
|
||||
if (exp instanceof Error) {
|
||||
logger.error(exp, "Error validating ACME challenge response");
|
||||
throw exp;
|
||||
}
|
||||
logger.error(exp, "Unknown error validating ACME challenge response");
|
||||
throw new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" });
|
||||
}
|
||||
};
|
||||
|
||||
return { validateChallengeResponse };
|
||||
return { markChallengeAsReady, validateChallengeResponse };
|
||||
};
|
||||
|
||||
67
backend/src/ee/services/pki-acme/pki-acme-queue.ts
Normal file
67
backend/src/ee/services/pki-acme/pki-acme-queue.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { logger } from "@app/lib/logger";
|
||||
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
|
||||
|
||||
import { TPkiAcmeChallengeServiceFactory } from "./pki-acme-types";
|
||||
|
||||
type TPkiAcmeQueueServiceFactoryDep = {
|
||||
queueService: TQueueServiceFactory;
|
||||
acmeChallengeService: TPkiAcmeChallengeServiceFactory;
|
||||
};
|
||||
|
||||
export type TPkiAcmeQueueServiceFactory = Awaited<ReturnType<typeof pkiAcmeQueueServiceFactory>>;
|
||||
|
||||
export const pkiAcmeQueueServiceFactory = async ({
|
||||
queueService,
|
||||
acmeChallengeService
|
||||
}: TPkiAcmeQueueServiceFactoryDep) => {
|
||||
const appCfg = getConfig();
|
||||
|
||||
// Initialize the worker to process challenge validation jobs
|
||||
await queueService.startPg<QueueName.PkiAcmeChallengeValidation>(
|
||||
QueueJobs.PkiAcmeChallengeValidation,
|
||||
async ([job]) => {
|
||||
const { challengeId } = job.data;
|
||||
const retryCount = job.retryCount || 0;
|
||||
try {
|
||||
logger.info({ challengeId, retryCount }, "Processing ACME challenge validation job");
|
||||
await acmeChallengeService.validateChallengeResponse(challengeId, retryCount);
|
||||
logger.info({ challengeId, retryCount }, "ACME challenge validation completed successfully");
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
logger.error(
|
||||
error,
|
||||
`Failed to validate ACME challenge ${challengeId} (retryCount ${retryCount}): ${errorMessage}`
|
||||
);
|
||||
// Re-throw to let pg-boss handle retries with exponential backoff
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
|
||||
const queueChallengeValidation = async (challengeId: string): Promise<void> => {
|
||||
if (appCfg.isSecondaryInstance) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info({ challengeId }, "Queueing ACME challenge validation");
|
||||
await queueService.queuePg(
|
||||
QueueJobs.PkiAcmeChallengeValidation,
|
||||
{ challengeId },
|
||||
{
|
||||
retryLimit: 3,
|
||||
retryDelay: 30, // Base delay of 30 seconds
|
||||
retryBackoff: true // Exponential backoff: 30s, 60s, 120s
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
return {
|
||||
queueChallengeValidation
|
||||
};
|
||||
};
|
||||
@@ -62,6 +62,7 @@ import {
|
||||
import { buildUrl, extractAccountIdFromKid, validateDnsIdentifier } from "./pki-acme-fns";
|
||||
import { TPkiAcmeOrderAuthDALFactory } from "./pki-acme-order-auth-dal";
|
||||
import { TPkiAcmeOrderDALFactory } from "./pki-acme-order-dal";
|
||||
import { TPkiAcmeQueueServiceFactory } from "./pki-acme-queue";
|
||||
import {
|
||||
AcmeAuthStatus,
|
||||
AcmeChallengeStatus,
|
||||
@@ -126,7 +127,8 @@ type TPkiAcmeServiceFactoryDep = {
|
||||
>;
|
||||
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
|
||||
certificateV3Service: Pick<TCertificateV3ServiceFactory, "signCertificateFromProfile">;
|
||||
acmeChallengeService: TPkiAcmeChallengeServiceFactory;
|
||||
acmeChallengeService: Pick<TPkiAcmeChallengeServiceFactory, "markChallengeAsReady">;
|
||||
pkiAcmeQueueService: Pick<TPkiAcmeQueueServiceFactory, "queueChallengeValidation">;
|
||||
};
|
||||
|
||||
export const pkiAcmeServiceFactory = ({
|
||||
@@ -147,7 +149,8 @@ export const pkiAcmeServiceFactory = ({
|
||||
kmsService,
|
||||
licenseService,
|
||||
certificateV3Service,
|
||||
acmeChallengeService
|
||||
acmeChallengeService,
|
||||
pkiAcmeQueueService
|
||||
}: TPkiAcmeServiceFactoryDep): TPkiAcmeServiceFactory => {
|
||||
const validateAcmeProfile = async (profileId: string): Promise<TCertificateProfileWithConfigs> => {
|
||||
const profile = await certificateProfileDAL.findByIdWithConfigs(profileId);
|
||||
@@ -975,7 +978,8 @@ export const pkiAcmeServiceFactory = ({
|
||||
if (!result) {
|
||||
throw new NotFoundError({ message: "ACME challenge not found" });
|
||||
}
|
||||
await acmeChallengeService.validateChallengeResponse(challengeId);
|
||||
await acmeChallengeService.markChallengeAsReady(challengeId);
|
||||
await pkiAcmeQueueService.queueChallengeValidation(challengeId);
|
||||
const challenge = (await acmeChallengeDAL.findByIdForChallengeValidation(challengeId))!;
|
||||
return {
|
||||
status: 200,
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { JWSHeaderParameters } from "jose";
|
||||
import { z } from "zod";
|
||||
|
||||
import { TPkiAcmeChallenges } from "@app/db/schemas/pki-acme-challenges";
|
||||
|
||||
import {
|
||||
AcmeOrderResourceSchema,
|
||||
CreateAcmeAccountBodySchema,
|
||||
@@ -176,5 +178,6 @@ export type TPkiAcmeServiceFactory = {
|
||||
};
|
||||
|
||||
export type TPkiAcmeChallengeServiceFactory = {
|
||||
validateChallengeResponse: (challengeId: string) => Promise<void>;
|
||||
markChallengeAsReady: (challengeId: string) => Promise<TPkiAcmeChallenges>;
|
||||
validateChallengeResponse: (challengeId: string, retryCount: number) => Promise<void>;
|
||||
};
|
||||
|
||||
@@ -81,7 +81,8 @@ export enum QueueName {
|
||||
UserNotification = "user-notification",
|
||||
HealthAlert = "health-alert",
|
||||
CertificateV3AutoRenewal = "certificate-v3-auto-renewal",
|
||||
PamAccountRotation = "pam-account-rotation"
|
||||
PamAccountRotation = "pam-account-rotation",
|
||||
PkiAcmeChallengeValidation = "pki-acme-challenge-validation"
|
||||
}
|
||||
|
||||
export enum QueueJobs {
|
||||
@@ -134,7 +135,8 @@ export enum QueueJobs {
|
||||
UserNotification = "user-notification-job",
|
||||
HealthAlert = "health-alert",
|
||||
CertificateV3DailyAutoRenewal = "certificate-v3-daily-auto-renewal",
|
||||
PamAccountRotation = "pam-account-rotation"
|
||||
PamAccountRotation = "pam-account-rotation",
|
||||
PkiAcmeChallengeValidation = "pki-acme-challenge-validation"
|
||||
}
|
||||
|
||||
export type TQueueJobTypes = {
|
||||
@@ -385,6 +387,10 @@ export type TQueueJobTypes = {
|
||||
name: QueueJobs.PamAccountRotation;
|
||||
payload: undefined;
|
||||
};
|
||||
[QueueName.PkiAcmeChallengeValidation]: {
|
||||
name: QueueJobs.PkiAcmeChallengeValidation;
|
||||
payload: { challengeId: string };
|
||||
};
|
||||
};
|
||||
|
||||
const SECRET_SCANNING_JOBS = [
|
||||
|
||||
@@ -81,6 +81,7 @@ import { pkiAcmeChallengeDALFactory } from "@app/ee/services/pki-acme/pki-acme-c
|
||||
import { pkiAcmeChallengeServiceFactory } from "@app/ee/services/pki-acme/pki-acme-challenge-service";
|
||||
import { pkiAcmeOrderAuthDALFactory } from "@app/ee/services/pki-acme/pki-acme-order-auth-dal";
|
||||
import { pkiAcmeOrderDALFactory } from "@app/ee/services/pki-acme/pki-acme-order-dal";
|
||||
import { pkiAcmeQueueServiceFactory } from "@app/ee/services/pki-acme/pki-acme-queue";
|
||||
import { pkiAcmeServiceFactory } from "@app/ee/services/pki-acme/pki-acme-service";
|
||||
import { projectTemplateDALFactory } from "@app/ee/services/project-template/project-template-dal";
|
||||
import { projectTemplateServiceFactory } from "@app/ee/services/project-template/project-template-service";
|
||||
@@ -2254,6 +2255,12 @@ export const registerRoutes = async (
|
||||
const acmeChallengeService = pkiAcmeChallengeServiceFactory({
|
||||
acmeChallengeDAL
|
||||
});
|
||||
|
||||
const pkiAcmeQueueService = await pkiAcmeQueueServiceFactory({
|
||||
queueService,
|
||||
acmeChallengeService
|
||||
});
|
||||
|
||||
const pkiAcmeService = pkiAcmeServiceFactory({
|
||||
projectDAL,
|
||||
appConnectionDAL,
|
||||
@@ -2272,7 +2279,8 @@ export const registerRoutes = async (
|
||||
kmsService,
|
||||
licenseService,
|
||||
certificateV3Service,
|
||||
acmeChallengeService
|
||||
acmeChallengeService,
|
||||
pkiAcmeQueueService
|
||||
});
|
||||
|
||||
const pkiSubscriberService = pkiSubscriberServiceFactory({
|
||||
|
||||
Reference in New Issue
Block a user