From d559d48e7ec15ba56db121b6a97c114c7d865d20 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 09:44:26 -0800 Subject: [PATCH 01/56] Handle external ca # Conflicts: # backend/src/ee/services/pki-acme/pki-acme-service.ts # Conflicts: # backend/src/ee/services/pki-acme/pki-acme-service.ts --- .../ee/services/pki-acme/pki-acme-service.ts | 66 +++++++++++++------ 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index 55e9cc43f..feb8ad782 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -29,7 +29,8 @@ import { TProjectDALFactory } from "@app/services/project/project-dal"; import { getProjectKmsCertificateKeyId } from "@app/services/project/project-fns"; import { getConfig } from "@app/lib/config/env"; -import { TLicenseServiceFactory } from "../license/license-service"; +import { TCertificateAuthorityDALFactory } from "@app/services/certificate-authority/certificate-authority-dal"; +import { CaType } from "@app/services/certificate-authority/certificate-authority-enums"; import { TPkiAcmeAccountDALFactory } from "./pki-acme-account-dal"; import { TPkiAcmeAuthDALFactory } from "./pki-acme-auth-dal"; import { TPkiAcmeChallengeDALFactory } from "./pki-acme-challenge-dal"; @@ -80,6 +81,7 @@ import { type TPkiAcmeServiceFactoryDep = { projectDAL: Pick; + certificateAuthorityDAL: Pick; certificateProfileDAL: Pick; certificateBodyDAL: Pick; acmeAccountDAL: Pick< @@ -110,6 +112,7 @@ type TPkiAcmeServiceFactoryDep = { export const pkiAcmeServiceFactory = ({ projectDAL, + certificateAuthorityDAL, certificateProfileDAL, certificateBodyDAL, acmeAccountDAL, @@ -622,6 +625,7 @@ export const pkiAcmeServiceFactory = ({ orderId: string; payload: TFinalizeAcmeOrderPayload; }): Promise> => { + const profile = (await certificateProfileDAL.findByIdWithConfigs(profileId))!; let order = await acmeOrderDAL.findByAccountAndOrderIdWithAuthorizations(accountId, orderId); if (!order) { throw new NotFoundError({ message: "ACME order not found" }); @@ -638,28 +642,48 @@ export const pkiAcmeServiceFactory = ({ throw new AcmeOrderNotReadyError({ message: "ACME order has expired" }); } const { csr } = payload; + const ca = await certificateAuthorityDAL.findByIdWithAssociatedCa(profile.caId); + if (!ca) { + throw new NotFoundError({ message: "Certificate Authority not found" }); + } + const caType = (ca.externalCa?.type as CaType) ?? CaType.INTERNAL; let errorToReturn: Error | undefined; try { - const { certificateId } = await certificateV3Service.signCertificateFromProfile({ - actor: ActorType.ACME_ACCOUNT, - actorId: accountId, - actorAuthMethod: null, - actorOrgId, - profileId, - csr, - notBefore: finalizingOrder.notBefore ? new Date(finalizingOrder.notBefore) : undefined, - notAfter: finalizingOrder.notAfter ? new Date(finalizingOrder.notAfter) : undefined, - validity: !finalizingOrder.notAfter - ? { - // 47 days, the default TTL comes with Let's Encrypt - // TODO: read config from the profile to get the expiration time instead - ttl: `${47}d` - } - : // ttl is not used if notAfter is provided - ({ ttl: "0d" } as const), - enrollmentType: EnrollmentType.ACME - }); - // TODO: associate the certificate with the order + const { certificateId } = await (async () => { + if (caType === CaType.INTERNAL) { + const result = await certificateV3Service.signCertificateFromProfile({ + actor: ActorType.ACME_ACCOUNT, + actorId: accountId, + actorAuthMethod: null, + actorOrgId, + profileId, + csr, + notBefore: finalizingOrder.notBefore ? new Date(finalizingOrder.notBefore) : undefined, + notAfter: finalizingOrder.notAfter ? new Date(finalizingOrder.notAfter) : undefined, + validity: !finalizingOrder.notAfter + ? { + // 47 days, the default TTL comes with Let's Encrypt + // TODO: read config from the profile to get the expiration time instead + ttl: `${47}d` + } + : // ttl is not used if notAfter is provided + ({ ttl: "0d" } as const), + enrollmentType: EnrollmentType.ACME + }); + return { certificateId: result.certificateId }; + } else { + const orderWithAuthorizations = (await acmeOrderDAL.findByAccountAndOrderIdWithAuthorizations( + accountId, + orderId, + tx + ))!; + const result = await orderCertificateForAcmeProfile( + profileId, + orderWithAuthorizations.authorizations[0].identifierValue + ); + return { certificateId: result }; + } + })(); await acmeOrderDAL.updateById( orderId, { From 490087dd90f713bf3db93a65975f24a07e524896 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 09:55:11 -0800 Subject: [PATCH 02/56] extract common func --- .../acme/acme-certificate-authority-fns.ts | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts index 0dea986d6..78df43b64 100644 --- a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts +++ b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts @@ -317,13 +317,16 @@ export const AcmeCertificateAuthorityFns = ({ return cas.map(castDbEntryToAcmeCertificateAuthority); }; - const orderSubscriberCertificate = async (subscriberId: string) => { - const subscriber = await pkiSubscriberDAL.findById(subscriberId); - if (!subscriber.caId) { - throw new BadRequestError({ message: "Subscriber does not have a CA" }); - } - - const ca = await certificateAuthorityDAL.findByIdWithAssociatedCa(subscriber.caId); + const orderCertificate = async ({ + caId, + commonName, + altNames + }: { + caId: string; + commonName: string; + altNames?: string[]; + }): Promise => { + const ca = await certificateAuthorityDAL.findByIdWithAssociatedCa(caId); if (!ca.externalCa || ca.externalCa.type !== CaType.ACME) { throw new BadRequestError({ message: "CA is not an ACME CA" }); } @@ -401,8 +404,8 @@ export const AcmeCertificateAuthorityFns = ({ const [, certificateCsr] = await acme.crypto.createCsr( { - altNames: subscriber.subjectAlternativeNames, - commonName: subscriber.commonName + altNames, + commonName }, skLeaf ); @@ -535,10 +538,21 @@ export const AcmeCertificateAuthorityFns = ({ await triggerAutoSyncForSubscriber(subscriber.id, { pkiSyncDAL, pkiSyncQueue }); }; + const orderCertificateForAcmeProfile = async (profileId: string, commonName: string): Promise => { + const profile = await certificateProfileDAL.findByIdWithConfigs(profileId); + if (!profile) { + throw new NotFoundError({ message: "Certificate profile not found" }); + } + if (profile.enrollmentType !== EnrollmentType.ACME) { + throw new NotFoundError({ message: "Certificate profile is not configured for ACME enrollment" }); + } + }; + return { createCertificateAuthority, updateCertificateAuthority, listCertificateAuthorities, - orderSubscriberCertificate + orderSubscriberCertificate, + orderCertificateForAcmeProfile }; }; From 8f199fecb818eadeb17491c3ec7880a83581109f Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 11:09:01 -0800 Subject: [PATCH 03/56] Refactor --- .../acme/acme-certificate-authority-fns.ts | 52 ++++++++++++------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts index 78df43b64..cfe734eb8 100644 --- a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts +++ b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts @@ -319,13 +319,19 @@ export const AcmeCertificateAuthorityFns = ({ const orderCertificate = async ({ caId, + subscriberId, commonName, - altNames + altNames, + keyUsages, + extendedKeyUsages }: { caId: string; + subscriberId?: string; commonName: string; altNames?: string[]; - }): Promise => { + keyUsages?: CertKeyUsage[]; + extendedKeyUsages?: CertExtendedKeyUsage[]; + }) => { const ca = await certificateAuthorityDAL.findByIdWithAssociatedCa(caId); if (!ca.externalCa || ca.externalCa.type !== CaType.ACME) { throw new BadRequestError({ message: "CA is not an ACME CA" }); @@ -498,20 +504,20 @@ export const AcmeCertificateAuthorityFns = ({ plainText: Buffer.from(skLeaf) }); - await certificateDAL.transaction(async (tx) => { + return certificateDAL.transaction(async (tx) => { const cert = await certificateDAL.create( { caId: ca.id, - pkiSubscriberId: subscriber.id, + pkiSubscriberId: subscriberId, status: CertStatus.ACTIVE, - friendlyName: subscriber.commonName, - commonName: subscriber.commonName, - altNames: subscriber.subjectAlternativeNames.join(","), + friendlyName: commonName, + commonName, + altNames: altNames?.join(","), serialNumber: certObj.serialNumber, notBefore: certObj.notBefore, notAfter: certObj.notAfter, - keyUsages: subscriber.keyUsages as CertKeyUsage[], - extendedKeyUsages: subscriber.extendedKeyUsages as CertExtendedKeyUsage[], + keyUsages: keyUsages, + extendedKeyUsages: extendedKeyUsages, projectId: ca.projectId }, tx @@ -533,26 +539,32 @@ export const AcmeCertificateAuthorityFns = ({ }, tx ); - }); - await triggerAutoSyncForSubscriber(subscriber.id, { pkiSyncDAL, pkiSyncQueue }); + return cert; + }); }; - const orderCertificateForAcmeProfile = async (profileId: string, commonName: string): Promise => { - const profile = await certificateProfileDAL.findByIdWithConfigs(profileId); - if (!profile) { - throw new NotFoundError({ message: "Certificate profile not found" }); - } - if (profile.enrollmentType !== EnrollmentType.ACME) { - throw new NotFoundError({ message: "Certificate profile is not configured for ACME enrollment" }); + const orderSubscriberCertificate = async (subscriberId: string) => { + const subscriber = await pkiSubscriberDAL.findById(subscriberId); + if (!subscriber.caId) { + throw new BadRequestError({ message: "Subscriber does not have a CA" }); } + await orderCertificate({ + caId: subscriber.caId, + subscriberId: subscriber.id, + commonName: subscriber.commonName, + altNames: subscriber.subjectAlternativeNames, + keyUsages: subscriber.keyUsages as CertKeyUsage[], + extendedKeyUsages: subscriber.extendedKeyUsages as CertExtendedKeyUsage[] + }); + await triggerAutoSyncForSubscriber(subscriber.id, { pkiSyncDAL, pkiSyncQueue }); }; return { createCertificateAuthority, updateCertificateAuthority, listCertificateAuthorities, - orderSubscriberCertificate, - orderCertificateForAcmeProfile + orderCertificate, + orderSubscriberCertificate }; }; From b2297a3ecd71a2ee3a039383f1046ce9c2bc285f Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 11:43:14 -0800 Subject: [PATCH 04/56] Check more for the CSR --- .../ee/services/pki-acme/pki-acme-service.ts | 58 +++++++++++++++---- .../certificate-profile-dal.ts | 15 ++--- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index feb8ad782..e2566f57c 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -31,6 +31,12 @@ import { getProjectKmsCertificateKeyId } from "@app/services/project/project-fns import { getConfig } from "@app/lib/config/env"; import { TCertificateAuthorityDALFactory } from "@app/services/certificate-authority/certificate-authority-dal"; import { CaType } from "@app/services/certificate-authority/certificate-authority-enums"; +import { extractCertificateRequestFromCSR } from "@app/services/certificate-common/certificate-csr-utils"; +import { + CertExtendedKeyUsage, + CertKeyUsage, + CertSubjectAlternativeNameType +} from "@app/services/certificate/certificate-types"; import { TPkiAcmeAccountDALFactory } from "./pki-acme-account-dal"; import { TPkiAcmeAuthDALFactory } from "./pki-acme-auth-dal"; import { TPkiAcmeChallengeDALFactory } from "./pki-acme-challenge-dal"; @@ -641,7 +647,40 @@ export const pkiAcmeServiceFactory = ({ if (finalizingOrder.expiresAt < new Date()) { throw new AcmeOrderNotReadyError({ message: "ACME order has expired" }); } + const { csr } = payload; + + // Check and validate the CSR + const certificateRequest = extractCertificateRequestFromCSR(csr); + if (!certificateRequest.commonName) { + throw new AcmeBadCSRError({ detail: "Invalid CSR: Common name is required" }); + } + if ( + certificateRequest.subjectAlternativeNames?.some( + (san) => san.type !== CertSubjectAlternativeNameType.DNS_NAME + ) + ) { + throw new AcmeBadCSRError({ detail: "Invalid CSR: Only DNS subject alternative names are supported" }); + } + const orderWithAuthorizations = (await acmeOrderDAL.findByAccountAndOrderIdWithAuthorizations( + accountId, + orderId, + tx + ))!; + const csrIdentifierValues = new Set( + orderWithAuthorizations.authorizations + .map((auth) => auth.identifierValue.toLowerCase()) + .concat([certificateRequest.commonName!.toLowerCase()]) + ); + if ( + csrIdentifierValues.size !== orderWithAuthorizations.authorizations.length || + !orderWithAuthorizations.authorizations.every((auth) => + csrIdentifierValues.has(auth.identifierValue.toLowerCase()) + ) + ) { + throw new AcmeBadCSRError({ detail: "Invalid CSR: Common name + SANs mismatch with order identifiers" }); + } + const ca = await certificateAuthorityDAL.findByIdWithAssociatedCa(profile.caId); if (!ca) { throw new NotFoundError({ message: "Certificate Authority not found" }); @@ -672,16 +711,15 @@ export const pkiAcmeServiceFactory = ({ }); return { certificateId: result.certificateId }; } else { - const orderWithAuthorizations = (await acmeOrderDAL.findByAccountAndOrderIdWithAuthorizations( - accountId, - orderId, - tx - ))!; - const result = await orderCertificateForAcmeProfile( - profileId, - orderWithAuthorizations.authorizations[0].identifierValue - ); - return { certificateId: result }; + const { certificateAuthority } = (await certificateProfileDAL.findByIdWithConfigs(profileId, tx))!; + const cert = await acmeCertificateAuthorityFns.orderCertificate({ + caId: certificateAuthority.id, + commonName: certificateRequest.commonName, + altNames: certificateRequest.subjectAlternativeNames?.map((san) => san.value), + keyUsages: [CertKeyUsage.DIGITAL_SIGNATURE, CertKeyUsage.KEY_ENCIPHERMENT, CertKeyUsage.KEY_AGREEMENT], + extendedKeyUsages: [CertExtendedKeyUsage.SERVER_AUTH] + }); + return { certificateId: cert.id }; } })(); await acmeOrderDAL.updateById( diff --git a/backend/src/services/certificate-profile/certificate-profile-dal.ts b/backend/src/services/certificate-profile/certificate-profile-dal.ts index 53172b744..d2f468248 100644 --- a/backend/src/services/certificate-profile/certificate-profile-dal.ts +++ b/backend/src/services/certificate-profile/certificate-profile-dal.ts @@ -168,15 +168,12 @@ export const certificateProfileDALFactory = (db: TDbClient) => { } as TCertificateProfileWithConfigs["acmeConfig"]) : undefined; - const certificateAuthority = - result.caId && result.caProjectId && result.caStatus && result.caName - ? ({ - id: result.caId, - projectId: result.caProjectId, - status: result.caStatus, - name: result.caName - } as TCertificateProfileWithConfigs["certificateAuthority"]) - : undefined; + const certificateAuthority = { + id: result.caId, + projectId: result.caProjectId, + status: result.caStatus, + name: result.caName + } as TCertificateProfileWithConfigs["certificateAuthority"]; const certificateTemplate = result.templateId && result.templateProjectId && result.templateName From 01b6c52788f8088656b54b1056062186900226b5 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 11:58:59 -0800 Subject: [PATCH 05/56] Call order cert --- .../ee/services/pki-acme/pki-acme-service.ts | 32 +- .../acme/acme-certificate-authority-fns.ts | 513 ++++++++++-------- 2 files changed, 302 insertions(+), 243 deletions(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index e2566f57c..546de859a 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -29,6 +29,7 @@ import { TProjectDALFactory } from "@app/services/project/project-dal"; import { getProjectKmsCertificateKeyId } from "@app/services/project/project-fns"; import { getConfig } from "@app/lib/config/env"; +import { orderCertificate } from "@app/services/certificate-authority/acme/acme-certificate-authority-fns"; import { TCertificateAuthorityDALFactory } from "@app/services/certificate-authority/certificate-authority-dal"; import { CaType } from "@app/services/certificate-authority/certificate-authority-enums"; import { extractCertificateRequestFromCSR } from "@app/services/certificate-common/certificate-csr-utils"; @@ -712,13 +713,30 @@ export const pkiAcmeServiceFactory = ({ return { certificateId: result.certificateId }; } else { const { certificateAuthority } = (await certificateProfileDAL.findByIdWithConfigs(profileId, tx))!; - const cert = await acmeCertificateAuthorityFns.orderCertificate({ - caId: certificateAuthority.id, - commonName: certificateRequest.commonName, - altNames: certificateRequest.subjectAlternativeNames?.map((san) => san.value), - keyUsages: [CertKeyUsage.DIGITAL_SIGNATURE, CertKeyUsage.KEY_ENCIPHERMENT, CertKeyUsage.KEY_AGREEMENT], - extendedKeyUsages: [CertExtendedKeyUsage.SERVER_AUTH] - }); + const cert = await orderCertificate( + { + caId: certificateAuthority.id, + commonName: certificateRequest.commonName!, + altNames: certificateRequest.subjectAlternativeNames?.map((san) => san.value), + // TODO: not 100% sure what are these columns for, but let's put the values for common website SSL certs for now + keyUsages: [ + CertKeyUsage.DIGITAL_SIGNATURE, + CertKeyUsage.KEY_ENCIPHERMENT, + CertKeyUsage.KEY_AGREEMENT + ], + extendedKeyUsages: [CertExtendedKeyUsage.SERVER_AUTH] + }, + { + appConnectionDAL, + certificateAuthorityDAL, + externalCertificateAuthorityDAL, + certificateDAL, + certificateBodyDAL, + certificateSecretDAL, + kmsService, + projectDAL + } + ); return { certificateId: cert.id }; } })(); diff --git a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts index cfe734eb8..f4fff2c41 100644 --- a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts +++ b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts @@ -29,6 +29,7 @@ import { triggerAutoSyncForSubscriber } from "@app/services/pki-sync/pki-sync-ut import { TProjectDALFactory } from "@app/services/project/project-dal"; import { getProjectKmsCertificateKeyId } from "@app/services/project/project-fns"; +import { Knex } from "knex"; import { TCertificateAuthorityDALFactory } from "../certificate-authority-dal"; import { CaStatus, CaType } from "../certificate-authority-enums"; import { keyAlgorithmToAlgCfg } from "../certificate-authority-fns"; @@ -64,6 +65,20 @@ type TAcmeCertificateAuthorityFnsDeps = { projectDAL: Pick; }; +type TOrderCertificateDeps = { + appConnectionDAL: Pick; + certificateAuthorityDAL: Pick; + externalCertificateAuthorityDAL: Pick; + certificateDAL: Pick; + certificateBodyDAL: Pick; + certificateSecretDAL: Pick; + kmsService: Pick< + TKmsServiceFactory, + "encryptWithKmsKey" | "generateKmsKey" | "createCipherPairWithDataKey" | "decryptWithKmsKey" + >; + projectDAL: Pick; +}; + type DBConfigurationColumn = { dnsProvider: string; directoryUrl: string; @@ -104,6 +119,248 @@ export const castDbEntryToAcmeCertificateAuthority = ( }; }; +export const orderCertificate = async ( + { + caId, + subscriberId, + commonName, + altNames, + keyUsages, + extendedKeyUsages + }: { + caId: string; + subscriberId?: string; + commonName: string; + altNames?: string[]; + keyUsages?: CertKeyUsage[]; + extendedKeyUsages?: CertExtendedKeyUsage[]; + }, + deps: TOrderCertificateDeps, + tx?: Knex +) => { + const { + appConnectionDAL, + certificateAuthorityDAL, + externalCertificateAuthorityDAL, + certificateDAL, + certificateBodyDAL, + certificateSecretDAL, + kmsService, + projectDAL + } = deps; + + const ca = await certificateAuthorityDAL.findByIdWithAssociatedCa(caId, tx); + if (!ca.externalCa || ca.externalCa.type !== CaType.ACME) { + throw new BadRequestError({ message: "CA is not an ACME CA" }); + } + + const acmeCa = castDbEntryToAcmeCertificateAuthority(ca); + if (acmeCa.status !== CaStatus.ACTIVE) { + throw new BadRequestError({ message: "CA is disabled" }); + } + + const certificateManagerKmsId = await getProjectKmsCertificateKeyId({ + projectId: ca.projectId, + projectDAL, + kmsService + }); + + const kmsEncryptor = await kmsService.encryptWithKmsKey({ + kmsId: certificateManagerKmsId + }); + + const kmsDecryptor = await kmsService.decryptWithKmsKey({ + kmsId: certificateManagerKmsId + }); + + let accountKey: Buffer | undefined; + if (acmeCa.credentials) { + const decryptedCredentials = await kmsDecryptor({ + cipherTextBlob: acmeCa.credentials as Buffer + }); + + const parsedCredentials = await AcmeCertificateAuthorityCredentialsSchema.parseAsync( + JSON.parse(decryptedCredentials.toString("utf8")) + ); + + accountKey = Buffer.from(parsedCredentials.accountKey, "base64"); + } + if (!accountKey) { + accountKey = await acme.crypto.createPrivateRsaKey(); + const newCredentials = { + accountKey: accountKey.toString("base64") + }; + const { cipherTextBlob: encryptedNewCredentials } = await kmsEncryptor({ + plainText: Buffer.from(JSON.stringify(newCredentials)) + }); + await externalCertificateAuthorityDAL.update( + { + caId: acmeCa.id + }, + { + credentials: encryptedNewCredentials + } + ); + } + + await blockLocalAndPrivateIpAddresses(acmeCa.configuration.directoryUrl); + + const acmeClientOptions: acme.ClientOptions = { + directoryUrl: acmeCa.configuration.directoryUrl, + accountKey + }; + + if (acmeCa.configuration.eabKid && acmeCa.configuration.eabHmacKey) { + acmeClientOptions.externalAccountBinding = { + kid: acmeCa.configuration.eabKid, + hmacKey: acmeCa.configuration.eabHmacKey + }; + } + + const acmeClient = new acme.Client(acmeClientOptions); + + const alg = keyAlgorithmToAlgCfg(CertKeyAlgorithm.RSA_2048); + + const leafKeys = await crypto.nativeCrypto.subtle.generateKey(alg, true, ["sign", "verify"]); + const skLeafObj = crypto.nativeCrypto.KeyObject.from(leafKeys.privateKey); + const skLeaf = skLeafObj.export({ format: "pem", type: "pkcs8" }) as string; + + const [, certificateCsr] = await acme.crypto.createCsr( + { + altNames, + commonName + }, + skLeaf + ); + + const appConnection = await appConnectionDAL.findById(acmeCa.configuration.dnsAppConnectionId); + const connection = await decryptAppConnection(appConnection, kmsService); + + const pem = await acmeClient.auto({ + csr: certificateCsr, + email: acmeCa.configuration.accountEmail, + challengePriority: ["dns-01"], + termsOfServiceAgreed: true, + + challengeCreateFn: async (authz, challenge, keyAuthorization) => { + if (challenge.type !== "dns-01") { + throw new Error("Unsupported challenge type"); + } + + const recordName = `_acme-challenge.${authz.identifier.value}`; // e.g., "_acme-challenge.example.com" + const recordValue = `"${keyAuthorization}"`; // must be double quoted + + switch (acmeCa.configuration.dnsProviderConfig.provider) { + case AcmeDnsProvider.Route53: { + await route53InsertTxtRecord( + connection as TAwsConnection, + acmeCa.configuration.dnsProviderConfig.hostedZoneId, + recordName, + recordValue + ); + break; + } + case AcmeDnsProvider.Cloudflare: { + await cloudflareInsertTxtRecord( + connection as TCloudflareConnection, + acmeCa.configuration.dnsProviderConfig.hostedZoneId, + recordName, + recordValue + ); + break; + } + default: { + throw new Error(`Unsupported DNS provider: ${acmeCa.configuration.dnsProviderConfig.provider as string}`); + } + } + }, + challengeRemoveFn: async (authz, challenge, keyAuthorization) => { + const recordName = `_acme-challenge.${authz.identifier.value}`; // e.g., "_acme-challenge.example.com" + const recordValue = `"${keyAuthorization}"`; // must be double quoted + + switch (acmeCa.configuration.dnsProviderConfig.provider) { + case AcmeDnsProvider.Route53: { + await route53DeleteTxtRecord( + connection as TAwsConnection, + acmeCa.configuration.dnsProviderConfig.hostedZoneId, + recordName, + recordValue + ); + break; + } + case AcmeDnsProvider.Cloudflare: { + await cloudflareDeleteTxtRecord( + connection as TCloudflareConnection, + acmeCa.configuration.dnsProviderConfig.hostedZoneId, + recordName, + recordValue + ); + break; + } + default: { + throw new Error(`Unsupported DNS provider: ${acmeCa.configuration.dnsProviderConfig.provider as string}`); + } + } + } + }); + + const [leafCert, parentCert] = acme.crypto.splitPemChain(pem); + const certObj = new x509.X509Certificate(leafCert); + + const { cipherTextBlob: encryptedCertificate } = await kmsEncryptor({ + plainText: Buffer.from(new Uint8Array(certObj.rawData)) + }); + + const certificateChainPem = parentCert.trim(); + + const { cipherTextBlob: encryptedCertificateChain } = await kmsEncryptor({ + plainText: Buffer.from(certificateChainPem) + }); + + const { cipherTextBlob: encryptedPrivateKey } = await kmsEncryptor({ + plainText: Buffer.from(skLeaf) + }); + + return (tx || certificateDAL).transaction(async (innerTx: Knex) => { + const cert = await certificateDAL.create( + { + caId: ca.id, + pkiSubscriberId: subscriberId, + status: CertStatus.ACTIVE, + friendlyName: commonName, + commonName, + altNames: altNames?.join(","), + serialNumber: certObj.serialNumber, + notBefore: certObj.notBefore, + notAfter: certObj.notAfter, + keyUsages: keyUsages, + extendedKeyUsages: extendedKeyUsages, + projectId: ca.projectId + }, + innerTx + ); + + await certificateBodyDAL.create( + { + certId: cert.id, + encryptedCertificate, + encryptedCertificateChain + }, + innerTx + ); + + await certificateSecretDAL.create( + { + certId: cert.id, + encryptedPrivateKey + }, + innerTx + ); + + return cert; + }); +}; + export const AcmeCertificateAuthorityFns = ({ appConnectionDAL, appConnectionService, @@ -317,246 +574,31 @@ export const AcmeCertificateAuthorityFns = ({ return cas.map(castDbEntryToAcmeCertificateAuthority); }; - const orderCertificate = async ({ - caId, - subscriberId, - commonName, - altNames, - keyUsages, - extendedKeyUsages - }: { - caId: string; - subscriberId?: string; - commonName: string; - altNames?: string[]; - keyUsages?: CertKeyUsage[]; - extendedKeyUsages?: CertExtendedKeyUsage[]; - }) => { - const ca = await certificateAuthorityDAL.findByIdWithAssociatedCa(caId); - if (!ca.externalCa || ca.externalCa.type !== CaType.ACME) { - throw new BadRequestError({ message: "CA is not an ACME CA" }); - } - - const acmeCa = castDbEntryToAcmeCertificateAuthority(ca); - if (acmeCa.status !== CaStatus.ACTIVE) { - throw new BadRequestError({ message: "CA is disabled" }); - } - - const certificateManagerKmsId = await getProjectKmsCertificateKeyId({ - projectId: ca.projectId, - projectDAL, - kmsService - }); - - const kmsEncryptor = await kmsService.encryptWithKmsKey({ - kmsId: certificateManagerKmsId - }); - - const kmsDecryptor = await kmsService.decryptWithKmsKey({ - kmsId: certificateManagerKmsId - }); - - let accountKey: Buffer | undefined; - if (acmeCa.credentials) { - const decryptedCredentials = await kmsDecryptor({ - cipherTextBlob: acmeCa.credentials as Buffer - }); - - const parsedCredentials = await AcmeCertificateAuthorityCredentialsSchema.parseAsync( - JSON.parse(decryptedCredentials.toString("utf8")) - ); - - accountKey = Buffer.from(parsedCredentials.accountKey, "base64"); - } - if (!accountKey) { - accountKey = await acme.crypto.createPrivateRsaKey(); - const newCredentials = { - accountKey: accountKey.toString("base64") - }; - const { cipherTextBlob: encryptedNewCredentials } = await kmsEncryptor({ - plainText: Buffer.from(JSON.stringify(newCredentials)) - }); - await externalCertificateAuthorityDAL.update( - { - caId: acmeCa.id - }, - { - credentials: encryptedNewCredentials - } - ); - } - - await blockLocalAndPrivateIpAddresses(acmeCa.configuration.directoryUrl); - - const acmeClientOptions: acme.ClientOptions = { - directoryUrl: acmeCa.configuration.directoryUrl, - accountKey - }; - - if (acmeCa.configuration.eabKid && acmeCa.configuration.eabHmacKey) { - acmeClientOptions.externalAccountBinding = { - kid: acmeCa.configuration.eabKid, - hmacKey: acmeCa.configuration.eabHmacKey - }; - } - - const acmeClient = new acme.Client(acmeClientOptions); - - const alg = keyAlgorithmToAlgCfg(CertKeyAlgorithm.RSA_2048); - - const leafKeys = await crypto.nativeCrypto.subtle.generateKey(alg, true, ["sign", "verify"]); - const skLeafObj = crypto.nativeCrypto.KeyObject.from(leafKeys.privateKey); - const skLeaf = skLeafObj.export({ format: "pem", type: "pkcs8" }) as string; - - const [, certificateCsr] = await acme.crypto.createCsr( - { - altNames, - commonName - }, - skLeaf - ); - - const appConnection = await appConnectionDAL.findById(acmeCa.configuration.dnsAppConnectionId); - const connection = await decryptAppConnection(appConnection, kmsService); - - const pem = await acmeClient.auto({ - csr: certificateCsr, - email: acmeCa.configuration.accountEmail, - challengePriority: ["dns-01"], - termsOfServiceAgreed: true, - - challengeCreateFn: async (authz, challenge, keyAuthorization) => { - if (challenge.type !== "dns-01") { - throw new Error("Unsupported challenge type"); - } - - const recordName = `_acme-challenge.${authz.identifier.value}`; // e.g., "_acme-challenge.example.com" - const recordValue = `"${keyAuthorization}"`; // must be double quoted - - switch (acmeCa.configuration.dnsProviderConfig.provider) { - case AcmeDnsProvider.Route53: { - await route53InsertTxtRecord( - connection as TAwsConnection, - acmeCa.configuration.dnsProviderConfig.hostedZoneId, - recordName, - recordValue - ); - break; - } - case AcmeDnsProvider.Cloudflare: { - await cloudflareInsertTxtRecord( - connection as TCloudflareConnection, - acmeCa.configuration.dnsProviderConfig.hostedZoneId, - recordName, - recordValue - ); - break; - } - default: { - throw new Error(`Unsupported DNS provider: ${acmeCa.configuration.dnsProviderConfig.provider as string}`); - } - } - }, - challengeRemoveFn: async (authz, challenge, keyAuthorization) => { - const recordName = `_acme-challenge.${authz.identifier.value}`; // e.g., "_acme-challenge.example.com" - const recordValue = `"${keyAuthorization}"`; // must be double quoted - - switch (acmeCa.configuration.dnsProviderConfig.provider) { - case AcmeDnsProvider.Route53: { - await route53DeleteTxtRecord( - connection as TAwsConnection, - acmeCa.configuration.dnsProviderConfig.hostedZoneId, - recordName, - recordValue - ); - break; - } - case AcmeDnsProvider.Cloudflare: { - await cloudflareDeleteTxtRecord( - connection as TCloudflareConnection, - acmeCa.configuration.dnsProviderConfig.hostedZoneId, - recordName, - recordValue - ); - break; - } - default: { - throw new Error(`Unsupported DNS provider: ${acmeCa.configuration.dnsProviderConfig.provider as string}`); - } - } - } - }); - - const [leafCert, parentCert] = acme.crypto.splitPemChain(pem); - const certObj = new x509.X509Certificate(leafCert); - - const { cipherTextBlob: encryptedCertificate } = await kmsEncryptor({ - plainText: Buffer.from(new Uint8Array(certObj.rawData)) - }); - - const certificateChainPem = parentCert.trim(); - - const { cipherTextBlob: encryptedCertificateChain } = await kmsEncryptor({ - plainText: Buffer.from(certificateChainPem) - }); - - const { cipherTextBlob: encryptedPrivateKey } = await kmsEncryptor({ - plainText: Buffer.from(skLeaf) - }); - - return certificateDAL.transaction(async (tx) => { - const cert = await certificateDAL.create( - { - caId: ca.id, - pkiSubscriberId: subscriberId, - status: CertStatus.ACTIVE, - friendlyName: commonName, - commonName, - altNames: altNames?.join(","), - serialNumber: certObj.serialNumber, - notBefore: certObj.notBefore, - notAfter: certObj.notAfter, - keyUsages: keyUsages, - extendedKeyUsages: extendedKeyUsages, - projectId: ca.projectId - }, - tx - ); - - await certificateBodyDAL.create( - { - certId: cert.id, - encryptedCertificate, - encryptedCertificateChain - }, - tx - ); - - await certificateSecretDAL.create( - { - certId: cert.id, - encryptedPrivateKey - }, - tx - ); - - return cert; - }); - }; - const orderSubscriberCertificate = async (subscriberId: string) => { const subscriber = await pkiSubscriberDAL.findById(subscriberId); if (!subscriber.caId) { throw new BadRequestError({ message: "Subscriber does not have a CA" }); } - await orderCertificate({ - caId: subscriber.caId, - subscriberId: subscriber.id, - commonName: subscriber.commonName, - altNames: subscriber.subjectAlternativeNames, - keyUsages: subscriber.keyUsages as CertKeyUsage[], - extendedKeyUsages: subscriber.extendedKeyUsages as CertExtendedKeyUsage[] - }); + await orderCertificate( + { + caId: subscriber.caId, + subscriberId: subscriber.id, + commonName: subscriber.commonName, + altNames: subscriber.subjectAlternativeNames, + keyUsages: subscriber.keyUsages as CertKeyUsage[], + extendedKeyUsages: subscriber.extendedKeyUsages as CertExtendedKeyUsage[] + }, + { + appConnectionDAL, + certificateAuthorityDAL, + externalCertificateAuthorityDAL, + certificateDAL, + certificateBodyDAL, + certificateSecretDAL, + kmsService, + projectDAL + } + ); await triggerAutoSyncForSubscriber(subscriber.id, { pkiSyncDAL, pkiSyncQueue }); }; @@ -564,7 +606,6 @@ export const AcmeCertificateAuthorityFns = ({ createCertificateAuthority, updateCertificateAuthority, listCertificateAuthorities, - orderCertificate, orderSubscriberCertificate }; }; From d80d3365bab8422b1204b09dbc7617f2a865d05e Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 12:05:59 -0800 Subject: [PATCH 06/56] Add missing deps --- .../src/ee/services/pki-acme/pki-acme-service.ts | 15 +++++++++++++-- backend/src/server/routes/index.ts | 5 +++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index 546de859a..0f4c081b0 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -32,7 +32,10 @@ import { getConfig } from "@app/lib/config/env"; import { orderCertificate } from "@app/services/certificate-authority/acme/acme-certificate-authority-fns"; import { TCertificateAuthorityDALFactory } from "@app/services/certificate-authority/certificate-authority-dal"; import { CaType } from "@app/services/certificate-authority/certificate-authority-enums"; +import { TExternalCertificateAuthorityDALFactory } from "@app/services/certificate-authority/external-certificate-authority-dal"; import { extractCertificateRequestFromCSR } from "@app/services/certificate-common/certificate-csr-utils"; +import { TCertificateDALFactory } from "@app/services/certificate/certificate-dal"; +import { TCertificateSecretDALFactory } from "@app/services/certificate/certificate-secret-dal"; import { CertExtendedKeyUsage, CertKeyUsage, @@ -87,10 +90,14 @@ import { } from "./pki-acme-types"; type TPkiAcmeServiceFactoryDep = { - projectDAL: Pick; + projectDAL: Pick; + appConnectionDAL: Pick; + certificateDAL: Pick; certificateAuthorityDAL: Pick; + externalCertificateAuthorityDAL: Pick; certificateProfileDAL: Pick; - certificateBodyDAL: Pick; + certificateBodyDAL: Pick; + certificateSecretDAL: Pick; acmeAccountDAL: Pick< TPkiAcmeAccountDALFactory, "findByProjectIdAndAccountId" | "findByProfileIdAndPublicKeyThumbprintAndAlg" | "create" @@ -119,9 +126,13 @@ type TPkiAcmeServiceFactoryDep = { export const pkiAcmeServiceFactory = ({ projectDAL, + appConnectionDAL, + certificateDAL, certificateAuthorityDAL, + externalCertificateAuthorityDAL, certificateProfileDAL, certificateBodyDAL, + certificateSecretDAL, acmeAccountDAL, acmeOrderDAL, acmeAuthDAL, diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index fab7ebde3..3195e670d 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -2244,8 +2244,13 @@ export const registerRoutes = async ( }); const pkiAcmeService = pkiAcmeServiceFactory({ projectDAL, + appConnectionDAL, + certificateDAL, + certificateAuthorityDAL, + externalCertificateAuthorityDAL, certificateProfileDAL, certificateBodyDAL, + certificateSecretDAL, acmeAccountDAL, acmeOrderDAL, acmeAuthDAL, From 204bf59b22a68e3d13e1f97097a9e8eebb8970f6 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 12:08:52 -0800 Subject: [PATCH 07/56] Add TODO --- backend/src/ee/services/pki-acme/pki-acme-service.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index 0f4c081b0..7cc9ae6f1 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -724,6 +724,8 @@ export const pkiAcmeServiceFactory = ({ return { certificateId: result.certificateId }; } else { const { certificateAuthority } = (await certificateProfileDAL.findByIdWithConfigs(profileId, tx))!; + // TODO: this is pretty slow, and we are holding the transaction open for a long time, + // we should queue the certificate issuance to a background job instead const cert = await orderCertificate( { caId: certificateAuthority.id, From d70fa8f40686b76ede0f821098c8ed05aa83fa84 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 12:14:48 -0800 Subject: [PATCH 08/56] Fix match logic --- .../src/ee/services/pki-acme/pki-acme-service.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index 7cc9ae6f1..e3b366d5b 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -680,15 +680,16 @@ export const pkiAcmeServiceFactory = ({ tx ))!; const csrIdentifierValues = new Set( - orderWithAuthorizations.authorizations - .map((auth) => auth.identifierValue.toLowerCase()) + (certificateRequest.subjectAlternativeNames ?? []) + .map((san) => san.value.toLowerCase()) .concat([certificateRequest.commonName!.toLowerCase()]) ); + const expectedIdentifierValues = new Set( + orderWithAuthorizations.authorizations.map((auth) => auth.identifierValue.toLowerCase()) + ); if ( - csrIdentifierValues.size !== orderWithAuthorizations.authorizations.length || - !orderWithAuthorizations.authorizations.every((auth) => - csrIdentifierValues.has(auth.identifierValue.toLowerCase()) - ) + csrIdentifierValues.size != expectedIdentifierValues.size || + !csrIdentifierValues.isSubsetOf(expectedIdentifierValues) ) { throw new AcmeBadCSRError({ detail: "Invalid CSR: Common name + SANs mismatch with order identifiers" }); } From 0593c71d4350f4802c513601239c640900a3a830 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 12:26:19 -0800 Subject: [PATCH 09/56] Fix the compare logic, isSubsetOf seems like not available --- backend/src/ee/services/pki-acme/pki-acme-service.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index e3b366d5b..4958308e1 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -684,12 +684,11 @@ export const pkiAcmeServiceFactory = ({ .map((san) => san.value.toLowerCase()) .concat([certificateRequest.commonName!.toLowerCase()]) ); - const expectedIdentifierValues = new Set( - orderWithAuthorizations.authorizations.map((auth) => auth.identifierValue.toLowerCase()) - ); if ( - csrIdentifierValues.size != expectedIdentifierValues.size || - !csrIdentifierValues.isSubsetOf(expectedIdentifierValues) + csrIdentifierValues.size !== orderWithAuthorizations.authorizations.length || + !orderWithAuthorizations.authorizations.every((auth) => + csrIdentifierValues.has(auth.identifierValue.toLowerCase()) + ) ) { throw new AcmeBadCSRError({ detail: "Invalid CSR: Common name + SANs mismatch with order identifiers" }); } From e31c8c364cfeeaa5121243f626bfaa89c3375e8b Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 15:33:09 -0800 Subject: [PATCH 10/56] Fix broken stuff --- backend/src/ee/services/pki-acme/pki-acme-service.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index 4958308e1..b2ba50d9f 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -29,6 +29,7 @@ import { TProjectDALFactory } from "@app/services/project/project-dal"; import { getProjectKmsCertificateKeyId } from "@app/services/project/project-fns"; import { getConfig } from "@app/lib/config/env"; +import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal"; import { orderCertificate } from "@app/services/certificate-authority/acme/acme-certificate-authority-fns"; import { TCertificateAuthorityDALFactory } from "@app/services/certificate-authority/certificate-authority-dal"; import { CaType } from "@app/services/certificate-authority/certificate-authority-enums"; @@ -665,14 +666,14 @@ export const pkiAcmeServiceFactory = ({ // Check and validate the CSR const certificateRequest = extractCertificateRequestFromCSR(csr); if (!certificateRequest.commonName) { - throw new AcmeBadCSRError({ detail: "Invalid CSR: Common name is required" }); + throw new AcmeBadCSRError({ message: "Invalid CSR: Common name is required" }); } if ( certificateRequest.subjectAlternativeNames?.some( (san) => san.type !== CertSubjectAlternativeNameType.DNS_NAME ) ) { - throw new AcmeBadCSRError({ detail: "Invalid CSR: Only DNS subject alternative names are supported" }); + throw new AcmeBadCSRError({ message: "Invalid CSR: Only DNS subject alternative names are supported" }); } const orderWithAuthorizations = (await acmeOrderDAL.findByAccountAndOrderIdWithAuthorizations( accountId, @@ -690,7 +691,7 @@ export const pkiAcmeServiceFactory = ({ csrIdentifierValues.has(auth.identifierValue.toLowerCase()) ) ) { - throw new AcmeBadCSRError({ detail: "Invalid CSR: Common name + SANs mismatch with order identifiers" }); + throw new AcmeBadCSRError({ message: "Invalid CSR: Common name + SANs mismatch with order identifiers" }); } const ca = await certificateAuthorityDAL.findByIdWithAssociatedCa(profile.caId); From 771b026175decdeb1cf99bb894d847aa863625a6 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 17:00:40 -0800 Subject: [PATCH 11/56] More test cases --- .../bdd/features/pki/acme/challenge.feature | 76 ++++++++++++++++- backend/bdd/features/steps/pki_acme.py | 81 ++++++++++++++----- 2 files changed, 135 insertions(+), 22 deletions(-) diff --git a/backend/bdd/features/pki/acme/challenge.feature b/backend/bdd/features/pki/acme/challenge.feature index bee46c3fb..c747d5d06 100644 --- a/backend/bdd/features/pki/acme/challenge.feature +++ b/backend/bdd/features/pki/acme/challenge.feature @@ -14,8 +14,82 @@ Feature: Challenge And I create a RSA private key pair as cert_key And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order - And I select challenge with type http-01 for domain localhost from order at order as challenge + And I select challenge with type http-01 for domain localhost from order in order as challenge And I serve challenge response for challenge at localhost And I tell ACME server that challenge is ready to be verified And I poll and finalize the ACME order order as finalized_order And the value finalized_order.body with jq ".status" should be equal to "valid" + + Scenario: Did not finish all challenges + Given I have an ACME cert profile as "acme_profile" + When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" + Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account + When I create certificate signing request as csr + Then I add names to certificate signing request csr + """ + { + "COMMON_NAME": "localhost" + } + """ + And I add subject alternative name to certificate signing request csr + """ + [ + "infisical.com" + ] + """ + And I create a RSA private key pair as cert_key + And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format + And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order + And I select challenge with type http-01 for domain localhost from order in order as challenge + And I serve challenge response for challenge at localhost + And I tell ACME server that challenge is ready to be verified + + # the localhost auth should be valid + And I memorize order with jq ".authorizations | map(select(.body.identifier.value == "localhost")) | first | .uri" as localhost_auth + And I peak and memorize the next nonce as nonce + When I send a raw ACME request to "{localhost_auth}" + """ + { + "protected": { + "alg": "RS256", + "nonce": "{nonce}", + "url": "{localhost_auth}", + "kid": "{acme_account.uri}" + } + } + """ + Then the value response.status_code should be equal to 200 + And the value response with jq ".status" should be equal to "valid" + + # the infisical.com auth should still be pending + And I memorize order with jq ".authorizations | map(select(.body.identifier.value == "infisical.com")) | first | .uri" as infisical_auth + And I memorize response.headers with jq ".["replay-nonce"]" as nonce + When I send a raw ACME request to "{infisical_auth}" + """ + { + "protected": { + "alg": "RS256", + "nonce": "{nonce}", + "url": "{infisical_auth}", + "kid": "{acme_account.uri}" + } + } + """ + Then the value response.status_code should be equal to 200 + And the value response with jq ".status" should be equal to "pending" + + # the order should be pending as well + And I memorize response.headers with jq ".["replay-nonce"]" as nonce + When I send a raw ACME request to "{order.uri}" + """ + { + "protected": { + "alg": "RS256", + "nonce": "{nonce}", + "url": "{order.uri}", + "kid": "{acme_account.uri}" + } + } + """ + Then the value response.status_code should be equal to 200 + And the value response with jq ".status" should be equal to "pending" diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index d9004e0ba..d05472ba2 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -8,6 +8,7 @@ import acme.client import httpx import jq import requests +import requests.structures import glom from faker import Faker from acme import client @@ -115,6 +116,8 @@ def eval_var(context: Context, var_path: str, as_json: bool = True): value = value.to_json() elif isinstance(value, requests.Response): value = value.json() + elif isinstance(value, requests.structures.CaseInsensitiveDict): + value = dict(value.lower_items()) elif isinstance(value, httpx.Response): value = value.json() return value @@ -569,51 +572,58 @@ def step_impl(context: Context, var_path: str): print(json.dumps(value.json(), indent=2)) -@then( - "I select challenge with type {challenge_type} for domain {domain} from order at {var_path} as {challenge_var}" -) -def step_impl( +def select_challenge( context: Context, challenge_type: str, + order_var_path: str, domain: str, - var_path: str, - challenge_var: str, ): - order = eval_var(context, var_path, as_json=False) + acme_client = context.acme_client + order = eval_var(context, order_var_path, as_json=False) + if isinstance(order, dict): + order_body = messages.Order.from_json(order) + order = messages.OrderResource( + body=order_body, + authorizations=[ + acme_client._authzr_from_response( + acme_client._post_as_get(url), uri=url + ) + for url in order_body.authorizations + ], + ) if not isinstance(order, messages.OrderResource): raise ValueError( - f"Expected OrderResource but got {type(order)!r} at {var_path!r}" + f"Expected OrderResource but got {type(order)!r} at {order_var_path!r}" ) auths = list( filter(lambda o: o.body.identifier.value == domain, order.authorizations) ) if not auths: raise ValueError( - f"Authorization for domain {domain!r} not found in {var_path!r}" + f"Authorization for domain {domain!r} not found in {order_var_path!r}" ) if len(auths) > 1: raise ValueError( - f"More than one order for domain {domain!r} found in {var_path!r}" + f"More than one order for domain {domain!r} found in {order_var_path!r}" ) auth = auths[0] challenges = list(filter(lambda a: a.typ == challenge_type, auth.body.challenges)) if not challenges: raise ValueError( - f"Authorization type {challenge_type!r} not found in {var_path!r}" + f"Authorization type {challenge_type!r} not found in {order_var_path!r}" ) if len(challenges) > 1: raise ValueError( - f"More than one authorization for type {challenge_type!r} found in {var_path!r}" + f"More than one authorization for type {challenge_type!r} found in {order_var_path!r}" ) - context.vars[challenge_var] = challenges[0] + return challenges[0] -@then("I serve challenge response for {var_path} at {hostname}") -def step_impl(context: Context, var_path: str, hostname: str): - if hostname != "localhost": - raise ValueError("Currently only localhost is supported") - challenge = eval_var(context, var_path, as_json=False) +def serve_challenge( + context: Context, + challenge: messages.ChallengeBody, +): response, validation = challenge.response_and_validation( context.acme_client.net.key ) @@ -629,14 +639,43 @@ def step_impl(context: Context, var_path: str, hostname: str): context.web_server = web_server -@then("I tell ACME server that {var_path} is ready to be verified") -def step_impl(context: Context, var_path: str): - challenge = eval_var(context, var_path, as_json=False) +def notify_challenge_ready(context: Context, challenge: messages.ChallengeBody): acme_client = context.acme_client response, validation = challenge.response_and_validation(acme_client.net.key) acme_client.answer_challenge(challenge, response) +@then( + "I select challenge with type {challenge_type} for domain {domain} from order in {var_path} as {challenge_var}" +) +def step_impl( + context: Context, + challenge_type: str, + domain: str, + var_path: str, + challenge_var: str, +): + challenge = select_challenge( + context=context, + challenge_type=challenge_type, + domain=domain, + order_var_path=var_path, + ) + context.vars[challenge_var] = challenge + + +@then("I serve challenge response for {var_path} at {hostname}") +def step_impl(context: Context, var_path: str, hostname: str): + challenge = eval_var(context, var_path, as_json=False) + serve_challenge(context=context, challenge=challenge) + + +@then("I tell ACME server that {var_path} is ready to be verified") +def step_impl(context: Context, var_path: str): + challenge = eval_var(context, var_path, as_json=False) + notify_challenge_ready(context=context, challenge=challenge) + + @then("I poll and finalize the ACME order {var_path} as {finalized_var}") def step_impl(context: Context, var_path: str, finalized_var: str): order = eval_var(context, var_path, as_json=False) From d7cf08790b2b62f752e3f7d07cb7087dd971b48b Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 17:17:55 -0800 Subject: [PATCH 12/56] More tests --- backend/bdd/features/environment.py | 7 +++++++ .../bdd/features/pki/acme/challenge.feature | 21 +++++++++++++++++++ backend/bdd/features/steps/pki_acme.py | 8 ++----- .../pki-acme/pki-acme-challenge-service.ts | 1 + .../ee/services/pki-acme/pki-acme-errors.ts | 2 +- 5 files changed, 32 insertions(+), 7 deletions(-) diff --git a/backend/bdd/features/environment.py b/backend/bdd/features/environment.py index 52615036a..631128f8b 100644 --- a/backend/bdd/features/environment.py +++ b/backend/bdd/features/environment.py @@ -2,6 +2,8 @@ import json import os import pathlib +import typing + import httpx from behave.runner import Context from dotenv import load_dotenv @@ -198,3 +200,8 @@ def before_all(context: Context): "AUTH_TOKEN": AUTH_TOKEN, } context.http_client = httpx.Client(base_url=BASE_URL) + + +def after_feature(context: Context, feature: typing.Any): + if hasattr(context, "web_server"): + context.web_server.shutdown_and_server_close() diff --git a/backend/bdd/features/pki/acme/challenge.feature b/backend/bdd/features/pki/acme/challenge.feature index c747d5d06..35790cc5f 100644 --- a/backend/bdd/features/pki/acme/challenge.feature +++ b/backend/bdd/features/pki/acme/challenge.feature @@ -93,3 +93,24 @@ Feature: Challenge """ Then the value response.status_code should be equal to 200 And the value response with jq ".status" should be equal to "pending" + + # finalize should not be allowed when all auths are not valid yet + And I memorize response.headers with jq ".["replay-nonce"]" as nonce + When I send a raw ACME request to "{order.body.finalize}" + """ + { + "protected": { + "alg": "RS256", + "nonce": "{nonce}", + "url": "{order.body.finalize}", + "kid": "{acme_account.uri}" + }, + "payload": { + "csr": "{csr_pem}" + } + } + """ + Then the value response.status_code should be equal to 400 + Then the value response with jq ".status" should be equal to 400 + Then the value response with jq ".type" should be equal to "urn:ietf:params:acme:error:orderNotReady" + Then the value response with jq ".detail" should be equal to "ACME order is not ready" diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index d05472ba2..170ada1de 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -1,7 +1,6 @@ import json import logging import re -import threading import urllib.parse import acme.client @@ -632,11 +631,8 @@ def serve_challenge( ) # TODO: make port configurable servers = standalone.HTTP01DualNetworkedServers(("0.0.0.0", 8087), {resource}) - # Start client standalone web server. - web_server = threading.Thread(name="web_server", target=servers.serve_forever) - web_server.daemon = True - web_server.start() - context.web_server = web_server + servers.serve_forever() + context.web_server = servers def notify_challenge_ready(context: Context, challenge: messages.ChallengeBody): diff --git a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts index 61bd0c110..8bfe360d4 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts @@ -107,6 +107,7 @@ export const pkiAcmeChallengeServiceFactory = ({ if (fetchError.code === "ENOTFOUND" || fetchError.message.includes("ENOTFOUND")) { return new AcmeDnsFailureError({ message: "Hostname could not be resolved (DNS failure)" }); } + logger.error(exp, "Unknown error validating ACME challenge response"); return new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" }); } } else if (exp instanceof DOMException) { diff --git a/backend/src/ee/services/pki-acme/pki-acme-errors.ts b/backend/src/ee/services/pki-acme/pki-acme-errors.ts index 837dec8be..9053be391 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-errors.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-errors.ts @@ -468,7 +468,7 @@ export class AcmeOrderNotReadyError extends AcmeError { super({ type: AcmeErrorType.OrderNotReady, message, - status: 403, + status: 400, error }); this.name = "AcmeOrderNotReadyError"; From f7c76abe827300eb947068cb673e9f5f246b939a Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 17:23:23 -0800 Subject: [PATCH 13/56] Fix web server cleanup --- backend/bdd/features/environment.py | 2 +- .../bdd/features/pki/acme/challenge.feature | 57 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/backend/bdd/features/environment.py b/backend/bdd/features/environment.py index 631128f8b..41de93bc0 100644 --- a/backend/bdd/features/environment.py +++ b/backend/bdd/features/environment.py @@ -202,6 +202,6 @@ def before_all(context: Context): context.http_client = httpx.Client(base_url=BASE_URL) -def after_feature(context: Context, feature: typing.Any): +def after_scenario(context: Context, scenario: typing.Any): if hasattr(context, "web_server"): context.web_server.shutdown_and_server_close() diff --git a/backend/bdd/features/pki/acme/challenge.feature b/backend/bdd/features/pki/acme/challenge.feature index 35790cc5f..78065a22d 100644 --- a/backend/bdd/features/pki/acme/challenge.feature +++ b/backend/bdd/features/pki/acme/challenge.feature @@ -114,3 +114,60 @@ Feature: Challenge Then the value response with jq ".status" should be equal to 400 Then the value response with jq ".type" should be equal to "urn:ietf:params:acme:error:orderNotReady" Then the value response with jq ".detail" should be equal to "ACME order is not ready" + +# Scenario: CSR names mismatch with order identifier +# Given I have an ACME cert profile as "acme_profile" +# When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" +# Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account +# When I create certificate signing request as csr +# Then I add names to certificate signing request csr +# """ +# { +# "COMMON_NAME": "example.com" +# } +# """ +# And I create a RSA private key pair as cert_key +# And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format +# Then I peak and memorize the next nonce as nonce +# When I send a raw ACME request to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/new-order" +# """ +# { +# "protected": { +# "alg": "RS256", +# "nonce": "{nonce}", +# "url": "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/new-order", +# "kid": "{acme_account.uri}" +# }, +# "payload": { +# "identifiers": [ +# { "type": "dns", "value": "localhost" }, +# { "type": "dns", "value": "infisical.com" } +# ] +# } +# } +# """ +# Then the value response.status_code should be equal to 201 +# And I memorize response with jq ".finalize" as finalize_url +# And I memorize response.headers with jq ".["replay-nonce"]" as nonce +# And I memorize response as order +# And I select challenge with type http-01 for domain localhost from order in order as challenge +# And I serve challenge response for challenge at localhost +# And I tell ACME server that challenge is ready to be verified +# When I send a raw ACME request to "{finalize_url}" +# """ +# { +# "protected": { +# "alg": "RS256", +# "nonce": "{nonce}", +# "url": "{finalize_url}", +# "kid": "{acme_account.uri}" +# }, +# "payload": { +# "csr": "{csr_pem}" +# } +# } +# """ +# Then the value response.status_code should be equal to 400 +# Then the value response with jq ".status" should be equal to 400 +# Then the value response with jq ".type" should be equal to "urn:ietf:params:acme:error:malformed" +# Then the value response with jq ".detail" should be equal to "" From a19c840060e54b7ab7dae3cce38c0e9b67dd1e80 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 19:48:05 -0800 Subject: [PATCH 14/56] More tests --- .../bdd/features/pki/acme/challenge.feature | 110 +++++++++--------- backend/bdd/features/steps/pki_acme.py | 59 ++++++++++ 2 files changed, 113 insertions(+), 56 deletions(-) diff --git a/backend/bdd/features/pki/acme/challenge.feature b/backend/bdd/features/pki/acme/challenge.feature index 78065a22d..543e4cea3 100644 --- a/backend/bdd/features/pki/acme/challenge.feature +++ b/backend/bdd/features/pki/acme/challenge.feature @@ -115,59 +115,57 @@ Feature: Challenge Then the value response with jq ".type" should be equal to "urn:ietf:params:acme:error:orderNotReady" Then the value response with jq ".detail" should be equal to "ACME order is not ready" -# Scenario: CSR names mismatch with order identifier -# Given I have an ACME cert profile as "acme_profile" -# When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" -# Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account -# When I create certificate signing request as csr -# Then I add names to certificate signing request csr -# """ -# { -# "COMMON_NAME": "example.com" -# } -# """ -# And I create a RSA private key pair as cert_key -# And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format -# Then I peak and memorize the next nonce as nonce -# When I send a raw ACME request to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/new-order" -# """ -# { -# "protected": { -# "alg": "RS256", -# "nonce": "{nonce}", -# "url": "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/new-order", -# "kid": "{acme_account.uri}" -# }, -# "payload": { -# "identifiers": [ -# { "type": "dns", "value": "localhost" }, -# { "type": "dns", "value": "infisical.com" } -# ] -# } -# } -# """ -# Then the value response.status_code should be equal to 201 -# And I memorize response with jq ".finalize" as finalize_url -# And I memorize response.headers with jq ".["replay-nonce"]" as nonce -# And I memorize response as order -# And I select challenge with type http-01 for domain localhost from order in order as challenge -# And I serve challenge response for challenge at localhost -# And I tell ACME server that challenge is ready to be verified -# When I send a raw ACME request to "{finalize_url}" -# """ -# { -# "protected": { -# "alg": "RS256", -# "nonce": "{nonce}", -# "url": "{finalize_url}", -# "kid": "{acme_account.uri}" -# }, -# "payload": { -# "csr": "{csr_pem}" -# } -# } -# """ -# Then the value response.status_code should be equal to 400 -# Then the value response with jq ".status" should be equal to 400 -# Then the value response with jq ".type" should be equal to "urn:ietf:params:acme:error:malformed" -# Then the value response with jq ".detail" should be equal to "" + Scenario: CSR names mismatch with order identifier + Given I have an ACME cert profile as "acme_profile" + When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" + Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account + When I create certificate signing request as csr + Then I add names to certificate signing request csr + """ + { + "COMMON_NAME": "example.com" + } + """ + And I create a RSA private key pair as cert_key + And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format + Then I peak and memorize the next nonce as nonce + When I send a raw ACME request to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/new-order" + """ + { + "protected": { + "alg": "RS256", + "nonce": "{nonce}", + "url": "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/new-order", + "kid": "{acme_account.uri}" + }, + "payload": { + "identifiers": [ + { "type": "dns", "value": "localhost" }, + { "type": "dns", "value": "infisical.com" } + ] + } + } + """ + Then the value response.status_code should be equal to 201 + And I memorize response with jq ".finalize" as finalize_url + And I memorize response.headers with jq ".["replay-nonce"]" as nonce + And I memorize response as order + And I pass all challenges with type http-01 for order in order + When I send a raw ACME request to "{finalize_url}" + """ + { + "protected": { + "alg": "RS256", + "nonce": "{nonce}", + "url": "{finalize_url}", + "kid": "{acme_account.uri}" + }, + "payload": { + "csr": "{csr_pem}" + } + } + """ + Then the value response.status_code should be equal to 400 + And the value response with jq ".status" should be equal to 400 + And the value response with jq ".type" should be equal to "urn:ietf:params:acme:error:malformed" + And the value response with jq ".detail" should be equal to "" diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index 170ada1de..f8e6da300 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -623,6 +623,9 @@ def serve_challenge( context: Context, challenge: messages.ChallengeBody, ): + if hasattr(context, "web_server"): + context.web_server.shutdown_and_server_close() + response, validation = challenge.response_and_validation( context.acme_client.net.key ) @@ -660,6 +663,62 @@ def step_impl( context.vars[challenge_var] = challenge +@then("I pass all challenges with type {challenge_type} for order in {order_var_path}") +def step_impl( + context: Context, + challenge_type: str, + order_var_path: str, +): + acme_client = context.acme_client + order = eval_var(context, order_var_path, as_json=False) + if isinstance(order, dict): + order_body = messages.Order.from_json(order) + order = messages.OrderResource( + body=order_body, + authorizations=[ + acme_client._authzr_from_response( + acme_client._post_as_get(url), uri=url + ) + for url in order_body.authorizations + ], + ) + if not isinstance(order, messages.OrderResource): + raise ValueError( + f"Expected OrderResource but got {type(order)!r} at {order_var_path!r}" + ) + + for domain in order.body.identifiers: + logger.info( + "Selecting challenge for domain %s with type %s ...", + domain.value, + challenge_type, + ) + challenge = select_challenge( + context=context, + challenge_type=challenge_type, + domain=domain.value, + order_var_path=order_var_path, + ) + logger.info( + "Found challenge for domain %s with type %s, challenge=%s", + domain.value, + challenge_type, + challenge.uri, + ) + + logger.info( + "Serving challenge for domain %s with type %s ...", + domain.value, + challenge_type, + ) + serve_challenge(context=context, challenge=challenge) + + logger.info( + "Notifying challenge for domain %s with type %s ...", domain, challenge_type + ) + notify_challenge_ready(context=context, challenge=challenge) + + @then("I serve challenge response for {var_path} at {hostname}") def step_impl(context: Context, var_path: str, hostname: str): challenge = eval_var(context, var_path, as_json=False) From f14df38eaaad387e8bac4d17312fe02269fe3a5b Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Tue, 11 Nov 2025 21:24:47 -0800 Subject: [PATCH 15/56] Fix challenge bdd tests --- backend/bdd/features/pki/acme/challenge.feature | 9 ++++++--- backend/bdd/features/steps/pki_acme.py | 9 +++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/backend/bdd/features/pki/acme/challenge.feature b/backend/bdd/features/pki/acme/challenge.feature index 543e4cea3..b4aaa1a34 100644 --- a/backend/bdd/features/pki/acme/challenge.feature +++ b/backend/bdd/features/pki/acme/challenge.feature @@ -20,6 +20,8 @@ Feature: Challenge And I poll and finalize the ACME order order as finalized_order And the value finalized_order.body with jq ".status" should be equal to "valid" + # TODO: add challenge with SANs + Scenario: Did not finish all challenges Given I have an ACME cert profile as "acme_profile" When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" @@ -151,6 +153,7 @@ Feature: Challenge And I memorize response.headers with jq ".["replay-nonce"]" as nonce And I memorize response as order And I pass all challenges with type http-01 for order in order + And I encode CSR csr_pem as JOSE Base-64 DER as base64_csr_der When I send a raw ACME request to "{finalize_url}" """ { @@ -161,11 +164,11 @@ Feature: Challenge "kid": "{acme_account.uri}" }, "payload": { - "csr": "{csr_pem}" + "csr": "{base64_csr_der}" } } """ Then the value response.status_code should be equal to 400 And the value response with jq ".status" should be equal to 400 - And the value response with jq ".type" should be equal to "urn:ietf:params:acme:error:malformed" - And the value response with jq ".detail" should be equal to "" + And the value response with jq ".type" should be equal to "urn:ietf:params:acme:error:badCSR" + And the value response with jq ".detail" should be equal to "Invalid CSR: Common name + SANs mismatch with order identifiers" diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index f8e6da300..604aa5e7a 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -389,6 +389,15 @@ def step_impl(context: Context, url: str): send_raw_acme_req(context, url) +@then( + "I encode CSR {pem_var} as JOSE Base-64 DER as {var_name}", +) +def step_impl(context: Context, pem_var: str, var_name: str): + csr = eval_var(context, pem_var) + parsed_csr = x509.load_pem_x509_csr(csr) + context.vars[var_name] = json_util.encode_csr(parsed_csr) + + @then( "I submit the certificate signing request PEM {pem_var} certificate order to the ACME server as {order_var}" ) From 35c30d40625254544ff18559a5487c9bf4652029 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 12:37:44 -0800 Subject: [PATCH 16/56] Fix imports --- backend/src/ee/services/pki-acme/pki-acme-service.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index b2ba50d9f..78cffe9dd 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -89,6 +89,7 @@ import { TRawJwsPayload, TRespondToAcmeChallengeResponse } from "./pki-acme-types"; +import { TLicenseServiceFactory } from "../license/license-service"; type TPkiAcmeServiceFactoryDep = { projectDAL: Pick; From 8838700cf1357f17c322bf01b26135f1e378ab31 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 12:39:17 -0800 Subject: [PATCH 17/56] Fix imports --- backend/src/ee/services/pki-acme/pki-acme-service.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index 78cffe9dd..93339eea9 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -42,6 +42,7 @@ import { CertKeyUsage, CertSubjectAlternativeNameType } from "@app/services/certificate/certificate-types"; +import { TLicenseServiceFactory } from "../license/license-service"; import { TPkiAcmeAccountDALFactory } from "./pki-acme-account-dal"; import { TPkiAcmeAuthDALFactory } from "./pki-acme-auth-dal"; import { TPkiAcmeChallengeDALFactory } from "./pki-acme-challenge-dal"; @@ -89,7 +90,6 @@ import { TRawJwsPayload, TRespondToAcmeChallengeResponse } from "./pki-acme-types"; -import { TLicenseServiceFactory } from "../license/license-service"; type TPkiAcmeServiceFactoryDep = { projectDAL: Pick; @@ -120,7 +120,10 @@ type TPkiAcmeServiceFactoryDep = { "create" | "transaction" | "updateById" | "findByAccountAuthAndChallengeId" | "findByIdForChallengeValidation" >; keyStore: Pick; - kmsService: Pick; + kmsService: Pick< + TKmsServiceFactory, + "decryptWithKmsKey" | "generateKmsKey" | "encryptWithKmsKey" | "createCipherPairWithDataKey" + >; licenseService: Pick; certificateV3Service: Pick; acmeChallengeService: TPkiAcmeChallengeServiceFactory; @@ -730,7 +733,7 @@ export const pkiAcmeServiceFactory = ({ // we should queue the certificate issuance to a background job instead const cert = await orderCertificate( { - caId: certificateAuthority.id, + caId: certificateAuthority!.id, commonName: certificateRequest.commonName!, altNames: certificateRequest.subjectAlternativeNames?.map((san) => san.value), // TODO: not 100% sure what are these columns for, but let's put the values for common website SSL certs for now From dda9c26c10a00c4a6e25ce611643b0c0dcfda945 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 12:42:10 -0800 Subject: [PATCH 18/56] Add more bdd tests --- .../bdd/features/pki/acme/challenge.feature | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/backend/bdd/features/pki/acme/challenge.feature b/backend/bdd/features/pki/acme/challenge.feature index b4aaa1a34..77405b4b7 100644 --- a/backend/bdd/features/pki/acme/challenge.feature +++ b/backend/bdd/features/pki/acme/challenge.feature @@ -20,7 +20,30 @@ Feature: Challenge And I poll and finalize the ACME order order as finalized_order And the value finalized_order.body with jq ".status" should be equal to "valid" - # TODO: add challenge with SANs + Scenario: Validate challenges for multiple domains + Given I have an ACME cert profile as "acme_profile" + When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" + Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account + When I create certificate signing request as csr + Then I add names to certificate signing request csr + """ + { + "COMMON_NAME": "localhost" + } + """ + And I add subject alternative name to certificate signing request csr + """ + [ + "infisical.com", + "example.com" + ] + """ + And I create a RSA private key pair as cert_key + And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format + And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order + And I pass all challenges with type http-01 for order in order + And I poll and finalize the ACME order order as finalized_order + And the value finalized_order.body with jq ".status" should be equal to "valid" Scenario: Did not finish all challenges Given I have an ACME cert profile as "acme_profile" From 5c69f3bbcca89c0714a6d542bcc7f1177b4b6a7f Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 14:04:57 -0800 Subject: [PATCH 19/56] Add x509 asserts --- backend/bdd/features/steps/pki_acme.py | 109 ++--------- backend/bdd/features/steps/utils.py | 257 +++++++++++++++++++++++++ 2 files changed, 269 insertions(+), 97 deletions(-) create mode 100644 backend/bdd/features/steps/utils.py diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index 604aa5e7a..aa8a110dd 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -4,11 +4,7 @@ import re import urllib.parse import acme.client -import httpx import jq -import requests -import requests.structures -import glom from faker import Faker from acme import client from acme import messages @@ -19,7 +15,6 @@ from behave import given from behave import when from behave import then from josepy.jwk import JWKRSA -from josepy import JSONObjectWithFields from josepy import json_util from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa @@ -27,6 +22,11 @@ from cryptography import x509 from cryptography.x509.oid import NameOID from cryptography.hazmat.primitives import hashes +from utils import replace_vars +from utils import eval_var +from utils import prepare_headers + + ACC_KEY_BITS = 2048 ACC_KEY_PUBLIC_EXPONENT = 65537 logger = logging.getLogger(__name__) @@ -40,98 +40,6 @@ class AcmeProfile: self.eab_secret = eab_secret -def replace_vars(payload: dict | list | int | float | str, vars: dict): - if isinstance(payload, dict): - return { - replace_vars(key, vars): replace_vars(value, vars) - for key, value in payload.items() - } - elif isinstance(payload, list): - return [replace_vars(item, vars) for item in payload] - elif isinstance(payload, str): - return payload.format(**vars) - else: - return payload - - -def parse_glom_path(path_str: str) -> glom.Path: - """ - Parse a glom path string with 'attr[index]' syntax into a Path object. - - Examples: - >>> parse_glom_path('authorizations[0]') == Path('authorizations', 0) - True - >>> parse_glom_path('data.items[1].name') == Path('data', 'items', 1, 'name') - True - >>> parse_glom_path('user.addresses[0].street') == Path('user', 'addresses', 0, 'street') - True - """ - parts = [] - - # Split by dots, but preserve bracketed content - tokens = re.split(r"(? dict | None: - headers = {} - auth_token = getattr(context, "auth_token", None) - if auth_token is not None: - headers["authorization"] = "Bearer {}".format(auth_token) - if not headers: - return None - return headers - - @given("I make a random {faker_type} as {var_name}") def step_impl(context: Context, faker_type: str, var_name: str): context.vars[var_name] = getattr(faker, faker_type)() @@ -746,3 +654,10 @@ def step_impl(context: Context, var_path: str, finalized_var: str): acme_client = context.acme_client finalized_order = acme_client.poll_and_finalize(order) context.vars[finalized_var] = finalized_order + + +@then("I parse the full-chain certificate from order {order_var_path} as {cert_var}") +def step_impl(context: Context, order_var_path: str, cert_var: str): + order = eval_var(context, order_var_path, as_json=False) + cert = x509.load_pem_x509_certificate(order.fullchain_pem.encode()) + context.vars[cert_var] = cert diff --git a/backend/bdd/features/steps/utils.py b/backend/bdd/features/steps/utils.py new file mode 100644 index 000000000..cd967267a --- /dev/null +++ b/backend/bdd/features/steps/utils.py @@ -0,0 +1,257 @@ +from cryptography import x509 +from cryptography.hazmat.primitives import hashes +from cryptography.x509.oid import NameOID +import logging +import re + +import httpx +import requests +import requests.structures +import glom +from faker import Faker +from behave.runner import Context +from josepy import JSONObjectWithFields + +ACC_KEY_BITS = 2048 +ACC_KEY_PUBLIC_EXPONENT = 65537 +logger = logging.getLogger(__name__) +faker = Faker() + + +class AcmeProfile: + def __init__(self, id: str, eab_kid: str, eab_secret: str): + self.id = id + self.eab_kid = eab_kid + self.eab_secret = eab_secret + + +def replace_vars(payload: dict | list | int | float | str, vars: dict): + if isinstance(payload, dict): + return { + replace_vars(key, vars): replace_vars(value, vars) + for key, value in payload.items() + } + elif isinstance(payload, list): + return [replace_vars(item, vars) for item in payload] + elif isinstance(payload, str): + return payload.format(**vars) + else: + return payload + + +def parse_glom_path(path_str: str) -> glom.Path: + """ + Parse a glom path string with 'attr[index]' syntax into a Path object. + + Examples: + >>> parse_glom_path('authorizations[0]') == Path('authorizations', 0) + True + >>> parse_glom_path('data.items[1].name') == Path('data', 'items', 1, 'name') + True + >>> parse_glom_path('user.addresses[0].street') == Path('user', 'addresses', 0, 'street') + True + """ + parts = [] + + # Split by dots, but preserve bracketed content + tokens = re.split(r"(? dict | None: + headers = {} + auth_token = getattr(context, "auth_token", None) + if auth_token is not None: + headers["authorization"] = "Bearer {}".format(auth_token) + if not headers: + return None + return headers + + +def x509_cert_to_dict(cert: x509.Certificate) -> dict: + """ + Convert a cryptography.x509.Certificate to a JSON-serializable nested dict + with human-readable keys. + """ + + def oid_to_name(oid): + # Map known OIDs to human-readable names + mapping = { + NameOID.COMMON_NAME: "common_name", + NameOID.ORGANIZATION_NAME: "organization", + NameOID.ORGANIZATIONAL_UNIT_NAME: "organizational_unit", + NameOID.COUNTRY_NAME: "country", + NameOID.LOCALITY_NAME: "locality", + NameOID.STATE_OR_PROVINCE_NAME: "state_or_province", + NameOID.EMAIL_ADDRESS: "email_address", + NameOID.SERIAL_NUMBER: "serial_number", + NameOID.SURNAME: "surname", + NameOID.GIVEN_NAME: "given_name", + NameOID.TITLE: "title", + NameOID.JURISDICTION_COUNTRY_NAME: "jurisdiction_country", + NameOID.JURISDICTION_STATE_OR_PROVINCE_NAME: "jurisdiction_state", + NameOID.JURISDICTION_LOCALITY_NAME: "jurisdiction_locality", + NameOID.BUSINESS_CATEGORY: "business_category", + NameOID.POSTAL_CODE: "postal_code", + NameOID.STREET_ADDRESS: "street_address", + NameOID.DOMAIN_COMPONENT: "domain_component", + NameOID.USER_ID: "user_id", + # Add more as needed + } + return mapping.get(oid, oid.dotted_string) + + def name_to_dict(name: x509.Name) -> dict: + return {oid_to_name(attr.oid): attr.value for attr in name} + + def extension_to_dict(ext): + if isinstance(ext.value, x509.SubjectAlternativeName): + return { + "critical": ext.critical, + "general_names": [str(gn) for gn in ext.value], + } + elif isinstance(ext.value, x509.BasicConstraints): + return { + "critical": ext.critical, + "ca": ext.value.ca, + "path_length": ext.value.path_length, + } + elif isinstance(ext.value, x509.KeyUsage): + return { + "critical": ext.critical, + **{ + field.lower(): getattr(ext.value, field) + for field in [ + "digital_signature", + "content_commitment", + "key_encipherment", + "data_encipherment", + "key_agreement", + "key_cert_sign", + "crl_sign", + "encipher_only", + "decipher_only", + ] + if getattr(ext.value, field) is not None + }, + } + elif isinstance(ext.value, x509.ExtendedKeyUsage): + return { + "critical": ext.critical, + "usages": [eku.dotted_string for eku in ext.value], + } + elif isinstance(ext.value, x509.CRLDistributionPoints): + return { + "critical": ext.critical, + "distribution_points": [ + { + "full_name": [str(uri) for uri in dp.full_name] + if dp.full_name + else None, + "crl_issuer": [str(issuer) for issuer in dp.crl_issuer] + if dp.crl_issuer + else None, + "reasons": [r.name for r in dp.reasons] if dp.reasons else None, + } + for dp in ext.value + ], + } + elif isinstance(ext.value, x509.AuthorityKeyIdentifier): + return { + "critical": ext.critical, + "key_identifier": ext.value.key_identifier.hex() + if ext.value.key_identifier + else None, + "authority_cert_issuer": [ + str(n) for n in ext.value.authority_cert_issuer + ] + if ext.value.authority_cert_issuer + else None, + "authority_cert_serial_number": ext.value.authority_cert_serial_number, + } + elif isinstance(ext.value, x509.SubjectKeyIdentifier): + return {"critical": ext.critical, "digest": ext.value.digest.hex()} + else: + return { + "critical": ext.critical, + "oid": ext.oid.dotted_string, + "value": str(ext.value), + } + + # Build the main dict + result = dict( + version=cert.version.name, + serial_number=cert.serial_number, + signature_algorithm=cert.signature_algorithm_oid._name, + issuer=name_to_dict(cert.issuer), + subject=name_to_dict(cert.subject), + validity={ + "not_valid_before": cert.not_valid_before.isoformat(), + "not_valid_after": cert.not_valid_after.isoformat(), + }, + public_key={ + "key_size": cert.public_key().key_size, + }, + extensions={ + ext.oid._name + if hasattr(ext.oid, "_name") and ext.oid._name + else ext.oid.dotted_string: extension_to_dict(ext) + for ext in cert.extensions + }, + fingerprint={ + "sha1": cert.fingerprint(hashes.SHA1()).hex(), + "sha256": cert.fingerprint(hashes.SHA256()).hex(), + }, + ) + + return result From 41725add64fc199f0cfc6f7bef992e18e63e90e9 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 14:19:56 -0800 Subject: [PATCH 20/56] Fix test cases --- backend/bdd/features/pki/acme/challenge.feature | 12 ++++++++++++ backend/bdd/features/steps/utils.py | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/backend/bdd/features/pki/acme/challenge.feature b/backend/bdd/features/pki/acme/challenge.feature index 77405b4b7..fdd5c8379 100644 --- a/backend/bdd/features/pki/acme/challenge.feature +++ b/backend/bdd/features/pki/acme/challenge.feature @@ -19,6 +19,8 @@ Feature: Challenge And I tell ACME server that challenge is ready to be verified And I poll and finalize the ACME order order as finalized_order And the value finalized_order.body with jq ".status" should be equal to "valid" + And I parse the full-chain certificate from order finalized_order as cert + And the value cert with jq ".subject.common_name" should be equal to "localhost" Scenario: Validate challenges for multiple domains Given I have an ACME cert profile as "acme_profile" @@ -44,6 +46,16 @@ Feature: Challenge And I pass all challenges with type http-01 for order in order And I poll and finalize the ACME order order as finalized_order And the value finalized_order.body with jq ".status" should be equal to "valid" + And I parse the full-chain certificate from order finalized_order as cert + And the value cert with jq ".subject.common_name" should be equal to "localhost" + And the value cert with jq "[.extensions.subjectAltName.general_names.[].value] | sort" should be equal to json + """ + [ + "example.com", + "infisical.com" + ] + """ + Scenario: Did not finish all challenges Given I have an ACME cert profile as "acme_profile" diff --git a/backend/bdd/features/steps/utils.py b/backend/bdd/features/steps/utils.py index cd967267a..df1a3f17f 100644 --- a/backend/bdd/features/steps/utils.py +++ b/backend/bdd/features/steps/utils.py @@ -154,11 +154,14 @@ def x509_cert_to_dict(cert: x509.Certificate) -> dict: def name_to_dict(name: x509.Name) -> dict: return {oid_to_name(attr.oid): attr.value for attr in name} + def dns_to_dict(dns: x509.DNSName) -> dict: + return dict(value=dns.value) + def extension_to_dict(ext): if isinstance(ext.value, x509.SubjectAlternativeName): return { "critical": ext.critical, - "general_names": [str(gn) for gn in ext.value], + "general_names": [dns_to_dict(gn) for gn in ext.value], } elif isinstance(ext.value, x509.BasicConstraints): return { From 6ff590e63383007daab49b21734b19ca2cf6e73d Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 14:29:53 -0800 Subject: [PATCH 21/56] Add pebble for BDD tests --- backend/bdd/configs/pebble.json | 0 docker-compose.bdd.yml | 12 ++++++++++++ 2 files changed, 12 insertions(+) create mode 100644 backend/bdd/configs/pebble.json diff --git a/backend/bdd/configs/pebble.json b/backend/bdd/configs/pebble.json new file mode 100644 index 000000000..e69de29bb diff --git a/docker-compose.bdd.yml b/docker-compose.bdd.yml index dfee5f6b2..2377b86d6 100644 --- a/docker-compose.bdd.yml +++ b/docker-compose.bdd.yml @@ -75,6 +75,18 @@ services: - ./frontend/public:/app/public env_file: .env + # ACME server for BDD tests + pebble: + image: ghcr.io/letsencrypt/pebble:2.8.0 + command: -config /var/data/pebble/pebble-config.json + ports: + - 14000:14000 # ACME port + - 15000:15000 # Management port + environment: + - PEBBLE_VA_NOSLEEP=1 + volumes: + - ./bdd/config/pebble-config.json:/var/data/pebble/pebble-config.json + volumes: postgres-data: driver: local From d20373e840cfe39b77a753374761ec6a51c73ddc Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 16:34:49 -0800 Subject: [PATCH 22/56] Add pebble config and certs --- backend/bdd/configs/pebble.json | 0 .../bdd/features/pki/acme/challenge.feature | 1 - .../bdd/features/pki/acme/external-ca.feature | 23 +++++++++++++++ backend/bdd/pebble/localhost/cert.pem | 13 +++++++++ backend/bdd/pebble/localhost/key.pem | 6 ++++ backend/bdd/pebble/pebble-config.json | 28 +++++++++++++++++++ backend/bdd/pebble/pebble.minica.key.pem | 6 ++++ backend/bdd/pebble/pebble.minica.pem | 13 +++++++++ docker-compose.bdd.yml | 9 +++++- 9 files changed, 97 insertions(+), 2 deletions(-) delete mode 100644 backend/bdd/configs/pebble.json create mode 100644 backend/bdd/features/pki/acme/external-ca.feature create mode 100644 backend/bdd/pebble/localhost/cert.pem create mode 100644 backend/bdd/pebble/localhost/key.pem create mode 100644 backend/bdd/pebble/pebble-config.json create mode 100644 backend/bdd/pebble/pebble.minica.key.pem create mode 100644 backend/bdd/pebble/pebble.minica.pem diff --git a/backend/bdd/configs/pebble.json b/backend/bdd/configs/pebble.json deleted file mode 100644 index e69de29bb..000000000 diff --git a/backend/bdd/features/pki/acme/challenge.feature b/backend/bdd/features/pki/acme/challenge.feature index fdd5c8379..67f73aab2 100644 --- a/backend/bdd/features/pki/acme/challenge.feature +++ b/backend/bdd/features/pki/acme/challenge.feature @@ -56,7 +56,6 @@ Feature: Challenge ] """ - Scenario: Did not finish all challenges Given I have an ACME cert profile as "acme_profile" When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" diff --git a/backend/bdd/features/pki/acme/external-ca.feature b/backend/bdd/features/pki/acme/external-ca.feature new file mode 100644 index 000000000..6d9ae02db --- /dev/null +++ b/backend/bdd/features/pki/acme/external-ca.feature @@ -0,0 +1,23 @@ +Feature: External CA + + Scenario: Issue a certificate from an external CA + Given I have an ACME cert profile as "acme_profile" + When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" + Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account + When I create certificate signing request as csr + Then I add names to certificate signing request csr + """ + { + "COMMON_NAME": "localhost" + } + """ + And I create a RSA private key pair as cert_key + And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format + And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order + And I select challenge with type http-01 for domain localhost from order in order as challenge + And I serve challenge response for challenge at localhost + And I tell ACME server that challenge is ready to be verified + And I poll and finalize the ACME order order as finalized_order + And the value finalized_order.body with jq ".status" should be equal to "valid" + And I parse the full-chain certificate from order finalized_order as cert + And the value cert with jq ".subject.common_name" should be equal to "localhost" diff --git a/backend/bdd/pebble/localhost/cert.pem b/backend/bdd/pebble/localhost/cert.pem new file mode 100644 index 000000000..9117526df --- /dev/null +++ b/backend/bdd/pebble/localhost/cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIICBDCCAYmgAwIBAgIIHZvNVJSPdsYwCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV +bWluaWNhIHJvb3QgY2EgN2ZlMDQwMB4XDTI1MTExMzAwMzAxMloXDTI3MTIxMzAw +MzAxMlowFDESMBAGA1UEAxMJbG9jYWxob3N0MHYwEAYHKoZIzj0CAQYFK4EEACID +YgAE2V5oM5JimqDjzEfH10cKu6L8eQ9rxzkULbIJRFFuuXtKQQwkcAW8L4UuMkmG +lu5hFCBR8saHDpISuAyYLYqsddxwndxmGT3zyw6oU+8oXWX0tThL0KgajmZckOfR +ysYpo4GbMIGYMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI +KwYBBQUHAwIwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBSIDfQe2L6+9aYyBFbd +t0S51xW3UDA4BgNVHREEMTAvgglsb2NhbGhvc3SCBnBlYmJsZYIUaG9zdC5kb2Nr +ZXIuaW50ZXJuYWyHBH8AAAEwCgYIKoZIzj0EAwMDaQAwZgIxAPkeGVzCDKuJYd/1 +87+lXXtlMHrW7F+Rn1kyR8SBud2hDt5r3a+ZZ8IQ9aHazRia/AIxAOI4I41jwxf0 +86i7fKx8of4s/CBc4+PF0hbCBkmen3aKuiZ7ueYuEsSNT6zHV2xc2w== +-----END CERTIFICATE----- diff --git a/backend/bdd/pebble/localhost/key.pem b/backend/bdd/pebble/localhost/key.pem new file mode 100644 index 000000000..93b93eada --- /dev/null +++ b/backend/bdd/pebble/localhost/key.pem @@ -0,0 +1,6 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDBx7d0VqxwTYcJajFgz +ja0PExBmxdZjEQRfGCMQY8GfHa0WpBUEwVtBD6XOGE5xZB2hZANiAATZXmgzkmKa +oOPMR8fXRwq7ovx5D2vHORQtsglEUW65e0pBDCRwBbwvhS4ySYaW7mEUIFHyxocO +khK4DJgtiqx13HCd3GYZPfPLDqhT7yhdZfS1OEvQqBqOZlyQ59HKxik= +-----END PRIVATE KEY----- diff --git a/backend/bdd/pebble/pebble-config.json b/backend/bdd/pebble/pebble-config.json new file mode 100644 index 000000000..013f6ff64 --- /dev/null +++ b/backend/bdd/pebble/pebble-config.json @@ -0,0 +1,28 @@ +{ + "pebble": { + "listenAddress": "0.0.0.0:14000", + "managementListenAddress": "0.0.0.0:15000", + "certificate": "/var/data/pebble/localhost/cert.pem", + "privateKey": "/var/data/pebble/localhost/key.pem", + "httpPort": 5002, + "tlsPort": 5001, + "ocspResponderURL": "", + "externalAccountBindingRequired": false, + "domainBlocklist": ["blocked-domain.example"], + "retryAfter": { + "authz": 3, + "order": 5 + }, + "keyAlgorithm": "ecdsa", + "profiles": { + "default": { + "description": "The profile you know and love", + "validityPeriod": 7776000 + }, + "shortlived": { + "description": "A short-lived cert profile, without actual enforcement", + "validityPeriod": 518400 + } + } + } +} \ No newline at end of file diff --git a/backend/bdd/pebble/pebble.minica.key.pem b/backend/bdd/pebble/pebble.minica.key.pem new file mode 100644 index 000000000..322b4e88e --- /dev/null +++ b/backend/bdd/pebble/pebble.minica.key.pem @@ -0,0 +1,6 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDDnPx90G0J4ba0CMTrh +AT0kJkRGyhv5ePWyobdT75za/I9MpU/VsC8BG5uJBraxiSOhZANiAAQWEiTINq0t +j+6Qiyzin74FU4/zLNuEs1FnipFn+Vb1W8qhvbBwLOGsANpaHIg4dpR+CghfccRQ +0kQm/AMgj08VXvta6vV7aQ8yk+/Cp6l4SVQ9GzizHiJ//Qb71vrXbco= +-----END PRIVATE KEY----- diff --git a/backend/bdd/pebble/pebble.minica.pem b/backend/bdd/pebble/pebble.minica.pem new file mode 100644 index 000000000..030ca32bb --- /dev/null +++ b/backend/bdd/pebble/pebble.minica.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB+zCCAYKgAwIBAgIIf+BA3XMRozcwCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV +bWluaWNhIHJvb3QgY2EgN2ZlMDQwMCAXDTI1MTExMzAwMzAxMloYDzIxMjUxMTEz +MDAzMDEyWjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSA3ZmUwNDAwdjAQBgcq +hkjOPQIBBgUrgQQAIgNiAAQWEiTINq0tj+6Qiyzin74FU4/zLNuEs1FnipFn+Vb1 +W8qhvbBwLOGsANpaHIg4dpR+CghfccRQ0kQm/AMgj08VXvta6vV7aQ8yk+/Cp6l4 +SVQ9GzizHiJ//Qb71vrXbcqjgYYwgYMwDgYDVR0PAQH/BAQDAgKEMB0GA1UdJQQW +MBQGCCsGAQUFBwMBBggrBgEFBQcDAjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1Ud +DgQWBBSIDfQe2L6+9aYyBFbdt0S51xW3UDAfBgNVHSMEGDAWgBSIDfQe2L6+9aYy +BFbdt0S51xW3UDAKBggqhkjOPQQDAwNnADBkAjAK2OUUVHs2LVqwyLEqIrXbc3gw +5r5p9TC9asqPN8vJxlTRStrXnJQRSQ2KoWztiSICMEV5jZGVk6TaUwlqcGmXEmGr +iFeQ3rXLaRw8XKMqj7+EiwaCD1o2wLgzny/21NFtxQ== +-----END CERTIFICATE----- diff --git a/docker-compose.bdd.yml b/docker-compose.bdd.yml index 2377b86d6..2984bf386 100644 --- a/docker-compose.bdd.yml +++ b/docker-compose.bdd.yml @@ -41,6 +41,12 @@ services: build: context: ./backend dockerfile: Dockerfile.dev + command: + - "/bin/bash" + - "-c" + - | + update-ca-certificates && \ + npm run dev:docker depends_on: db: condition: service_started @@ -57,6 +63,7 @@ services: - TELEMETRY_ENABLED=false volumes: - ./backend/src:/app/src + - ./backend/bdd/pebble/pebble.minica.pem:/usr/local/share/ca-certificates/pebble.minica.crt:ro - softhsm_tokens:/etc/softhsm2/tokens # SoftHSM tokens are stored in a volume to persist across container restarts extra_hosts: - "host.docker.internal:host-gateway" @@ -85,7 +92,7 @@ services: environment: - PEBBLE_VA_NOSLEEP=1 volumes: - - ./bdd/config/pebble-config.json:/var/data/pebble/pebble-config.json + - ./backend/bdd/pebble/:/var/data/pebble:ro volumes: postgres-data: From 8246483e8adfed120eaa2e8150481a881696c0ed Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 16:58:17 -0800 Subject: [PATCH 23/56] Use `NODE_EXTRA_CA_CERTS` to fix the problem --- docker-compose.bdd.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docker-compose.bdd.yml b/docker-compose.bdd.yml index 2984bf386..7264ddae1 100644 --- a/docker-compose.bdd.yml +++ b/docker-compose.bdd.yml @@ -41,12 +41,6 @@ services: build: context: ./backend dockerfile: Dockerfile.dev - command: - - "/bin/bash" - - "-c" - - | - update-ca-certificates && \ - npm run dev:docker depends_on: db: condition: service_started @@ -61,8 +55,11 @@ services: - NODE_ENV=development - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable - TELEMETRY_ENABLED=false + # This is needed to trust the Pebble CA certificate, which is used for the BDD tests + - NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/pebble.minica.crt volumes: - ./backend/src:/app/src + # This is needed to trust the Pebble CA certificate, which is used for the BDD tests - ./backend/bdd/pebble/pebble.minica.pem:/usr/local/share/ca-certificates/pebble.minica.crt:ro - softhsm_tokens:/etc/softhsm2/tokens # SoftHSM tokens are stored in a volume to persist across container restarts extra_hosts: From 1a18096ac887467c078440797145a34726561407 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 19:56:33 -0800 Subject: [PATCH 24/56] Mock third party calls --- backend/package-lock.json | 97 +++++++++++++++++++++++ backend/package.json | 1 + backend/src/lib/config/env.ts | 2 + backend/src/mock-third-party-api-calls.ts | 87 ++++++++++++++++++++ backend/src/server/app.ts | 7 ++ 5 files changed, 194 insertions(+) create mode 100644 backend/src/mock-third-party-api-calls.ts diff --git a/backend/package-lock.json b/backend/package-lock.json index cfe22d916..a871c38b1 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -177,6 +177,7 @@ "eslint-plugin-import": "^2.29.1", "eslint-plugin-prettier": "^5.1.3", "eslint-plugin-simple-import-sort": "^10.0.0", + "nock": "^14.0.10", "nodemon": "^3.0.2", "pino-pretty": "^10.2.3", "prompt-sync": "^4.2.0", @@ -9705,6 +9706,24 @@ "win32" ] }, + "node_modules/@mswjs/interceptors": { + "version": "0.39.8", + "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.39.8.tgz", + "integrity": "sha512-2+BzZbjRO7Ct61k8fMNHEtoKjeWI9pIlHFTqBwZ5icHpqszIgEZbjb1MW5Z0+bITTCTl3gk4PDBxs9tA/csXvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@open-draft/deferred-promise": "^2.2.0", + "@open-draft/logger": "^0.3.0", + "@open-draft/until": "^2.0.0", + "is-node-process": "^1.2.0", + "outvariant": "^1.4.3", + "strict-event-emitter": "^0.5.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@next/env": { "version": "15.5.2", "resolved": "https://registry.npmjs.org/@next/env/-/env-15.5.2.tgz", @@ -10714,6 +10733,31 @@ "urijs": "^1.19.11" } }, + "node_modules/@open-draft/deferred-promise": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", + "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@open-draft/logger": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz", + "integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-node-process": "^1.2.0", + "outvariant": "^1.4.0" + } + }, + "node_modules/@open-draft/until": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz", + "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", + "dev": true, + "license": "MIT" + }, "node_modules/@opentelemetry/api": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", @@ -22958,6 +23002,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-node-process": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz", + "integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==", + "dev": true, + "license": "MIT" + }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -23507,6 +23558,13 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "dev": true, + "license": "ISC" + }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -25074,6 +25132,21 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/nock": { + "version": "14.0.10", + "resolved": "https://registry.npmjs.org/nock/-/nock-14.0.10.tgz", + "integrity": "sha512-Q7HjkpyPeLa0ZVZC5qpxBt5EyLczFJ91MEewQiIi9taWuA0KB/MDJlUWtON+7dGouVdADTQsf9RA7TZk6D8VMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@mswjs/interceptors": "^0.39.5", + "json-stringify-safe": "^5.0.1", + "propagate": "^2.0.0" + }, + "engines": { + "node": ">=18.20.0 <20 || >=20.12.1" + } + }, "node_modules/node-abi": { "version": "3.65.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.65.0.tgz", @@ -27702,6 +27775,13 @@ "@otplib/preset-v11": "^12.0.1" } }, + "node_modules/outvariant": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/outvariant/-/outvariant-1.4.3.tgz", + "integrity": "sha512-+Sl2UErvtsoajRDKCE5/dBz4DIvHXQQnAxtQTF04OJxY0+DyZXSo5P5Bb7XYWOh81syohlYL24hbDwxedPUJCA==", + "dev": true, + "license": "MIT" + }, "node_modules/p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", @@ -29103,6 +29183,16 @@ "node": ">= 6" } }, + "node_modules/propagate": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz", + "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/proto3-json-serializer": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.2.tgz", @@ -31601,6 +31691,13 @@ "node": ">=4.0.0" } }, + "node_modules/strict-event-emitter": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz", + "integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==", + "dev": true, + "license": "MIT" + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", diff --git a/backend/package.json b/backend/package.json index 7a5efcb78..aa97de2ed 100644 --- a/backend/package.json +++ b/backend/package.json @@ -123,6 +123,7 @@ "eslint-plugin-import": "^2.29.1", "eslint-plugin-prettier": "^5.1.3", "eslint-plugin-simple-import-sort": "^10.0.0", + "nock": "^14.0.10", "nodemon": "^3.0.2", "pino-pretty": "^10.2.3", "prompt-sync": "^4.2.0", diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 6f0502184..fa649dd8e 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -106,6 +106,7 @@ const envSchema = z HTTPS_ENABLED: zodStrBool, ROTATION_DEVELOPMENT_MODE: zodStrBool.default("false").optional(), DAILY_RESOURCE_CLEAN_UP_DEVELOPMENT_MODE: zodStrBool.default("false").optional(), + BDD_MOCK_THIRD_PARTY_API_CALLS: zodStrBool.default("false").optional(), ACME_DEVELOPMENT_MODE: zodStrBool.default("false").optional(), ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES: zpStr( z @@ -398,6 +399,7 @@ const envSchema = z isAcmeDevelopmentMode: data.NODE_ENV === "development" && data.ACME_DEVELOPMENT_MODE, isProductionMode: data.NODE_ENV === "production" || IS_PACKAGED, isRedisSentinelMode: Boolean(data.REDIS_SENTINEL_HOSTS), + shouldMockThirdPartyApiCalls: data.NODE_ENV === "development" && data.BDD_MOCK_THIRD_PARTY_API_CALLS, REDIS_SENTINEL_HOSTS: data.REDIS_SENTINEL_HOSTS?.trim() ?.split(",") .map((el) => { diff --git a/backend/src/mock-third-party-api-calls.ts b/backend/src/mock-third-party-api-calls.ts new file mode 100644 index 000000000..521fc77bb --- /dev/null +++ b/backend/src/mock-third-party-api-calls.ts @@ -0,0 +1,87 @@ +import nock from "nock"; + +export const mockThirdPartyApiCalls = () => { + nock("https://api.cloudflare.com:443") + .get("/client/v4/accounts/MOCK_ACCOUNT_ID") + .reply(200, { + result: { + id: "A2A6347F-88B5-442D-9798-95E408BC7701", + name: "Mock Account", + type: "standard", + settings: { + enforce_twofactor: true, + api_access_enabled: null, + access_approval_expiry: null, + abuse_contact_email: null, + user_groups_ui_beta: false + }, + legacy_flags: { + enterprise_zone_quota: { maximum: 0, current: 0, available: 0 } + }, + created_on: "2013-04-18T00:41:02.215243Z" + }, + success: true, + errors: [], + messages: [] + }); + + nock("https://api.cloudflare.com:443") + .get("/client/v4/zones") + .reply(200, { + result: [ + { + id: "2DF47D5E-7FE2-4BD9-8503-BF27ACE6EBE5", + name: "example.com", + status: "active", + paused: false, + type: "full", + development_mode: 0, + name_servers: ["abby.ns.cloudflare.com", "cody.ns.cloudflare.com"], + original_name_servers: ["ns1gmz.name.com", "ns2fgp.name.com", "ns3jwx.name.com", "ns4lny.name.com"], + original_registrar: "name.com, inc. (id: 625)", + original_dnshost: null, + modified_on: "2025-11-05T18:08:57.046348Z", + created_on: "2025-11-05T18:05:52.536690Z", + activated_on: "2025-11-05T18:08:57.046348Z", + vanity_name_servers: [], + vanity_name_servers_ips: null, + meta: { + step: 2, + custom_certificate_quota: 0, + page_rule_quota: 3, + phishing_detected: false + }, + owner: { id: null, type: "user", email: null }, + account: { + id: "A2A6347F-88B5-442D-9798-95E408BC7701", + name: "Mock Account" + }, + tenant: { id: null, name: null }, + tenant_unit: { id: null }, + permissions: ["#dns_records:edit", "#dns_records:read", "#zone:read"], + plan: { + id: "0feeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + name: "Free Website", + price: 0, + currency: "USD", + frequency: "", + is_subscribed: false, + can_subscribe: false, + legacy_id: "free", + legacy_discount: false, + externally_managed: false + } + } + ], + result_info: { + page: 1, + per_page: 20, + total_pages: 1, + count: 1, + total_count: 1 + }, + success: true, + errors: [], + messages: [] + }); +}; diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index 60b678f63..3ced74160 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -26,6 +26,7 @@ import { TSmtpService } from "@app/services/smtp/smtp-service"; import { TSuperAdminDALFactory } from "@app/services/super-admin/super-admin-dal"; import { globalRateLimiterCfg } from "./config/rateLimiter"; +import { mockThirdPartyApiCalls } from "./mock-third-party-api-calls"; import { addErrorsToResponseSchemas } from "./plugins/add-errors-to-response-schemas"; import { apiMetrics } from "./plugins/api-metrics"; import { fastifyErrHandler } from "./plugins/error-handler"; @@ -66,6 +67,12 @@ export const main = async ({ }: TMain) => { const appCfg = getConfig(); + if (appCfg.shouldMockThirdPartyApiCalls) { + logger?.info("Mocking third party API calls for BDD"); + // Note: to make BDD tests much easier, we mock some third party API calls here + mockThirdPartyApiCalls(); + } + const server = fastify({ logger: appCfg.NODE_ENV === "test" ? false : logger, genReqId: () => `req-${alphaNumericNanoId(14)}`, From 873deff71bb95ec941c8285a22e3b55bfeadd4fb Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 19:58:20 -0800 Subject: [PATCH 25/56] more mock --- backend/src/mock-third-party-api-calls.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/src/mock-third-party-api-calls.ts b/backend/src/mock-third-party-api-calls.ts index 521fc77bb..97805f3f4 100644 --- a/backend/src/mock-third-party-api-calls.ts +++ b/backend/src/mock-third-party-api-calls.ts @@ -1,11 +1,15 @@ import nock from "nock"; -export const mockThirdPartyApiCalls = () => { +const CLOUDFLARE_ACCOUNT_ID = "A2A6347F-88B5-442D-9798-95E408BC7701"; + +// We mock the Cloudflare account API calls to let adding app connection much easier in BDD tests. +// It's mainly for the DNS provider validation in ACME challenge validation. +const mockCloudflare = () => { nock("https://api.cloudflare.com:443") .get("/client/v4/accounts/MOCK_ACCOUNT_ID") .reply(200, { result: { - id: "A2A6347F-88B5-442D-9798-95E408BC7701", + id: CLOUDFLARE_ACCOUNT_ID, name: "Mock Account", type: "standard", settings: { @@ -53,7 +57,7 @@ export const mockThirdPartyApiCalls = () => { }, owner: { id: null, type: "user", email: null }, account: { - id: "A2A6347F-88B5-442D-9798-95E408BC7701", + id: CLOUDFLARE_ACCOUNT_ID, name: "Mock Account" }, tenant: { id: null, name: null }, @@ -85,3 +89,7 @@ export const mockThirdPartyApiCalls = () => { messages: [] }); }; + +export const mockThirdPartyApiCalls = () => { + mockCloudflare(); +}; From 79ddc8465c6af7ef096d9c93478044b7dba9849f Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 20:02:17 -0800 Subject: [PATCH 26/56] Move files --- backend/src/{ => server}/mock-third-party-api-calls.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename backend/src/{ => server}/mock-third-party-api-calls.ts (100%) diff --git a/backend/src/mock-third-party-api-calls.ts b/backend/src/server/mock-third-party-api-calls.ts similarity index 100% rename from backend/src/mock-third-party-api-calls.ts rename to backend/src/server/mock-third-party-api-calls.ts From 89baed638d012f848eb645e7e99e2e03957c0d48 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 20:34:07 -0800 Subject: [PATCH 27/56] Provide nock api --- backend/src/server/app.ts | 7 -- .../src/server/mock-third-party-api-calls.ts | 95 ------------------- .../src/server/routes/v1/bdd-nock-router.ts | 33 +++++++ backend/src/server/routes/v1/index.ts | 8 ++ 4 files changed, 41 insertions(+), 102 deletions(-) delete mode 100644 backend/src/server/mock-third-party-api-calls.ts create mode 100644 backend/src/server/routes/v1/bdd-nock-router.ts diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index 3ced74160..60b678f63 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -26,7 +26,6 @@ import { TSmtpService } from "@app/services/smtp/smtp-service"; import { TSuperAdminDALFactory } from "@app/services/super-admin/super-admin-dal"; import { globalRateLimiterCfg } from "./config/rateLimiter"; -import { mockThirdPartyApiCalls } from "./mock-third-party-api-calls"; import { addErrorsToResponseSchemas } from "./plugins/add-errors-to-response-schemas"; import { apiMetrics } from "./plugins/api-metrics"; import { fastifyErrHandler } from "./plugins/error-handler"; @@ -67,12 +66,6 @@ export const main = async ({ }: TMain) => { const appCfg = getConfig(); - if (appCfg.shouldMockThirdPartyApiCalls) { - logger?.info("Mocking third party API calls for BDD"); - // Note: to make BDD tests much easier, we mock some third party API calls here - mockThirdPartyApiCalls(); - } - const server = fastify({ logger: appCfg.NODE_ENV === "test" ? false : logger, genReqId: () => `req-${alphaNumericNanoId(14)}`, diff --git a/backend/src/server/mock-third-party-api-calls.ts b/backend/src/server/mock-third-party-api-calls.ts deleted file mode 100644 index 97805f3f4..000000000 --- a/backend/src/server/mock-third-party-api-calls.ts +++ /dev/null @@ -1,95 +0,0 @@ -import nock from "nock"; - -const CLOUDFLARE_ACCOUNT_ID = "A2A6347F-88B5-442D-9798-95E408BC7701"; - -// We mock the Cloudflare account API calls to let adding app connection much easier in BDD tests. -// It's mainly for the DNS provider validation in ACME challenge validation. -const mockCloudflare = () => { - nock("https://api.cloudflare.com:443") - .get("/client/v4/accounts/MOCK_ACCOUNT_ID") - .reply(200, { - result: { - id: CLOUDFLARE_ACCOUNT_ID, - name: "Mock Account", - type: "standard", - settings: { - enforce_twofactor: true, - api_access_enabled: null, - access_approval_expiry: null, - abuse_contact_email: null, - user_groups_ui_beta: false - }, - legacy_flags: { - enterprise_zone_quota: { maximum: 0, current: 0, available: 0 } - }, - created_on: "2013-04-18T00:41:02.215243Z" - }, - success: true, - errors: [], - messages: [] - }); - - nock("https://api.cloudflare.com:443") - .get("/client/v4/zones") - .reply(200, { - result: [ - { - id: "2DF47D5E-7FE2-4BD9-8503-BF27ACE6EBE5", - name: "example.com", - status: "active", - paused: false, - type: "full", - development_mode: 0, - name_servers: ["abby.ns.cloudflare.com", "cody.ns.cloudflare.com"], - original_name_servers: ["ns1gmz.name.com", "ns2fgp.name.com", "ns3jwx.name.com", "ns4lny.name.com"], - original_registrar: "name.com, inc. (id: 625)", - original_dnshost: null, - modified_on: "2025-11-05T18:08:57.046348Z", - created_on: "2025-11-05T18:05:52.536690Z", - activated_on: "2025-11-05T18:08:57.046348Z", - vanity_name_servers: [], - vanity_name_servers_ips: null, - meta: { - step: 2, - custom_certificate_quota: 0, - page_rule_quota: 3, - phishing_detected: false - }, - owner: { id: null, type: "user", email: null }, - account: { - id: CLOUDFLARE_ACCOUNT_ID, - name: "Mock Account" - }, - tenant: { id: null, name: null }, - tenant_unit: { id: null }, - permissions: ["#dns_records:edit", "#dns_records:read", "#zone:read"], - plan: { - id: "0feeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - name: "Free Website", - price: 0, - currency: "USD", - frequency: "", - is_subscribed: false, - can_subscribe: false, - legacy_id: "free", - legacy_discount: false, - externally_managed: false - } - } - ], - result_info: { - page: 1, - per_page: 20, - total_pages: 1, - count: 1, - total_count: 1 - }, - success: true, - errors: [], - messages: [] - }); -}; - -export const mockThirdPartyApiCalls = () => { - mockCloudflare(); -}; diff --git a/backend/src/server/routes/v1/bdd-nock-router.ts b/backend/src/server/routes/v1/bdd-nock-router.ts new file mode 100644 index 000000000..5237ce6ae --- /dev/null +++ b/backend/src/server/routes/v1/bdd-nock-router.ts @@ -0,0 +1,33 @@ +import { z } from "zod"; + +import { getConfig } from "@app/lib/config/env"; +import { ForbiddenRequestError } from "@app/lib/errors"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AuthMode } from "@app/services/auth/auth-type"; +import nock, { Definition } from "nock"; + +export const registerBddNockRouter = async (server: FastifyZodProvider) => { + server.route({ + method: "POST", + url: "/define", + schema: { + body: z.object({ definition: z.string() }), + response: { + 200: z.object({ status: z.string() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const appCfg = getConfig(); + // Note: Please note that this API is only available in development mode and only for BDD tests. + // This endpoint should NEVER BE ENABLED IN PRODUCTION! + if (appCfg.NODE_ENV !== "development" || !appCfg.isBddNockApiEnabled) { + throw new ForbiddenRequestError({ message: "BDD Nock API is not enabled" }); + } + const { body } = req; + const { definition } = body; + nock.define(definition as unknown as Definition[]); + return { status: "ok" }; + } + }); +}; diff --git a/backend/src/server/routes/v1/index.ts b/backend/src/server/routes/v1/index.ts index b480a5144..4d8b87e60 100644 --- a/backend/src/server/routes/v1/index.ts +++ b/backend/src/server/routes/v1/index.ts @@ -6,8 +6,10 @@ import { registerCmekRouter } from "@app/server/routes/v1/cmek-router"; import { registerDashboardRouter } from "@app/server/routes/v1/dashboard-router"; import { registerSecretSyncRouter, SECRET_SYNC_REGISTER_ROUTER_MAP } from "@app/server/routes/v1/secret-sync-routers"; +import { getConfig } from "@app/lib/config/env"; import { registerAdminRouter } from "./admin-router"; import { registerAuthRoutes } from "./auth-router"; +import { registerBddNockRouter } from "./bdd-nock-router"; import { registerProjectBotRouter } from "./bot-router"; import { registerCaRouter } from "./certificate-authority-router"; import { CERTIFICATE_AUTHORITY_REGISTER_ROUTER_MAP } from "./certificate-authority-routers"; @@ -237,4 +239,10 @@ export const registerV1Routes = async (server: FastifyZodProvider) => { await server.register(registerEventRouter, { prefix: "/events" }); await server.register(registerUpgradePathRouter, { prefix: "/upgrade-path" }); + + // Note: This is a special route for BDD tests. It's only available in development mode and only for BDD tests. + // This route should NEVER BE ENABLED IN PRODUCTION! + if (getConfig().isBddNockApiEnabled) { + await server.register(registerBddNockRouter, { prefix: "/bdd-nock" }); + } }; From 146524ac2d8a60cb0e6df714198374a21e23de6f Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 20:55:17 -0800 Subject: [PATCH 28/56] More mock --- .../src/server/routes/v1/bdd-nock-router.ts | 48 ++++++++++++++++--- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/backend/src/server/routes/v1/bdd-nock-router.ts b/backend/src/server/routes/v1/bdd-nock-router.ts index 5237ce6ae..7b492df9f 100644 --- a/backend/src/server/routes/v1/bdd-nock-router.ts +++ b/backend/src/server/routes/v1/bdd-nock-router.ts @@ -7,6 +7,15 @@ import { AuthMode } from "@app/services/auth/auth-type"; import nock, { Definition } from "nock"; export const registerBddNockRouter = async (server: FastifyZodProvider) => { + const checkIfBddNockApiEnabled = () => { + const appCfg = getConfig(); + // Note: Please note that this API is only available in development mode and only for BDD tests. + // This endpoint should NEVER BE ENABLED IN PRODUCTION! + if (appCfg.NODE_ENV !== "development" || !appCfg.isBddNockApiEnabled) { + throw new ForbiddenRequestError({ message: "BDD Nock API is not enabled" }); + } + }; + server.route({ method: "POST", url: "/define", @@ -18,16 +27,43 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { - const appCfg = getConfig(); - // Note: Please note that this API is only available in development mode and only for BDD tests. - // This endpoint should NEVER BE ENABLED IN PRODUCTION! - if (appCfg.NODE_ENV !== "development" || !appCfg.isBddNockApiEnabled) { - throw new ForbiddenRequestError({ message: "BDD Nock API is not enabled" }); - } + checkIfBddNockApiEnabled(); const { body } = req; const { definition } = body; nock.define(definition as unknown as Definition[]); return { status: "ok" }; } }); + + server.route({ + method: "POST", + url: "/restore", + schema: { + response: { + 200: z.object({ status: z.string() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + checkIfBddNockApiEnabled(); + nock.restore(); + return { status: "ok" }; + } + }); + + server.route({ + method: "POST", + url: "/clear-all", + schema: { + response: { + 200: z.object({ status: z.string() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + checkIfBddNockApiEnabled(); + nock.cleanAll(); + return { status: "ok" }; + } + }); }; From 1b90a1dca0a21d0ba5e54bc831affe608f56011f Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 21:26:43 -0800 Subject: [PATCH 29/56] Use mock api call --- backend/bdd/features/environment.py | 4 + backend/bdd/features/steps/pki_acme.py | 103 +++++++++++++++++- backend/bdd/features/steps/utils.py | 41 +++++++ .../src/server/routes/v1/bdd-nock-router.ts | 10 +- 4 files changed, 154 insertions(+), 4 deletions(-) diff --git a/backend/bdd/features/environment.py b/backend/bdd/features/environment.py index 41de93bc0..f63422caa 100644 --- a/backend/bdd/features/environment.py +++ b/backend/bdd/features/environment.py @@ -10,6 +10,8 @@ from dotenv import load_dotenv from faker import Faker import logging +from features.steps.utils import clear_all_nock, restore_nock + load_dotenv() logger = logging.getLogger(__name__) @@ -205,3 +207,5 @@ def before_all(context: Context): def after_scenario(context: Context, scenario: typing.Any): if hasattr(context, "web_server"): context.web_server.shutdown_and_server_close() + clear_all_nock(context) + restore_nock(context) diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index aa8a110dd..274fabbdf 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -22,7 +22,7 @@ from cryptography import x509 from cryptography.x509.oid import NameOID from cryptography.hazmat.primitives import hashes -from utils import replace_vars +from utils import replace_vars, with_nocks from utils import eval_var from utils import prepare_headers @@ -87,6 +87,107 @@ def step_impl(context: Context, profile_var: str): ) +@given("I create a Cloudflare connection as {var_name}") +def step_impl(context: Context, var_name: str): + jwt_token = context.vars["AUTH_TOKEN"] + conn_slug = faker.slug() + mock_account_id = "MOCK_ACCOUNT_ID" + with with_nocks( + context, + definitions=[ + { + "scope": "https://api.cloudflare.com:443", + "method": "GET", + "path": f"/client/v4/accounts/{mock_account_id}", + "status": 200, + "response": { + "result": { + "id": "A2A6347F-88B5-442D-9798-95E408BC7701", + "name": "Mock Account", + "type": "standard", + "settings": { + "enforce_twofactor": True, + "api_access_enabled": None, + "access_approval_expiry": None, + "abuse_contact_email": None, + "user_groups_ui_beta": False, + }, + "legacy_flags": { + "enterprise_zone_quota": { + "maximum": 0, + "current": 0, + "available": 0, + } + }, + "created_on": "2013-04-18T00:41:02.215243Z", + }, + "success": True, + "errors": [], + "messages": [], + }, + "responseIsBinary": False, + } + ], + ): + response = context.http_client.post( + "/api/v1/app-connections/cloudflare", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json={ + "name": conn_slug, + "description": "", + "method": "api-token", + "credentials": { + "apiToken": "MOCK_API_TOKEN", + "accountId": mock_account_id, + }, + }, + ) + response.raise_for_status() + context.vars[var_name] = response + + +@given('I have an ACME cert profile with external ACME CA as "{profile_var}"') +def step_impl(context: Context, profile_var: str): + profile_id = context.vars.get("PROFILE_ID") + secret = context.vars.get("EAB_SECRET") + if profile_id is not None and secret is not None: + kid = profile_id + else: + profile_slug = faker.slug() + jwt_token = context.vars["AUTH_TOKEN"] + response = context.http_client.post( + "/api/v1/pki/certificate-profiles", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json={ + "projectId": context.vars["PROJECT_ID"], + "slug": profile_slug, + "description": "ACME Profile created by BDD test", + "enrollmentType": "acme", + "caId": context.vars["CERT_CA_ID"], + "certificateTemplateId": context.vars["CERT_TEMPLATE_ID"], + "acmeConfig": {}, + }, + ) + response.raise_for_status() + resp_json = response.json() + profile_id = resp_json["certificateProfile"]["id"] + kid = profile_id + + response = context.http_client.get( + f"/api/v1/pki/certificate-profiles/{profile_id}/acme/eab-secret/reveal", + headers=dict(authorization="Bearer {}".format(jwt_token)), + ) + response.raise_for_status() + resp_json = response.json() + secret = resp_json["eabSecret"] + + context.vars[profile_var] = AcmeProfile( + profile_id, + eab_kid=kid, + eab_secret=secret, + ) + + @given("I use {token_var} for authentication") def step_impl(context: Context, token_var: str): context.auth_token = eval_var(context, token_var) diff --git a/backend/bdd/features/steps/utils.py b/backend/bdd/features/steps/utils.py index df1a3f17f..49f9850de 100644 --- a/backend/bdd/features/steps/utils.py +++ b/backend/bdd/features/steps/utils.py @@ -3,6 +3,7 @@ from cryptography.hazmat.primitives import hashes from cryptography.x509.oid import NameOID import logging import re +import contextlib import httpx import requests @@ -258,3 +259,43 @@ def x509_cert_to_dict(cert: x509.Certificate) -> dict: ) return result + + +def define_nock(context: Context, definitions: list[dict]): + jwt_token = context.vars["AUTH_TOKEN"] + response = context.http_client.post( + "/api/v1/bdd-nock/define", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json=dict(definitions=definitions), + ) + response.raise_for_status() + + +def restore_nock(context: Context): + jwt_token = context.vars["AUTH_TOKEN"] + response = context.http_client.post( + "/api/v1/bdd-nock/restore", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json=dict(), + ) + response.raise_for_status() + + +def clear_all_nock(context: Context): + jwt_token = context.vars["AUTH_TOKEN"] + response = context.http_client.post( + "/api/v1/bdd-nock/clear-all", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json=dict(), + ) + response.raise_for_status() + + +@contextlib.contextmanager +def with_nocks(context: Context, definitions: list[dict]): + try: + define_nock(context, definitions) + yield + finally: + clear_all_nock(context) + restore_nock(context) diff --git a/backend/src/server/routes/v1/bdd-nock-router.ts b/backend/src/server/routes/v1/bdd-nock-router.ts index 7b492df9f..1ec68bea3 100644 --- a/backend/src/server/routes/v1/bdd-nock-router.ts +++ b/backend/src/server/routes/v1/bdd-nock-router.ts @@ -4,6 +4,7 @@ import { getConfig } from "@app/lib/config/env"; import { ForbiddenRequestError } from "@app/lib/errors"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; +import { logger } from "@app/lib/logger"; import nock, { Definition } from "nock"; export const registerBddNockRouter = async (server: FastifyZodProvider) => { @@ -20,7 +21,7 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { method: "POST", url: "/define", schema: { - body: z.object({ definition: z.string() }), + body: z.object({ definitions: z.unknown().array() }), response: { 200: z.object({ status: z.string() }) } @@ -29,8 +30,9 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { handler: async (req) => { checkIfBddNockApiEnabled(); const { body } = req; - const { definition } = body; - nock.define(definition as unknown as Definition[]); + const { definitions } = body; + logger.info(definitions, "Defining nock"); + nock.define(definitions as Definition[]); return { status: "ok" }; } }); @@ -46,6 +48,7 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { checkIfBddNockApiEnabled(); + logger.info("Restore network requests from nock"); nock.restore(); return { status: "ok" }; } @@ -62,6 +65,7 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { checkIfBddNockApiEnabled(); + logger.info("Cleaning all nocks"); nock.cleanAll(); return { status: "ok" }; } From 3742d23d60a0d6336cc1688df679dd8e491d248e Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 21:33:41 -0800 Subject: [PATCH 30/56] order --- .../src/server/routes/v1/bdd-nock-router.ts | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/backend/src/server/routes/v1/bdd-nock-router.ts b/backend/src/server/routes/v1/bdd-nock-router.ts index 1ec68bea3..0a9fdbd44 100644 --- a/backend/src/server/routes/v1/bdd-nock-router.ts +++ b/backend/src/server/routes/v1/bdd-nock-router.ts @@ -37,23 +37,6 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { } }); - server.route({ - method: "POST", - url: "/restore", - schema: { - response: { - 200: z.object({ status: z.string() }) - } - }, - onRequest: verifyAuth([AuthMode.JWT]), - handler: async (req) => { - checkIfBddNockApiEnabled(); - logger.info("Restore network requests from nock"); - nock.restore(); - return { status: "ok" }; - } - }); - server.route({ method: "POST", url: "/clear-all", @@ -70,4 +53,21 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { return { status: "ok" }; } }); + + server.route({ + method: "POST", + url: "/restore", + schema: { + response: { + 200: z.object({ status: z.string() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + checkIfBddNockApiEnabled(); + logger.info("Restore network requests from nock"); + nock.restore(); + return { status: "ok" }; + } + }); }; From 4b70f73362607dab6cc8537c54b073ffd0a5b1cf Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 21:35:23 -0800 Subject: [PATCH 31/56] Clean all --- backend/bdd/features/environment.py | 4 ++-- backend/bdd/features/steps/utils.py | 6 +++--- backend/src/server/routes/v1/bdd-nock-router.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/bdd/features/environment.py b/backend/bdd/features/environment.py index f63422caa..99595338b 100644 --- a/backend/bdd/features/environment.py +++ b/backend/bdd/features/environment.py @@ -10,7 +10,7 @@ from dotenv import load_dotenv from faker import Faker import logging -from features.steps.utils import clear_all_nock, restore_nock +from features.steps.utils import clean_all_nock, restore_nock load_dotenv() logger = logging.getLogger(__name__) @@ -207,5 +207,5 @@ def before_all(context: Context): def after_scenario(context: Context, scenario: typing.Any): if hasattr(context, "web_server"): context.web_server.shutdown_and_server_close() - clear_all_nock(context) + clean_all_nock(context) restore_nock(context) diff --git a/backend/bdd/features/steps/utils.py b/backend/bdd/features/steps/utils.py index 49f9850de..a678c121e 100644 --- a/backend/bdd/features/steps/utils.py +++ b/backend/bdd/features/steps/utils.py @@ -281,10 +281,10 @@ def restore_nock(context: Context): response.raise_for_status() -def clear_all_nock(context: Context): +def clean_all_nock(context: Context): jwt_token = context.vars["AUTH_TOKEN"] response = context.http_client.post( - "/api/v1/bdd-nock/clear-all", + "/api/v1/bdd-nock/clean-all", headers=dict(authorization="Bearer {}".format(jwt_token)), json=dict(), ) @@ -297,5 +297,5 @@ def with_nocks(context: Context, definitions: list[dict]): define_nock(context, definitions) yield finally: - clear_all_nock(context) + clean_all_nock(context) restore_nock(context) diff --git a/backend/src/server/routes/v1/bdd-nock-router.ts b/backend/src/server/routes/v1/bdd-nock-router.ts index 0a9fdbd44..44d68ddae 100644 --- a/backend/src/server/routes/v1/bdd-nock-router.ts +++ b/backend/src/server/routes/v1/bdd-nock-router.ts @@ -39,7 +39,7 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { server.route({ method: "POST", - url: "/clear-all", + url: "/clean-all", schema: { response: { 200: z.object({ status: z.string() }) From 907cdd94c53ae368dc2c8c09e843a247b8dd6522 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 21:37:44 -0800 Subject: [PATCH 32/56] activate... --- backend/src/server/routes/v1/bdd-nock-router.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/server/routes/v1/bdd-nock-router.ts b/backend/src/server/routes/v1/bdd-nock-router.ts index 44d68ddae..91e74919c 100644 --- a/backend/src/server/routes/v1/bdd-nock-router.ts +++ b/backend/src/server/routes/v1/bdd-nock-router.ts @@ -33,6 +33,8 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { const { definitions } = body; logger.info(definitions, "Defining nock"); nock.define(definitions as Definition[]); + // Ensure we are activating the nocks, because we could have called `nock.restore()` before this call. + nock.activate(); return { status: "ok" }; } }); From 668abeb47b2021632d9e812b63cda9456aa036d4 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 21:38:15 -0800 Subject: [PATCH 33/56] Activate --- backend/src/server/routes/v1/bdd-nock-router.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/src/server/routes/v1/bdd-nock-router.ts b/backend/src/server/routes/v1/bdd-nock-router.ts index 91e74919c..e8e0b06a1 100644 --- a/backend/src/server/routes/v1/bdd-nock-router.ts +++ b/backend/src/server/routes/v1/bdd-nock-router.ts @@ -34,7 +34,9 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { logger.info(definitions, "Defining nock"); nock.define(definitions as Definition[]); // Ensure we are activating the nocks, because we could have called `nock.restore()` before this call. - nock.activate(); + if (!nock.isActive()) { + nock.activate(); + } return { status: "ok" }; } }); From 7d97f3263b7420a49a8368a7b8ad23cc43825e7c Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 21:48:06 -0800 Subject: [PATCH 34/56] Create ext ca --- backend/bdd/features/steps/pki_acme.py | 21 +++++++++++++++++++++ backend/src/lib/config/env.ts | 4 ++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index 274fabbdf..ba4507b65 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -146,6 +146,27 @@ def step_impl(context: Context, var_name: str): context.vars[var_name] = response +@given("I create a external ACME CA with the following config as {var_name}") +def step_impl(context: Context, var_name: str): + jwt_token = context.vars["AUTH_TOKEN"] + ca_slug = faker.slug() + config = replace_vars(json.loads(context.text), context.vars) + response = context.http_client.post( + "/api/v1/pki/ca/acme", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json={ + "projectId": context.vars["PROJECT_ID"], + "name": ca_slug, + "type": "acme", + "status": "active", + "enableDirectIssuance": True, + "configuration": config, + }, + ) + response.raise_for_status() + context.vars[var_name] = response + + @given('I have an ACME cert profile with external ACME CA as "{profile_var}"') def step_impl(context: Context, profile_var: str): profile_id = context.vars.get("PROFILE_ID") diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index fa649dd8e..b02a9d4dc 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -106,7 +106,7 @@ const envSchema = z HTTPS_ENABLED: zodStrBool, ROTATION_DEVELOPMENT_MODE: zodStrBool.default("false").optional(), DAILY_RESOURCE_CLEAN_UP_DEVELOPMENT_MODE: zodStrBool.default("false").optional(), - BDD_MOCK_THIRD_PARTY_API_CALLS: zodStrBool.default("false").optional(), + BDD_NOCK_API_ENABLED: zodStrBool.default("false").optional(), ACME_DEVELOPMENT_MODE: zodStrBool.default("false").optional(), ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES: zpStr( z @@ -399,7 +399,7 @@ const envSchema = z isAcmeDevelopmentMode: data.NODE_ENV === "development" && data.ACME_DEVELOPMENT_MODE, isProductionMode: data.NODE_ENV === "production" || IS_PACKAGED, isRedisSentinelMode: Boolean(data.REDIS_SENTINEL_HOSTS), - shouldMockThirdPartyApiCalls: data.NODE_ENV === "development" && data.BDD_MOCK_THIRD_PARTY_API_CALLS, + isBddNockApiEnabled: data.NODE_ENV === "development" && data.BDD_NOCK_API_ENABLED, REDIS_SENTINEL_HOSTS: data.REDIS_SENTINEL_HOSTS?.trim() ?.split(",") .map((el) => { From 4f17e23a9546270b0f3b9f27b3dbcccc5041869c Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 21:55:09 -0800 Subject: [PATCH 35/56] Add todo --- backend/bdd/features/steps/pki_acme.py | 19 +++++++++++++++++++ .../ee/services/pki-acme/pki-acme-service.ts | 2 ++ 2 files changed, 21 insertions(+) diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index ba4507b65..5aab5531e 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -167,6 +167,25 @@ def step_impl(context: Context, var_name: str): context.vars[var_name] = response +@given("I create a certificate template with the following config as {var_name}") +def step_impl(context: Context, var_name: str): + jwt_token = context.vars["AUTH_TOKEN"] + template_slug = faker.slug() + config = replace_vars(json.loads(context.text), context.vars) + response = context.http_client.post( + "/api/v2/certificate-templates", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json={ + "projectId": context.vars["PROJECT_ID"], + "name": template_slug, + "description": "", + } + | config, + ) + response.raise_for_status() + context.vars[var_name] = response + + @given('I have an ACME cert profile with external ACME CA as "{profile_var}"') def step_impl(context: Context, profile_var: str): profile_id = context.vars.get("PROFILE_ID") diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index 93339eea9..76277c0bd 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -729,6 +729,8 @@ export const pkiAcmeServiceFactory = ({ return { certificateId: result.certificateId }; } else { const { certificateAuthority } = (await certificateProfileDAL.findByIdWithConfigs(profileId, tx))!; + // TODO: for internal CA, we rely on the internal certificate authority service to check CSR against the template + // we should check the CSR against the template here // TODO: this is pretty slow, and we are holding the transaction open for a long time, // we should queue the certificate issuance to a background job instead const cert = await orderCertificate( From 95f909ebc28424e2b747b26a4e7379182eb67033 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 22:00:10 -0800 Subject: [PATCH 36/56] More ext bdd tests --- .../bdd/features/pki/acme/external-ca.feature | 119 +++++++++++++++--- backend/bdd/features/steps/pki_acme.py | 39 ++++++ 2 files changed, 142 insertions(+), 16 deletions(-) diff --git a/backend/bdd/features/pki/acme/external-ca.feature b/backend/bdd/features/pki/acme/external-ca.feature index 6d9ae02db..2ddf8f539 100644 --- a/backend/bdd/features/pki/acme/external-ca.feature +++ b/backend/bdd/features/pki/acme/external-ca.feature @@ -1,23 +1,110 @@ Feature: External CA Scenario: Issue a certificate from an external CA - Given I have an ACME cert profile as "acme_profile" - When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" - Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account - When I create certificate signing request as csr - Then I add names to certificate signing request csr + Given I create a Cloudflare connection as cloudflare + Then I memorize cloudflare with jq ".appConnection.id" as app_conn_id + Given I create a external ACME CA with the following config as ext_ca """ { - "COMMON_NAME": "localhost" + "dnsProviderConfig": { + "provider": "cloudflare", + "hostedZoneId": "MOCK_ZONE_ID" + }, + "directoryUrl": "https://acme-v02.api.letsencrypt.org/directory", + "accountEmail": "fangpen@infisical.com", + "dnsAppConnectionId": "{app_conn_id}", + "eabKid": "", + "eabHmacKey": "" } """ - And I create a RSA private key pair as cert_key - And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format - And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order - And I select challenge with type http-01 for domain localhost from order in order as challenge - And I serve challenge response for challenge at localhost - And I tell ACME server that challenge is ready to be verified - And I poll and finalize the ACME order order as finalized_order - And the value finalized_order.body with jq ".status" should be equal to "valid" - And I parse the full-chain certificate from order finalized_order as cert - And the value cert with jq ".subject.common_name" should be equal to "localhost" + Then I memorize ext_ca with jq ".id" as ext_ca_id + Given I create a certificate template with the following config as cert_template + """ + { + "subject": [ + { + "type": "common_name", + "allowed": [ + "*" + ] + } + ], + "sans": [ + { + "type": "dns_name", + "allowed": [ + "*" + ] + } + ], + "keyUsages": { + "required": [], + "allowed": [ + "digital_signature", + "key_encipherment", + "non_repudiation", + "data_encipherment", + "key_agreement", + "key_cert_sign", + "crl_sign", + "encipher_only", + "decipher_only" + ] + }, + "extendedKeyUsages": { + "required": [], + "allowed": [ + "client_auth", + "server_auth", + "code_signing", + "email_protection", + "ocsp_signing", + "time_stamping" + ] + }, + "algorithms": { + "signature": [ + "SHA256-RSA", + "SHA512-RSA", + "SHA384-ECDSA", + "SHA384-RSA", + "SHA256-ECDSA", + "SHA512-ECDSA" + ], + "keyAlgorithm": [ + "RSA-2048", + "RSA-4096", + "ECDSA-P384", + "RSA-3072", + "ECDSA-P256", + "ECDSA-P521" + ] + }, + "validity": { + "max": "365d" + } + } + """ + Then I memorize cert_template with jq ".certificateTemplate.id" as cert_template_id + Given I create an ACME profile with ca ext_ca_id and template cert_template_id as "acme_profile" + +# Given I have an ACME cert profile as "acme_profile" +# When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" +# Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account +# When I create certificate signing request as csr +# Then I add names to certificate signing request csr +# """ +# { +# "COMMON_NAME": "localhost" +# } +# """ +# And I create a RSA private key pair as cert_key +# And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format +# And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order +# And I select challenge with type http-01 for domain localhost from order in order as challenge +# And I serve challenge response for challenge at localhost +# And I tell ACME server that challenge is ready to be verified +# And I poll and finalize the ACME order order as finalized_order +# And the value finalized_order.body with jq ".status" should be equal to "valid" +# And I parse the full-chain certificate from order finalized_order as cert +# And the value cert with jq ".subject.common_name" should be equal to "localhost" diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index 5aab5531e..6c6e55852 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -186,6 +186,45 @@ def step_impl(context: Context, var_name: str): context.vars[var_name] = response +@given( + 'I create an ACME profile with ca {ca_id} and template {template_id} as "{profile_var}"' +) +def step_impl(context: Context, ca_id: str, template_id: str, profile_var: str): + profile_slug = faker.slug() + jwt_token = context.vars["AUTH_TOKEN"] + response = context.http_client.post( + "/api/v1/pki/certificate-profiles", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json={ + "projectId": context.vars["PROJECT_ID"], + "slug": profile_slug, + "description": "ACME Profile created by BDD test", + "enrollmentType": "acme", + "caId": replace_vars(ca_id, context.vars), + "certificateTemplateId": replace_vars(template_id, context.vars), + "acmeConfig": {}, + }, + ) + response.raise_for_status() + resp_json = response.json() + profile_id = resp_json["certificateProfile"]["id"] + kid = profile_id + + response = context.http_client.get( + f"/api/v1/pki/certificate-profiles/{profile_id}/acme/eab-secret/reveal", + headers=dict(authorization="Bearer {}".format(jwt_token)), + ) + response.raise_for_status() + resp_json = response.json() + secret = resp_json["eabSecret"] + + context.vars[profile_var] = AcmeProfile( + profile_id, + eab_kid=kid, + eab_secret=secret, + ) + + @given('I have an ACME cert profile with external ACME CA as "{profile_var}"') def step_impl(context: Context, profile_var: str): profile_id = context.vars.get("PROFILE_ID") From 833061f080e37dd1232f524adb0502ee2619b8af Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 22:02:13 -0800 Subject: [PATCH 37/56] Ext ca bdd --- backend/bdd/features/environment.py | 1 + .../bdd/features/pki/acme/external-ca.feature | 44 +++++++++---------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/backend/bdd/features/environment.py b/backend/bdd/features/environment.py index 99595338b..69637b850 100644 --- a/backend/bdd/features/environment.py +++ b/backend/bdd/features/environment.py @@ -16,6 +16,7 @@ load_dotenv() logger = logging.getLogger(__name__) BASE_URL = os.environ.get("INFISICAL_API_URL", "http://localhost:8080") +PEBBLE_URL = os.environ.get("PEBBLE_URL", "https://pebble:14000/dir") PROJECT_ID = os.environ.get("PROJECT_ID") CERT_CA_ID = os.environ.get("CERT_CA_ID") CERT_TEMPLATE_ID = os.environ.get("CERT_TEMPLATE_ID") diff --git a/backend/bdd/features/pki/acme/external-ca.feature b/backend/bdd/features/pki/acme/external-ca.feature index 2ddf8f539..64b0bee6b 100644 --- a/backend/bdd/features/pki/acme/external-ca.feature +++ b/backend/bdd/features/pki/acme/external-ca.feature @@ -10,7 +10,7 @@ Feature: External CA "provider": "cloudflare", "hostedZoneId": "MOCK_ZONE_ID" }, - "directoryUrl": "https://acme-v02.api.letsencrypt.org/directory", + "directoryUrl": "{PEBBLE_URL}", "accountEmail": "fangpen@infisical.com", "dnsAppConnectionId": "{app_conn_id}", "eabKid": "", @@ -86,25 +86,23 @@ Feature: External CA } """ Then I memorize cert_template with jq ".certificateTemplate.id" as cert_template_id - Given I create an ACME profile with ca ext_ca_id and template cert_template_id as "acme_profile" - -# Given I have an ACME cert profile as "acme_profile" -# When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" -# Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account -# When I create certificate signing request as csr -# Then I add names to certificate signing request csr -# """ -# { -# "COMMON_NAME": "localhost" -# } -# """ -# And I create a RSA private key pair as cert_key -# And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format -# And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order -# And I select challenge with type http-01 for domain localhost from order in order as challenge -# And I serve challenge response for challenge at localhost -# And I tell ACME server that challenge is ready to be verified -# And I poll and finalize the ACME order order as finalized_order -# And the value finalized_order.body with jq ".status" should be equal to "valid" -# And I parse the full-chain certificate from order finalized_order as cert -# And the value cert with jq ".subject.common_name" should be equal to "localhost" + Given I create an ACME profile with ca {ext_ca_id} and template {cert_template_id} as "acme_profile" + When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" + Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account + When I create certificate signing request as csr + Then I add names to certificate signing request csr + """ + { + "COMMON_NAME": "localhost" + } + """ + And I create a RSA private key pair as cert_key + And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format + And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order + And I select challenge with type http-01 for domain localhost from order in order as challenge + And I serve challenge response for challenge at localhost + And I tell ACME server that challenge is ready to be verified + And I poll and finalize the ACME order order as finalized_order + And the value finalized_order.body with jq ".status" should be equal to "valid" + And I parse the full-chain certificate from order finalized_order as cert + And the value cert with jq ".subject.common_name" should be equal to "localhost" From 3b9c881586dab1e0a852bc3d7298f840e0c4a29a Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 22:03:54 -0800 Subject: [PATCH 38/56] Add missing env --- backend/bdd/features/environment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/bdd/features/environment.py b/backend/bdd/features/environment.py index 69637b850..bd1683dd3 100644 --- a/backend/bdd/features/environment.py +++ b/backend/bdd/features/environment.py @@ -189,6 +189,7 @@ def before_all(context: Context): details = bootstrap_infisical(context) context.vars = { "BASE_URL": BASE_URL, + "PEBBLE_URL": PEBBLE_URL, "PROJECT_ID": details["project"]["id"], "CERT_CA_ID": details["ca"]["id"], "CERT_TEMPLATE_ID": details["cert_template"]["id"], @@ -197,6 +198,7 @@ def before_all(context: Context): else: context.vars = { "BASE_URL": BASE_URL, + "PEBBLE_URL": PEBBLE_URL, "PROJECT_ID": PROJECT_ID, "CERT_CA_ID": CERT_CA_ID, "CERT_TEMPLATE_ID": CERT_TEMPLATE_ID, From 2f8700fdcdea1c0ee0c624009c6a0c55d05c1128 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 23:02:01 -0800 Subject: [PATCH 39/56] Allow regex for path --- .../bdd/features/pki/acme/external-ca.feature | 61 ++++++++++++++++++- backend/bdd/features/steps/pki_acme.py | 13 ++++ .../src/server/routes/v1/bdd-nock-router.ts | 13 +++- .../acme/dns-providers/cloudflare.ts | 4 +- docker-compose.bdd.yml | 3 + 5 files changed, 91 insertions(+), 3 deletions(-) diff --git a/backend/bdd/features/pki/acme/external-ca.feature b/backend/bdd/features/pki/acme/external-ca.feature index 64b0bee6b..eac3b9c22 100644 --- a/backend/bdd/features/pki/acme/external-ca.feature +++ b/backend/bdd/features/pki/acme/external-ca.feature @@ -102,7 +102,66 @@ Feature: External CA And I select challenge with type http-01 for domain localhost from order in order as challenge And I serve challenge response for challenge at localhost And I tell ACME server that challenge is ready to be verified - And I poll and finalize the ACME order order as finalized_order + Given I intercept outgoing requests + """ + [ + { + "scope": "https://api.cloudflare.com:443", + "method": "POST", + "path": "/client/v4/zones/MOCK_ZONE_ID/dns_records", + "status": 200, + "response": { + "result": { + "id": "A2A6347F-88B5-442D-9798-95E408BC7701", + "name": "Mock Account", + "type": "standard", + "settings": { + "enforce_twofactor": false, + "api_access_enabled": null, + "access_approval_expiry": null, + "abuse_contact_email": null, + "user_groups_ui_beta": false + }, + "legacy_flags": { + "enterprise_zone_quota": { + "maximum": 0, + "current": 0, + "available": 0 + } + }, + "created_on": "2013-04-18T00:41:02.215243Z" + }, + "success": true, + "errors": [], + "messages": [] + }, + "responseIsBinary": false + }, + { + "scope": "https://api.cloudflare.com:443", + "method": "GET", + "path": { + "regex": "/client/v4/zones/[^/]+/dns_records\\?" + }, + "status": 200, + "response": { + "result": [], + "success": true, + "errors": [], + "messages": [], + "result_info": { + "page": 1, + "per_page": 100, + "count": 0, + "total_count": 0, + "total_pages": 1 + } + }, + "responseIsBinary": false + } + ] + """ + Then I poll and finalize the ACME order order as finalized_order And the value finalized_order.body with jq ".status" should be equal to "valid" And I parse the full-chain certificate from order finalized_order as cert And the value cert with jq ".subject.common_name" should be equal to "localhost" diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index 6c6e55852..46b10c13e 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -22,6 +22,7 @@ from cryptography import x509 from cryptography.x509.oid import NameOID from cryptography.hazmat.primitives import hashes +from features.steps.utils import define_nock, clean_all_nock, restore_nock from utils import replace_vars, with_nocks from utils import eval_var from utils import prepare_headers @@ -267,6 +268,18 @@ def step_impl(context: Context, profile_var: str): ) +@given("I intercept outgoing requests") +def step_impl(context: Context): + definitions = replace_vars(json.loads(context.text), context.vars) + define_nock(context, definitions) + + +@then("I reset requests interceptions") +def step_impl(context: Context): + clean_all_nock(context) + restore_nock(context) + + @given("I use {token_var} for authentication") def step_impl(context: Context, token_var: str): context.auth_token = eval_var(context, token_var) diff --git a/backend/src/server/routes/v1/bdd-nock-router.ts b/backend/src/server/routes/v1/bdd-nock-router.ts index e8e0b06a1..b5b2aa448 100644 --- a/backend/src/server/routes/v1/bdd-nock-router.ts +++ b/backend/src/server/routes/v1/bdd-nock-router.ts @@ -32,7 +32,18 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { const { body } = req; const { definitions } = body; logger.info(definitions, "Defining nock"); - nock.define(definitions as Definition[]); + const processedDefinitions = definitions.map((definition: unknown) => { + const { path, ...rest } = definition as Definition; + return { + ...rest, + path: + path !== undefined && typeof path === "string" + ? path + : new RegExp((path as unknown as { regex: string }).regex ?? "") + } as Definition; + }); + + nock.define(processedDefinitions as Definition[]); // Ensure we are activating the nocks, because we could have called `nock.restore()` before this call. if (!nock.isActive()) { nock.activate(); diff --git a/backend/src/services/certificate-authority/acme/dns-providers/cloudflare.ts b/backend/src/services/certificate-authority/acme/dns-providers/cloudflare.ts index f4b12e657..cf7725fdd 100644 --- a/backend/src/services/certificate-authority/acme/dns-providers/cloudflare.ts +++ b/backend/src/services/certificate-authority/acme/dns-providers/cloudflare.ts @@ -68,7 +68,9 @@ export const cloudflareDeleteTxtRecord = async ( }, params: { type: "TXT", - name: domain, + // TODO: this is incorrect. The domain seems need to be fqdn, but we are passing just the record name here. + // as a result, we are not deleting the record correctly. + // name: domain, content: value } }); diff --git a/docker-compose.bdd.yml b/docker-compose.bdd.yml index 7264ddae1..b73683867 100644 --- a/docker-compose.bdd.yml +++ b/docker-compose.bdd.yml @@ -87,7 +87,10 @@ services: - 14000:14000 # ACME port - 15000:15000 # Management port environment: + # Do not perform validation sleep to make the BDD tests faster - PEBBLE_VA_NOSLEEP=1 + # Skip validation for now to make the BDD tests easier to write + - PEBBLE_VA_ALWAYS_VALID=1 volumes: - ./backend/bdd/pebble/:/var/data/pebble:ro From 40532290f80fea0784332ad1a5f7fcc39595070e Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 23:16:43 -0800 Subject: [PATCH 40/56] Skip challenge verification --- .../acme/acme-certificate-authority-fns.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts index f4fff2c41..03ce9c642 100644 --- a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts +++ b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts @@ -43,6 +43,7 @@ import { } from "./acme-certificate-authority-types"; import { cloudflareDeleteTxtRecord, cloudflareInsertTxtRecord } from "./dns-providers/cloudflare"; import { route53DeleteTxtRecord, route53InsertTxtRecord } from "./dns-providers/route54"; +import { getConfig } from "@app/lib/config/env"; type TAcmeCertificateAuthorityFnsDeps = { appConnectionDAL: Pick; @@ -240,6 +241,9 @@ export const orderCertificate = async ( csr: certificateCsr, email: acmeCa.configuration.accountEmail, challengePriority: ["dns-01"], + // For ACME development mode, we mock the DNS challenge API calls. So, no real DNS records are created. + // We need to disable the challenge verification to avoid errors. + skipChallengeVerification: getConfig().isAcmeDevelopmentMode, termsOfServiceAgreed: true, challengeCreateFn: async (authz, challenge, keyAuthorization) => { From 839fe9fa6b0dd78477d05ed6169eb3c3d58b529d Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 23:22:55 -0800 Subject: [PATCH 41/56] Fix import --- backend/bdd/features/steps/utils.py | 5 +++-- .../acme/acme-certificate-authority-fns.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/bdd/features/steps/utils.py b/backend/bdd/features/steps/utils.py index a678c121e..4ee7c8921 100644 --- a/backend/bdd/features/steps/utils.py +++ b/backend/bdd/features/steps/utils.py @@ -183,8 +183,9 @@ def x509_cert_to_dict(cert: x509.Certificate) -> dict: "key_agreement", "key_cert_sign", "crl_sign", - "encipher_only", - "decipher_only", + # TODO: deal with error: "ValueError: encipher_only is undefined unless key_agreement is true" + # "encipher_only", + # "decipher_only", ] if getattr(ext.value, field) is not None }, diff --git a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts index 03ce9c642..ecbf5ddbb 100644 --- a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts +++ b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts @@ -29,6 +29,7 @@ import { triggerAutoSyncForSubscriber } from "@app/services/pki-sync/pki-sync-ut import { TProjectDALFactory } from "@app/services/project/project-dal"; import { getProjectKmsCertificateKeyId } from "@app/services/project/project-fns"; +import { getConfig } from "@app/lib/config/env"; import { Knex } from "knex"; import { TCertificateAuthorityDALFactory } from "../certificate-authority-dal"; import { CaStatus, CaType } from "../certificate-authority-enums"; @@ -43,7 +44,6 @@ import { } from "./acme-certificate-authority-types"; import { cloudflareDeleteTxtRecord, cloudflareInsertTxtRecord } from "./dns-providers/cloudflare"; import { route53DeleteTxtRecord, route53InsertTxtRecord } from "./dns-providers/route54"; -import { getConfig } from "@app/lib/config/env"; type TAcmeCertificateAuthorityFnsDeps = { appConnectionDAL: Pick; From edd9fa3eef18e7d52cc7cb7f1851b7460fda1f75 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 23:24:34 -0800 Subject: [PATCH 42/56] Fix assert --- backend/bdd/features/pki/acme/external-ca.feature | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/bdd/features/pki/acme/external-ca.feature b/backend/bdd/features/pki/acme/external-ca.feature index eac3b9c22..6b4123b91 100644 --- a/backend/bdd/features/pki/acme/external-ca.feature +++ b/backend/bdd/features/pki/acme/external-ca.feature @@ -164,4 +164,10 @@ Feature: External CA Then I poll and finalize the ACME order order as finalized_order And the value finalized_order.body with jq ".status" should be equal to "valid" And I parse the full-chain certificate from order finalized_order as cert - And the value cert with jq ".subject.common_name" should be equal to "localhost" + # Note: somehow Pebble is issuing a cert without common name but just SANs + And the value cert with jq "[.extensions.subjectAltName.general_names.[].value] | sort" should be equal to json + """ + [ + "localhost" + ] + """ \ No newline at end of file From 3a3a6f315222dd4f14596a9c4c669349d9468d9c Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 23:28:21 -0800 Subject: [PATCH 43/56] Keep the domain for now --- .../certificate-authority/acme/dns-providers/cloudflare.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/services/certificate-authority/acme/dns-providers/cloudflare.ts b/backend/src/services/certificate-authority/acme/dns-providers/cloudflare.ts index cf7725fdd..ab87ee113 100644 --- a/backend/src/services/certificate-authority/acme/dns-providers/cloudflare.ts +++ b/backend/src/services/certificate-authority/acme/dns-providers/cloudflare.ts @@ -70,7 +70,7 @@ export const cloudflareDeleteTxtRecord = async ( type: "TXT", // TODO: this is incorrect. The domain seems need to be fqdn, but we are passing just the record name here. // as a result, we are not deleting the record correctly. - // name: domain, + name: domain, content: value } }); From 58963b8185665b5c206a223d9ea9e8dd173c2a38 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 23:37:36 -0800 Subject: [PATCH 44/56] Lint --- .../ee/services/pki-acme/pki-acme-service.ts | 59 +++++++++---------- .../src/server/routes/v1/bdd-nock-router.ts | 4 +- .../acme/acme-certificate-authority-fns.ts | 4 +- 3 files changed, 31 insertions(+), 36 deletions(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index 76277c0bd..c0031e0ff 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -687,7 +687,7 @@ export const pkiAcmeServiceFactory = ({ const csrIdentifierValues = new Set( (certificateRequest.subjectAlternativeNames ?? []) .map((san) => san.value.toLowerCase()) - .concat([certificateRequest.commonName!.toLowerCase()]) + .concat([certificateRequest.commonName.toLowerCase()]) ); if ( csrIdentifierValues.size !== orderWithAuthorizations.authorizations.length || @@ -727,38 +727,33 @@ export const pkiAcmeServiceFactory = ({ enrollmentType: EnrollmentType.ACME }); return { certificateId: result.certificateId }; - } else { - const { certificateAuthority } = (await certificateProfileDAL.findByIdWithConfigs(profileId, tx))!; - // TODO: for internal CA, we rely on the internal certificate authority service to check CSR against the template - // we should check the CSR against the template here - // TODO: this is pretty slow, and we are holding the transaction open for a long time, - // we should queue the certificate issuance to a background job instead - const cert = await orderCertificate( - { - caId: certificateAuthority!.id, - commonName: certificateRequest.commonName!, - altNames: certificateRequest.subjectAlternativeNames?.map((san) => san.value), - // TODO: not 100% sure what are these columns for, but let's put the values for common website SSL certs for now - keyUsages: [ - CertKeyUsage.DIGITAL_SIGNATURE, - CertKeyUsage.KEY_ENCIPHERMENT, - CertKeyUsage.KEY_AGREEMENT - ], - extendedKeyUsages: [CertExtendedKeyUsage.SERVER_AUTH] - }, - { - appConnectionDAL, - certificateAuthorityDAL, - externalCertificateAuthorityDAL, - certificateDAL, - certificateBodyDAL, - certificateSecretDAL, - kmsService, - projectDAL - } - ); - return { certificateId: cert.id }; } + const { certificateAuthority } = (await certificateProfileDAL.findByIdWithConfigs(profileId, tx))!; + // TODO: for internal CA, we rely on the internal certificate authority service to check CSR against the template + // we should check the CSR against the template here + // TODO: this is pretty slow, and we are holding the transaction open for a long time, + // we should queue the certificate issuance to a background job instead + const cert = await orderCertificate( + { + caId: certificateAuthority!.id, + commonName: certificateRequest.commonName!, + altNames: certificateRequest.subjectAlternativeNames?.map((san) => san.value), + // TODO: not 100% sure what are these columns for, but let's put the values for common website SSL certs for now + keyUsages: [CertKeyUsage.DIGITAL_SIGNATURE, CertKeyUsage.KEY_ENCIPHERMENT, CertKeyUsage.KEY_AGREEMENT], + extendedKeyUsages: [CertExtendedKeyUsage.SERVER_AUTH] + }, + { + appConnectionDAL, + certificateAuthorityDAL, + externalCertificateAuthorityDAL, + certificateDAL, + certificateBodyDAL, + certificateSecretDAL, + kmsService, + projectDAL + } + ); + return { certificateId: cert.id }; })(); await acmeOrderDAL.updateById( orderId, diff --git a/backend/src/server/routes/v1/bdd-nock-router.ts b/backend/src/server/routes/v1/bdd-nock-router.ts index b5b2aa448..4faaf53d8 100644 --- a/backend/src/server/routes/v1/bdd-nock-router.ts +++ b/backend/src/server/routes/v1/bdd-nock-router.ts @@ -61,7 +61,7 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { } }, onRequest: verifyAuth([AuthMode.JWT]), - handler: async (req) => { + handler: async () => { checkIfBddNockApiEnabled(); logger.info("Cleaning all nocks"); nock.cleanAll(); @@ -78,7 +78,7 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { } }, onRequest: verifyAuth([AuthMode.JWT]), - handler: async (req) => { + handler: async () => { checkIfBddNockApiEnabled(); logger.info("Restore network requests from nock"); nock.restore(); diff --git a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts index ecbf5ddbb..ff06afde9 100644 --- a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts +++ b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts @@ -337,8 +337,8 @@ export const orderCertificate = async ( serialNumber: certObj.serialNumber, notBefore: certObj.notBefore, notAfter: certObj.notAfter, - keyUsages: keyUsages, - extendedKeyUsages: extendedKeyUsages, + keyUsages, + extendedKeyUsages, projectId: ca.projectId }, innerTx From 48601a5e8f1a03ccfc9384b3645d1b9cdc087e7b Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 23:38:42 -0800 Subject: [PATCH 45/56] BDD nock enabled on CI --- .github/workflows/run-backend-bdd-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-backend-bdd-tests.yml b/.github/workflows/run-backend-bdd-tests.yml index bf2075864..5abd95082 100644 --- a/.github/workflows/run-backend-bdd-tests.yml +++ b/.github/workflows/run-backend-bdd-tests.yml @@ -50,6 +50,7 @@ jobs: cp .env.example .env echo "ACME_DEVELOPMENT_MODE=true" >> .env echo "ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES={\"localhost\": \"host.docker.internal:8087\"}" >> .env + echo "BDD_NOCK_API_ENABLED=true" >> .env # Enable ACME feature in license for BDD tests sed -i 's/pkiAcme: .*/pkiAcme: true,/g' backend/src/ee/services/license/license-fns.ts - name: Set up Docker Buildx From d4f15a39318af12460686691292a8eeaf28f6d00 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 23:42:23 -0800 Subject: [PATCH 46/56] Do not enforce empty payload as some client somehow may not follow it and it's probably not defined in the RFC --- backend/src/ee/routes/v1/pki-acme-router.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/backend/src/ee/routes/v1/pki-acme-router.ts b/backend/src/ee/routes/v1/pki-acme-router.ts index d58790039..1ee6a3486 100644 --- a/backend/src/ee/routes/v1/pki-acme-router.ts +++ b/backend/src/ee/routes/v1/pki-acme-router.ts @@ -2,7 +2,6 @@ import { FastifyReply, FastifyRequest } from "fastify"; import { z } from "zod"; -import { AcmeMalformedError } from "@app/ee/services/pki-acme/pki-acme-errors"; import { AcmeOrderResourceSchema, CreateAcmeAccountResponseSchema, @@ -260,9 +259,6 @@ export const registerPkiAcmeRouter = async (server: FastifyZodProvider) => { const { profileId, accountId, payload } = await validateExistingAccount({ req }); - if (payload !== "") { - throw new AcmeMalformedError({ message: "Payload should be empty" }); - } return sendAcmeResponse( res, profileId, @@ -372,9 +368,6 @@ export const registerPkiAcmeRouter = async (server: FastifyZodProvider) => { const { profileId, accountId, payload } = await validateExistingAccount({ req }); - if (payload !== "") { - throw new AcmeMalformedError({ message: "Payload should be empty" }); - } res.type("application/pem-certificate-chain"); return sendAcmeResponse( res, @@ -406,9 +399,6 @@ export const registerPkiAcmeRouter = async (server: FastifyZodProvider) => { }, handler: async (req, res) => { const { profileId, accountId, payload } = await validateExistingAccount({ req }); - if (payload !== "") { - throw new AcmeMalformedError({ message: "Payload should be empty" }); - } return sendAcmeResponse( res, profileId, From 0b7f691f08ddb954874f22c411cb6b49cc3b3e66 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 23:52:03 -0800 Subject: [PATCH 47/56] Fix linter issues --- backend/package-lock.json | 12 +----------- backend/package.json | 2 +- backend/src/ee/routes/v1/pki-acme-router.ts | 6 +++--- backend/src/server/routes/v1/bdd-nock-router.ts | 2 +- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index a871c38b1..8f1112cb7 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -98,6 +98,7 @@ "ms": "^2.1.3", "mysql2": "^3.9.8", "nanoid": "^3.3.8", + "nock": "^14.0.10", "node-forge": "^1.3.1", "nodemailer": "^6.9.9", "oci-sdk": "^2.108.0", @@ -177,7 +178,6 @@ "eslint-plugin-import": "^2.29.1", "eslint-plugin-prettier": "^5.1.3", "eslint-plugin-simple-import-sort": "^10.0.0", - "nock": "^14.0.10", "nodemon": "^3.0.2", "pino-pretty": "^10.2.3", "prompt-sync": "^4.2.0", @@ -9710,7 +9710,6 @@ "version": "0.39.8", "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.39.8.tgz", "integrity": "sha512-2+BzZbjRO7Ct61k8fMNHEtoKjeWI9pIlHFTqBwZ5icHpqszIgEZbjb1MW5Z0+bITTCTl3gk4PDBxs9tA/csXvA==", - "dev": true, "license": "MIT", "dependencies": { "@open-draft/deferred-promise": "^2.2.0", @@ -10737,14 +10736,12 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==", - "dev": true, "license": "MIT" }, "node_modules/@open-draft/logger": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz", "integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==", - "dev": true, "license": "MIT", "dependencies": { "is-node-process": "^1.2.0", @@ -10755,7 +10752,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz", "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", - "dev": true, "license": "MIT" }, "node_modules/@opentelemetry/api": { @@ -23006,7 +23002,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz", "integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==", - "dev": true, "license": "MIT" }, "node_modules/is-number": { @@ -23562,7 +23557,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", - "dev": true, "license": "ISC" }, "node_modules/json5": { @@ -25136,7 +25130,6 @@ "version": "14.0.10", "resolved": "https://registry.npmjs.org/nock/-/nock-14.0.10.tgz", "integrity": "sha512-Q7HjkpyPeLa0ZVZC5qpxBt5EyLczFJ91MEewQiIi9taWuA0KB/MDJlUWtON+7dGouVdADTQsf9RA7TZk6D8VMw==", - "dev": true, "license": "MIT", "dependencies": { "@mswjs/interceptors": "^0.39.5", @@ -27779,7 +27772,6 @@ "version": "1.4.3", "resolved": "https://registry.npmjs.org/outvariant/-/outvariant-1.4.3.tgz", "integrity": "sha512-+Sl2UErvtsoajRDKCE5/dBz4DIvHXQQnAxtQTF04OJxY0+DyZXSo5P5Bb7XYWOh81syohlYL24hbDwxedPUJCA==", - "dev": true, "license": "MIT" }, "node_modules/p-finally": { @@ -29187,7 +29179,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz", "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -31695,7 +31686,6 @@ "version": "0.5.1", "resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz", "integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==", - "dev": true, "license": "MIT" }, "node_modules/string_decoder": { diff --git a/backend/package.json b/backend/package.json index aa97de2ed..db94de681 100644 --- a/backend/package.json +++ b/backend/package.json @@ -123,7 +123,6 @@ "eslint-plugin-import": "^2.29.1", "eslint-plugin-prettier": "^5.1.3", "eslint-plugin-simple-import-sort": "^10.0.0", - "nock": "^14.0.10", "nodemon": "^3.0.2", "pino-pretty": "^10.2.3", "prompt-sync": "^4.2.0", @@ -227,6 +226,7 @@ "ms": "^2.1.3", "mysql2": "^3.9.8", "nanoid": "^3.3.8", + "nock": "^14.0.10", "node-forge": "^1.3.1", "nodemailer": "^6.9.9", "oci-sdk": "^2.108.0", diff --git a/backend/src/ee/routes/v1/pki-acme-router.ts b/backend/src/ee/routes/v1/pki-acme-router.ts index 1ee6a3486..c4ccf6be5 100644 --- a/backend/src/ee/routes/v1/pki-acme-router.ts +++ b/backend/src/ee/routes/v1/pki-acme-router.ts @@ -256,7 +256,7 @@ export const registerPkiAcmeRouter = async (server: FastifyZodProvider) => { } }, handler: async (req, res) => { - const { profileId, accountId, payload } = await validateExistingAccount({ + const { profileId, accountId } = await validateExistingAccount({ req }); return sendAcmeResponse( @@ -365,7 +365,7 @@ export const registerPkiAcmeRouter = async (server: FastifyZodProvider) => { } }, handler: async (req, res) => { - const { profileId, accountId, payload } = await validateExistingAccount({ + const { profileId, accountId } = await validateExistingAccount({ req }); res.type("application/pem-certificate-chain"); @@ -398,7 +398,7 @@ export const registerPkiAcmeRouter = async (server: FastifyZodProvider) => { } }, handler: async (req, res) => { - const { profileId, accountId, payload } = await validateExistingAccount({ req }); + const { profileId, accountId } = await validateExistingAccount({ req }); return sendAcmeResponse( res, profileId, diff --git a/backend/src/server/routes/v1/bdd-nock-router.ts b/backend/src/server/routes/v1/bdd-nock-router.ts index 4faaf53d8..ad4777772 100644 --- a/backend/src/server/routes/v1/bdd-nock-router.ts +++ b/backend/src/server/routes/v1/bdd-nock-router.ts @@ -43,7 +43,7 @@ export const registerBddNockRouter = async (server: FastifyZodProvider) => { } as Definition; }); - nock.define(processedDefinitions as Definition[]); + nock.define(processedDefinitions); // Ensure we are activating the nocks, because we could have called `nock.restore()` before this call. if (!nock.isActive()) { nock.activate(); From 25102757e8e218b9186fd70b626bc07b875f17c0 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Wed, 12 Nov 2025 23:58:48 -0800 Subject: [PATCH 48/56] Return more msg in challenge --- .../src/ee/services/pki-acme/pki-acme-challenge-service.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts index 8bfe360d4..9148b0336 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts @@ -76,7 +76,9 @@ export const pkiAcmeChallengeServiceFactory = ({ // challenge validation at the same time, it should be fine. const challengeResponse = await fetch(challengeUrl, { signal: AbortSignal.timeout(timeoutMs) }); if (challengeResponse.status !== 200) { - throw new BadRequestError({ message: "ACME challenge response is not 200" }); + throw new AcmeIncorrectResponseError({ + message: `ACME challenge response is not 200: ${challengeResponse.status}` + }); } const challengeResponseBody = await challengeResponse.text(); const thumbprint = challenge.auth.account.publicKeyThumbprint; From a7c785f1689fc01ee0cafc7d2b5e9609e83d55f0 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Thu, 13 Nov 2025 00:10:40 -0800 Subject: [PATCH 49/56] Fix BDD --- .github/workflows/run-backend-bdd-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-backend-bdd-tests.yml b/.github/workflows/run-backend-bdd-tests.yml index 5abd95082..8d81a40d4 100644 --- a/.github/workflows/run-backend-bdd-tests.yml +++ b/.github/workflows/run-backend-bdd-tests.yml @@ -49,7 +49,7 @@ jobs: run: | cp .env.example .env echo "ACME_DEVELOPMENT_MODE=true" >> .env - echo "ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES={\"localhost\": \"host.docker.internal:8087\"}" >> .env + echo "ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES={\"localhost\": \"host.docker.internal:8087\", \"infisical.com\": \"host.docker.internal:8087\", \"example.com\": \"host.docker.internal:8087\"}" >> .env echo "BDD_NOCK_API_ENABLED=true" >> .env # Enable ACME feature in license for BDD tests sed -i 's/pkiAcme: .*/pkiAcme: true,/g' backend/src/ee/services/license/license-fns.ts From 5d3a730443476fb68499feb04d9a94b9efe6fabd Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Thu, 13 Nov 2025 09:03:59 -0800 Subject: [PATCH 50/56] Fix bdd tests --- backend/bdd/features/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/bdd/features/environment.py b/backend/bdd/features/environment.py index bd1683dd3..9a2e9f90b 100644 --- a/backend/bdd/features/environment.py +++ b/backend/bdd/features/environment.py @@ -121,7 +121,7 @@ def bootstrap_infisical(context: Context): "name": cert_template_slug, "description": "", "subject": [{"type": "common_name", "allowed": ["*"]}], - "sans": [], + "sans": [{"type": "dns_name", "allowed": ["*"]}], "keyUsages": { "required": [], "allowed": [ From 73a3ee6b61a8d9bfa42995a853613b34847029df Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Thu, 13 Nov 2025 09:38:18 -0800 Subject: [PATCH 51/56] Fix BDD --- .github/workflows/run-backend-bdd-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/run-backend-bdd-tests.yml b/.github/workflows/run-backend-bdd-tests.yml index 8d81a40d4..0ce5baea6 100644 --- a/.github/workflows/run-backend-bdd-tests.yml +++ b/.github/workflows/run-backend-bdd-tests.yml @@ -51,6 +51,9 @@ jobs: echo "ACME_DEVELOPMENT_MODE=true" >> .env echo "ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES={\"localhost\": \"host.docker.internal:8087\", \"infisical.com\": \"host.docker.internal:8087\", \"example.com\": \"host.docker.internal:8087\"}" >> .env echo "BDD_NOCK_API_ENABLED=true" >> .env + # We are not using FIPS mode, need a different encryption key for BDD tests + NEW_ENCRYPTION_KEY=6c1fe4e407b8911c104518103505b218 + sed -i "s#ENCRYPTION_KEY=.*#ENCRYPTION_KEY=$NEW_ENCRYPTION_KEY#" .env # Enable ACME feature in license for BDD tests sed -i 's/pkiAcme: .*/pkiAcme: true,/g' backend/src/ee/services/license/license-fns.ts - name: Set up Docker Buildx From c183d91257ba972b395cc0b0704af7e3458037aa Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Thu, 13 Nov 2025 11:36:51 -0800 Subject: [PATCH 52/56] Fix upstream skip validation --- .github/workflows/run-backend-bdd-tests.yml | 3 +++ backend/src/lib/config/env.ts | 1 + .../acme/acme-certificate-authority-fns.ts | 2 +- .../certificate-authority/acme/dns-providers/cloudflare.ts | 2 -- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-backend-bdd-tests.yml b/.github/workflows/run-backend-bdd-tests.yml index 0ce5baea6..7b54aa44e 100644 --- a/.github/workflows/run-backend-bdd-tests.yml +++ b/.github/workflows/run-backend-bdd-tests.yml @@ -51,6 +51,9 @@ jobs: echo "ACME_DEVELOPMENT_MODE=true" >> .env echo "ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES={\"localhost\": \"host.docker.internal:8087\", \"infisical.com\": \"host.docker.internal:8087\", \"example.com\": \"host.docker.internal:8087\"}" >> .env echo "BDD_NOCK_API_ENABLED=true" >> .env + # Skip upstream validation, otherwise the ACME client for the upstream will try to + # validate the DNS records, which will fail because the DNS records are not actually created. + echo "ACME_SKIP_UPSTREAM_VALIDATION=true" >> .env # We are not using FIPS mode, need a different encryption key for BDD tests NEW_ENCRYPTION_KEY=6c1fe4e407b8911c104518103505b218 sed -i "s#ENCRYPTION_KEY=.*#ENCRYPTION_KEY=$NEW_ENCRYPTION_KEY#" .env diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index b02a9d4dc..96107306f 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -108,6 +108,7 @@ const envSchema = z DAILY_RESOURCE_CLEAN_UP_DEVELOPMENT_MODE: zodStrBool.default("false").optional(), BDD_NOCK_API_ENABLED: zodStrBool.default("false").optional(), ACME_DEVELOPMENT_MODE: zodStrBool.default("false").optional(), + ACME_SKIP_UPSTREAM_VALIDATION: zodStrBool.default("false").optional(), ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES: zpStr( z .string() diff --git a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts index ff06afde9..436e37f3b 100644 --- a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts +++ b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts @@ -243,7 +243,7 @@ export const orderCertificate = async ( challengePriority: ["dns-01"], // For ACME development mode, we mock the DNS challenge API calls. So, no real DNS records are created. // We need to disable the challenge verification to avoid errors. - skipChallengeVerification: getConfig().isAcmeDevelopmentMode, + skipChallengeVerification: getConfig().isAcmeDevelopmentMode && getConfig().ACME_SKIP_UPSTREAM_VALIDATION, termsOfServiceAgreed: true, challengeCreateFn: async (authz, challenge, keyAuthorization) => { diff --git a/backend/src/services/certificate-authority/acme/dns-providers/cloudflare.ts b/backend/src/services/certificate-authority/acme/dns-providers/cloudflare.ts index ab87ee113..f4b12e657 100644 --- a/backend/src/services/certificate-authority/acme/dns-providers/cloudflare.ts +++ b/backend/src/services/certificate-authority/acme/dns-providers/cloudflare.ts @@ -68,8 +68,6 @@ export const cloudflareDeleteTxtRecord = async ( }, params: { type: "TXT", - // TODO: this is incorrect. The domain seems need to be fqdn, but we are passing just the record name here. - // as a result, we are not deleting the record correctly. name: domain, content: value } From 8895c520318234912f234949efaac0d564b05111 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Thu, 13 Nov 2025 11:50:30 -0800 Subject: [PATCH 53/56] Passing CSR --- .../ee/services/pki-acme/pki-acme-service.ts | 1 + .../acme/acme-certificate-authority-fns.ts | 60 +++++++++++-------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index c0031e0ff..a41f1ee5f 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -738,6 +738,7 @@ export const pkiAcmeServiceFactory = ({ caId: certificateAuthority!.id, commonName: certificateRequest.commonName!, altNames: certificateRequest.subjectAlternativeNames?.map((san) => san.value), + csr: Buffer.from(csr), // TODO: not 100% sure what are these columns for, but let's put the values for common website SSL certs for now keyUsages: [CertKeyUsage.DIGITAL_SIGNATURE, CertKeyUsage.KEY_ENCIPHERMENT, CertKeyUsage.KEY_AGREEMENT], extendedKeyUsages: [CertExtendedKeyUsage.SERVER_AUTH] diff --git a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts index 436e37f3b..2dc2b045e 100644 --- a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts +++ b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts @@ -1,5 +1,5 @@ import * as x509 from "@peculiar/x509"; -import acme from "acme-client"; +import acme, { CsrBuffer } from "acme-client"; import { TableName } from "@app/db/schemas"; import { crypto } from "@app/lib/crypto/cryptography"; @@ -126,6 +126,8 @@ export const orderCertificate = async ( subscriberId, commonName, altNames, + csr, + csrPrivateKey, keyUsages, extendedKeyUsages }: { @@ -133,6 +135,8 @@ export const orderCertificate = async ( subscriberId?: string; commonName: string; altNames?: string[]; + csr: CsrBuffer; + csrPrivateKey?: string; keyUsages?: CertKeyUsage[]; extendedKeyUsages?: CertExtendedKeyUsage[]; }, @@ -220,25 +224,11 @@ export const orderCertificate = async ( const acmeClient = new acme.Client(acmeClientOptions); - const alg = keyAlgorithmToAlgCfg(CertKeyAlgorithm.RSA_2048); - - const leafKeys = await crypto.nativeCrypto.subtle.generateKey(alg, true, ["sign", "verify"]); - const skLeafObj = crypto.nativeCrypto.KeyObject.from(leafKeys.privateKey); - const skLeaf = skLeafObj.export({ format: "pem", type: "pkcs8" }) as string; - - const [, certificateCsr] = await acme.crypto.createCsr( - { - altNames, - commonName - }, - skLeaf - ); - const appConnection = await appConnectionDAL.findById(acmeCa.configuration.dnsAppConnectionId); const connection = await decryptAppConnection(appConnection, kmsService); const pem = await acmeClient.auto({ - csr: certificateCsr, + csr, email: acmeCa.configuration.accountEmail, challengePriority: ["dns-01"], // For ACME development mode, we mock the DNS challenge API calls. So, no real DNS records are created. @@ -321,9 +311,11 @@ export const orderCertificate = async ( plainText: Buffer.from(certificateChainPem) }); - const { cipherTextBlob: encryptedPrivateKey } = await kmsEncryptor({ - plainText: Buffer.from(skLeaf) - }); + const { cipherTextBlob: encryptedPrivateKey } = csrPrivateKey + ? await kmsEncryptor({ + plainText: Buffer.from(csrPrivateKey) + }) + : { cipherTextBlob: undefined }; return (tx || certificateDAL).transaction(async (innerTx: Knex) => { const cert = await certificateDAL.create( @@ -353,13 +345,15 @@ export const orderCertificate = async ( innerTx ); - await certificateSecretDAL.create( - { - certId: cert.id, - encryptedPrivateKey - }, - innerTx - ); + if (encryptedPrivateKey !== undefined) { + await certificateSecretDAL.create( + { + certId: cert.id, + encryptedPrivateKey + }, + innerTx + ); + } return cert; }); @@ -583,12 +577,26 @@ export const AcmeCertificateAuthorityFns = ({ if (!subscriber.caId) { throw new BadRequestError({ message: "Subscriber does not have a CA" }); } + const alg = keyAlgorithmToAlgCfg(CertKeyAlgorithm.RSA_2048); + + const leafKeys = await crypto.nativeCrypto.subtle.generateKey(alg, true, ["sign", "verify"]); + const skLeafObj = crypto.nativeCrypto.KeyObject.from(leafKeys.privateKey); + const skLeaf = skLeafObj.export({ format: "pem", type: "pkcs8" }) as string; + + const [, certificateCsr] = await acme.crypto.createCsr({ + altNames: subscriber.subjectAlternativeNames, + commonName: subscriber.commonName, + key: skLeaf + }); + await orderCertificate( { caId: subscriber.caId, subscriberId: subscriber.id, commonName: subscriber.commonName, altNames: subscriber.subjectAlternativeNames, + csr: certificateCsr, + csrPrivateKey: skLeaf, keyUsages: subscriber.keyUsages as CertKeyUsage[], extendedKeyUsages: subscriber.extendedKeyUsages as CertExtendedKeyUsage[] }, From c510d841b8ab0e8df10b63333f4c3555e0f427a8 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Thu, 13 Nov 2025 12:02:05 -0800 Subject: [PATCH 54/56] Fix wrong CSR format --- backend/src/ee/services/pki-acme/pki-acme-service.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index a41f1ee5f..dadb809d0 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -729,6 +729,8 @@ export const pkiAcmeServiceFactory = ({ return { certificateId: result.certificateId }; } const { certificateAuthority } = (await certificateProfileDAL.findByIdWithConfigs(profileId, tx))!; + const csrObj = new x509.Pkcs10CertificateRequest(csr); + const csrPem = csrObj.toString("pem"); // TODO: for internal CA, we rely on the internal certificate authority service to check CSR against the template // we should check the CSR against the template here // TODO: this is pretty slow, and we are holding the transaction open for a long time, @@ -738,7 +740,7 @@ export const pkiAcmeServiceFactory = ({ caId: certificateAuthority!.id, commonName: certificateRequest.commonName!, altNames: certificateRequest.subjectAlternativeNames?.map((san) => san.value), - csr: Buffer.from(csr), + csr: Buffer.from(csrPem), // TODO: not 100% sure what are these columns for, but let's put the values for common website SSL certs for now keyUsages: [CertKeyUsage.DIGITAL_SIGNATURE, CertKeyUsage.KEY_ENCIPHERMENT, CertKeyUsage.KEY_AGREEMENT], extendedKeyUsages: [CertExtendedKeyUsage.SERVER_AUTH] From 698986080d0cd6b6ef12e4ad02901dca471140b7 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Thu, 13 Nov 2025 12:18:17 -0800 Subject: [PATCH 55/56] Fix wrong arg --- .../acme/acme-certificate-authority-fns.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts index 2dc2b045e..52761e6a0 100644 --- a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts +++ b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts @@ -583,11 +583,13 @@ export const AcmeCertificateAuthorityFns = ({ const skLeafObj = crypto.nativeCrypto.KeyObject.from(leafKeys.privateKey); const skLeaf = skLeafObj.export({ format: "pem", type: "pkcs8" }) as string; - const [, certificateCsr] = await acme.crypto.createCsr({ - altNames: subscriber.subjectAlternativeNames, - commonName: subscriber.commonName, - key: skLeaf - }); + const [, certificateCsr] = await acme.crypto.createCsr( + { + altNames: subscriber.subjectAlternativeNames, + commonName: subscriber.commonName + }, + skLeaf + ); await orderCertificate( { From 95e2d7078161ac8b8856db86befbd32bbdfe3500 Mon Sep 17 00:00:00 2001 From: Fang-Pen Lin Date: Thu, 13 Nov 2025 12:29:41 -0800 Subject: [PATCH 56/56] Fix BDD test caused by strict SANs only rule of Pebble --- backend/bdd/features/pki/acme/external-ca.feature | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backend/bdd/features/pki/acme/external-ca.feature b/backend/bdd/features/pki/acme/external-ca.feature index 6b4123b91..26bfd84ad 100644 --- a/backend/bdd/features/pki/acme/external-ca.feature +++ b/backend/bdd/features/pki/acme/external-ca.feature @@ -96,6 +96,13 @@ Feature: External CA "COMMON_NAME": "localhost" } """ + # Pebble has a strict rule to only takes SANs + Then I add subject alternative name to certificate signing request csr + """ + [ + "localhost" + ] + """ And I create a RSA private key pair as cert_key And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order