diff --git a/.github/workflows/run-backend-bdd-tests.yml b/.github/workflows/run-backend-bdd-tests.yml index bf2075864..7b54aa44e 100644 --- a/.github/workflows/run-backend-bdd-tests.yml +++ b/.github/workflows/run-backend-bdd-tests.yml @@ -49,7 +49,14 @@ jobs: run: | cp .env.example .env echo "ACME_DEVELOPMENT_MODE=true" >> .env - echo "ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES={\"localhost\": \"host.docker.internal:8087\"}" >> .env + echo "ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES={\"localhost\": \"host.docker.internal:8087\", \"infisical.com\": \"host.docker.internal:8087\", \"example.com\": \"host.docker.internal:8087\"}" >> .env + echo "BDD_NOCK_API_ENABLED=true" >> .env + # Skip upstream validation, otherwise the ACME client for the upstream will try to + # validate the DNS records, which will fail because the DNS records are not actually created. + echo "ACME_SKIP_UPSTREAM_VALIDATION=true" >> .env + # We are not using FIPS mode, need a different encryption key for BDD tests + NEW_ENCRYPTION_KEY=6c1fe4e407b8911c104518103505b218 + sed -i "s#ENCRYPTION_KEY=.*#ENCRYPTION_KEY=$NEW_ENCRYPTION_KEY#" .env # Enable ACME feature in license for BDD tests sed -i 's/pkiAcme: .*/pkiAcme: true,/g' backend/src/ee/services/license/license-fns.ts - name: Set up Docker Buildx diff --git a/backend/bdd/features/environment.py b/backend/bdd/features/environment.py index 52615036a..9a2e9f90b 100644 --- a/backend/bdd/features/environment.py +++ b/backend/bdd/features/environment.py @@ -2,16 +2,21 @@ import json import os import pathlib +import typing + import httpx from behave.runner import Context from dotenv import load_dotenv from faker import Faker import logging +from features.steps.utils import clean_all_nock, restore_nock + load_dotenv() logger = logging.getLogger(__name__) BASE_URL = os.environ.get("INFISICAL_API_URL", "http://localhost:8080") +PEBBLE_URL = os.environ.get("PEBBLE_URL", "https://pebble:14000/dir") PROJECT_ID = os.environ.get("PROJECT_ID") CERT_CA_ID = os.environ.get("CERT_CA_ID") CERT_TEMPLATE_ID = os.environ.get("CERT_TEMPLATE_ID") @@ -116,7 +121,7 @@ def bootstrap_infisical(context: Context): "name": cert_template_slug, "description": "", "subject": [{"type": "common_name", "allowed": ["*"]}], - "sans": [], + "sans": [{"type": "dns_name", "allowed": ["*"]}], "keyUsages": { "required": [], "allowed": [ @@ -184,6 +189,7 @@ def before_all(context: Context): details = bootstrap_infisical(context) context.vars = { "BASE_URL": BASE_URL, + "PEBBLE_URL": PEBBLE_URL, "PROJECT_ID": details["project"]["id"], "CERT_CA_ID": details["ca"]["id"], "CERT_TEMPLATE_ID": details["cert_template"]["id"], @@ -192,9 +198,17 @@ def before_all(context: Context): else: context.vars = { "BASE_URL": BASE_URL, + "PEBBLE_URL": PEBBLE_URL, "PROJECT_ID": PROJECT_ID, "CERT_CA_ID": CERT_CA_ID, "CERT_TEMPLATE_ID": CERT_TEMPLATE_ID, "AUTH_TOKEN": AUTH_TOKEN, } context.http_client = httpx.Client(base_url=BASE_URL) + + +def after_scenario(context: Context, scenario: typing.Any): + if hasattr(context, "web_server"): + context.web_server.shutdown_and_server_close() + clean_all_nock(context) + restore_nock(context) diff --git a/backend/bdd/features/pki/acme/challenge.feature b/backend/bdd/features/pki/acme/challenge.feature index bee46c3fb..67f73aab2 100644 --- a/backend/bdd/features/pki/acme/challenge.feature +++ b/backend/bdd/features/pki/acme/challenge.feature @@ -14,8 +14,195 @@ Feature: Challenge And I create a RSA private key pair as cert_key And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order - And I select challenge with type http-01 for domain localhost from order at order as challenge + And I select challenge with type http-01 for domain localhost from order in order as challenge And I serve challenge response for challenge at localhost And I tell ACME server that challenge is ready to be verified And I poll and finalize the ACME order order as finalized_order And the value finalized_order.body with jq ".status" should be equal to "valid" + And I parse the full-chain certificate from order finalized_order as cert + And the value cert with jq ".subject.common_name" should be equal to "localhost" + + Scenario: Validate challenges for multiple domains + Given I have an ACME cert profile as "acme_profile" + When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" + Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account + When I create certificate signing request as csr + Then I add names to certificate signing request csr + """ + { + "COMMON_NAME": "localhost" + } + """ + And I add subject alternative name to certificate signing request csr + """ + [ + "infisical.com", + "example.com" + ] + """ + And I create a RSA private key pair as cert_key + And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format + And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order + And I pass all challenges with type http-01 for order in order + And I poll and finalize the ACME order order as finalized_order + And the value finalized_order.body with jq ".status" should be equal to "valid" + And I parse the full-chain certificate from order finalized_order as cert + And the value cert with jq ".subject.common_name" should be equal to "localhost" + And the value cert with jq "[.extensions.subjectAltName.general_names.[].value] | sort" should be equal to json + """ + [ + "example.com", + "infisical.com" + ] + """ + + Scenario: Did not finish all challenges + Given I have an ACME cert profile as "acme_profile" + When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" + Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account + When I create certificate signing request as csr + Then I add names to certificate signing request csr + """ + { + "COMMON_NAME": "localhost" + } + """ + And I add subject alternative name to certificate signing request csr + """ + [ + "infisical.com" + ] + """ + And I create a RSA private key pair as cert_key + And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format + And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order + And I select challenge with type http-01 for domain localhost from order in order as challenge + And I serve challenge response for challenge at localhost + And I tell ACME server that challenge is ready to be verified + + # the localhost auth should be valid + And I memorize order with jq ".authorizations | map(select(.body.identifier.value == "localhost")) | first | .uri" as localhost_auth + And I peak and memorize the next nonce as nonce + When I send a raw ACME request to "{localhost_auth}" + """ + { + "protected": { + "alg": "RS256", + "nonce": "{nonce}", + "url": "{localhost_auth}", + "kid": "{acme_account.uri}" + } + } + """ + Then the value response.status_code should be equal to 200 + And the value response with jq ".status" should be equal to "valid" + + # the infisical.com auth should still be pending + And I memorize order with jq ".authorizations | map(select(.body.identifier.value == "infisical.com")) | first | .uri" as infisical_auth + And I memorize response.headers with jq ".["replay-nonce"]" as nonce + When I send a raw ACME request to "{infisical_auth}" + """ + { + "protected": { + "alg": "RS256", + "nonce": "{nonce}", + "url": "{infisical_auth}", + "kid": "{acme_account.uri}" + } + } + """ + Then the value response.status_code should be equal to 200 + And the value response with jq ".status" should be equal to "pending" + + # the order should be pending as well + And I memorize response.headers with jq ".["replay-nonce"]" as nonce + When I send a raw ACME request to "{order.uri}" + """ + { + "protected": { + "alg": "RS256", + "nonce": "{nonce}", + "url": "{order.uri}", + "kid": "{acme_account.uri}" + } + } + """ + Then the value response.status_code should be equal to 200 + And the value response with jq ".status" should be equal to "pending" + + # finalize should not be allowed when all auths are not valid yet + And I memorize response.headers with jq ".["replay-nonce"]" as nonce + When I send a raw ACME request to "{order.body.finalize}" + """ + { + "protected": { + "alg": "RS256", + "nonce": "{nonce}", + "url": "{order.body.finalize}", + "kid": "{acme_account.uri}" + }, + "payload": { + "csr": "{csr_pem}" + } + } + """ + Then the value response.status_code should be equal to 400 + Then the value response with jq ".status" should be equal to 400 + Then the value response with jq ".type" should be equal to "urn:ietf:params:acme:error:orderNotReady" + Then the value response with jq ".detail" should be equal to "ACME order is not ready" + + Scenario: CSR names mismatch with order identifier + Given I have an ACME cert profile as "acme_profile" + When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" + Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account + When I create certificate signing request as csr + Then I add names to certificate signing request csr + """ + { + "COMMON_NAME": "example.com" + } + """ + And I create a RSA private key pair as cert_key + And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format + Then I peak and memorize the next nonce as nonce + When I send a raw ACME request to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/new-order" + """ + { + "protected": { + "alg": "RS256", + "nonce": "{nonce}", + "url": "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/new-order", + "kid": "{acme_account.uri}" + }, + "payload": { + "identifiers": [ + { "type": "dns", "value": "localhost" }, + { "type": "dns", "value": "infisical.com" } + ] + } + } + """ + Then the value response.status_code should be equal to 201 + And I memorize response with jq ".finalize" as finalize_url + And I memorize response.headers with jq ".["replay-nonce"]" as nonce + And I memorize response as order + And I pass all challenges with type http-01 for order in order + And I encode CSR csr_pem as JOSE Base-64 DER as base64_csr_der + When I send a raw ACME request to "{finalize_url}" + """ + { + "protected": { + "alg": "RS256", + "nonce": "{nonce}", + "url": "{finalize_url}", + "kid": "{acme_account.uri}" + }, + "payload": { + "csr": "{base64_csr_der}" + } + } + """ + Then the value response.status_code should be equal to 400 + And the value response with jq ".status" should be equal to 400 + And the value response with jq ".type" should be equal to "urn:ietf:params:acme:error:badCSR" + And the value response with jq ".detail" should be equal to "Invalid CSR: Common name + SANs mismatch with order identifiers" diff --git a/backend/bdd/features/pki/acme/external-ca.feature b/backend/bdd/features/pki/acme/external-ca.feature new file mode 100644 index 000000000..26bfd84ad --- /dev/null +++ b/backend/bdd/features/pki/acme/external-ca.feature @@ -0,0 +1,180 @@ +Feature: External CA + + Scenario: Issue a certificate from an external CA + Given I create a Cloudflare connection as cloudflare + Then I memorize cloudflare with jq ".appConnection.id" as app_conn_id + Given I create a external ACME CA with the following config as ext_ca + """ + { + "dnsProviderConfig": { + "provider": "cloudflare", + "hostedZoneId": "MOCK_ZONE_ID" + }, + "directoryUrl": "{PEBBLE_URL}", + "accountEmail": "fangpen@infisical.com", + "dnsAppConnectionId": "{app_conn_id}", + "eabKid": "", + "eabHmacKey": "" + } + """ + Then I memorize ext_ca with jq ".id" as ext_ca_id + Given I create a certificate template with the following config as cert_template + """ + { + "subject": [ + { + "type": "common_name", + "allowed": [ + "*" + ] + } + ], + "sans": [ + { + "type": "dns_name", + "allowed": [ + "*" + ] + } + ], + "keyUsages": { + "required": [], + "allowed": [ + "digital_signature", + "key_encipherment", + "non_repudiation", + "data_encipherment", + "key_agreement", + "key_cert_sign", + "crl_sign", + "encipher_only", + "decipher_only" + ] + }, + "extendedKeyUsages": { + "required": [], + "allowed": [ + "client_auth", + "server_auth", + "code_signing", + "email_protection", + "ocsp_signing", + "time_stamping" + ] + }, + "algorithms": { + "signature": [ + "SHA256-RSA", + "SHA512-RSA", + "SHA384-ECDSA", + "SHA384-RSA", + "SHA256-ECDSA", + "SHA512-ECDSA" + ], + "keyAlgorithm": [ + "RSA-2048", + "RSA-4096", + "ECDSA-P384", + "RSA-3072", + "ECDSA-P256", + "ECDSA-P521" + ] + }, + "validity": { + "max": "365d" + } + } + """ + Then I memorize cert_template with jq ".certificateTemplate.id" as cert_template_id + Given I create an ACME profile with ca {ext_ca_id} and template {cert_template_id} as "acme_profile" + When I have an ACME client connecting to "{BASE_URL}/api/v1/pki/acme/profiles/{acme_profile.id}/directory" + Then I register a new ACME account with email fangpen@infisical.com and EAB key id "{acme_profile.eab_kid}" with secret "{acme_profile.eab_secret}" as acme_account + When I create certificate signing request as csr + Then I add names to certificate signing request csr + """ + { + "COMMON_NAME": "localhost" + } + """ + # Pebble has a strict rule to only takes SANs + Then I add subject alternative name to certificate signing request csr + """ + [ + "localhost" + ] + """ + And I create a RSA private key pair as cert_key + And I sign the certificate signing request csr with private key cert_key and output it as csr_pem in PEM format + And I submit the certificate signing request PEM csr_pem certificate order to the ACME server as order + And I select challenge with type http-01 for domain localhost from order in order as challenge + And I serve challenge response for challenge at localhost + And I tell ACME server that challenge is ready to be verified + Given I intercept outgoing requests + """ + [ + { + "scope": "https://api.cloudflare.com:443", + "method": "POST", + "path": "/client/v4/zones/MOCK_ZONE_ID/dns_records", + "status": 200, + "response": { + "result": { + "id": "A2A6347F-88B5-442D-9798-95E408BC7701", + "name": "Mock Account", + "type": "standard", + "settings": { + "enforce_twofactor": false, + "api_access_enabled": null, + "access_approval_expiry": null, + "abuse_contact_email": null, + "user_groups_ui_beta": false + }, + "legacy_flags": { + "enterprise_zone_quota": { + "maximum": 0, + "current": 0, + "available": 0 + } + }, + "created_on": "2013-04-18T00:41:02.215243Z" + }, + "success": true, + "errors": [], + "messages": [] + }, + "responseIsBinary": false + }, + { + "scope": "https://api.cloudflare.com:443", + "method": "GET", + "path": { + "regex": "/client/v4/zones/[^/]+/dns_records\\?" + }, + "status": 200, + "response": { + "result": [], + "success": true, + "errors": [], + "messages": [], + "result_info": { + "page": 1, + "per_page": 100, + "count": 0, + "total_count": 0, + "total_pages": 1 + } + }, + "responseIsBinary": false + } + ] + """ + Then I poll and finalize the ACME order order as finalized_order + And the value finalized_order.body with jq ".status" should be equal to "valid" + And I parse the full-chain certificate from order finalized_order as cert + # Note: somehow Pebble is issuing a cert without common name but just SANs + And the value cert with jq "[.extensions.subjectAltName.general_names.[].value] | sort" should be equal to json + """ + [ + "localhost" + ] + """ \ No newline at end of file diff --git a/backend/bdd/features/steps/pki_acme.py b/backend/bdd/features/steps/pki_acme.py index d9004e0ba..46b10c13e 100644 --- a/backend/bdd/features/steps/pki_acme.py +++ b/backend/bdd/features/steps/pki_acme.py @@ -1,14 +1,10 @@ import json import logging import re -import threading import urllib.parse import acme.client -import httpx import jq -import requests -import glom from faker import Faker from acme import client from acme import messages @@ -19,7 +15,6 @@ from behave import given from behave import when from behave import then from josepy.jwk import JWKRSA -from josepy import JSONObjectWithFields from josepy import json_util from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa @@ -27,6 +22,12 @@ from cryptography import x509 from cryptography.x509.oid import NameOID from cryptography.hazmat.primitives import hashes +from features.steps.utils import define_nock, clean_all_nock, restore_nock +from utils import replace_vars, with_nocks +from utils import eval_var +from utils import prepare_headers + + ACC_KEY_BITS = 2048 ACC_KEY_PUBLIC_EXPONENT = 65537 logger = logging.getLogger(__name__) @@ -40,96 +41,6 @@ class AcmeProfile: self.eab_secret = eab_secret -def replace_vars(payload: dict | list | int | float | str, vars: dict): - if isinstance(payload, dict): - return { - replace_vars(key, vars): replace_vars(value, vars) - for key, value in payload.items() - } - elif isinstance(payload, list): - return [replace_vars(item, vars) for item in payload] - elif isinstance(payload, str): - return payload.format(**vars) - else: - return payload - - -def parse_glom_path(path_str: str) -> glom.Path: - """ - Parse a glom path string with 'attr[index]' syntax into a Path object. - - Examples: - >>> parse_glom_path('authorizations[0]') == Path('authorizations', 0) - True - >>> parse_glom_path('data.items[1].name') == Path('data', 'items', 1, 'name') - True - >>> parse_glom_path('user.addresses[0].street') == Path('user', 'addresses', 0, 'street') - True - """ - parts = [] - - # Split by dots, but preserve bracketed content - tokens = re.split(r"(? dict | None: - headers = {} - auth_token = getattr(context, "auth_token", None) - if auth_token is not None: - headers["authorization"] = "Bearer {}".format(auth_token) - if not headers: - return None - return headers - - @given("I make a random {faker_type} as {var_name}") def step_impl(context: Context, faker_type: str, var_name: str): context.vars[var_name] = getattr(faker, faker_type)() @@ -177,6 +88,198 @@ def step_impl(context: Context, profile_var: str): ) +@given("I create a Cloudflare connection as {var_name}") +def step_impl(context: Context, var_name: str): + jwt_token = context.vars["AUTH_TOKEN"] + conn_slug = faker.slug() + mock_account_id = "MOCK_ACCOUNT_ID" + with with_nocks( + context, + definitions=[ + { + "scope": "https://api.cloudflare.com:443", + "method": "GET", + "path": f"/client/v4/accounts/{mock_account_id}", + "status": 200, + "response": { + "result": { + "id": "A2A6347F-88B5-442D-9798-95E408BC7701", + "name": "Mock Account", + "type": "standard", + "settings": { + "enforce_twofactor": True, + "api_access_enabled": None, + "access_approval_expiry": None, + "abuse_contact_email": None, + "user_groups_ui_beta": False, + }, + "legacy_flags": { + "enterprise_zone_quota": { + "maximum": 0, + "current": 0, + "available": 0, + } + }, + "created_on": "2013-04-18T00:41:02.215243Z", + }, + "success": True, + "errors": [], + "messages": [], + }, + "responseIsBinary": False, + } + ], + ): + response = context.http_client.post( + "/api/v1/app-connections/cloudflare", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json={ + "name": conn_slug, + "description": "", + "method": "api-token", + "credentials": { + "apiToken": "MOCK_API_TOKEN", + "accountId": mock_account_id, + }, + }, + ) + response.raise_for_status() + context.vars[var_name] = response + + +@given("I create a external ACME CA with the following config as {var_name}") +def step_impl(context: Context, var_name: str): + jwt_token = context.vars["AUTH_TOKEN"] + ca_slug = faker.slug() + config = replace_vars(json.loads(context.text), context.vars) + response = context.http_client.post( + "/api/v1/pki/ca/acme", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json={ + "projectId": context.vars["PROJECT_ID"], + "name": ca_slug, + "type": "acme", + "status": "active", + "enableDirectIssuance": True, + "configuration": config, + }, + ) + response.raise_for_status() + context.vars[var_name] = response + + +@given("I create a certificate template with the following config as {var_name}") +def step_impl(context: Context, var_name: str): + jwt_token = context.vars["AUTH_TOKEN"] + template_slug = faker.slug() + config = replace_vars(json.loads(context.text), context.vars) + response = context.http_client.post( + "/api/v2/certificate-templates", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json={ + "projectId": context.vars["PROJECT_ID"], + "name": template_slug, + "description": "", + } + | config, + ) + response.raise_for_status() + context.vars[var_name] = response + + +@given( + 'I create an ACME profile with ca {ca_id} and template {template_id} as "{profile_var}"' +) +def step_impl(context: Context, ca_id: str, template_id: str, profile_var: str): + profile_slug = faker.slug() + jwt_token = context.vars["AUTH_TOKEN"] + response = context.http_client.post( + "/api/v1/pki/certificate-profiles", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json={ + "projectId": context.vars["PROJECT_ID"], + "slug": profile_slug, + "description": "ACME Profile created by BDD test", + "enrollmentType": "acme", + "caId": replace_vars(ca_id, context.vars), + "certificateTemplateId": replace_vars(template_id, context.vars), + "acmeConfig": {}, + }, + ) + response.raise_for_status() + resp_json = response.json() + profile_id = resp_json["certificateProfile"]["id"] + kid = profile_id + + response = context.http_client.get( + f"/api/v1/pki/certificate-profiles/{profile_id}/acme/eab-secret/reveal", + headers=dict(authorization="Bearer {}".format(jwt_token)), + ) + response.raise_for_status() + resp_json = response.json() + secret = resp_json["eabSecret"] + + context.vars[profile_var] = AcmeProfile( + profile_id, + eab_kid=kid, + eab_secret=secret, + ) + + +@given('I have an ACME cert profile with external ACME CA as "{profile_var}"') +def step_impl(context: Context, profile_var: str): + profile_id = context.vars.get("PROFILE_ID") + secret = context.vars.get("EAB_SECRET") + if profile_id is not None and secret is not None: + kid = profile_id + else: + profile_slug = faker.slug() + jwt_token = context.vars["AUTH_TOKEN"] + response = context.http_client.post( + "/api/v1/pki/certificate-profiles", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json={ + "projectId": context.vars["PROJECT_ID"], + "slug": profile_slug, + "description": "ACME Profile created by BDD test", + "enrollmentType": "acme", + "caId": context.vars["CERT_CA_ID"], + "certificateTemplateId": context.vars["CERT_TEMPLATE_ID"], + "acmeConfig": {}, + }, + ) + response.raise_for_status() + resp_json = response.json() + profile_id = resp_json["certificateProfile"]["id"] + kid = profile_id + + response = context.http_client.get( + f"/api/v1/pki/certificate-profiles/{profile_id}/acme/eab-secret/reveal", + headers=dict(authorization="Bearer {}".format(jwt_token)), + ) + response.raise_for_status() + resp_json = response.json() + secret = resp_json["eabSecret"] + + context.vars[profile_var] = AcmeProfile( + profile_id, + eab_kid=kid, + eab_secret=secret, + ) + + +@given("I intercept outgoing requests") +def step_impl(context: Context): + definitions = replace_vars(json.loads(context.text), context.vars) + define_nock(context, definitions) + + +@then("I reset requests interceptions") +def step_impl(context: Context): + clean_all_nock(context) + restore_nock(context) + + @given("I use {token_var} for authentication") def step_impl(context: Context, token_var: str): context.auth_token = eval_var(context, token_var) @@ -387,6 +490,15 @@ def step_impl(context: Context, url: str): send_raw_acme_req(context, url) +@then( + "I encode CSR {pem_var} as JOSE Base-64 DER as {var_name}", +) +def step_impl(context: Context, pem_var: str, var_name: str): + csr = eval_var(context, pem_var) + parsed_csr = x509.load_pem_x509_csr(csr) + context.vars[var_name] = json_util.encode_csr(parsed_csr) + + @then( "I submit the certificate signing request PEM {pem_var} certificate order to the ACME server as {order_var}" ) @@ -569,51 +681,61 @@ def step_impl(context: Context, var_path: str): print(json.dumps(value.json(), indent=2)) -@then( - "I select challenge with type {challenge_type} for domain {domain} from order at {var_path} as {challenge_var}" -) -def step_impl( +def select_challenge( context: Context, challenge_type: str, + order_var_path: str, domain: str, - var_path: str, - challenge_var: str, ): - order = eval_var(context, var_path, as_json=False) + acme_client = context.acme_client + order = eval_var(context, order_var_path, as_json=False) + if isinstance(order, dict): + order_body = messages.Order.from_json(order) + order = messages.OrderResource( + body=order_body, + authorizations=[ + acme_client._authzr_from_response( + acme_client._post_as_get(url), uri=url + ) + for url in order_body.authorizations + ], + ) if not isinstance(order, messages.OrderResource): raise ValueError( - f"Expected OrderResource but got {type(order)!r} at {var_path!r}" + f"Expected OrderResource but got {type(order)!r} at {order_var_path!r}" ) auths = list( filter(lambda o: o.body.identifier.value == domain, order.authorizations) ) if not auths: raise ValueError( - f"Authorization for domain {domain!r} not found in {var_path!r}" + f"Authorization for domain {domain!r} not found in {order_var_path!r}" ) if len(auths) > 1: raise ValueError( - f"More than one order for domain {domain!r} found in {var_path!r}" + f"More than one order for domain {domain!r} found in {order_var_path!r}" ) auth = auths[0] challenges = list(filter(lambda a: a.typ == challenge_type, auth.body.challenges)) if not challenges: raise ValueError( - f"Authorization type {challenge_type!r} not found in {var_path!r}" + f"Authorization type {challenge_type!r} not found in {order_var_path!r}" ) if len(challenges) > 1: raise ValueError( - f"More than one authorization for type {challenge_type!r} found in {var_path!r}" + f"More than one authorization for type {challenge_type!r} found in {order_var_path!r}" ) - context.vars[challenge_var] = challenges[0] + return challenges[0] -@then("I serve challenge response for {var_path} at {hostname}") -def step_impl(context: Context, var_path: str, hostname: str): - if hostname != "localhost": - raise ValueError("Currently only localhost is supported") - challenge = eval_var(context, var_path, as_json=False) +def serve_challenge( + context: Context, + challenge: messages.ChallengeBody, +): + if hasattr(context, "web_server"): + context.web_server.shutdown_and_server_close() + response, validation = challenge.response_and_validation( context.acme_client.net.key ) @@ -622,19 +744,101 @@ def step_impl(context: Context, var_path: str, hostname: str): ) # TODO: make port configurable servers = standalone.HTTP01DualNetworkedServers(("0.0.0.0", 8087), {resource}) - # Start client standalone web server. - web_server = threading.Thread(name="web_server", target=servers.serve_forever) - web_server.daemon = True - web_server.start() - context.web_server = web_server + servers.serve_forever() + context.web_server = servers + + +def notify_challenge_ready(context: Context, challenge: messages.ChallengeBody): + acme_client = context.acme_client + response, validation = challenge.response_and_validation(acme_client.net.key) + acme_client.answer_challenge(challenge, response) + + +@then( + "I select challenge with type {challenge_type} for domain {domain} from order in {var_path} as {challenge_var}" +) +def step_impl( + context: Context, + challenge_type: str, + domain: str, + var_path: str, + challenge_var: str, +): + challenge = select_challenge( + context=context, + challenge_type=challenge_type, + domain=domain, + order_var_path=var_path, + ) + context.vars[challenge_var] = challenge + + +@then("I pass all challenges with type {challenge_type} for order in {order_var_path}") +def step_impl( + context: Context, + challenge_type: str, + order_var_path: str, +): + acme_client = context.acme_client + order = eval_var(context, order_var_path, as_json=False) + if isinstance(order, dict): + order_body = messages.Order.from_json(order) + order = messages.OrderResource( + body=order_body, + authorizations=[ + acme_client._authzr_from_response( + acme_client._post_as_get(url), uri=url + ) + for url in order_body.authorizations + ], + ) + if not isinstance(order, messages.OrderResource): + raise ValueError( + f"Expected OrderResource but got {type(order)!r} at {order_var_path!r}" + ) + + for domain in order.body.identifiers: + logger.info( + "Selecting challenge for domain %s with type %s ...", + domain.value, + challenge_type, + ) + challenge = select_challenge( + context=context, + challenge_type=challenge_type, + domain=domain.value, + order_var_path=order_var_path, + ) + logger.info( + "Found challenge for domain %s with type %s, challenge=%s", + domain.value, + challenge_type, + challenge.uri, + ) + + logger.info( + "Serving challenge for domain %s with type %s ...", + domain.value, + challenge_type, + ) + serve_challenge(context=context, challenge=challenge) + + logger.info( + "Notifying challenge for domain %s with type %s ...", domain, challenge_type + ) + notify_challenge_ready(context=context, challenge=challenge) + + +@then("I serve challenge response for {var_path} at {hostname}") +def step_impl(context: Context, var_path: str, hostname: str): + challenge = eval_var(context, var_path, as_json=False) + serve_challenge(context=context, challenge=challenge) @then("I tell ACME server that {var_path} is ready to be verified") def step_impl(context: Context, var_path: str): challenge = eval_var(context, var_path, as_json=False) - acme_client = context.acme_client - response, validation = challenge.response_and_validation(acme_client.net.key) - acme_client.answer_challenge(challenge, response) + notify_challenge_ready(context=context, challenge=challenge) @then("I poll and finalize the ACME order {var_path} as {finalized_var}") @@ -643,3 +847,10 @@ def step_impl(context: Context, var_path: str, finalized_var: str): acme_client = context.acme_client finalized_order = acme_client.poll_and_finalize(order) context.vars[finalized_var] = finalized_order + + +@then("I parse the full-chain certificate from order {order_var_path} as {cert_var}") +def step_impl(context: Context, order_var_path: str, cert_var: str): + order = eval_var(context, order_var_path, as_json=False) + cert = x509.load_pem_x509_certificate(order.fullchain_pem.encode()) + context.vars[cert_var] = cert diff --git a/backend/bdd/features/steps/utils.py b/backend/bdd/features/steps/utils.py new file mode 100644 index 000000000..4ee7c8921 --- /dev/null +++ b/backend/bdd/features/steps/utils.py @@ -0,0 +1,302 @@ +from cryptography import x509 +from cryptography.hazmat.primitives import hashes +from cryptography.x509.oid import NameOID +import logging +import re +import contextlib + +import httpx +import requests +import requests.structures +import glom +from faker import Faker +from behave.runner import Context +from josepy import JSONObjectWithFields + +ACC_KEY_BITS = 2048 +ACC_KEY_PUBLIC_EXPONENT = 65537 +logger = logging.getLogger(__name__) +faker = Faker() + + +class AcmeProfile: + def __init__(self, id: str, eab_kid: str, eab_secret: str): + self.id = id + self.eab_kid = eab_kid + self.eab_secret = eab_secret + + +def replace_vars(payload: dict | list | int | float | str, vars: dict): + if isinstance(payload, dict): + return { + replace_vars(key, vars): replace_vars(value, vars) + for key, value in payload.items() + } + elif isinstance(payload, list): + return [replace_vars(item, vars) for item in payload] + elif isinstance(payload, str): + return payload.format(**vars) + else: + return payload + + +def parse_glom_path(path_str: str) -> glom.Path: + """ + Parse a glom path string with 'attr[index]' syntax into a Path object. + + Examples: + >>> parse_glom_path('authorizations[0]') == Path('authorizations', 0) + True + >>> parse_glom_path('data.items[1].name') == Path('data', 'items', 1, 'name') + True + >>> parse_glom_path('user.addresses[0].street') == Path('user', 'addresses', 0, 'street') + True + """ + parts = [] + + # Split by dots, but preserve bracketed content + tokens = re.split(r"(? dict | None: + headers = {} + auth_token = getattr(context, "auth_token", None) + if auth_token is not None: + headers["authorization"] = "Bearer {}".format(auth_token) + if not headers: + return None + return headers + + +def x509_cert_to_dict(cert: x509.Certificate) -> dict: + """ + Convert a cryptography.x509.Certificate to a JSON-serializable nested dict + with human-readable keys. + """ + + def oid_to_name(oid): + # Map known OIDs to human-readable names + mapping = { + NameOID.COMMON_NAME: "common_name", + NameOID.ORGANIZATION_NAME: "organization", + NameOID.ORGANIZATIONAL_UNIT_NAME: "organizational_unit", + NameOID.COUNTRY_NAME: "country", + NameOID.LOCALITY_NAME: "locality", + NameOID.STATE_OR_PROVINCE_NAME: "state_or_province", + NameOID.EMAIL_ADDRESS: "email_address", + NameOID.SERIAL_NUMBER: "serial_number", + NameOID.SURNAME: "surname", + NameOID.GIVEN_NAME: "given_name", + NameOID.TITLE: "title", + NameOID.JURISDICTION_COUNTRY_NAME: "jurisdiction_country", + NameOID.JURISDICTION_STATE_OR_PROVINCE_NAME: "jurisdiction_state", + NameOID.JURISDICTION_LOCALITY_NAME: "jurisdiction_locality", + NameOID.BUSINESS_CATEGORY: "business_category", + NameOID.POSTAL_CODE: "postal_code", + NameOID.STREET_ADDRESS: "street_address", + NameOID.DOMAIN_COMPONENT: "domain_component", + NameOID.USER_ID: "user_id", + # Add more as needed + } + return mapping.get(oid, oid.dotted_string) + + def name_to_dict(name: x509.Name) -> dict: + return {oid_to_name(attr.oid): attr.value for attr in name} + + def dns_to_dict(dns: x509.DNSName) -> dict: + return dict(value=dns.value) + + def extension_to_dict(ext): + if isinstance(ext.value, x509.SubjectAlternativeName): + return { + "critical": ext.critical, + "general_names": [dns_to_dict(gn) for gn in ext.value], + } + elif isinstance(ext.value, x509.BasicConstraints): + return { + "critical": ext.critical, + "ca": ext.value.ca, + "path_length": ext.value.path_length, + } + elif isinstance(ext.value, x509.KeyUsage): + return { + "critical": ext.critical, + **{ + field.lower(): getattr(ext.value, field) + for field in [ + "digital_signature", + "content_commitment", + "key_encipherment", + "data_encipherment", + "key_agreement", + "key_cert_sign", + "crl_sign", + # TODO: deal with error: "ValueError: encipher_only is undefined unless key_agreement is true" + # "encipher_only", + # "decipher_only", + ] + if getattr(ext.value, field) is not None + }, + } + elif isinstance(ext.value, x509.ExtendedKeyUsage): + return { + "critical": ext.critical, + "usages": [eku.dotted_string for eku in ext.value], + } + elif isinstance(ext.value, x509.CRLDistributionPoints): + return { + "critical": ext.critical, + "distribution_points": [ + { + "full_name": [str(uri) for uri in dp.full_name] + if dp.full_name + else None, + "crl_issuer": [str(issuer) for issuer in dp.crl_issuer] + if dp.crl_issuer + else None, + "reasons": [r.name for r in dp.reasons] if dp.reasons else None, + } + for dp in ext.value + ], + } + elif isinstance(ext.value, x509.AuthorityKeyIdentifier): + return { + "critical": ext.critical, + "key_identifier": ext.value.key_identifier.hex() + if ext.value.key_identifier + else None, + "authority_cert_issuer": [ + str(n) for n in ext.value.authority_cert_issuer + ] + if ext.value.authority_cert_issuer + else None, + "authority_cert_serial_number": ext.value.authority_cert_serial_number, + } + elif isinstance(ext.value, x509.SubjectKeyIdentifier): + return {"critical": ext.critical, "digest": ext.value.digest.hex()} + else: + return { + "critical": ext.critical, + "oid": ext.oid.dotted_string, + "value": str(ext.value), + } + + # Build the main dict + result = dict( + version=cert.version.name, + serial_number=cert.serial_number, + signature_algorithm=cert.signature_algorithm_oid._name, + issuer=name_to_dict(cert.issuer), + subject=name_to_dict(cert.subject), + validity={ + "not_valid_before": cert.not_valid_before.isoformat(), + "not_valid_after": cert.not_valid_after.isoformat(), + }, + public_key={ + "key_size": cert.public_key().key_size, + }, + extensions={ + ext.oid._name + if hasattr(ext.oid, "_name") and ext.oid._name + else ext.oid.dotted_string: extension_to_dict(ext) + for ext in cert.extensions + }, + fingerprint={ + "sha1": cert.fingerprint(hashes.SHA1()).hex(), + "sha256": cert.fingerprint(hashes.SHA256()).hex(), + }, + ) + + return result + + +def define_nock(context: Context, definitions: list[dict]): + jwt_token = context.vars["AUTH_TOKEN"] + response = context.http_client.post( + "/api/v1/bdd-nock/define", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json=dict(definitions=definitions), + ) + response.raise_for_status() + + +def restore_nock(context: Context): + jwt_token = context.vars["AUTH_TOKEN"] + response = context.http_client.post( + "/api/v1/bdd-nock/restore", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json=dict(), + ) + response.raise_for_status() + + +def clean_all_nock(context: Context): + jwt_token = context.vars["AUTH_TOKEN"] + response = context.http_client.post( + "/api/v1/bdd-nock/clean-all", + headers=dict(authorization="Bearer {}".format(jwt_token)), + json=dict(), + ) + response.raise_for_status() + + +@contextlib.contextmanager +def with_nocks(context: Context, definitions: list[dict]): + try: + define_nock(context, definitions) + yield + finally: + clean_all_nock(context) + restore_nock(context) diff --git a/backend/bdd/pebble/localhost/cert.pem b/backend/bdd/pebble/localhost/cert.pem new file mode 100644 index 000000000..9117526df --- /dev/null +++ b/backend/bdd/pebble/localhost/cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIICBDCCAYmgAwIBAgIIHZvNVJSPdsYwCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV +bWluaWNhIHJvb3QgY2EgN2ZlMDQwMB4XDTI1MTExMzAwMzAxMloXDTI3MTIxMzAw +MzAxMlowFDESMBAGA1UEAxMJbG9jYWxob3N0MHYwEAYHKoZIzj0CAQYFK4EEACID +YgAE2V5oM5JimqDjzEfH10cKu6L8eQ9rxzkULbIJRFFuuXtKQQwkcAW8L4UuMkmG +lu5hFCBR8saHDpISuAyYLYqsddxwndxmGT3zyw6oU+8oXWX0tThL0KgajmZckOfR +ysYpo4GbMIGYMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI +KwYBBQUHAwIwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBSIDfQe2L6+9aYyBFbd +t0S51xW3UDA4BgNVHREEMTAvgglsb2NhbGhvc3SCBnBlYmJsZYIUaG9zdC5kb2Nr +ZXIuaW50ZXJuYWyHBH8AAAEwCgYIKoZIzj0EAwMDaQAwZgIxAPkeGVzCDKuJYd/1 +87+lXXtlMHrW7F+Rn1kyR8SBud2hDt5r3a+ZZ8IQ9aHazRia/AIxAOI4I41jwxf0 +86i7fKx8of4s/CBc4+PF0hbCBkmen3aKuiZ7ueYuEsSNT6zHV2xc2w== +-----END CERTIFICATE----- diff --git a/backend/bdd/pebble/localhost/key.pem b/backend/bdd/pebble/localhost/key.pem new file mode 100644 index 000000000..93b93eada --- /dev/null +++ b/backend/bdd/pebble/localhost/key.pem @@ -0,0 +1,6 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDBx7d0VqxwTYcJajFgz +ja0PExBmxdZjEQRfGCMQY8GfHa0WpBUEwVtBD6XOGE5xZB2hZANiAATZXmgzkmKa +oOPMR8fXRwq7ovx5D2vHORQtsglEUW65e0pBDCRwBbwvhS4ySYaW7mEUIFHyxocO +khK4DJgtiqx13HCd3GYZPfPLDqhT7yhdZfS1OEvQqBqOZlyQ59HKxik= +-----END PRIVATE KEY----- diff --git a/backend/bdd/pebble/pebble-config.json b/backend/bdd/pebble/pebble-config.json new file mode 100644 index 000000000..013f6ff64 --- /dev/null +++ b/backend/bdd/pebble/pebble-config.json @@ -0,0 +1,28 @@ +{ + "pebble": { + "listenAddress": "0.0.0.0:14000", + "managementListenAddress": "0.0.0.0:15000", + "certificate": "/var/data/pebble/localhost/cert.pem", + "privateKey": "/var/data/pebble/localhost/key.pem", + "httpPort": 5002, + "tlsPort": 5001, + "ocspResponderURL": "", + "externalAccountBindingRequired": false, + "domainBlocklist": ["blocked-domain.example"], + "retryAfter": { + "authz": 3, + "order": 5 + }, + "keyAlgorithm": "ecdsa", + "profiles": { + "default": { + "description": "The profile you know and love", + "validityPeriod": 7776000 + }, + "shortlived": { + "description": "A short-lived cert profile, without actual enforcement", + "validityPeriod": 518400 + } + } + } +} \ No newline at end of file diff --git a/backend/bdd/pebble/pebble.minica.key.pem b/backend/bdd/pebble/pebble.minica.key.pem new file mode 100644 index 000000000..322b4e88e --- /dev/null +++ b/backend/bdd/pebble/pebble.minica.key.pem @@ -0,0 +1,6 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDDnPx90G0J4ba0CMTrh +AT0kJkRGyhv5ePWyobdT75za/I9MpU/VsC8BG5uJBraxiSOhZANiAAQWEiTINq0t +j+6Qiyzin74FU4/zLNuEs1FnipFn+Vb1W8qhvbBwLOGsANpaHIg4dpR+CghfccRQ +0kQm/AMgj08VXvta6vV7aQ8yk+/Cp6l4SVQ9GzizHiJ//Qb71vrXbco= +-----END PRIVATE KEY----- diff --git a/backend/bdd/pebble/pebble.minica.pem b/backend/bdd/pebble/pebble.minica.pem new file mode 100644 index 000000000..030ca32bb --- /dev/null +++ b/backend/bdd/pebble/pebble.minica.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB+zCCAYKgAwIBAgIIf+BA3XMRozcwCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV +bWluaWNhIHJvb3QgY2EgN2ZlMDQwMCAXDTI1MTExMzAwMzAxMloYDzIxMjUxMTEz +MDAzMDEyWjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSA3ZmUwNDAwdjAQBgcq +hkjOPQIBBgUrgQQAIgNiAAQWEiTINq0tj+6Qiyzin74FU4/zLNuEs1FnipFn+Vb1 +W8qhvbBwLOGsANpaHIg4dpR+CghfccRQ0kQm/AMgj08VXvta6vV7aQ8yk+/Cp6l4 +SVQ9GzizHiJ//Qb71vrXbcqjgYYwgYMwDgYDVR0PAQH/BAQDAgKEMB0GA1UdJQQW +MBQGCCsGAQUFBwMBBggrBgEFBQcDAjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1Ud +DgQWBBSIDfQe2L6+9aYyBFbdt0S51xW3UDAfBgNVHSMEGDAWgBSIDfQe2L6+9aYy +BFbdt0S51xW3UDAKBggqhkjOPQQDAwNnADBkAjAK2OUUVHs2LVqwyLEqIrXbc3gw +5r5p9TC9asqPN8vJxlTRStrXnJQRSQ2KoWztiSICMEV5jZGVk6TaUwlqcGmXEmGr +iFeQ3rXLaRw8XKMqj7+EiwaCD1o2wLgzny/21NFtxQ== +-----END CERTIFICATE----- diff --git a/backend/package-lock.json b/backend/package-lock.json index cfe22d916..8f1112cb7 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -98,6 +98,7 @@ "ms": "^2.1.3", "mysql2": "^3.9.8", "nanoid": "^3.3.8", + "nock": "^14.0.10", "node-forge": "^1.3.1", "nodemailer": "^6.9.9", "oci-sdk": "^2.108.0", @@ -9705,6 +9706,23 @@ "win32" ] }, + "node_modules/@mswjs/interceptors": { + "version": "0.39.8", + "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.39.8.tgz", + "integrity": "sha512-2+BzZbjRO7Ct61k8fMNHEtoKjeWI9pIlHFTqBwZ5icHpqszIgEZbjb1MW5Z0+bITTCTl3gk4PDBxs9tA/csXvA==", + "license": "MIT", + "dependencies": { + "@open-draft/deferred-promise": "^2.2.0", + "@open-draft/logger": "^0.3.0", + "@open-draft/until": "^2.0.0", + "is-node-process": "^1.2.0", + "outvariant": "^1.4.3", + "strict-event-emitter": "^0.5.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@next/env": { "version": "15.5.2", "resolved": "https://registry.npmjs.org/@next/env/-/env-15.5.2.tgz", @@ -10714,6 +10732,28 @@ "urijs": "^1.19.11" } }, + "node_modules/@open-draft/deferred-promise": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", + "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==", + "license": "MIT" + }, + "node_modules/@open-draft/logger": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz", + "integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==", + "license": "MIT", + "dependencies": { + "is-node-process": "^1.2.0", + "outvariant": "^1.4.0" + } + }, + "node_modules/@open-draft/until": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz", + "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", + "license": "MIT" + }, "node_modules/@opentelemetry/api": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", @@ -22958,6 +22998,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-node-process": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz", + "integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==", + "license": "MIT" + }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -23507,6 +23553,12 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "license": "ISC" + }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -25074,6 +25126,20 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/nock": { + "version": "14.0.10", + "resolved": "https://registry.npmjs.org/nock/-/nock-14.0.10.tgz", + "integrity": "sha512-Q7HjkpyPeLa0ZVZC5qpxBt5EyLczFJ91MEewQiIi9taWuA0KB/MDJlUWtON+7dGouVdADTQsf9RA7TZk6D8VMw==", + "license": "MIT", + "dependencies": { + "@mswjs/interceptors": "^0.39.5", + "json-stringify-safe": "^5.0.1", + "propagate": "^2.0.0" + }, + "engines": { + "node": ">=18.20.0 <20 || >=20.12.1" + } + }, "node_modules/node-abi": { "version": "3.65.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.65.0.tgz", @@ -27702,6 +27768,12 @@ "@otplib/preset-v11": "^12.0.1" } }, + "node_modules/outvariant": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/outvariant/-/outvariant-1.4.3.tgz", + "integrity": "sha512-+Sl2UErvtsoajRDKCE5/dBz4DIvHXQQnAxtQTF04OJxY0+DyZXSo5P5Bb7XYWOh81syohlYL24hbDwxedPUJCA==", + "license": "MIT" + }, "node_modules/p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", @@ -29103,6 +29175,15 @@ "node": ">= 6" } }, + "node_modules/propagate": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz", + "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/proto3-json-serializer": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.2.tgz", @@ -31601,6 +31682,12 @@ "node": ">=4.0.0" } }, + "node_modules/strict-event-emitter": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz", + "integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==", + "license": "MIT" + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", diff --git a/backend/package.json b/backend/package.json index 7a5efcb78..db94de681 100644 --- a/backend/package.json +++ b/backend/package.json @@ -226,6 +226,7 @@ "ms": "^2.1.3", "mysql2": "^3.9.8", "nanoid": "^3.3.8", + "nock": "^14.0.10", "node-forge": "^1.3.1", "nodemailer": "^6.9.9", "oci-sdk": "^2.108.0", diff --git a/backend/src/ee/routes/v1/pki-acme-router.ts b/backend/src/ee/routes/v1/pki-acme-router.ts index d58790039..c4ccf6be5 100644 --- a/backend/src/ee/routes/v1/pki-acme-router.ts +++ b/backend/src/ee/routes/v1/pki-acme-router.ts @@ -2,7 +2,6 @@ import { FastifyReply, FastifyRequest } from "fastify"; import { z } from "zod"; -import { AcmeMalformedError } from "@app/ee/services/pki-acme/pki-acme-errors"; import { AcmeOrderResourceSchema, CreateAcmeAccountResponseSchema, @@ -257,12 +256,9 @@ export const registerPkiAcmeRouter = async (server: FastifyZodProvider) => { } }, handler: async (req, res) => { - const { profileId, accountId, payload } = await validateExistingAccount({ + const { profileId, accountId } = await validateExistingAccount({ req }); - if (payload !== "") { - throw new AcmeMalformedError({ message: "Payload should be empty" }); - } return sendAcmeResponse( res, profileId, @@ -369,12 +365,9 @@ export const registerPkiAcmeRouter = async (server: FastifyZodProvider) => { } }, handler: async (req, res) => { - const { profileId, accountId, payload } = await validateExistingAccount({ + const { profileId, accountId } = await validateExistingAccount({ req }); - if (payload !== "") { - throw new AcmeMalformedError({ message: "Payload should be empty" }); - } res.type("application/pem-certificate-chain"); return sendAcmeResponse( res, @@ -405,10 +398,7 @@ export const registerPkiAcmeRouter = async (server: FastifyZodProvider) => { } }, handler: async (req, res) => { - const { profileId, accountId, payload } = await validateExistingAccount({ req }); - if (payload !== "") { - throw new AcmeMalformedError({ message: "Payload should be empty" }); - } + const { profileId, accountId } = await validateExistingAccount({ req }); return sendAcmeResponse( res, profileId, diff --git a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts index 61bd0c110..9148b0336 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-challenge-service.ts @@ -76,7 +76,9 @@ export const pkiAcmeChallengeServiceFactory = ({ // challenge validation at the same time, it should be fine. const challengeResponse = await fetch(challengeUrl, { signal: AbortSignal.timeout(timeoutMs) }); if (challengeResponse.status !== 200) { - throw new BadRequestError({ message: "ACME challenge response is not 200" }); + throw new AcmeIncorrectResponseError({ + message: `ACME challenge response is not 200: ${challengeResponse.status}` + }); } const challengeResponseBody = await challengeResponse.text(); const thumbprint = challenge.auth.account.publicKeyThumbprint; @@ -107,6 +109,7 @@ export const pkiAcmeChallengeServiceFactory = ({ if (fetchError.code === "ENOTFOUND" || fetchError.message.includes("ENOTFOUND")) { return new AcmeDnsFailureError({ message: "Hostname could not be resolved (DNS failure)" }); } + logger.error(exp, "Unknown error validating ACME challenge response"); return new AcmeServerInternalError({ message: "Unknown error validating ACME challenge response" }); } } else if (exp instanceof DOMException) { diff --git a/backend/src/ee/services/pki-acme/pki-acme-errors.ts b/backend/src/ee/services/pki-acme/pki-acme-errors.ts index 837dec8be..9053be391 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-errors.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-errors.ts @@ -468,7 +468,7 @@ export class AcmeOrderNotReadyError extends AcmeError { super({ type: AcmeErrorType.OrderNotReady, message, - status: 403, + status: 400, error }); this.name = "AcmeOrderNotReadyError"; diff --git a/backend/src/ee/services/pki-acme/pki-acme-service.ts b/backend/src/ee/services/pki-acme/pki-acme-service.ts index 55e9cc43f..dadb809d0 100644 --- a/backend/src/ee/services/pki-acme/pki-acme-service.ts +++ b/backend/src/ee/services/pki-acme/pki-acme-service.ts @@ -29,6 +29,19 @@ import { TProjectDALFactory } from "@app/services/project/project-dal"; import { getProjectKmsCertificateKeyId } from "@app/services/project/project-fns"; import { getConfig } from "@app/lib/config/env"; +import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal"; +import { orderCertificate } from "@app/services/certificate-authority/acme/acme-certificate-authority-fns"; +import { TCertificateAuthorityDALFactory } from "@app/services/certificate-authority/certificate-authority-dal"; +import { CaType } from "@app/services/certificate-authority/certificate-authority-enums"; +import { TExternalCertificateAuthorityDALFactory } from "@app/services/certificate-authority/external-certificate-authority-dal"; +import { extractCertificateRequestFromCSR } from "@app/services/certificate-common/certificate-csr-utils"; +import { TCertificateDALFactory } from "@app/services/certificate/certificate-dal"; +import { TCertificateSecretDALFactory } from "@app/services/certificate/certificate-secret-dal"; +import { + CertExtendedKeyUsage, + CertKeyUsage, + CertSubjectAlternativeNameType +} from "@app/services/certificate/certificate-types"; import { TLicenseServiceFactory } from "../license/license-service"; import { TPkiAcmeAccountDALFactory } from "./pki-acme-account-dal"; import { TPkiAcmeAuthDALFactory } from "./pki-acme-auth-dal"; @@ -79,9 +92,14 @@ import { } from "./pki-acme-types"; type TPkiAcmeServiceFactoryDep = { - projectDAL: Pick; + projectDAL: Pick; + appConnectionDAL: Pick; + certificateDAL: Pick; + certificateAuthorityDAL: Pick; + externalCertificateAuthorityDAL: Pick; certificateProfileDAL: Pick; - certificateBodyDAL: Pick; + certificateBodyDAL: Pick; + certificateSecretDAL: Pick; acmeAccountDAL: Pick< TPkiAcmeAccountDALFactory, "findByProjectIdAndAccountId" | "findByProfileIdAndPublicKeyThumbprintAndAlg" | "create" @@ -102,7 +120,10 @@ type TPkiAcmeServiceFactoryDep = { "create" | "transaction" | "updateById" | "findByAccountAuthAndChallengeId" | "findByIdForChallengeValidation" >; keyStore: Pick; - kmsService: Pick; + kmsService: Pick< + TKmsServiceFactory, + "decryptWithKmsKey" | "generateKmsKey" | "encryptWithKmsKey" | "createCipherPairWithDataKey" + >; licenseService: Pick; certificateV3Service: Pick; acmeChallengeService: TPkiAcmeChallengeServiceFactory; @@ -110,8 +131,13 @@ type TPkiAcmeServiceFactoryDep = { export const pkiAcmeServiceFactory = ({ projectDAL, + appConnectionDAL, + certificateDAL, + certificateAuthorityDAL, + externalCertificateAuthorityDAL, certificateProfileDAL, certificateBodyDAL, + certificateSecretDAL, acmeAccountDAL, acmeOrderDAL, acmeAuthDAL, @@ -622,6 +648,7 @@ export const pkiAcmeServiceFactory = ({ orderId: string; payload: TFinalizeAcmeOrderPayload; }): Promise> => { + const profile = (await certificateProfileDAL.findByIdWithConfigs(profileId))!; let order = await acmeOrderDAL.findByAccountAndOrderIdWithAuthorizations(accountId, orderId); if (!order) { throw new NotFoundError({ message: "ACME order not found" }); @@ -637,29 +664,100 @@ export const pkiAcmeServiceFactory = ({ if (finalizingOrder.expiresAt < new Date()) { throw new AcmeOrderNotReadyError({ message: "ACME order has expired" }); } + const { csr } = payload; + + // Check and validate the CSR + const certificateRequest = extractCertificateRequestFromCSR(csr); + if (!certificateRequest.commonName) { + throw new AcmeBadCSRError({ message: "Invalid CSR: Common name is required" }); + } + if ( + certificateRequest.subjectAlternativeNames?.some( + (san) => san.type !== CertSubjectAlternativeNameType.DNS_NAME + ) + ) { + throw new AcmeBadCSRError({ message: "Invalid CSR: Only DNS subject alternative names are supported" }); + } + const orderWithAuthorizations = (await acmeOrderDAL.findByAccountAndOrderIdWithAuthorizations( + accountId, + orderId, + tx + ))!; + const csrIdentifierValues = new Set( + (certificateRequest.subjectAlternativeNames ?? []) + .map((san) => san.value.toLowerCase()) + .concat([certificateRequest.commonName.toLowerCase()]) + ); + if ( + csrIdentifierValues.size !== orderWithAuthorizations.authorizations.length || + !orderWithAuthorizations.authorizations.every((auth) => + csrIdentifierValues.has(auth.identifierValue.toLowerCase()) + ) + ) { + throw new AcmeBadCSRError({ message: "Invalid CSR: Common name + SANs mismatch with order identifiers" }); + } + + const ca = await certificateAuthorityDAL.findByIdWithAssociatedCa(profile.caId); + if (!ca) { + throw new NotFoundError({ message: "Certificate Authority not found" }); + } + const caType = (ca.externalCa?.type as CaType) ?? CaType.INTERNAL; let errorToReturn: Error | undefined; try { - const { certificateId } = await certificateV3Service.signCertificateFromProfile({ - actor: ActorType.ACME_ACCOUNT, - actorId: accountId, - actorAuthMethod: null, - actorOrgId, - profileId, - csr, - notBefore: finalizingOrder.notBefore ? new Date(finalizingOrder.notBefore) : undefined, - notAfter: finalizingOrder.notAfter ? new Date(finalizingOrder.notAfter) : undefined, - validity: !finalizingOrder.notAfter - ? { - // 47 days, the default TTL comes with Let's Encrypt - // TODO: read config from the profile to get the expiration time instead - ttl: `${47}d` - } - : // ttl is not used if notAfter is provided - ({ ttl: "0d" } as const), - enrollmentType: EnrollmentType.ACME - }); - // TODO: associate the certificate with the order + const { certificateId } = await (async () => { + if (caType === CaType.INTERNAL) { + const result = await certificateV3Service.signCertificateFromProfile({ + actor: ActorType.ACME_ACCOUNT, + actorId: accountId, + actorAuthMethod: null, + actorOrgId, + profileId, + csr, + notBefore: finalizingOrder.notBefore ? new Date(finalizingOrder.notBefore) : undefined, + notAfter: finalizingOrder.notAfter ? new Date(finalizingOrder.notAfter) : undefined, + validity: !finalizingOrder.notAfter + ? { + // 47 days, the default TTL comes with Let's Encrypt + // TODO: read config from the profile to get the expiration time instead + ttl: `${47}d` + } + : // ttl is not used if notAfter is provided + ({ ttl: "0d" } as const), + enrollmentType: EnrollmentType.ACME + }); + return { certificateId: result.certificateId }; + } + const { certificateAuthority } = (await certificateProfileDAL.findByIdWithConfigs(profileId, tx))!; + const csrObj = new x509.Pkcs10CertificateRequest(csr); + const csrPem = csrObj.toString("pem"); + // TODO: for internal CA, we rely on the internal certificate authority service to check CSR against the template + // we should check the CSR against the template here + // TODO: this is pretty slow, and we are holding the transaction open for a long time, + // we should queue the certificate issuance to a background job instead + const cert = await orderCertificate( + { + caId: certificateAuthority!.id, + commonName: certificateRequest.commonName!, + altNames: certificateRequest.subjectAlternativeNames?.map((san) => san.value), + csr: Buffer.from(csrPem), + // TODO: not 100% sure what are these columns for, but let's put the values for common website SSL certs for now + keyUsages: [CertKeyUsage.DIGITAL_SIGNATURE, CertKeyUsage.KEY_ENCIPHERMENT, CertKeyUsage.KEY_AGREEMENT], + extendedKeyUsages: [CertExtendedKeyUsage.SERVER_AUTH] + }, + { + appConnectionDAL, + certificateAuthorityDAL, + externalCertificateAuthorityDAL, + certificateDAL, + certificateBodyDAL, + certificateSecretDAL, + kmsService, + projectDAL + } + ); + return { certificateId: cert.id }; + })(); await acmeOrderDAL.updateById( orderId, { diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 6f0502184..96107306f 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -106,7 +106,9 @@ const envSchema = z HTTPS_ENABLED: zodStrBool, ROTATION_DEVELOPMENT_MODE: zodStrBool.default("false").optional(), DAILY_RESOURCE_CLEAN_UP_DEVELOPMENT_MODE: zodStrBool.default("false").optional(), + BDD_NOCK_API_ENABLED: zodStrBool.default("false").optional(), ACME_DEVELOPMENT_MODE: zodStrBool.default("false").optional(), + ACME_SKIP_UPSTREAM_VALIDATION: zodStrBool.default("false").optional(), ACME_DEVELOPMENT_HTTP01_CHALLENGE_HOST_OVERRIDES: zpStr( z .string() @@ -398,6 +400,7 @@ const envSchema = z isAcmeDevelopmentMode: data.NODE_ENV === "development" && data.ACME_DEVELOPMENT_MODE, isProductionMode: data.NODE_ENV === "production" || IS_PACKAGED, isRedisSentinelMode: Boolean(data.REDIS_SENTINEL_HOSTS), + isBddNockApiEnabled: data.NODE_ENV === "development" && data.BDD_NOCK_API_ENABLED, REDIS_SENTINEL_HOSTS: data.REDIS_SENTINEL_HOSTS?.trim() ?.split(",") .map((el) => { diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index fab7ebde3..3195e670d 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -2244,8 +2244,13 @@ export const registerRoutes = async ( }); const pkiAcmeService = pkiAcmeServiceFactory({ projectDAL, + appConnectionDAL, + certificateDAL, + certificateAuthorityDAL, + externalCertificateAuthorityDAL, certificateProfileDAL, certificateBodyDAL, + certificateSecretDAL, acmeAccountDAL, acmeOrderDAL, acmeAuthDAL, diff --git a/backend/src/server/routes/v1/bdd-nock-router.ts b/backend/src/server/routes/v1/bdd-nock-router.ts new file mode 100644 index 000000000..ad4777772 --- /dev/null +++ b/backend/src/server/routes/v1/bdd-nock-router.ts @@ -0,0 +1,88 @@ +import { z } from "zod"; + +import { getConfig } from "@app/lib/config/env"; +import { ForbiddenRequestError } from "@app/lib/errors"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AuthMode } from "@app/services/auth/auth-type"; +import { logger } from "@app/lib/logger"; +import nock, { Definition } from "nock"; + +export const registerBddNockRouter = async (server: FastifyZodProvider) => { + const checkIfBddNockApiEnabled = () => { + const appCfg = getConfig(); + // Note: Please note that this API is only available in development mode and only for BDD tests. + // This endpoint should NEVER BE ENABLED IN PRODUCTION! + if (appCfg.NODE_ENV !== "development" || !appCfg.isBddNockApiEnabled) { + throw new ForbiddenRequestError({ message: "BDD Nock API is not enabled" }); + } + }; + + server.route({ + method: "POST", + url: "/define", + schema: { + body: z.object({ definitions: z.unknown().array() }), + response: { + 200: z.object({ status: z.string() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + checkIfBddNockApiEnabled(); + const { body } = req; + const { definitions } = body; + logger.info(definitions, "Defining nock"); + const processedDefinitions = definitions.map((definition: unknown) => { + const { path, ...rest } = definition as Definition; + return { + ...rest, + path: + path !== undefined && typeof path === "string" + ? path + : new RegExp((path as unknown as { regex: string }).regex ?? "") + } as Definition; + }); + + nock.define(processedDefinitions); + // Ensure we are activating the nocks, because we could have called `nock.restore()` before this call. + if (!nock.isActive()) { + nock.activate(); + } + return { status: "ok" }; + } + }); + + server.route({ + method: "POST", + url: "/clean-all", + schema: { + response: { + 200: z.object({ status: z.string() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async () => { + checkIfBddNockApiEnabled(); + logger.info("Cleaning all nocks"); + nock.cleanAll(); + return { status: "ok" }; + } + }); + + server.route({ + method: "POST", + url: "/restore", + schema: { + response: { + 200: z.object({ status: z.string() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async () => { + checkIfBddNockApiEnabled(); + logger.info("Restore network requests from nock"); + nock.restore(); + return { status: "ok" }; + } + }); +}; diff --git a/backend/src/server/routes/v1/index.ts b/backend/src/server/routes/v1/index.ts index b480a5144..4d8b87e60 100644 --- a/backend/src/server/routes/v1/index.ts +++ b/backend/src/server/routes/v1/index.ts @@ -6,8 +6,10 @@ import { registerCmekRouter } from "@app/server/routes/v1/cmek-router"; import { registerDashboardRouter } from "@app/server/routes/v1/dashboard-router"; import { registerSecretSyncRouter, SECRET_SYNC_REGISTER_ROUTER_MAP } from "@app/server/routes/v1/secret-sync-routers"; +import { getConfig } from "@app/lib/config/env"; import { registerAdminRouter } from "./admin-router"; import { registerAuthRoutes } from "./auth-router"; +import { registerBddNockRouter } from "./bdd-nock-router"; import { registerProjectBotRouter } from "./bot-router"; import { registerCaRouter } from "./certificate-authority-router"; import { CERTIFICATE_AUTHORITY_REGISTER_ROUTER_MAP } from "./certificate-authority-routers"; @@ -237,4 +239,10 @@ export const registerV1Routes = async (server: FastifyZodProvider) => { await server.register(registerEventRouter, { prefix: "/events" }); await server.register(registerUpgradePathRouter, { prefix: "/upgrade-path" }); + + // Note: This is a special route for BDD tests. It's only available in development mode and only for BDD tests. + // This route should NEVER BE ENABLED IN PRODUCTION! + if (getConfig().isBddNockApiEnabled) { + await server.register(registerBddNockRouter, { prefix: "/bdd-nock" }); + } }; diff --git a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts index 0dea986d6..52761e6a0 100644 --- a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts +++ b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts @@ -1,5 +1,5 @@ import * as x509 from "@peculiar/x509"; -import acme from "acme-client"; +import acme, { CsrBuffer } from "acme-client"; import { TableName } from "@app/db/schemas"; import { crypto } from "@app/lib/crypto/cryptography"; @@ -29,6 +29,8 @@ import { triggerAutoSyncForSubscriber } from "@app/services/pki-sync/pki-sync-ut import { TProjectDALFactory } from "@app/services/project/project-dal"; import { getProjectKmsCertificateKeyId } from "@app/services/project/project-fns"; +import { getConfig } from "@app/lib/config/env"; +import { Knex } from "knex"; import { TCertificateAuthorityDALFactory } from "../certificate-authority-dal"; import { CaStatus, CaType } from "../certificate-authority-enums"; import { keyAlgorithmToAlgCfg } from "../certificate-authority-fns"; @@ -64,6 +66,20 @@ type TAcmeCertificateAuthorityFnsDeps = { projectDAL: Pick; }; +type TOrderCertificateDeps = { + appConnectionDAL: Pick; + certificateAuthorityDAL: Pick; + externalCertificateAuthorityDAL: Pick; + certificateDAL: Pick; + certificateBodyDAL: Pick; + certificateSecretDAL: Pick; + kmsService: Pick< + TKmsServiceFactory, + "encryptWithKmsKey" | "generateKmsKey" | "createCipherPairWithDataKey" | "decryptWithKmsKey" + >; + projectDAL: Pick; +}; + type DBConfigurationColumn = { dnsProvider: string; directoryUrl: string; @@ -104,6 +120,245 @@ export const castDbEntryToAcmeCertificateAuthority = ( }; }; +export const orderCertificate = async ( + { + caId, + subscriberId, + commonName, + altNames, + csr, + csrPrivateKey, + keyUsages, + extendedKeyUsages + }: { + caId: string; + subscriberId?: string; + commonName: string; + altNames?: string[]; + csr: CsrBuffer; + csrPrivateKey?: string; + keyUsages?: CertKeyUsage[]; + extendedKeyUsages?: CertExtendedKeyUsage[]; + }, + deps: TOrderCertificateDeps, + tx?: Knex +) => { + const { + appConnectionDAL, + certificateAuthorityDAL, + externalCertificateAuthorityDAL, + certificateDAL, + certificateBodyDAL, + certificateSecretDAL, + kmsService, + projectDAL + } = deps; + + const ca = await certificateAuthorityDAL.findByIdWithAssociatedCa(caId, tx); + if (!ca.externalCa || ca.externalCa.type !== CaType.ACME) { + throw new BadRequestError({ message: "CA is not an ACME CA" }); + } + + const acmeCa = castDbEntryToAcmeCertificateAuthority(ca); + if (acmeCa.status !== CaStatus.ACTIVE) { + throw new BadRequestError({ message: "CA is disabled" }); + } + + const certificateManagerKmsId = await getProjectKmsCertificateKeyId({ + projectId: ca.projectId, + projectDAL, + kmsService + }); + + const kmsEncryptor = await kmsService.encryptWithKmsKey({ + kmsId: certificateManagerKmsId + }); + + const kmsDecryptor = await kmsService.decryptWithKmsKey({ + kmsId: certificateManagerKmsId + }); + + let accountKey: Buffer | undefined; + if (acmeCa.credentials) { + const decryptedCredentials = await kmsDecryptor({ + cipherTextBlob: acmeCa.credentials as Buffer + }); + + const parsedCredentials = await AcmeCertificateAuthorityCredentialsSchema.parseAsync( + JSON.parse(decryptedCredentials.toString("utf8")) + ); + + accountKey = Buffer.from(parsedCredentials.accountKey, "base64"); + } + if (!accountKey) { + accountKey = await acme.crypto.createPrivateRsaKey(); + const newCredentials = { + accountKey: accountKey.toString("base64") + }; + const { cipherTextBlob: encryptedNewCredentials } = await kmsEncryptor({ + plainText: Buffer.from(JSON.stringify(newCredentials)) + }); + await externalCertificateAuthorityDAL.update( + { + caId: acmeCa.id + }, + { + credentials: encryptedNewCredentials + } + ); + } + + await blockLocalAndPrivateIpAddresses(acmeCa.configuration.directoryUrl); + + const acmeClientOptions: acme.ClientOptions = { + directoryUrl: acmeCa.configuration.directoryUrl, + accountKey + }; + + if (acmeCa.configuration.eabKid && acmeCa.configuration.eabHmacKey) { + acmeClientOptions.externalAccountBinding = { + kid: acmeCa.configuration.eabKid, + hmacKey: acmeCa.configuration.eabHmacKey + }; + } + + const acmeClient = new acme.Client(acmeClientOptions); + + const appConnection = await appConnectionDAL.findById(acmeCa.configuration.dnsAppConnectionId); + const connection = await decryptAppConnection(appConnection, kmsService); + + const pem = await acmeClient.auto({ + csr, + email: acmeCa.configuration.accountEmail, + challengePriority: ["dns-01"], + // For ACME development mode, we mock the DNS challenge API calls. So, no real DNS records are created. + // We need to disable the challenge verification to avoid errors. + skipChallengeVerification: getConfig().isAcmeDevelopmentMode && getConfig().ACME_SKIP_UPSTREAM_VALIDATION, + termsOfServiceAgreed: true, + + challengeCreateFn: async (authz, challenge, keyAuthorization) => { + if (challenge.type !== "dns-01") { + throw new Error("Unsupported challenge type"); + } + + const recordName = `_acme-challenge.${authz.identifier.value}`; // e.g., "_acme-challenge.example.com" + const recordValue = `"${keyAuthorization}"`; // must be double quoted + + switch (acmeCa.configuration.dnsProviderConfig.provider) { + case AcmeDnsProvider.Route53: { + await route53InsertTxtRecord( + connection as TAwsConnection, + acmeCa.configuration.dnsProviderConfig.hostedZoneId, + recordName, + recordValue + ); + break; + } + case AcmeDnsProvider.Cloudflare: { + await cloudflareInsertTxtRecord( + connection as TCloudflareConnection, + acmeCa.configuration.dnsProviderConfig.hostedZoneId, + recordName, + recordValue + ); + break; + } + default: { + throw new Error(`Unsupported DNS provider: ${acmeCa.configuration.dnsProviderConfig.provider as string}`); + } + } + }, + challengeRemoveFn: async (authz, challenge, keyAuthorization) => { + const recordName = `_acme-challenge.${authz.identifier.value}`; // e.g., "_acme-challenge.example.com" + const recordValue = `"${keyAuthorization}"`; // must be double quoted + + switch (acmeCa.configuration.dnsProviderConfig.provider) { + case AcmeDnsProvider.Route53: { + await route53DeleteTxtRecord( + connection as TAwsConnection, + acmeCa.configuration.dnsProviderConfig.hostedZoneId, + recordName, + recordValue + ); + break; + } + case AcmeDnsProvider.Cloudflare: { + await cloudflareDeleteTxtRecord( + connection as TCloudflareConnection, + acmeCa.configuration.dnsProviderConfig.hostedZoneId, + recordName, + recordValue + ); + break; + } + default: { + throw new Error(`Unsupported DNS provider: ${acmeCa.configuration.dnsProviderConfig.provider as string}`); + } + } + } + }); + + const [leafCert, parentCert] = acme.crypto.splitPemChain(pem); + const certObj = new x509.X509Certificate(leafCert); + + const { cipherTextBlob: encryptedCertificate } = await kmsEncryptor({ + plainText: Buffer.from(new Uint8Array(certObj.rawData)) + }); + + const certificateChainPem = parentCert.trim(); + + const { cipherTextBlob: encryptedCertificateChain } = await kmsEncryptor({ + plainText: Buffer.from(certificateChainPem) + }); + + const { cipherTextBlob: encryptedPrivateKey } = csrPrivateKey + ? await kmsEncryptor({ + plainText: Buffer.from(csrPrivateKey) + }) + : { cipherTextBlob: undefined }; + + return (tx || certificateDAL).transaction(async (innerTx: Knex) => { + const cert = await certificateDAL.create( + { + caId: ca.id, + pkiSubscriberId: subscriberId, + status: CertStatus.ACTIVE, + friendlyName: commonName, + commonName, + altNames: altNames?.join(","), + serialNumber: certObj.serialNumber, + notBefore: certObj.notBefore, + notAfter: certObj.notAfter, + keyUsages, + extendedKeyUsages, + projectId: ca.projectId + }, + innerTx + ); + + await certificateBodyDAL.create( + { + certId: cert.id, + encryptedCertificate, + encryptedCertificateChain + }, + innerTx + ); + + if (encryptedPrivateKey !== undefined) { + await certificateSecretDAL.create( + { + certId: cert.id, + encryptedPrivateKey + }, + innerTx + ); + } + + return cert; + }); +}; + export const AcmeCertificateAuthorityFns = ({ appConnectionDAL, appConnectionService, @@ -322,77 +577,6 @@ export const AcmeCertificateAuthorityFns = ({ if (!subscriber.caId) { throw new BadRequestError({ message: "Subscriber does not have a CA" }); } - - const ca = await certificateAuthorityDAL.findByIdWithAssociatedCa(subscriber.caId); - if (!ca.externalCa || ca.externalCa.type !== CaType.ACME) { - throw new BadRequestError({ message: "CA is not an ACME CA" }); - } - - const acmeCa = castDbEntryToAcmeCertificateAuthority(ca); - if (acmeCa.status !== CaStatus.ACTIVE) { - throw new BadRequestError({ message: "CA is disabled" }); - } - - const certificateManagerKmsId = await getProjectKmsCertificateKeyId({ - projectId: ca.projectId, - projectDAL, - kmsService - }); - - const kmsEncryptor = await kmsService.encryptWithKmsKey({ - kmsId: certificateManagerKmsId - }); - - const kmsDecryptor = await kmsService.decryptWithKmsKey({ - kmsId: certificateManagerKmsId - }); - - let accountKey: Buffer | undefined; - if (acmeCa.credentials) { - const decryptedCredentials = await kmsDecryptor({ - cipherTextBlob: acmeCa.credentials as Buffer - }); - - const parsedCredentials = await AcmeCertificateAuthorityCredentialsSchema.parseAsync( - JSON.parse(decryptedCredentials.toString("utf8")) - ); - - accountKey = Buffer.from(parsedCredentials.accountKey, "base64"); - } - if (!accountKey) { - accountKey = await acme.crypto.createPrivateRsaKey(); - const newCredentials = { - accountKey: accountKey.toString("base64") - }; - const { cipherTextBlob: encryptedNewCredentials } = await kmsEncryptor({ - plainText: Buffer.from(JSON.stringify(newCredentials)) - }); - await externalCertificateAuthorityDAL.update( - { - caId: acmeCa.id - }, - { - credentials: encryptedNewCredentials - } - ); - } - - await blockLocalAndPrivateIpAddresses(acmeCa.configuration.directoryUrl); - - const acmeClientOptions: acme.ClientOptions = { - directoryUrl: acmeCa.configuration.directoryUrl, - accountKey - }; - - if (acmeCa.configuration.eabKid && acmeCa.configuration.eabHmacKey) { - acmeClientOptions.externalAccountBinding = { - kid: acmeCa.configuration.eabKid, - hmacKey: acmeCa.configuration.eabHmacKey - }; - } - - const acmeClient = new acme.Client(acmeClientOptions); - const alg = keyAlgorithmToAlgCfg(CertKeyAlgorithm.RSA_2048); const leafKeys = await crypto.nativeCrypto.subtle.generateKey(alg, true, ["sign", "verify"]); @@ -407,131 +591,28 @@ export const AcmeCertificateAuthorityFns = ({ skLeaf ); - const appConnection = await appConnectionDAL.findById(acmeCa.configuration.dnsAppConnectionId); - const connection = await decryptAppConnection(appConnection, kmsService); - - const pem = await acmeClient.auto({ - csr: certificateCsr, - email: acmeCa.configuration.accountEmail, - challengePriority: ["dns-01"], - termsOfServiceAgreed: true, - - challengeCreateFn: async (authz, challenge, keyAuthorization) => { - if (challenge.type !== "dns-01") { - throw new Error("Unsupported challenge type"); - } - - const recordName = `_acme-challenge.${authz.identifier.value}`; // e.g., "_acme-challenge.example.com" - const recordValue = `"${keyAuthorization}"`; // must be double quoted - - switch (acmeCa.configuration.dnsProviderConfig.provider) { - case AcmeDnsProvider.Route53: { - await route53InsertTxtRecord( - connection as TAwsConnection, - acmeCa.configuration.dnsProviderConfig.hostedZoneId, - recordName, - recordValue - ); - break; - } - case AcmeDnsProvider.Cloudflare: { - await cloudflareInsertTxtRecord( - connection as TCloudflareConnection, - acmeCa.configuration.dnsProviderConfig.hostedZoneId, - recordName, - recordValue - ); - break; - } - default: { - throw new Error(`Unsupported DNS provider: ${acmeCa.configuration.dnsProviderConfig.provider as string}`); - } - } + await orderCertificate( + { + caId: subscriber.caId, + subscriberId: subscriber.id, + commonName: subscriber.commonName, + altNames: subscriber.subjectAlternativeNames, + csr: certificateCsr, + csrPrivateKey: skLeaf, + keyUsages: subscriber.keyUsages as CertKeyUsage[], + extendedKeyUsages: subscriber.extendedKeyUsages as CertExtendedKeyUsage[] }, - challengeRemoveFn: async (authz, challenge, keyAuthorization) => { - const recordName = `_acme-challenge.${authz.identifier.value}`; // e.g., "_acme-challenge.example.com" - const recordValue = `"${keyAuthorization}"`; // must be double quoted - - switch (acmeCa.configuration.dnsProviderConfig.provider) { - case AcmeDnsProvider.Route53: { - await route53DeleteTxtRecord( - connection as TAwsConnection, - acmeCa.configuration.dnsProviderConfig.hostedZoneId, - recordName, - recordValue - ); - break; - } - case AcmeDnsProvider.Cloudflare: { - await cloudflareDeleteTxtRecord( - connection as TCloudflareConnection, - acmeCa.configuration.dnsProviderConfig.hostedZoneId, - recordName, - recordValue - ); - break; - } - default: { - throw new Error(`Unsupported DNS provider: ${acmeCa.configuration.dnsProviderConfig.provider as string}`); - } - } + { + appConnectionDAL, + certificateAuthorityDAL, + externalCertificateAuthorityDAL, + certificateDAL, + certificateBodyDAL, + certificateSecretDAL, + kmsService, + projectDAL } - }); - - const [leafCert, parentCert] = acme.crypto.splitPemChain(pem); - const certObj = new x509.X509Certificate(leafCert); - - const { cipherTextBlob: encryptedCertificate } = await kmsEncryptor({ - plainText: Buffer.from(new Uint8Array(certObj.rawData)) - }); - - const certificateChainPem = parentCert.trim(); - - const { cipherTextBlob: encryptedCertificateChain } = await kmsEncryptor({ - plainText: Buffer.from(certificateChainPem) - }); - - const { cipherTextBlob: encryptedPrivateKey } = await kmsEncryptor({ - plainText: Buffer.from(skLeaf) - }); - - await certificateDAL.transaction(async (tx) => { - const cert = await certificateDAL.create( - { - caId: ca.id, - pkiSubscriberId: subscriber.id, - status: CertStatus.ACTIVE, - friendlyName: subscriber.commonName, - commonName: subscriber.commonName, - altNames: subscriber.subjectAlternativeNames.join(","), - serialNumber: certObj.serialNumber, - notBefore: certObj.notBefore, - notAfter: certObj.notAfter, - keyUsages: subscriber.keyUsages as CertKeyUsage[], - extendedKeyUsages: subscriber.extendedKeyUsages as CertExtendedKeyUsage[], - projectId: ca.projectId - }, - tx - ); - - await certificateBodyDAL.create( - { - certId: cert.id, - encryptedCertificate, - encryptedCertificateChain - }, - tx - ); - - await certificateSecretDAL.create( - { - certId: cert.id, - encryptedPrivateKey - }, - tx - ); - }); - + ); await triggerAutoSyncForSubscriber(subscriber.id, { pkiSyncDAL, pkiSyncQueue }); }; diff --git a/backend/src/services/certificate-profile/certificate-profile-dal.ts b/backend/src/services/certificate-profile/certificate-profile-dal.ts index 53172b744..d2f468248 100644 --- a/backend/src/services/certificate-profile/certificate-profile-dal.ts +++ b/backend/src/services/certificate-profile/certificate-profile-dal.ts @@ -168,15 +168,12 @@ export const certificateProfileDALFactory = (db: TDbClient) => { } as TCertificateProfileWithConfigs["acmeConfig"]) : undefined; - const certificateAuthority = - result.caId && result.caProjectId && result.caStatus && result.caName - ? ({ - id: result.caId, - projectId: result.caProjectId, - status: result.caStatus, - name: result.caName - } as TCertificateProfileWithConfigs["certificateAuthority"]) - : undefined; + const certificateAuthority = { + id: result.caId, + projectId: result.caProjectId, + status: result.caStatus, + name: result.caName + } as TCertificateProfileWithConfigs["certificateAuthority"]; const certificateTemplate = result.templateId && result.templateProjectId && result.templateName diff --git a/docker-compose.bdd.yml b/docker-compose.bdd.yml index dfee5f6b2..b73683867 100644 --- a/docker-compose.bdd.yml +++ b/docker-compose.bdd.yml @@ -55,8 +55,12 @@ services: - NODE_ENV=development - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable - TELEMETRY_ENABLED=false + # This is needed to trust the Pebble CA certificate, which is used for the BDD tests + - NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/pebble.minica.crt volumes: - ./backend/src:/app/src + # This is needed to trust the Pebble CA certificate, which is used for the BDD tests + - ./backend/bdd/pebble/pebble.minica.pem:/usr/local/share/ca-certificates/pebble.minica.crt:ro - softhsm_tokens:/etc/softhsm2/tokens # SoftHSM tokens are stored in a volume to persist across container restarts extra_hosts: - "host.docker.internal:host-gateway" @@ -75,6 +79,21 @@ services: - ./frontend/public:/app/public env_file: .env + # ACME server for BDD tests + pebble: + image: ghcr.io/letsencrypt/pebble:2.8.0 + command: -config /var/data/pebble/pebble-config.json + ports: + - 14000:14000 # ACME port + - 15000:15000 # Management port + environment: + # Do not perform validation sleep to make the BDD tests faster + - PEBBLE_VA_NOSLEEP=1 + # Skip validation for now to make the BDD tests easier to write + - PEBBLE_VA_ALWAYS_VALID=1 + volumes: + - ./backend/bdd/pebble/:/var/data/pebble:ro + volumes: postgres-data: driver: local