Merge pull request #4874 from Infisical/dynamic-import-nock

Move nock to dev deps and include the route only for dev/test builds

.gitignore

@@ -74,3 +74,4 @@ cli/test/infisical-merge
 backend/bdd/.bdd-infisical-bootstrap-result.json
 
 /npm/bin
+__pycache__

backend/bdd/features/steps/pki_acme.py

@@ -615,13 +615,14 @@ def step_impl(context: Context, var_path: str, jq_query: str, expected: str):
 
 @then('the value {var_path} with jq "{jq_query}" should match pattern {regex}')
 def step_impl(context: Context, var_path: str, jq_query: str, regex: str):
+    actual_regex = replace_vars(regex, context.vars)
     value, result = apply_value_with_jq(
         context=context,
         var_path=var_path,
         jq_query=jq_query,
     )
-    assert re.match(replace_vars(regex, context.vars), result), (
+    assert re.match(actual_regex, result), (
-        f"{json.dumps(value)!r} with jq {jq_query!r}, the result {json.dumps(result)!r} does not match {regex!r}"
+        f"{json.dumps(value)!r} with jq {jq_query!r}, the result {json.dumps(result)!r} does not match {actual_regex!r}"
     )
 
 

backend/bdd/features/steps/utils.py

@@ -15,6 +15,7 @@ from josepy import JSONObjectWithFields
 
 ACC_KEY_BITS = 2048
 ACC_KEY_PUBLIC_EXPONENT = 65537
+NOCK_API_PREFIX = "/api/__bdd_nock__"
 logger = logging.getLogger(__name__)
 faker = Faker()
 
@@ -265,7 +266,7 @@ def x509_cert_to_dict(cert: x509.Certificate) -> dict:
 def define_nock(context: Context, definitions: list[dict]):
     jwt_token = context.vars["AUTH_TOKEN"]
     response = context.http_client.post(
-        "/api/v1/bdd-nock/define",
+        f"{NOCK_API_PREFIX}/define",
         headers=dict(authorization="Bearer {}".format(jwt_token)),
         json=dict(definitions=definitions),
     )
@@ -275,7 +276,7 @@ def define_nock(context: Context, definitions: list[dict]):
 def restore_nock(context: Context):
     jwt_token = context.vars["AUTH_TOKEN"]
     response = context.http_client.post(
-        "/api/v1/bdd-nock/restore",
+        f"{NOCK_API_PREFIX}/restore",
         headers=dict(authorization="Bearer {}".format(jwt_token)),
         json=dict(),
     )
@@ -285,7 +286,7 @@ def restore_nock(context: Context):
 def clean_all_nock(context: Context):
     jwt_token = context.vars["AUTH_TOKEN"]
     response = context.http_client.post(
-        "/api/v1/bdd-nock/clean-all",
+        f"{NOCK_API_PREFIX}/clean-all",
         headers=dict(authorization="Bearer {}".format(jwt_token)),
         json=dict(),
     )

backend/nodemon.json

@@ -1,6 +1,8 @@
 {
-  "watch": ["src"],
+  "watch": [
+    "src"
+  ],
   "ext": ".ts,.js",
   "ignore": [],
-  "exec": "tsx ./src/main.ts | pino-pretty --colorize --colorizeObjects --singleLine"
+  "exec": "tsx --tsconfig=./tsconfig.dev.json --inspect=0.0.0.0:9229 ./src/main.ts | pino-pretty --colorize --colorizeObjects --singleLine"
-}
+}

backend/package.json

@@ -32,7 +32,7 @@
     "binary:clean": "rm -rf ./dist && rm -rf ./binary",
     "binary:rename-imports": "ts-node ./scripts/rename-mjs.ts",
     "test": "echo \"Error: no test specified\" && exit 1",
-    "dev": "tsx watch --clear-screen=false ./src/main.ts  | pino-pretty --colorize --colorizeObjects --singleLine",
+    "dev": "tsx watch --clear-screen=false ./src/main.ts  --config tsconfig.dev.json | pino-pretty --colorize --colorizeObjects --singleLine",
     "dev:docker": "nodemon",
     "build": "tsup --sourcemap",
     "build:frontend": "npm run build --prefix ../frontend",
@@ -266,4 +266,4 @@
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.24.5"
   }
-}
+}

backend/src/lib/config/env.ts

@@ -400,7 +400,7 @@ const envSchema = z
     isAcmeDevelopmentMode: data.NODE_ENV === "development" && data.ACME_DEVELOPMENT_MODE,
     isProductionMode: data.NODE_ENV === "production" || IS_PACKAGED,
     isRedisSentinelMode: Boolean(data.REDIS_SENTINEL_HOSTS),
-    isBddNockApiEnabled: data.NODE_ENV === "development" && data.BDD_NOCK_API_ENABLED,
+    isBddNockApiEnabled: data.NODE_ENV !== "production" && data.BDD_NOCK_API_ENABLED,
     REDIS_SENTINEL_HOSTS: data.REDIS_SENTINEL_HOSTS?.trim()
       ?.split(",")
       .map((el) => {

backend/src/server/routes/bdd/bdd-nock-router.dev.ts

@@ -0,0 +1,104 @@
+import type { Definition } from "nock";
+import { z } from "zod";
+
+import { getConfig } from "@app/lib/config/env";
+import { ForbiddenRequestError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
+import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
+import { AuthMode } from "@app/services/auth/auth-type";
+
+// When running in production, we don't want to even import nock, because it's not needed and it increases memory usage a lots.
+// It once caused an outage in the production environment.
+// This is why we would rather to crash the app if it's not in development mode (in that case, Kubernetes should stop it from rolling out).
+if (process.env.NODE_ENV === "production") {
+  throw new Error("BDD Nock API can only be enabled in development or test mode");
+}
+
+export const registerBddNockRouter = async (server: FastifyZodProvider) => {
+  const appCfg = getConfig();
+  const importNock = async () => {
+    // eslint-disable-next-line import/no-extraneous-dependencies
+    const { default: nock } = await import("nock");
+    return nock;
+  };
+
+  const checkIfBddNockApiEnabled = () => {
+    // Note: Please note that this API is only available in development mode and only for BDD tests.
+    // This endpoint should NEVER BE ENABLED IN PRODUCTION!
+    if (appCfg.NODE_ENV === "production" || !appCfg.isBddNockApiEnabled) {
+      throw new ForbiddenRequestError({ message: "BDD Nock API is not enabled" });
+    }
+  };
+
+  server.route({
+    method: "POST",
+    url: "/define",
+    schema: {
+      body: z.object({ definitions: z.unknown().array() }),
+      response: {
+        200: z.object({ status: z.string() })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      checkIfBddNockApiEnabled();
+      const { body } = req;
+      const { definitions } = body;
+      logger.info(definitions, "Defining nock");
+      const processedDefinitions = definitions.map((definition: unknown) => {
+        const { path, ...rest } = definition as Definition;
+        return {
+          ...rest,
+          path:
+            path !== undefined && typeof path === "string"
+              ? path
+              : new RegExp((path as unknown as { regex: string }).regex ?? "")
+        } as Definition;
+      });
+
+      const nock = await importNock();
+      nock.define(processedDefinitions);
+      // Ensure we are activating the nocks, because we could have called `nock.restore()` before this call.
+      if (!nock.isActive()) {
+        nock.activate();
+      }
+      return { status: "ok" };
+    }
+  });
+
+  server.route({
+    method: "POST",
+    url: "/clean-all",
+    schema: {
+      response: {
+        200: z.object({ status: z.string() })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async () => {
+      checkIfBddNockApiEnabled();
+      logger.info("Cleaning all nocks");
+      const nock = await importNock();
+      nock.cleanAll();
+      return { status: "ok" };
+    }
+  });
+
+  server.route({
+    method: "POST",
+    url: "/restore",
+    schema: {
+      response: {
+        200: z.object({ status: z.string() })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async () => {
+      checkIfBddNockApiEnabled();
+      logger.info("Restore network requests from nock");
+      const nock = await importNock();
+      nock.restore();
+      return { status: "ok" };
+    }
+  });
+};

backend/src/server/routes/bdd/bdd-nock-router.ts

@@ -0,0 +1,6 @@
+export const registerBddNockRouter = async () => {
+  // This route is only available in development or test mode.
+  // The actual implementation is in the dev.ts file and will be aliased to that file in development or test mode.
+  // And if somehow we try to enable it in production, we will throw an error.
+  throw new Error("BDD Nock should not be enabled in production");
+};

backend/src/server/routes/index.ts

@@ -1,3 +1,4 @@
+import { registerBddNockRouter } from "@bdd_routes/bdd-nock-router";
 import { CronJob } from "cron";
 import { Knex } from "knex";
 import { monitorEventLoopDelay } from "perf_hooks";
@@ -2698,6 +2699,12 @@ export const registerRoutes = async (
   await server.register(registerV3Routes, { prefix: "/api/v3" });
   await server.register(registerV4Routes, { prefix: "/api/v4" });
 
+  // Note: This is a special route for BDD tests. It's only available in development mode and only for BDD tests.
+  // This route should NEVER BE ENABLED IN PRODUCTION!
+  if (getConfig().isBddNockApiEnabled) {
+    await server.register(registerBddNockRouter, { prefix: "/api/__bdd_nock__" });
+  }
+
   server.addHook("onClose", async () => {
     cronJobs.forEach((job) => job.stop());
     await telemetryService.flushAll();

backend/src/server/routes/v1/bdd-nock-router.ts

@@ -1,87 +0,0 @@
-// import { z } from "zod";
-
-// import { getConfig } from "@app/lib/config/env";
-// import { ForbiddenRequestError } from "@app/lib/errors";
-// import { logger } from "@app/lib/logger";
-// import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
-// import { AuthMode } from "@app/services/auth/auth-type";
-
-// export const registerBddNockRouter = async (server: FastifyZodProvider) => {
-//   const checkIfBddNockApiEnabled = () => {
-//     const appCfg = getConfig();
-//     // Note: Please note that this API is only available in development mode and only for BDD tests.
-//     // This endpoint should NEVER BE ENABLED IN PRODUCTION!
-//     if (appCfg.NODE_ENV !== "development" || !appCfg.isBddNockApiEnabled) {
-//       throw new ForbiddenRequestError({ message: "BDD Nock API is not enabled" });
-//     }
-//   };
-
-//   server.route({
-//     method: "POST",
-//     url: "/define",
-//     schema: {
-//       body: z.object({ definitions: z.unknown().array() }),
-//       response: {
-//         200: z.object({ status: z.string() })
-//       }
-//     },
-//     onRequest: verifyAuth([AuthMode.JWT]),
-//     handler: async (req) => {
-//       checkIfBddNockApiEnabled();
-//       const { body } = req;
-//       const { definitions } = body;
-//       logger.info(definitions, "Defining nock");
-//       const processedDefinitions = definitions.map((definition: unknown) => {
-//         const { path, ...rest } = definition as Definition;
-//         return {
-//           ...rest,
-//           path:
-//             path !== undefined && typeof path === "string"
-//               ? path
-//               : new RegExp((path as unknown as { regex: string }).regex ?? "")
-//         } as Definition;
-//       });
-
-//       nock.define(processedDefinitions);
-//       // Ensure we are activating the nocks, because we could have called `nock.restore()` before this call.
-//       if (!nock.isActive()) {
-//         nock.activate();
-//       }
-//       return { status: "ok" };
-//     }
-//   });
-
-//   server.route({
-//     method: "POST",
-//     url: "/clean-all",
-//     schema: {
-//       response: {
-//         200: z.object({ status: z.string() })
-//       }
-//     },
-//     onRequest: verifyAuth([AuthMode.JWT]),
-//     handler: async () => {
-//       checkIfBddNockApiEnabled();
-//       logger.info("Cleaning all nocks");
-//       nock.cleanAll();
-//       return { status: "ok" };
-//     }
-//   });
-
-//   server.route({
-//     method: "POST",
-//     url: "/restore",
-//     schema: {
-//       response: {
-//         200: z.object({ status: z.string() })
-//       }
-//     },
-//     onRequest: verifyAuth([AuthMode.JWT]),
-//     handler: async () => {
-//       checkIfBddNockApiEnabled();
-//       logger.info("Restore network requests from nock");
-//       nock.restore();
-//       return { status: "ok" };
-//     }
-//   });
-// };

backend/src/server/routes/v1/index.ts

@@ -8,7 +8,6 @@ import { registerSecretSyncRouter, SECRET_SYNC_REGISTER_ROUTER_MAP } from "@app/
 
 import { registerAdminRouter } from "./admin-router";
 import { registerAuthRoutes } from "./auth-router";
-// import { registerBddNockRouter } from "./bdd-nock-router";
 import { registerProjectBotRouter } from "./bot-router";
 import { registerCaRouter } from "./certificate-authority-router";
 import { CERTIFICATE_AUTHORITY_REGISTER_ROUTER_MAP } from "./certificate-authority-routers";
@@ -238,10 +237,4 @@ export const registerV1Routes = async (server: FastifyZodProvider) => {
 
   await server.register(registerEventRouter, { prefix: "/events" });
   await server.register(registerUpgradePathRouter, { prefix: "/upgrade-path" });
-
-  // Note: This is a special route for BDD tests. It's only available in development mode and only for BDD tests.
-  // This route should NEVER BE ENABLED IN PRODUCTION!
-  // if (getConfig().isBddNockApiEnabled) {
-  //   await server.register(registerBddNockRouter, { prefix: "/bdd-nock" });
-  // }
 };

backend/src/services/certificate-profile/certificate-profile-service.test.ts

@@ -428,7 +428,13 @@ describe("CertificateProfileService", () => {
         service.createProfile({
           ...mockActor,
           projectId: "project-123",
-          data: validProfileData
+          data: {
+            ...validProfileData,
+            enrollmentType: EnrollmentType.ACME,
+            acmeConfig: {},
+            apiConfig: undefined,
+            estConfig: undefined
+          }
         })
       ).rejects.toThrowError(
         new BadRequestError({

backend/tsconfig.dev.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "paths": {
+      "@app/*": ["./src/*"],
+      "@bdd_routes/bdd-nock-router": ["./src/server/routes/bdd/bdd-nock-router.dev.ts"]
+    }
+  }
+}

backend/tsconfig.json

@@ -24,7 +24,8 @@
     "skipLibCheck": true,
     "baseUrl": ".",
     "paths": {
-      "@app/*": ["./src/*"]
+      "@app/*": ["./src/*"],
+      "@bdd_routes/*": ["./src/server/routes/bdd/*"]
     },
     "jsx": "react-jsx"
   },

backend/tsup.config.js

@@ -2,8 +2,8 @@
 import path from "node:path";
 
 import fs from "fs/promises";
-import {replaceTscAliasPaths} from "tsc-alias";
+import { replaceTscAliasPaths } from "tsc-alias";
-import {defineConfig} from "tsup";
+import { defineConfig } from "tsup";
 
 // Instead of using tsx or tsc for building, consider using tsup.
 // TSX serves as an alternative to Node.js, allowing you to build directly on the Node.js runtime.
@@ -29,7 +29,7 @@ export default defineConfig({
   external: ["../../../frontend/node_modules/next/dist/server/next-server.js"],
   outDir: "dist",
   tsconfig: "./tsconfig.json",
-  entry: ["./src"],
+  entry: ["./src", "!./src/**/*.dev.ts"],
   sourceMap: true,
   skipNodeModulesBundle: true,
   esbuildPlugins: [
@@ -45,22 +45,22 @@ export default defineConfig({
               const isRelativePath = args.path.startsWith(".");
               const absPath = isRelativePath
                 ? path.join(args.resolveDir, args.path)
-                : path.join(args.path.replace("@app", "./src"));
+                : path.join(args.path.replace("@app", "./src").replace("@bdd_routes", "./src/server/routes/bdd"));
 
               const isFile = await fs
                 .stat(`${absPath}.ts`)
                 .then((el) => el.isFile)
-                  .catch(async (err) => {
+                .catch(async (err) => {
-                    if (err.code === "ENOTDIR") {
+                  if (err.code === "ENOTDIR") {
-                      return true;
+                    return true;
-                    }
+                  }
 
-                    // If .ts file doesn't exist, try checking for .tsx file
+                  // If .ts file doesn't exist, try checking for .tsx file
-                    return fs
+                  return fs
-                        .stat(`${absPath}.tsx`)
+                    .stat(`${absPath}.tsx`)
-                        .then((el) => el.isFile)
+                    .then((el) => el.isFile)
-                        .catch((err) => err.code === "ENOTDIR");
+                    .catch((err) => err.code === "ENOTDIR");
-                  });
+                });
 
               return {
                 path: isFile ? `${args.path}.mjs` : `${args.path}/index.mjs`,

backend/vitest.e2e.config.mts

@@ -28,7 +28,8 @@ export default defineConfig({
   },
   resolve: {
     alias: {
-      "@app": path.resolve(__dirname, "./src")
+      "@app": path.resolve(__dirname, "./src"),
+      "@bdd_routes/bdd-nock-router": path.resolve(__dirname, "./src/server/routes/bdd/bdd-nock-router.dev.ts")
     }
   }
 });

backend/vitest.unit.config.mts

@@ -11,7 +11,8 @@ export default defineConfig({
   },
   resolve: {
     alias: {
-      "@app": path.resolve(__dirname, "./src")
+      "@app": path.resolve(__dirname, "./src"),
+      "@bdd_routes/bdd-nock-router": path.resolve(__dirname, "./src/server/routes/bdd/bdd-nock-router.dev.ts")
     }
   }
 });

docker-compose.dev.yml

@@ -71,6 +71,7 @@ services:
     ports:
       - 4000:4000
       - 9464:9464 # for OTEL collection of Prometheus metrics
+      - 9229:9229 # For debugger access
     environment:
       - NODE_ENV=development
       - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable