Merge pull request #4211 from Infisical/daniel/vault-import

feat(external-migrations): vault migrations

backend/src/server/routes/v3/external-migration-router.ts

@@ -1,9 +1,11 @@
 import fastifyMultipart from "@fastify/multipart";
+import { z } from "zod";
 
 import { BadRequestError } from "@app/lib/errors";
-import { readLimit } from "@app/server/config/rateLimiter";
+import { writeLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
+import { VaultMappingType } from "@app/services/external-migration/external-migration-types";
 
 const MB25_IN_BYTES = 26214400;
 
@@ -15,7 +17,7 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider
     bodyLimit: MB25_IN_BYTES,
     url: "/env-key",
     config: {
-      rateLimit: readLimit
+      rateLimit: writeLimit
     },
     onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
     handler: async (req) => {
@@ -52,4 +54,30 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider
       });
     }
   });
+
+  server.route({
+    method: "POST",
+    url: "/vault",
+    config: {
+      rateLimit: writeLimit
+    },
+    schema: {
+      body: z.object({
+        vaultAccessToken: z.string(),
+        vaultNamespace: z.string().trim().optional(),
+        vaultUrl: z.string(),
+        mappingType: z.nativeEnum(VaultMappingType)
+      })
+    },
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
+    handler: async (req) => {
+      await server.services.migration.importVaultData({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorOrgId: req.permission.orgId,
+        actorAuthMethod: req.permission.authMethod,
+        ...req.body
+      });
+    }
+  });
 };

backend/src/server/routes/v3/index.ts

@@ -11,5 +11,5 @@ export const registerV3Routes = async (server: FastifyZodProvider) => {
   await server.register(registerUserRouter, { prefix: "/users" });
   await server.register(registerSecretRouter, { prefix: "/secrets" });
   await server.register(registerSecretBlindIndexRouter, { prefix: "/workspaces" });
-  await server.register(registerExternalMigrationRouter, { prefix: "/migrate" });
+  await server.register(registerExternalMigrationRouter, { prefix: "/external-migration" });
 };

backend/src/services/external-migration/external-migration-fns/envkey.ts

@@ -1,32 +1,26 @@
-import slugify from "@sindresorhus/slugify";
 import sjcl from "sjcl";
 import tweetnacl from "tweetnacl";
 import tweetnaclUtil from "tweetnacl-util";
 
-import { SecretType, TSecretFolders } from "@app/db/schemas";
 import { crypto } from "@app/lib/crypto/cryptography";
-import { BadRequestError, NotFoundError } from "@app/lib/errors";
-import { chunkArray } from "@app/lib/fn";
+import { BadRequestError } from "@app/lib/errors";
 import { logger } from "@app/lib/logger";
-import { alphaNumericNanoId } from "@app/lib/nanoid";
 
-import { CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
-import { TKmsServiceFactory } from "../kms/kms-service";
-import { KmsDataKey } from "../kms/kms-types";
-import { TProjectDALFactory } from "../project/project-dal";
-import { TProjectServiceFactory } from "../project/project-service";
-import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
-import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
-import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal";
-import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
-import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal";
-import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
-import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
-import { fnSecretBulkInsert, getAllSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns";
-import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service";
-import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal";
-import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
-import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types";
+import { TFolderCommitServiceFactory } from "../../folder-commit/folder-commit-service";
+import { TKmsServiceFactory } from "../../kms/kms-service";
+import { TProjectDALFactory } from "../../project/project-dal";
+import { TProjectServiceFactory } from "../../project/project-service";
+import { TProjectEnvDALFactory } from "../../project-env/project-env-dal";
+import { TProjectEnvServiceFactory } from "../../project-env/project-env-service";
+import { TResourceMetadataDALFactory } from "../../resource-metadata/resource-metadata-dal";
+import { TSecretFolderDALFactory } from "../../secret-folder/secret-folder-dal";
+import { TSecretFolderVersionDALFactory } from "../../secret-folder/secret-folder-version-dal";
+import { TSecretTagDALFactory } from "../../secret-tag/secret-tag-dal";
+import { TSecretV2BridgeDALFactory } from "../../secret-v2-bridge/secret-v2-bridge-dal";
+import type { TSecretV2BridgeServiceFactory } from "../../secret-v2-bridge/secret-v2-bridge-service";
+import { TSecretVersionV2DALFactory } from "../../secret-v2-bridge/secret-version-dal";
+import { TSecretVersionV2TagDALFactory } from "../../secret-v2-bridge/secret-version-tag-dal";
+import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "../external-migration-types";
 
 export type TImportDataIntoInfisicalDTO = {
   projectDAL: Pick<TProjectDALFactory, "transaction">;
@@ -499,326 +493,3 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
 
   return infisicalImportData;
 };
-
-export const importDataIntoInfisicalFn = async ({
-  projectService,
-  projectEnvDAL,
-  projectDAL,
-  secretDAL,
-  kmsService,
-  secretVersionDAL,
-  secretTagDAL,
-  secretVersionTagDAL,
-  folderDAL,
-  resourceMetadataDAL,
-  folderVersionDAL,
-  folderCommitService,
-  input: { data, actor, actorId, actorOrgId, actorAuthMethod }
-}: TImportDataIntoInfisicalDTO) => {
-  // Import data to infisical
-  if (!data || !data.projects) {
-    throw new BadRequestError({ message: "No projects found in data" });
-  }
-
-  const originalToNewProjectId = new Map<string, string>();
-  const originalToNewEnvironmentId = new Map<
-    string,
-    { envId: string; envSlug: string; rootFolderId: string; projectId: string }
-  >();
-  const originalToNewFolderId = new Map<
-    string,
-    {
-      folderId: string;
-      projectId: string;
-    }
-  >();
-  const projectsNotImported: string[] = [];
-
-  await projectDAL.transaction(async (tx) => {
-    for await (const project of data.projects) {
-      const newProject = await projectService
-        .createProject({
-          actor,
-          actorId,
-          actorOrgId,
-          actorAuthMethod,
-          workspaceName: project.name,
-          createDefaultEnvs: false,
-          tx
-        })
-        .catch((e) => {
-          logger.error(e, `Failed to import to project [name:${project.name}]`);
-          throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` });
-        });
-      originalToNewProjectId.set(project.id, newProject.id);
-    }
-
-    // Import environments
-    if (data.environments) {
-      for await (const environment of data.environments) {
-        const projectId = originalToNewProjectId.get(environment.projectId);
-        const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`);
-
-        if (!projectId) {
-          projectsNotImported.push(environment.projectId);
-          // eslint-disable-next-line no-continue
-          continue;
-        }
-
-        const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx);
-
-        if (existingEnv) {
-          throw new BadRequestError({
-            message: `Environment with slug '${slug}' already exist`,
-            name: "CreateEnvironment"
-          });
-        }
-
-        const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx);
-        const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx);
-        const folder = await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx);
-
-        originalToNewEnvironmentId.set(environment.id, {
-          envSlug: doc.slug,
-          envId: doc.id,
-          rootFolderId: folder.id,
-          projectId
-        });
-      }
-    }
-
-    if (data.folders) {
-      for await (const folder of data.folders) {
-        const parentEnv = originalToNewEnvironmentId.get(folder.parentFolderId as string);
-
-        if (!parentEnv) {
-          // eslint-disable-next-line no-continue
-          continue;
-        }
-
-        const newFolder = await folderDAL.create(
-          {
-            name: folder.name,
-            envId: parentEnv.envId,
-            parentId: parentEnv.rootFolderId
-          },
-          tx
-        );
-
-        const newFolderVersion = await folderVersionDAL.create(
-          {
-            name: newFolder.name,
-            envId: newFolder.envId,
-            version: newFolder.version,
-            folderId: newFolder.id
-          },
-          tx
-        );
-
-        await folderCommitService.createCommit(
-          {
-            actor: {
-              type: actor,
-              metadata: {
-                id: actorId
-              }
-            },
-            message: "Changed by external migration",
-            folderId: parentEnv.rootFolderId,
-            changes: [
-              {
-                type: CommitType.ADD,
-                folderVersionId: newFolderVersion.id
-              }
-            ]
-          },
-          tx
-        );
-
-        originalToNewFolderId.set(folder.id, {
-          folderId: newFolder.id,
-          projectId: parentEnv.projectId
-        });
-      }
-    }
-
-    // Useful for debugging:
-    // console.log("data.secrets", data.secrets);
-    // console.log("data.folders", data.folders);
-    // console.log("data.environment", data.environments);
-
-    if (data.secrets && data.secrets.length > 0) {
-      const mappedToEnvironmentId = new Map<
-        string,
-        {
-          secretKey: string;
-          secretValue: string;
-          folderId?: string;
-          isFromBlock?: boolean;
-        }[]
-      >();
-
-      for (const secret of data.secrets) {
-        const targetId = secret.folderId || secret.environmentId;
-
-        // Skip if we can't find either an environment or folder mapping for this secret
-        if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(targetId)) {
-          logger.info({ secret }, "[importDataIntoInfisicalFn]: Could not find environment or folder for secret");
-
-          // eslint-disable-next-line no-continue
-          continue;
-        }
-
-        if (!mappedToEnvironmentId.has(targetId)) {
-          mappedToEnvironmentId.set(targetId, []);
-        }
-
-        const alreadyHasSecret = mappedToEnvironmentId
-          .get(targetId)!
-          .find((el) => el.secretKey === secret.name && el.folderId === secret.folderId);
-
-        if (alreadyHasSecret && alreadyHasSecret.isFromBlock) {
-          // remove the existing secret if any
-          mappedToEnvironmentId
-            .get(targetId)!
-            .splice(mappedToEnvironmentId.get(targetId)!.indexOf(alreadyHasSecret), 1);
-        }
-        mappedToEnvironmentId.get(targetId)!.push({
-          secretKey: secret.name,
-          secretValue: secret.value || "",
-          folderId: secret.folderId,
-          isFromBlock: secret.appBlockOrderIndex !== undefined
-        });
-      }
-
-      // for each of the mappedEnvironmentId
-      for await (const [targetId, secrets] of mappedToEnvironmentId) {
-        logger.info("[importDataIntoInfisicalFn]: Processing secrets for targetId", targetId);
-
-        let selectedFolder: TSecretFolders | undefined;
-        let selectedProjectId: string | undefined;
-
-        // Case 1: Secret belongs to a folder / branch / branch of a block
-        const foundFolder = originalToNewFolderId.get(targetId);
-        if (foundFolder) {
-          logger.info("[importDataIntoInfisicalFn]: Processing secrets for folder");
-          selectedFolder = await folderDAL.findById(foundFolder.folderId, tx);
-          selectedProjectId = foundFolder.projectId;
-        } else {
-          logger.info("[importDataIntoInfisicalFn]: Processing secrets for normal environment");
-          const environment = data.environments.find((env) => env.id === targetId);
-          if (!environment) {
-            logger.info(
-              {
-                targetId
-              },
-              "[importDataIntoInfisicalFn]: Could not find environment for secret"
-            );
-            // eslint-disable-next-line no-continue
-            continue;
-          }
-
-          const projectId = originalToNewProjectId.get(environment.projectId)!;
-
-          if (!projectId) {
-            throw new BadRequestError({ message: `Failed to import secret, project not found` });
-          }
-
-          const env = originalToNewEnvironmentId.get(targetId);
-          if (!env) {
-            logger.info(
-              {
-                targetId
-              },
-              "[importDataIntoInfisicalFn]: Could not find environment for secret"
-            );
-
-            // eslint-disable-next-line no-continue
-            continue;
-          }
-
-          const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx);
-
-          if (!folder) {
-            throw new NotFoundError({
-              message: `Folder not found for the given environment slug (${env.envSlug}) & secret path (/)`,
-              name: "Create secret"
-            });
-          }
-
-          selectedFolder = folder;
-          selectedProjectId = projectId;
-        }
-
-        if (!selectedFolder) {
-          throw new NotFoundError({
-            message: `Folder not found for the given environment slug & secret path`,
-            name: "CreateSecret"
-          });
-        }
-
-        if (!selectedProjectId) {
-          throw new NotFoundError({
-            message: `Project not found for the given environment slug & secret path`,
-            name: "CreateSecret"
-          });
-        }
-
-        const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey(
-          {
-            type: KmsDataKey.SecretManager,
-            projectId: selectedProjectId
-          },
-          tx
-        );
-
-        const secretBatches = chunkArray(secrets, 2500);
-        for await (const secretBatch of secretBatches) {
-          const secretsByKeys = await secretDAL.findBySecretKeys(
-            selectedFolder.id,
-            secretBatch.map((el) => ({
-              key: el.secretKey,
-              type: SecretType.Shared
-            })),
-            tx
-          );
-          if (secretsByKeys.length) {
-            throw new BadRequestError({
-              message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}`
-            });
-          }
-          await fnSecretBulkInsert({
-            inputSecrets: secretBatch.map((el) => {
-              const references = getAllSecretReferences(el.secretValue).nestedReferences;
-
-              return {
-                version: 1,
-                encryptedValue: el.secretValue
-                  ? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob
-                  : undefined,
-                key: el.secretKey,
-                references,
-                type: SecretType.Shared
-              };
-            }),
-            folderId: selectedFolder.id,
-            orgId: actorOrgId,
-            resourceMetadataDAL,
-            secretDAL,
-            secretVersionDAL,
-            secretTagDAL,
-            secretVersionTagDAL,
-            folderCommitService,
-            actor: {
-              type: actor,
-              actorId
-            },
-            tx
-          });
-        }
-      }
-    }
-  });
-
-  return { projectsNotImported };
-};

backend/src/services/external-migration/external-migration-fns/import.ts

@@ -0,0 +1,352 @@
+import slugify from "@sindresorhus/slugify";
+
+import { SecretType, TSecretFolders } from "@app/db/schemas";
+import { BadRequestError, NotFoundError } from "@app/lib/errors";
+import { chunkArray } from "@app/lib/fn";
+import { logger } from "@app/lib/logger";
+import { alphaNumericNanoId } from "@app/lib/nanoid";
+import { CommitType } from "@app/services/folder-commit/folder-commit-service";
+import { KmsDataKey } from "@app/services/kms/kms-types";
+import { fnSecretBulkInsert, getAllSecretReferences } from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";
+
+import { TImportDataIntoInfisicalDTO } from "./envkey";
+
+export const importDataIntoInfisicalFn = async ({
+  projectService,
+  projectEnvDAL,
+  projectDAL,
+  secretDAL,
+  kmsService,
+  secretVersionDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  folderDAL,
+  resourceMetadataDAL,
+  folderVersionDAL,
+  folderCommitService,
+  input: { data, actor, actorId, actorOrgId, actorAuthMethod }
+}: TImportDataIntoInfisicalDTO) => {
+  // Import data to infisical
+  if (!data || !data.projects) {
+    throw new BadRequestError({ message: "No projects found in data" });
+  }
+
+  const originalToNewProjectId = new Map<string, string>();
+  const originalToNewEnvironmentId = new Map<
+    string,
+    { envId: string; envSlug: string; rootFolderId?: string; projectId: string }
+  >();
+  const originalToNewFolderId = new Map<
+    string,
+    {
+      envId: string;
+      envSlug: string;
+      folderId: string;
+      projectId: string;
+    }
+  >();
+  const projectsNotImported: string[] = [];
+
+  await projectDAL.transaction(async (tx) => {
+    for await (const project of data.projects) {
+      const newProject = await projectService
+        .createProject({
+          actor,
+          actorId,
+          actorOrgId,
+          actorAuthMethod,
+          workspaceName: project.name,
+          createDefaultEnvs: false,
+          tx
+        })
+        .catch((e) => {
+          logger.error(e, `Failed to import to project [name:${project.name}]`);
+          throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` });
+        });
+      originalToNewProjectId.set(project.id, newProject.id);
+    }
+
+    // Import environments
+    if (data.environments) {
+      for await (const environment of data.environments) {
+        const projectId = originalToNewProjectId.get(environment.projectId);
+        const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`);
+
+        if (!projectId) {
+          projectsNotImported.push(environment.projectId);
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx);
+
+        if (existingEnv) {
+          throw new BadRequestError({
+            message: `Environment with slug '${slug}' already exist`,
+            name: "CreateEnvironment"
+          });
+        }
+
+        const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx);
+        const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx);
+        const folder = await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx);
+
+        originalToNewEnvironmentId.set(environment.id, {
+          envSlug: doc.slug,
+          envId: doc.id,
+          rootFolderId: folder.id,
+          projectId
+        });
+      }
+    }
+
+    if (data.folders) {
+      for await (const folder of data.folders) {
+        const parentEnv = originalToNewEnvironmentId.get(folder.parentFolderId as string);
+        const parentFolder = originalToNewFolderId.get(folder.parentFolderId as string);
+
+        let newFolder: TSecretFolders;
+
+        if (parentEnv?.rootFolderId) {
+          newFolder = await folderDAL.create(
+            {
+              name: folder.name,
+              envId: parentEnv.envId,
+              parentId: parentEnv.rootFolderId
+            },
+            tx
+          );
+        } else if (parentFolder) {
+          newFolder = await folderDAL.create(
+            {
+              name: folder.name,
+              envId: parentFolder.envId,
+              parentId: parentFolder.folderId
+            },
+            tx
+          );
+        } else {
+          logger.info({ folder }, "No parent environment found for folder");
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        const newFolderVersion = await folderVersionDAL.create(
+          {
+            name: newFolder.name,
+            envId: newFolder.envId,
+            version: newFolder.version,
+            folderId: newFolder.id
+          },
+          tx
+        );
+
+        await folderCommitService.createCommit(
+          {
+            actor: {
+              type: actor,
+              metadata: {
+                id: actorId
+              }
+            },
+            message: "Changed by external migration",
+            folderId: parentEnv?.rootFolderId || parentFolder?.folderId || "",
+            changes: [
+              {
+                type: CommitType.ADD,
+                folderVersionId: newFolderVersion.id
+              }
+            ]
+          },
+          tx
+        );
+
+        originalToNewFolderId.set(folder.id, {
+          folderId: newFolder.id,
+          envId: parentEnv?.envId || parentFolder?.envId || "",
+          envSlug: parentEnv?.envSlug || parentFolder?.envSlug || "",
+          projectId: parentEnv?.projectId || parentFolder?.projectId || ""
+        });
+      }
+    }
+
+    // Useful for debugging:
+    // console.log("data.secrets", data.secrets);
+    // console.log("data.folders", data.folders);
+    // console.log("data.environment", data.environments);
+
+    if (data.secrets && data.secrets.length > 0) {
+      const mappedToEnvironmentId = new Map<
+        string,
+        {
+          secretKey: string;
+          secretValue: string;
+          folderId?: string;
+          isFromBlock?: boolean;
+        }[]
+      >();
+
+      for (const secret of data.secrets) {
+        const targetId = secret.folderId || secret.environmentId;
+
+        // Skip if we can't find either an environment or folder mapping for this secret
+        if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(targetId)) {
+          logger.info({ secret }, "[importDataIntoInfisicalFn]: Could not find environment or folder for secret");
+
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        if (!mappedToEnvironmentId.has(targetId)) {
+          mappedToEnvironmentId.set(targetId, []);
+        }
+
+        const alreadyHasSecret = mappedToEnvironmentId
+          .get(targetId)!
+          .find((el) => el.secretKey === secret.name && el.folderId === secret.folderId);
+
+        if (alreadyHasSecret && alreadyHasSecret.isFromBlock) {
+          // remove the existing secret if any
+          mappedToEnvironmentId
+            .get(targetId)!
+            .splice(mappedToEnvironmentId.get(targetId)!.indexOf(alreadyHasSecret), 1);
+        }
+        mappedToEnvironmentId.get(targetId)!.push({
+          secretKey: secret.name,
+          secretValue: secret.value || "",
+          folderId: secret.folderId,
+          isFromBlock: secret.appBlockOrderIndex !== undefined
+        });
+      }
+
+      // for each of the mappedEnvironmentId
+      for await (const [targetId, secrets] of mappedToEnvironmentId) {
+        logger.info("[importDataIntoInfisicalFn]: Processing secrets for targetId", targetId);
+
+        let selectedFolder: TSecretFolders | undefined;
+        let selectedProjectId: string | undefined;
+
+        // Case 1: Secret belongs to a folder / branch / branch of a block
+        const foundFolder = originalToNewFolderId.get(targetId);
+        if (foundFolder) {
+          logger.info("[importDataIntoInfisicalFn]: Processing secrets for folder");
+          selectedFolder = await folderDAL.findById(foundFolder.folderId, tx);
+          selectedProjectId = foundFolder.projectId;
+        } else {
+          logger.info("[importDataIntoInfisicalFn]: Processing secrets for normal environment");
+          const environment = data.environments.find((env) => env.id === targetId);
+          if (!environment) {
+            logger.info(
+              {
+                targetId
+              },
+              "[importDataIntoInfisicalFn]: Could not find environment for secret"
+            );
+            // eslint-disable-next-line no-continue
+            continue;
+          }
+
+          const projectId = originalToNewProjectId.get(environment.projectId)!;
+
+          if (!projectId) {
+            throw new BadRequestError({ message: `Failed to import secret, project not found` });
+          }
+
+          const env = originalToNewEnvironmentId.get(targetId);
+          if (!env) {
+            logger.info(
+              {
+                targetId
+              },
+              "[importDataIntoInfisicalFn]: Could not find environment for secret"
+            );
+
+            // eslint-disable-next-line no-continue
+            continue;
+          }
+
+          const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx);
+
+          if (!folder) {
+            throw new NotFoundError({
+              message: `Folder not found for the given environment slug (${env.envSlug}) & secret path (/)`,
+              name: "Create secret"
+            });
+          }
+
+          selectedFolder = folder;
+          selectedProjectId = projectId;
+        }
+
+        if (!selectedFolder) {
+          throw new NotFoundError({
+            message: `Folder not found for the given environment slug & secret path`,
+            name: "CreateSecret"
+          });
+        }
+
+        if (!selectedProjectId) {
+          throw new NotFoundError({
+            message: `Project not found for the given environment slug & secret path`,
+            name: "CreateSecret"
+          });
+        }
+
+        const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey(
+          {
+            type: KmsDataKey.SecretManager,
+            projectId: selectedProjectId
+          },
+          tx
+        );
+
+        const secretBatches = chunkArray(secrets, 2500);
+        for await (const secretBatch of secretBatches) {
+          const secretsByKeys = await secretDAL.findBySecretKeys(
+            selectedFolder.id,
+            secretBatch.map((el) => ({
+              key: el.secretKey,
+              type: SecretType.Shared
+            })),
+            tx
+          );
+          if (secretsByKeys.length) {
+            throw new BadRequestError({
+              message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}`
+            });
+          }
+          await fnSecretBulkInsert({
+            inputSecrets: secretBatch.map((el) => {
+              const references = getAllSecretReferences(el.secretValue).nestedReferences;
+
+              return {
+                version: 1,
+                encryptedValue: el.secretValue
+                  ? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob
+                  : undefined,
+                key: el.secretKey,
+                references,
+                type: SecretType.Shared
+              };
+            }),
+            folderId: selectedFolder.id,
+            orgId: actorOrgId,
+            resourceMetadataDAL,
+            secretDAL,
+            secretVersionDAL,
+            secretTagDAL,
+            secretVersionTagDAL,
+            folderCommitService,
+            actor: {
+              type: actor,
+              actorId
+            },
+            tx
+          });
+        }
+      }
+    }
+  });
+
+  return { projectsNotImported };
+};

backend/src/services/external-migration/external-migration-fns/index.ts

@@ -0,0 +1,3 @@
+export * from "./envkey";
+export * from "./import";
+export * from "./vault";

backend/src/services/external-migration/external-migration-fns/vault.ts

@@ -0,0 +1,341 @@
+import axios, { AxiosInstance } from "axios";
+import { v4 as uuidv4 } from "uuid";
+
+import { BadRequestError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
+import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
+
+import { InfisicalImportData, VaultMappingType } from "../external-migration-types";
+
+type VaultData = {
+  namespace: string;
+  mount: string;
+  path: string;
+  secretData: Record<string, string>;
+};
+
+const vaultFactory = () => {
+  const getMounts = async (request: AxiosInstance) => {
+    const response = await request
+      .get<
+        Record<
+          string,
+          {
+            accessor: string;
+            options: {
+              version?: string;
+            } | null;
+            type: string;
+          }
+        >
+      >("/v1/sys/mounts")
+      .catch((err) => {
+        if (axios.isAxiosError(err)) {
+          logger.error(err.response?.data, "External migration: Failed to get Vault mounts");
+        }
+        throw err;
+      });
+    return response.data;
+  };
+
+  const getPaths = async (
+    request: AxiosInstance,
+    { mountPath, secretPath = "" }: { mountPath: string; secretPath?: string }
+  ) => {
+    try {
+      // For KV v2: /v1/{mount}/metadata/{path}?list=true
+      const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`;
+      const response = await request.get<{
+        data: {
+          keys: string[];
+        };
+      }>(`/v1/${path}?list=true`);
+
+      return response.data.data.keys;
+    } catch (err) {
+      if (axios.isAxiosError(err)) {
+        logger.error(err.response?.data, "External migration: Failed to get Vault paths");
+        if (err.response?.status === 404) {
+          return null;
+        }
+      }
+      throw err;
+    }
+  };
+
+  const getSecrets = async (
+    request: AxiosInstance,
+    { mountPath, secretPath }: { mountPath: string; secretPath: string }
+  ) => {
+    // For KV v2: /v1/{mount}/data/{path}
+    const response = await request
+      .get<{
+        data: {
+          data: Record<string, string>; // KV v2 has nested data structure
+          metadata: {
+            created_time: string;
+            deletion_time: string;
+            destroyed: boolean;
+            version: number;
+          };
+        };
+      }>(`/v1/${mountPath}/data/${secretPath}`)
+      .catch((err) => {
+        if (axios.isAxiosError(err)) {
+          logger.error(err.response?.data, "External migration: Failed to get Vault secret");
+        }
+        throw err;
+      });
+
+    return response.data.data.data;
+  };
+
+  // helper function to check if a mount is KV v2 (will be useful if we add support for Vault KV v1)
+  // const isKvV2Mount = (mountInfo: { type: string; options?: { version?: string } | null }) => {
+  //   return mountInfo.type === "kv" && mountInfo.options?.version === "2";
+  // };
+
+  const recursivelyGetAllPaths = async (
+    request: AxiosInstance,
+    mountPath: string,
+    currentPath: string = ""
+  ): Promise<string[]> => {
+    const paths = await getPaths(request, { mountPath, secretPath: currentPath });
+
+    if (paths === null || paths.length === 0) {
+      return [];
+    }
+
+    const allSecrets: string[] = [];
+
+    for await (const path of paths) {
+      const cleanPath = path.endsWith("/") ? path.slice(0, -1) : path;
+      const fullItemPath = currentPath ? `${currentPath}/${cleanPath}` : cleanPath;
+
+      if (path.endsWith("/")) {
+        // it's a folder so we recurse into it
+        const subSecrets = await recursivelyGetAllPaths(request, mountPath, fullItemPath);
+        allSecrets.push(...subSecrets);
+      } else {
+        // it's a secret so we add it to our results
+        allSecrets.push(`${mountPath}/${fullItemPath}`);
+      }
+    }
+
+    return allSecrets;
+  };
+
+  async function collectVaultData({
+    baseUrl,
+    namespace,
+    accessToken
+  }: {
+    baseUrl: string;
+    namespace?: string;
+    accessToken: string;
+  }): Promise<VaultData[]> {
+    const request = axios.create({
+      baseURL: baseUrl,
+      headers: {
+        "X-Vault-Token": accessToken,
+        ...(namespace ? { "X-Vault-Namespace": namespace } : {})
+      }
+    });
+
+    const allData: VaultData[] = [];
+
+    // Get all mounts in this namespace
+    const mounts = await getMounts(request);
+
+    for (const mount of Object.keys(mounts)) {
+      if (!mount.endsWith("/")) {
+        delete mounts[mount];
+      }
+    }
+
+    for await (const [mountPath, mountInfo] of Object.entries(mounts)) {
+      // skip non-KV mounts
+      if (!mountInfo.type.startsWith("kv")) {
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+
+      // get all paths in this mount
+      const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`);
+
+      const cleanMountPath = mountPath.replace(/\/$/, "");
+
+      for await (const secretPath of paths) {
+        // get the actual secret data
+        const secretData = await getSecrets(request, {
+          mountPath: cleanMountPath,
+          secretPath: secretPath.replace(`${cleanMountPath}/`, "")
+        });
+
+        allData.push({
+          namespace: namespace || "",
+          mount: mountPath.replace(/\/$/, ""),
+          path: secretPath.replace(`${cleanMountPath}/`, ""),
+          secretData
+        });
+      }
+    }
+
+    return allData;
+  }
+
+  return {
+    collectVaultData,
+    getMounts,
+    getPaths,
+    getSecrets,
+    recursivelyGetAllPaths
+  };
+};
+
+export const transformToInfisicalFormatNamespaceToProjects = (
+  vaultData: VaultData[],
+  mappingType: VaultMappingType
+): InfisicalImportData => {
+  const projects: Array<{ name: string; id: string }> = [];
+  const environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }> = [];
+  const folders: Array<{ id: string; name: string; environmentId: string; parentFolderId?: string }> = [];
+  const secrets: Array<{ id: string; name: string; environmentId: string; value: string; folderId?: string }> = [];
+
+  // track created entities to avoid duplicates
+  const projectMap = new Map<string, string>(); // namespace -> projectId
+  const environmentMap = new Map<string, string>(); // namespace:mount -> environmentId
+  const folderMap = new Map<string, string>(); // namespace:mount:folderPath -> folderId
+
+  let environmentId: string = "";
+  for (const data of vaultData) {
+    const { namespace, mount, path, secretData } = data;
+
+    if (mappingType === VaultMappingType.Namespace) {
+      // create project (namespace)
+      if (!projectMap.has(namespace)) {
+        const projectId = uuidv4();
+        projectMap.set(namespace, projectId);
+        projects.push({
+          name: namespace,
+          id: projectId
+        });
+      }
+      const projectId = projectMap.get(namespace)!;
+
+      // create environment (mount)
+      const envKey = `${namespace}:${mount}`;
+      if (!environmentMap.has(envKey)) {
+        environmentId = uuidv4();
+        environmentMap.set(envKey, environmentId);
+        environments.push({
+          name: mount,
+          id: environmentId,
+          projectId
+        });
+      }
+      environmentId = environmentMap.get(envKey)!;
+    } else if (mappingType === VaultMappingType.KeyVault) {
+      if (!projectMap.has(mount)) {
+        const projectId = uuidv4();
+        projectMap.set(mount, projectId);
+        projects.push({
+          name: mount,
+          id: projectId
+        });
+      }
+      const projectId = projectMap.get(mount)!;
+
+      // create single "Production" environment per project, because we have no good way of determining environments from vault
+      if (!environmentMap.has(mount)) {
+        environmentId = uuidv4();
+        environmentMap.set(mount, environmentId);
+        environments.push({
+          name: "Production",
+          id: environmentId,
+          projectId
+        });
+      }
+      environmentId = environmentMap.get(mount)!;
+    }
+
+    // create folder structure
+    let currentFolderId: string | undefined;
+    let currentPath = "";
+
+    if (path.includes("/")) {
+      const pathParts = path.split("/").filter(Boolean);
+
+      const folderParts = pathParts;
+
+      // create nested folder structure for the entire path
+      for (const folderName of folderParts) {
+        currentPath = currentPath ? `${currentPath}/${folderName}` : folderName;
+        const folderKey = `${namespace}:${mount}:${currentPath}`;
+
+        if (!folderMap.has(folderKey)) {
+          const folderId = uuidv4();
+          folderMap.set(folderKey, folderId);
+          folders.push({
+            id: folderId,
+            name: folderName,
+            environmentId,
+            parentFolderId: currentFolderId || environmentId
+          });
+          currentFolderId = folderId;
+        } else {
+          currentFolderId = folderMap.get(folderKey)!;
+        }
+      }
+    }
+
+    for (const [key, value] of Object.entries(secretData)) {
+      secrets.push({
+        id: uuidv4(),
+        name: key,
+        environmentId,
+        value: String(value),
+        folderId: currentFolderId
+      });
+    }
+  }
+
+  return {
+    projects,
+    environments,
+    folders,
+    secrets
+  };
+};
+
+export const importVaultDataFn = async ({
+  vaultAccessToken,
+  vaultNamespace,
+  vaultUrl,
+  mappingType
+}: {
+  vaultAccessToken: string;
+  vaultNamespace?: string;
+  vaultUrl: string;
+  mappingType: VaultMappingType;
+}) => {
+  await blockLocalAndPrivateIpAddresses(vaultUrl);
+
+  if (mappingType === VaultMappingType.Namespace && !vaultNamespace) {
+    throw new BadRequestError({
+      message: "Vault namespace is required when project mapping type is set to namespace."
+    });
+  }
+
+  const vaultApi = vaultFactory();
+
+  const vaultData = await vaultApi.collectVaultData({
+    accessToken: vaultAccessToken,
+    baseUrl: vaultUrl,
+    namespace: vaultNamespace
+  });
+
+  const infisicalData = transformToInfisicalFormatNamespaceToProjects(vaultData, mappingType);
+
+  return infisicalData;
+};

backend/src/services/external-migration/external-migration-queue.ts

@@ -19,7 +19,7 @@ import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-d
 import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
 import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
 import { importDataIntoInfisicalFn } from "./external-migration-fns";
-import { ExternalPlatforms, TImportInfisicalDataCreate } from "./external-migration-types";
+import { ExternalPlatforms, ImportType, TImportInfisicalDataCreate } from "./external-migration-types";
 
 export type TExternalMigrationQueueFactoryDep = {
   smtpService: TSmtpService;
@@ -67,6 +67,7 @@ export const externalMigrationQueueFactory = ({
   const startImport = async (dto: {
     actorEmail: string;
     data: {
+      importType: ImportType;
       iv: string;
       tag: string;
       ciphertext: string;

backend/src/services/external-migration/external-migration-service.ts

@@ -4,9 +4,9 @@ import { crypto } from "@app/lib/crypto/cryptography";
 import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors";
 
 import { TUserDALFactory } from "../user/user-dal";
-import { decryptEnvKeyDataFn, parseEnvKeyDataFn } from "./external-migration-fns";
+import { decryptEnvKeyDataFn, importVaultDataFn, parseEnvKeyDataFn } from "./external-migration-fns";
 import { TExternalMigrationQueueFactory } from "./external-migration-queue";
-import { TImportEnvKeyDataCreate } from "./external-migration-types";
+import { ImportType, TImportEnvKeyDataDTO, TImportVaultDataDTO } from "./external-migration-types";
 
 type TExternalMigrationServiceFactoryDep = {
   permissionService: TPermissionServiceFactory;
@@ -28,7 +28,7 @@ export const externalMigrationServiceFactory = ({
     actorId,
     actorOrgId,
     actorAuthMethod
-  }: TImportEnvKeyDataCreate) => {
+  }: TImportEnvKeyDataDTO) => {
     if (crypto.isFipsModeEnabled()) {
       throw new BadRequestError({ message: "EnvKey migration is not supported when running in FIPS mode." });
     }
@@ -60,11 +60,65 @@ export const externalMigrationServiceFactory = ({
 
     await externalMigrationQueue.startImport({
       actorEmail: user.email!,
-      data: encrypted
+      data: {
+        importType: ImportType.EnvKey,
+        ...encrypted
+      }
+    });
+  };
+
+  const importVaultData = async ({
+    vaultAccessToken,
+    vaultNamespace,
+    mappingType,
+    vaultUrl,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod
+  }: TImportVaultDataDTO) => {
+    const { membership } = await permissionService.getOrgPermission(
+      actor,
+      actorId,
+      actorOrgId,
+      actorAuthMethod,
+      actorOrgId
+    );
+
+    if (membership.role !== OrgMembershipRole.Admin) {
+      throw new ForbiddenRequestError({ message: "Only admins can import data" });
+    }
+
+    const user = await userDAL.findById(actorId);
+
+    const vaultData = await importVaultDataFn({
+      vaultAccessToken,
+      vaultNamespace,
+      vaultUrl,
+      mappingType
+    });
+
+    const stringifiedJson = JSON.stringify({
+      data: vaultData,
+      actor,
+      actorId,
+      actorOrgId,
+      actorAuthMethod
+    });
+
+    const encrypted = crypto.encryption().symmetric().encryptWithRootEncryptionKey(stringifiedJson);
+
+    await externalMigrationQueue.startImport({
+      actorEmail: user.email!,
+      data: {
+        importType: ImportType.Vault,
+        ...encrypted
+      }
     });
   };
 
   return {
-    importEnvKeyData
+    importEnvKeyData,
+    importVaultData
   };
 };

backend/src/services/external-migration/external-migration-types.ts

@@ -1,5 +1,17 @@
+import { TOrgPermission } from "@app/lib/types";
+
 import { ActorAuthMethod, ActorType } from "../auth/auth-type";
 
+export enum ImportType {
+  EnvKey = "envkey",
+  Vault = "vault"
+}
+
+export enum VaultMappingType {
+  Namespace = "namespace",
+  KeyVault = "key-vault"
+}
+
 export type InfisicalImportData = {
   projects: Array<{ name: string; id: string }>;
   environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }>;
@@ -14,14 +26,17 @@ export type InfisicalImportData = {
   }>;
 };
 
-export type TImportEnvKeyDataCreate = {
+export type TImportEnvKeyDataDTO = {
   decryptionKey: string;
   encryptedJson: { nonce: string; data: string };
-  actor: ActorType;
-  actorId: string;
-  actorOrgId: string;
-  actorAuthMethod: ActorAuthMethod;
-};
+} & Omit<TOrgPermission, "orgId">;
+
+export type TImportVaultDataDTO = {
+  vaultAccessToken: string;
+  vaultNamespace?: string;
+  mappingType: VaultMappingType;
+  vaultUrl: string;
+} & Omit<TOrgPermission, "orgId">;
 
 export type TImportInfisicalDataCreate = {
   data: InfisicalImportData;