diff --git a/backend/src/server/routes/v3/external-migration-router.ts b/backend/src/server/routes/v3/external-migration-router.ts index 865287157..89621811d 100644 --- a/backend/src/server/routes/v3/external-migration-router.ts +++ b/backend/src/server/routes/v3/external-migration-router.ts @@ -1,9 +1,11 @@ import fastifyMultipart from "@fastify/multipart"; +import { z } from "zod"; import { BadRequestError } from "@app/lib/errors"; -import { readLimit } from "@app/server/config/rateLimiter"; +import { writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; +import { VaultMappingType } from "@app/services/external-migration/external-migration-types"; const MB25_IN_BYTES = 26214400; @@ -15,7 +17,7 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider bodyLimit: MB25_IN_BYTES, url: "/env-key", config: { - rateLimit: readLimit + rateLimit: writeLimit }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { @@ -52,4 +54,30 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider }); } }); + + server.route({ + method: "POST", + url: "/vault", + config: { + rateLimit: writeLimit + }, + schema: { + body: z.object({ + vaultAccessToken: z.string(), + vaultNamespace: z.string().trim().optional(), + vaultUrl: z.string(), + mappingType: z.nativeEnum(VaultMappingType) + }) + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + await server.services.migration.importVaultData({ + actorId: req.permission.id, + actor: req.permission.type, + actorOrgId: req.permission.orgId, + actorAuthMethod: req.permission.authMethod, + ...req.body + }); + } + }); }; diff --git a/backend/src/server/routes/v3/index.ts b/backend/src/server/routes/v3/index.ts index ed8401560..1c31741a1 100644 --- a/backend/src/server/routes/v3/index.ts +++ b/backend/src/server/routes/v3/index.ts @@ -11,5 +11,5 @@ export const registerV3Routes = async (server: FastifyZodProvider) => { await server.register(registerUserRouter, { prefix: "/users" }); await server.register(registerSecretRouter, { prefix: "/secrets" }); await server.register(registerSecretBlindIndexRouter, { prefix: "/workspaces" }); - await server.register(registerExternalMigrationRouter, { prefix: "/migrate" }); + await server.register(registerExternalMigrationRouter, { prefix: "/external-migration" }); }; diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns/envkey.ts similarity index 61% rename from backend/src/services/external-migration/external-migration-fns.ts rename to backend/src/services/external-migration/external-migration-fns/envkey.ts index 8af22d858..ffdf80e9c 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns/envkey.ts @@ -1,32 +1,26 @@ -import slugify from "@sindresorhus/slugify"; import sjcl from "sjcl"; import tweetnacl from "tweetnacl"; import tweetnaclUtil from "tweetnacl-util"; -import { SecretType, TSecretFolders } from "@app/db/schemas"; import { crypto } from "@app/lib/crypto/cryptography"; -import { BadRequestError, NotFoundError } from "@app/lib/errors"; -import { chunkArray } from "@app/lib/fn"; +import { BadRequestError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; -import { alphaNumericNanoId } from "@app/lib/nanoid"; -import { CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; -import { TKmsServiceFactory } from "../kms/kms-service"; -import { KmsDataKey } from "../kms/kms-types"; -import { TProjectDALFactory } from "../project/project-dal"; -import { TProjectServiceFactory } from "../project/project-service"; -import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; -import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; -import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; -import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; -import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; -import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; -import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; -import { fnSecretBulkInsert, getAllSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns"; -import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; -import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; -import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; -import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types"; +import { TFolderCommitServiceFactory } from "../../folder-commit/folder-commit-service"; +import { TKmsServiceFactory } from "../../kms/kms-service"; +import { TProjectDALFactory } from "../../project/project-dal"; +import { TProjectServiceFactory } from "../../project/project-service"; +import { TProjectEnvDALFactory } from "../../project-env/project-env-dal"; +import { TProjectEnvServiceFactory } from "../../project-env/project-env-service"; +import { TResourceMetadataDALFactory } from "../../resource-metadata/resource-metadata-dal"; +import { TSecretFolderDALFactory } from "../../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../../secret-folder/secret-folder-version-dal"; +import { TSecretTagDALFactory } from "../../secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "../../secret-v2-bridge/secret-v2-bridge-dal"; +import type { TSecretV2BridgeServiceFactory } from "../../secret-v2-bridge/secret-v2-bridge-service"; +import { TSecretVersionV2DALFactory } from "../../secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "../../secret-v2-bridge/secret-version-tag-dal"; +import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "../external-migration-types"; export type TImportDataIntoInfisicalDTO = { projectDAL: Pick; @@ -499,326 +493,3 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise { - // Import data to infisical - if (!data || !data.projects) { - throw new BadRequestError({ message: "No projects found in data" }); - } - - const originalToNewProjectId = new Map(); - const originalToNewEnvironmentId = new Map< - string, - { envId: string; envSlug: string; rootFolderId: string; projectId: string } - >(); - const originalToNewFolderId = new Map< - string, - { - folderId: string; - projectId: string; - } - >(); - const projectsNotImported: string[] = []; - - await projectDAL.transaction(async (tx) => { - for await (const project of data.projects) { - const newProject = await projectService - .createProject({ - actor, - actorId, - actorOrgId, - actorAuthMethod, - workspaceName: project.name, - createDefaultEnvs: false, - tx - }) - .catch((e) => { - logger.error(e, `Failed to import to project [name:${project.name}]`); - throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` }); - }); - originalToNewProjectId.set(project.id, newProject.id); - } - - // Import environments - if (data.environments) { - for await (const environment of data.environments) { - const projectId = originalToNewProjectId.get(environment.projectId); - const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`); - - if (!projectId) { - projectsNotImported.push(environment.projectId); - // eslint-disable-next-line no-continue - continue; - } - - const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx); - - if (existingEnv) { - throw new BadRequestError({ - message: `Environment with slug '${slug}' already exist`, - name: "CreateEnvironment" - }); - } - - const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); - const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx); - const folder = await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); - - originalToNewEnvironmentId.set(environment.id, { - envSlug: doc.slug, - envId: doc.id, - rootFolderId: folder.id, - projectId - }); - } - } - - if (data.folders) { - for await (const folder of data.folders) { - const parentEnv = originalToNewEnvironmentId.get(folder.parentFolderId as string); - - if (!parentEnv) { - // eslint-disable-next-line no-continue - continue; - } - - const newFolder = await folderDAL.create( - { - name: folder.name, - envId: parentEnv.envId, - parentId: parentEnv.rootFolderId - }, - tx - ); - - const newFolderVersion = await folderVersionDAL.create( - { - name: newFolder.name, - envId: newFolder.envId, - version: newFolder.version, - folderId: newFolder.id - }, - tx - ); - - await folderCommitService.createCommit( - { - actor: { - type: actor, - metadata: { - id: actorId - } - }, - message: "Changed by external migration", - folderId: parentEnv.rootFolderId, - changes: [ - { - type: CommitType.ADD, - folderVersionId: newFolderVersion.id - } - ] - }, - tx - ); - - originalToNewFolderId.set(folder.id, { - folderId: newFolder.id, - projectId: parentEnv.projectId - }); - } - } - - // Useful for debugging: - // console.log("data.secrets", data.secrets); - // console.log("data.folders", data.folders); - // console.log("data.environment", data.environments); - - if (data.secrets && data.secrets.length > 0) { - const mappedToEnvironmentId = new Map< - string, - { - secretKey: string; - secretValue: string; - folderId?: string; - isFromBlock?: boolean; - }[] - >(); - - for (const secret of data.secrets) { - const targetId = secret.folderId || secret.environmentId; - - // Skip if we can't find either an environment or folder mapping for this secret - if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(targetId)) { - logger.info({ secret }, "[importDataIntoInfisicalFn]: Could not find environment or folder for secret"); - - // eslint-disable-next-line no-continue - continue; - } - - if (!mappedToEnvironmentId.has(targetId)) { - mappedToEnvironmentId.set(targetId, []); - } - - const alreadyHasSecret = mappedToEnvironmentId - .get(targetId)! - .find((el) => el.secretKey === secret.name && el.folderId === secret.folderId); - - if (alreadyHasSecret && alreadyHasSecret.isFromBlock) { - // remove the existing secret if any - mappedToEnvironmentId - .get(targetId)! - .splice(mappedToEnvironmentId.get(targetId)!.indexOf(alreadyHasSecret), 1); - } - mappedToEnvironmentId.get(targetId)!.push({ - secretKey: secret.name, - secretValue: secret.value || "", - folderId: secret.folderId, - isFromBlock: secret.appBlockOrderIndex !== undefined - }); - } - - // for each of the mappedEnvironmentId - for await (const [targetId, secrets] of mappedToEnvironmentId) { - logger.info("[importDataIntoInfisicalFn]: Processing secrets for targetId", targetId); - - let selectedFolder: TSecretFolders | undefined; - let selectedProjectId: string | undefined; - - // Case 1: Secret belongs to a folder / branch / branch of a block - const foundFolder = originalToNewFolderId.get(targetId); - if (foundFolder) { - logger.info("[importDataIntoInfisicalFn]: Processing secrets for folder"); - selectedFolder = await folderDAL.findById(foundFolder.folderId, tx); - selectedProjectId = foundFolder.projectId; - } else { - logger.info("[importDataIntoInfisicalFn]: Processing secrets for normal environment"); - const environment = data.environments.find((env) => env.id === targetId); - if (!environment) { - logger.info( - { - targetId - }, - "[importDataIntoInfisicalFn]: Could not find environment for secret" - ); - // eslint-disable-next-line no-continue - continue; - } - - const projectId = originalToNewProjectId.get(environment.projectId)!; - - if (!projectId) { - throw new BadRequestError({ message: `Failed to import secret, project not found` }); - } - - const env = originalToNewEnvironmentId.get(targetId); - if (!env) { - logger.info( - { - targetId - }, - "[importDataIntoInfisicalFn]: Could not find environment for secret" - ); - - // eslint-disable-next-line no-continue - continue; - } - - const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx); - - if (!folder) { - throw new NotFoundError({ - message: `Folder not found for the given environment slug (${env.envSlug}) & secret path (/)`, - name: "Create secret" - }); - } - - selectedFolder = folder; - selectedProjectId = projectId; - } - - if (!selectedFolder) { - throw new NotFoundError({ - message: `Folder not found for the given environment slug & secret path`, - name: "CreateSecret" - }); - } - - if (!selectedProjectId) { - throw new NotFoundError({ - message: `Project not found for the given environment slug & secret path`, - name: "CreateSecret" - }); - } - - const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey( - { - type: KmsDataKey.SecretManager, - projectId: selectedProjectId - }, - tx - ); - - const secretBatches = chunkArray(secrets, 2500); - for await (const secretBatch of secretBatches) { - const secretsByKeys = await secretDAL.findBySecretKeys( - selectedFolder.id, - secretBatch.map((el) => ({ - key: el.secretKey, - type: SecretType.Shared - })), - tx - ); - if (secretsByKeys.length) { - throw new BadRequestError({ - message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}` - }); - } - await fnSecretBulkInsert({ - inputSecrets: secretBatch.map((el) => { - const references = getAllSecretReferences(el.secretValue).nestedReferences; - - return { - version: 1, - encryptedValue: el.secretValue - ? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob - : undefined, - key: el.secretKey, - references, - type: SecretType.Shared - }; - }), - folderId: selectedFolder.id, - orgId: actorOrgId, - resourceMetadataDAL, - secretDAL, - secretVersionDAL, - secretTagDAL, - secretVersionTagDAL, - folderCommitService, - actor: { - type: actor, - actorId - }, - tx - }); - } - } - } - }); - - return { projectsNotImported }; -}; diff --git a/backend/src/services/external-migration/external-migration-fns/import.ts b/backend/src/services/external-migration/external-migration-fns/import.ts new file mode 100644 index 000000000..5728bf1c0 --- /dev/null +++ b/backend/src/services/external-migration/external-migration-fns/import.ts @@ -0,0 +1,352 @@ +import slugify from "@sindresorhus/slugify"; + +import { SecretType, TSecretFolders } from "@app/db/schemas"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { chunkArray } from "@app/lib/fn"; +import { logger } from "@app/lib/logger"; +import { alphaNumericNanoId } from "@app/lib/nanoid"; +import { CommitType } from "@app/services/folder-commit/folder-commit-service"; +import { KmsDataKey } from "@app/services/kms/kms-types"; +import { fnSecretBulkInsert, getAllSecretReferences } from "@app/services/secret-v2-bridge/secret-v2-bridge-fns"; + +import { TImportDataIntoInfisicalDTO } from "./envkey"; + +export const importDataIntoInfisicalFn = async ({ + projectService, + projectEnvDAL, + projectDAL, + secretDAL, + kmsService, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL, + resourceMetadataDAL, + folderVersionDAL, + folderCommitService, + input: { data, actor, actorId, actorOrgId, actorAuthMethod } +}: TImportDataIntoInfisicalDTO) => { + // Import data to infisical + if (!data || !data.projects) { + throw new BadRequestError({ message: "No projects found in data" }); + } + + const originalToNewProjectId = new Map(); + const originalToNewEnvironmentId = new Map< + string, + { envId: string; envSlug: string; rootFolderId?: string; projectId: string } + >(); + const originalToNewFolderId = new Map< + string, + { + envId: string; + envSlug: string; + folderId: string; + projectId: string; + } + >(); + const projectsNotImported: string[] = []; + + await projectDAL.transaction(async (tx) => { + for await (const project of data.projects) { + const newProject = await projectService + .createProject({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + workspaceName: project.name, + createDefaultEnvs: false, + tx + }) + .catch((e) => { + logger.error(e, `Failed to import to project [name:${project.name}]`); + throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` }); + }); + originalToNewProjectId.set(project.id, newProject.id); + } + + // Import environments + if (data.environments) { + for await (const environment of data.environments) { + const projectId = originalToNewProjectId.get(environment.projectId); + const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`); + + if (!projectId) { + projectsNotImported.push(environment.projectId); + // eslint-disable-next-line no-continue + continue; + } + + const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx); + + if (existingEnv) { + throw new BadRequestError({ + message: `Environment with slug '${slug}' already exist`, + name: "CreateEnvironment" + }); + } + + const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); + const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx); + const folder = await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); + + originalToNewEnvironmentId.set(environment.id, { + envSlug: doc.slug, + envId: doc.id, + rootFolderId: folder.id, + projectId + }); + } + } + + if (data.folders) { + for await (const folder of data.folders) { + const parentEnv = originalToNewEnvironmentId.get(folder.parentFolderId as string); + const parentFolder = originalToNewFolderId.get(folder.parentFolderId as string); + + let newFolder: TSecretFolders; + + if (parentEnv?.rootFolderId) { + newFolder = await folderDAL.create( + { + name: folder.name, + envId: parentEnv.envId, + parentId: parentEnv.rootFolderId + }, + tx + ); + } else if (parentFolder) { + newFolder = await folderDAL.create( + { + name: folder.name, + envId: parentFolder.envId, + parentId: parentFolder.folderId + }, + tx + ); + } else { + logger.info({ folder }, "No parent environment found for folder"); + // eslint-disable-next-line no-continue + continue; + } + + const newFolderVersion = await folderVersionDAL.create( + { + name: newFolder.name, + envId: newFolder.envId, + version: newFolder.version, + folderId: newFolder.id + }, + tx + ); + + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Changed by external migration", + folderId: parentEnv?.rootFolderId || parentFolder?.folderId || "", + changes: [ + { + type: CommitType.ADD, + folderVersionId: newFolderVersion.id + } + ] + }, + tx + ); + + originalToNewFolderId.set(folder.id, { + folderId: newFolder.id, + envId: parentEnv?.envId || parentFolder?.envId || "", + envSlug: parentEnv?.envSlug || parentFolder?.envSlug || "", + projectId: parentEnv?.projectId || parentFolder?.projectId || "" + }); + } + } + + // Useful for debugging: + // console.log("data.secrets", data.secrets); + // console.log("data.folders", data.folders); + // console.log("data.environment", data.environments); + + if (data.secrets && data.secrets.length > 0) { + const mappedToEnvironmentId = new Map< + string, + { + secretKey: string; + secretValue: string; + folderId?: string; + isFromBlock?: boolean; + }[] + >(); + + for (const secret of data.secrets) { + const targetId = secret.folderId || secret.environmentId; + + // Skip if we can't find either an environment or folder mapping for this secret + if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(targetId)) { + logger.info({ secret }, "[importDataIntoInfisicalFn]: Could not find environment or folder for secret"); + + // eslint-disable-next-line no-continue + continue; + } + + if (!mappedToEnvironmentId.has(targetId)) { + mappedToEnvironmentId.set(targetId, []); + } + + const alreadyHasSecret = mappedToEnvironmentId + .get(targetId)! + .find((el) => el.secretKey === secret.name && el.folderId === secret.folderId); + + if (alreadyHasSecret && alreadyHasSecret.isFromBlock) { + // remove the existing secret if any + mappedToEnvironmentId + .get(targetId)! + .splice(mappedToEnvironmentId.get(targetId)!.indexOf(alreadyHasSecret), 1); + } + mappedToEnvironmentId.get(targetId)!.push({ + secretKey: secret.name, + secretValue: secret.value || "", + folderId: secret.folderId, + isFromBlock: secret.appBlockOrderIndex !== undefined + }); + } + + // for each of the mappedEnvironmentId + for await (const [targetId, secrets] of mappedToEnvironmentId) { + logger.info("[importDataIntoInfisicalFn]: Processing secrets for targetId", targetId); + + let selectedFolder: TSecretFolders | undefined; + let selectedProjectId: string | undefined; + + // Case 1: Secret belongs to a folder / branch / branch of a block + const foundFolder = originalToNewFolderId.get(targetId); + if (foundFolder) { + logger.info("[importDataIntoInfisicalFn]: Processing secrets for folder"); + selectedFolder = await folderDAL.findById(foundFolder.folderId, tx); + selectedProjectId = foundFolder.projectId; + } else { + logger.info("[importDataIntoInfisicalFn]: Processing secrets for normal environment"); + const environment = data.environments.find((env) => env.id === targetId); + if (!environment) { + logger.info( + { + targetId + }, + "[importDataIntoInfisicalFn]: Could not find environment for secret" + ); + // eslint-disable-next-line no-continue + continue; + } + + const projectId = originalToNewProjectId.get(environment.projectId)!; + + if (!projectId) { + throw new BadRequestError({ message: `Failed to import secret, project not found` }); + } + + const env = originalToNewEnvironmentId.get(targetId); + if (!env) { + logger.info( + { + targetId + }, + "[importDataIntoInfisicalFn]: Could not find environment for secret" + ); + + // eslint-disable-next-line no-continue + continue; + } + + const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx); + + if (!folder) { + throw new NotFoundError({ + message: `Folder not found for the given environment slug (${env.envSlug}) & secret path (/)`, + name: "Create secret" + }); + } + + selectedFolder = folder; + selectedProjectId = projectId; + } + + if (!selectedFolder) { + throw new NotFoundError({ + message: `Folder not found for the given environment slug & secret path`, + name: "CreateSecret" + }); + } + + if (!selectedProjectId) { + throw new NotFoundError({ + message: `Project not found for the given environment slug & secret path`, + name: "CreateSecret" + }); + } + + const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey( + { + type: KmsDataKey.SecretManager, + projectId: selectedProjectId + }, + tx + ); + + const secretBatches = chunkArray(secrets, 2500); + for await (const secretBatch of secretBatches) { + const secretsByKeys = await secretDAL.findBySecretKeys( + selectedFolder.id, + secretBatch.map((el) => ({ + key: el.secretKey, + type: SecretType.Shared + })), + tx + ); + if (secretsByKeys.length) { + throw new BadRequestError({ + message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}` + }); + } + await fnSecretBulkInsert({ + inputSecrets: secretBatch.map((el) => { + const references = getAllSecretReferences(el.secretValue).nestedReferences; + + return { + version: 1, + encryptedValue: el.secretValue + ? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob + : undefined, + key: el.secretKey, + references, + type: SecretType.Shared + }; + }), + folderId: selectedFolder.id, + orgId: actorOrgId, + resourceMetadataDAL, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + folderCommitService, + actor: { + type: actor, + actorId + }, + tx + }); + } + } + } + }); + + return { projectsNotImported }; +}; diff --git a/backend/src/services/external-migration/external-migration-fns/index.ts b/backend/src/services/external-migration/external-migration-fns/index.ts new file mode 100644 index 000000000..4af82bf22 --- /dev/null +++ b/backend/src/services/external-migration/external-migration-fns/index.ts @@ -0,0 +1,3 @@ +export * from "./envkey"; +export * from "./import"; +export * from "./vault"; diff --git a/backend/src/services/external-migration/external-migration-fns/vault.ts b/backend/src/services/external-migration/external-migration-fns/vault.ts new file mode 100644 index 000000000..197102f82 --- /dev/null +++ b/backend/src/services/external-migration/external-migration-fns/vault.ts @@ -0,0 +1,341 @@ +import axios, { AxiosInstance } from "axios"; +import { v4 as uuidv4 } from "uuid"; + +import { BadRequestError } from "@app/lib/errors"; +import { logger } from "@app/lib/logger"; +import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator"; + +import { InfisicalImportData, VaultMappingType } from "../external-migration-types"; + +type VaultData = { + namespace: string; + mount: string; + path: string; + secretData: Record; +}; + +const vaultFactory = () => { + const getMounts = async (request: AxiosInstance) => { + const response = await request + .get< + Record< + string, + { + accessor: string; + options: { + version?: string; + } | null; + type: string; + } + > + >("/v1/sys/mounts") + .catch((err) => { + if (axios.isAxiosError(err)) { + logger.error(err.response?.data, "External migration: Failed to get Vault mounts"); + } + throw err; + }); + return response.data; + }; + + const getPaths = async ( + request: AxiosInstance, + { mountPath, secretPath = "" }: { mountPath: string; secretPath?: string } + ) => { + try { + // For KV v2: /v1/{mount}/metadata/{path}?list=true + const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`; + const response = await request.get<{ + data: { + keys: string[]; + }; + }>(`/v1/${path}?list=true`); + + return response.data.data.keys; + } catch (err) { + if (axios.isAxiosError(err)) { + logger.error(err.response?.data, "External migration: Failed to get Vault paths"); + if (err.response?.status === 404) { + return null; + } + } + throw err; + } + }; + + const getSecrets = async ( + request: AxiosInstance, + { mountPath, secretPath }: { mountPath: string; secretPath: string } + ) => { + // For KV v2: /v1/{mount}/data/{path} + const response = await request + .get<{ + data: { + data: Record; // KV v2 has nested data structure + metadata: { + created_time: string; + deletion_time: string; + destroyed: boolean; + version: number; + }; + }; + }>(`/v1/${mountPath}/data/${secretPath}`) + .catch((err) => { + if (axios.isAxiosError(err)) { + logger.error(err.response?.data, "External migration: Failed to get Vault secret"); + } + throw err; + }); + + return response.data.data.data; + }; + + // helper function to check if a mount is KV v2 (will be useful if we add support for Vault KV v1) + // const isKvV2Mount = (mountInfo: { type: string; options?: { version?: string } | null }) => { + // return mountInfo.type === "kv" && mountInfo.options?.version === "2"; + // }; + + const recursivelyGetAllPaths = async ( + request: AxiosInstance, + mountPath: string, + currentPath: string = "" + ): Promise => { + const paths = await getPaths(request, { mountPath, secretPath: currentPath }); + + if (paths === null || paths.length === 0) { + return []; + } + + const allSecrets: string[] = []; + + for await (const path of paths) { + const cleanPath = path.endsWith("/") ? path.slice(0, -1) : path; + const fullItemPath = currentPath ? `${currentPath}/${cleanPath}` : cleanPath; + + if (path.endsWith("/")) { + // it's a folder so we recurse into it + const subSecrets = await recursivelyGetAllPaths(request, mountPath, fullItemPath); + allSecrets.push(...subSecrets); + } else { + // it's a secret so we add it to our results + allSecrets.push(`${mountPath}/${fullItemPath}`); + } + } + + return allSecrets; + }; + + async function collectVaultData({ + baseUrl, + namespace, + accessToken + }: { + baseUrl: string; + namespace?: string; + accessToken: string; + }): Promise { + const request = axios.create({ + baseURL: baseUrl, + headers: { + "X-Vault-Token": accessToken, + ...(namespace ? { "X-Vault-Namespace": namespace } : {}) + } + }); + + const allData: VaultData[] = []; + + // Get all mounts in this namespace + const mounts = await getMounts(request); + + for (const mount of Object.keys(mounts)) { + if (!mount.endsWith("/")) { + delete mounts[mount]; + } + } + + for await (const [mountPath, mountInfo] of Object.entries(mounts)) { + // skip non-KV mounts + if (!mountInfo.type.startsWith("kv")) { + // eslint-disable-next-line no-continue + continue; + } + + // get all paths in this mount + const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`); + + const cleanMountPath = mountPath.replace(/\/$/, ""); + + for await (const secretPath of paths) { + // get the actual secret data + const secretData = await getSecrets(request, { + mountPath: cleanMountPath, + secretPath: secretPath.replace(`${cleanMountPath}/`, "") + }); + + allData.push({ + namespace: namespace || "", + mount: mountPath.replace(/\/$/, ""), + path: secretPath.replace(`${cleanMountPath}/`, ""), + secretData + }); + } + } + + return allData; + } + + return { + collectVaultData, + getMounts, + getPaths, + getSecrets, + recursivelyGetAllPaths + }; +}; + +export const transformToInfisicalFormatNamespaceToProjects = ( + vaultData: VaultData[], + mappingType: VaultMappingType +): InfisicalImportData => { + const projects: Array<{ name: string; id: string }> = []; + const environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }> = []; + const folders: Array<{ id: string; name: string; environmentId: string; parentFolderId?: string }> = []; + const secrets: Array<{ id: string; name: string; environmentId: string; value: string; folderId?: string }> = []; + + // track created entities to avoid duplicates + const projectMap = new Map(); // namespace -> projectId + const environmentMap = new Map(); // namespace:mount -> environmentId + const folderMap = new Map(); // namespace:mount:folderPath -> folderId + + let environmentId: string = ""; + for (const data of vaultData) { + const { namespace, mount, path, secretData } = data; + + if (mappingType === VaultMappingType.Namespace) { + // create project (namespace) + if (!projectMap.has(namespace)) { + const projectId = uuidv4(); + projectMap.set(namespace, projectId); + projects.push({ + name: namespace, + id: projectId + }); + } + const projectId = projectMap.get(namespace)!; + + // create environment (mount) + const envKey = `${namespace}:${mount}`; + if (!environmentMap.has(envKey)) { + environmentId = uuidv4(); + environmentMap.set(envKey, environmentId); + environments.push({ + name: mount, + id: environmentId, + projectId + }); + } + environmentId = environmentMap.get(envKey)!; + } else if (mappingType === VaultMappingType.KeyVault) { + if (!projectMap.has(mount)) { + const projectId = uuidv4(); + projectMap.set(mount, projectId); + projects.push({ + name: mount, + id: projectId + }); + } + const projectId = projectMap.get(mount)!; + + // create single "Production" environment per project, because we have no good way of determining environments from vault + if (!environmentMap.has(mount)) { + environmentId = uuidv4(); + environmentMap.set(mount, environmentId); + environments.push({ + name: "Production", + id: environmentId, + projectId + }); + } + environmentId = environmentMap.get(mount)!; + } + + // create folder structure + let currentFolderId: string | undefined; + let currentPath = ""; + + if (path.includes("/")) { + const pathParts = path.split("/").filter(Boolean); + + const folderParts = pathParts; + + // create nested folder structure for the entire path + for (const folderName of folderParts) { + currentPath = currentPath ? `${currentPath}/${folderName}` : folderName; + const folderKey = `${namespace}:${mount}:${currentPath}`; + + if (!folderMap.has(folderKey)) { + const folderId = uuidv4(); + folderMap.set(folderKey, folderId); + folders.push({ + id: folderId, + name: folderName, + environmentId, + parentFolderId: currentFolderId || environmentId + }); + currentFolderId = folderId; + } else { + currentFolderId = folderMap.get(folderKey)!; + } + } + } + + for (const [key, value] of Object.entries(secretData)) { + secrets.push({ + id: uuidv4(), + name: key, + environmentId, + value: String(value), + folderId: currentFolderId + }); + } + } + + return { + projects, + environments, + folders, + secrets + }; +}; + +export const importVaultDataFn = async ({ + vaultAccessToken, + vaultNamespace, + vaultUrl, + mappingType +}: { + vaultAccessToken: string; + vaultNamespace?: string; + vaultUrl: string; + mappingType: VaultMappingType; +}) => { + await blockLocalAndPrivateIpAddresses(vaultUrl); + + if (mappingType === VaultMappingType.Namespace && !vaultNamespace) { + throw new BadRequestError({ + message: "Vault namespace is required when project mapping type is set to namespace." + }); + } + + const vaultApi = vaultFactory(); + + const vaultData = await vaultApi.collectVaultData({ + accessToken: vaultAccessToken, + baseUrl: vaultUrl, + namespace: vaultNamespace + }); + + const infisicalData = transformToInfisicalFormatNamespaceToProjects(vaultData, mappingType); + + return infisicalData; +}; diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts index c4e6b43c5..8bde91aa5 100644 --- a/backend/src/services/external-migration/external-migration-queue.ts +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -19,7 +19,7 @@ import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-d import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service"; import { importDataIntoInfisicalFn } from "./external-migration-fns"; -import { ExternalPlatforms, TImportInfisicalDataCreate } from "./external-migration-types"; +import { ExternalPlatforms, ImportType, TImportInfisicalDataCreate } from "./external-migration-types"; export type TExternalMigrationQueueFactoryDep = { smtpService: TSmtpService; @@ -67,6 +67,7 @@ export const externalMigrationQueueFactory = ({ const startImport = async (dto: { actorEmail: string; data: { + importType: ImportType; iv: string; tag: string; ciphertext: string; diff --git a/backend/src/services/external-migration/external-migration-service.ts b/backend/src/services/external-migration/external-migration-service.ts index c310fd273..766b6cef4 100644 --- a/backend/src/services/external-migration/external-migration-service.ts +++ b/backend/src/services/external-migration/external-migration-service.ts @@ -4,9 +4,9 @@ import { crypto } from "@app/lib/crypto/cryptography"; import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors"; import { TUserDALFactory } from "../user/user-dal"; -import { decryptEnvKeyDataFn, parseEnvKeyDataFn } from "./external-migration-fns"; +import { decryptEnvKeyDataFn, importVaultDataFn, parseEnvKeyDataFn } from "./external-migration-fns"; import { TExternalMigrationQueueFactory } from "./external-migration-queue"; -import { TImportEnvKeyDataCreate } from "./external-migration-types"; +import { ImportType, TImportEnvKeyDataDTO, TImportVaultDataDTO } from "./external-migration-types"; type TExternalMigrationServiceFactoryDep = { permissionService: TPermissionServiceFactory; @@ -28,7 +28,7 @@ export const externalMigrationServiceFactory = ({ actorId, actorOrgId, actorAuthMethod - }: TImportEnvKeyDataCreate) => { + }: TImportEnvKeyDataDTO) => { if (crypto.isFipsModeEnabled()) { throw new BadRequestError({ message: "EnvKey migration is not supported when running in FIPS mode." }); } @@ -60,11 +60,65 @@ export const externalMigrationServiceFactory = ({ await externalMigrationQueue.startImport({ actorEmail: user.email!, - data: encrypted + data: { + importType: ImportType.EnvKey, + ...encrypted + } + }); + }; + + const importVaultData = async ({ + vaultAccessToken, + vaultNamespace, + mappingType, + vaultUrl, + actor, + actorId, + actorOrgId, + actorAuthMethod + }: TImportVaultDataDTO) => { + const { membership } = await permissionService.getOrgPermission( + actor, + actorId, + actorOrgId, + actorAuthMethod, + actorOrgId + ); + + if (membership.role !== OrgMembershipRole.Admin) { + throw new ForbiddenRequestError({ message: "Only admins can import data" }); + } + + const user = await userDAL.findById(actorId); + + const vaultData = await importVaultDataFn({ + vaultAccessToken, + vaultNamespace, + vaultUrl, + mappingType + }); + + const stringifiedJson = JSON.stringify({ + data: vaultData, + actor, + actorId, + actorOrgId, + actorAuthMethod + }); + + const encrypted = crypto.encryption().symmetric().encryptWithRootEncryptionKey(stringifiedJson); + + await externalMigrationQueue.startImport({ + actorEmail: user.email!, + data: { + importType: ImportType.Vault, + ...encrypted + } }); }; return { - importEnvKeyData + importEnvKeyData, + importVaultData }; }; diff --git a/backend/src/services/external-migration/external-migration-types.ts b/backend/src/services/external-migration/external-migration-types.ts index 32c70a688..89589d674 100644 --- a/backend/src/services/external-migration/external-migration-types.ts +++ b/backend/src/services/external-migration/external-migration-types.ts @@ -1,5 +1,17 @@ +import { TOrgPermission } from "@app/lib/types"; + import { ActorAuthMethod, ActorType } from "../auth/auth-type"; +export enum ImportType { + EnvKey = "envkey", + Vault = "vault" +} + +export enum VaultMappingType { + Namespace = "namespace", + KeyVault = "key-vault" +} + export type InfisicalImportData = { projects: Array<{ name: string; id: string }>; environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }>; @@ -14,14 +26,17 @@ export type InfisicalImportData = { }>; }; -export type TImportEnvKeyDataCreate = { +export type TImportEnvKeyDataDTO = { decryptionKey: string; encryptedJson: { nonce: string; data: string }; - actor: ActorType; - actorId: string; - actorOrgId: string; - actorAuthMethod: ActorAuthMethod; -}; +} & Omit; + +export type TImportVaultDataDTO = { + vaultAccessToken: string; + vaultNamespace?: string; + mappingType: VaultMappingType; + vaultUrl: string; +} & Omit; export type TImportInfisicalDataCreate = { data: InfisicalImportData; diff --git a/docs/docs.json b/docs/docs.json index 13ec5367e..e0d7a2031 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -198,6 +198,21 @@ "documentation/platform/workflow-integrations/microsoft-teams-integration" ] }, + { + "group": "External Migrations", + "pages": [ + "documentation/platform/external-migrations/overview", + "documentation/platform/external-migrations/envkey", + "documentation/platform/external-migrations/vault" + ] + }, + { + "group": "External Migrations", + "pages": [ + "documentation/platform/workflow-integrations/slack-integration", + "documentation/platform/workflow-integrations/microsoft-teams-integration" + ] + }, { "group": "Admin Consoles", "pages": [ diff --git a/docs/documentation/guides/migrating-from-envkey.mdx b/docs/documentation/guides/migrating-from-envkey.mdx deleted file mode 100644 index e1d75bab0..000000000 --- a/docs/documentation/guides/migrating-from-envkey.mdx +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: "Migrating from EnvKey to Infisical" -sidebarTitle: "Migration" -description: "Learn how to migrate from EnvKey to Infisical in the easiest way possible." ---- - -## What is Infisical? - -[Infisical](https://infisical.com) is an open-source all-in-one secret management platform that helps developers manage secrets (e.g., API-keys, DB access tokens, [certificates](https://infisical.com/docs/documentation/platform/pki/overview)) across their infrastructure. In addition, Infisical provides [secret sharing](https://infisical.com/docs/documentation/platform/secret-sharing) functionality, ability to [prevent secret leaks](https://infisical.com/docs/cli/scanning-overview), and more. - -Infisical is used by 10,000+ organizations across all industries including First American Financial Corporation, Delivery Hero, and [Hugging Face](https://infisical.com/customers/hugging-face). - -## Migrating from EnvKey - - - -Open the EnvKey dashboard and go to My Org. -![EnvKey Dashboard](../../images/guides/import-envkey/envkey-dashboard.png) - - -Go to Import/Export on the top right corner, Click on Export Org and save the exported file. -![Export organization](../../images/guides/import-envkey/envkey-export.png) - - -Click on copy to copy the encryption key and save it. -![Copy encryption key](../../images/guides/import-envkey/copy-encryption-key.png) - - -Open the Infisical dashboard and go to Organization Settings > Import. -![Infisical Organization settings](../../images/guides/import-envkey/infisical-import-dashboard.png) - - -Upload the exported file from EnvKey, paste the encryption key and click Import. -![Infisical Import EnvKey](../../images/guides/import-envkey/infisical-import-envkey.png) - - - - -## Talk to our team - -To make the migration process even more seamless, you can [schedule a meeting with our team](https://infisical.cal.com/vlad/migration-from-envkey-to-infisical) to learn more about how Infisical compares to EnvKey and discuss unique needs of your organization. You are also welcome to email us at [support@infisical.com](mailto:support@infisical.com) to ask any questions or get any technical help. diff --git a/docs/documentation/platform/external-migrations/envkey.mdx b/docs/documentation/platform/external-migrations/envkey.mdx new file mode 100644 index 000000000..46051cd49 --- /dev/null +++ b/docs/documentation/platform/external-migrations/envkey.mdx @@ -0,0 +1,44 @@ +--- +title: "Migrating from EnvKey to Infisical" +sidebarTitle: "EnvKey" +description: "Learn how to migrate secrets from EnvKey to Infisical." +--- + +## Migrating from EnvKey + + + + ![EnvKey Dashboard](/images/platform/external-migrations/envkey-dashboard.png) + + + Go to Import/Export on the top right corner, Click on Export Org and save the exported file. + ![Export organization](/images/platform/external-migrations/envkey-export.png) + + + Click on copy to copy the encryption key and save it. + ![Copy encryption key](/images/platform/external-migrations/envkey-copy-encryption-key.png) + + + Open the Infisical dashboard and go to Organization Settings > External Migrations. + ![Infisical Organization settings](/images/platform/external-migrations/infisical-external-migration-dashboard.png) + + + + Select the EnvKey platform and click on Next. + ![Select EnvKey platform](/images/platform/external-migrations/infisical-import-envkey-modal.png) + + + + Upload the exported file from EnvKey, paste the encryption key and click Import data. + ![Infisical Import EnvKey](/images/platform/external-migrations/infisical-import-envkey.png) + + + + + It may take several minutes to complete the migration. You will receive an email when the migration is complete, or if there were any errors during the migration process. + + + +## Talk to our team + +To make the migration process even more seamless, you can [schedule a meeting with our team](https://infisical.cal.com/vlad/migration-from-envkey-to-infisical) to learn more about how Infisical compares to EnvKey and discuss unique needs of your organization. You are also welcome to email us at [support@infisical.com](mailto:support@infisical.com) to ask any questions or get any technical help. diff --git a/docs/documentation/platform/external-migrations/overview.mdx b/docs/documentation/platform/external-migrations/overview.mdx new file mode 100644 index 000000000..cf6c5e5a1 --- /dev/null +++ b/docs/documentation/platform/external-migrations/overview.mdx @@ -0,0 +1,16 @@ +--- +title: "External Migrations" +sidebarTitle: "Overview" +description: "Learn how to migrate secrets from third-party secrets management platforms to Infisical." +--- + +## Overview + +Infisical supports migrating secrets from third-party secrets management platforms to Infisical. This is useful if you're looking to easily switch to Infisical and wish to move over your existing secrets from a different platform. + +## Supported Platforms + +- [EnvKey](./envkey) +- [Vault](./vault) + +We're always looking to add more migration paths for other providers. If we're missing a platform, please open an issue on our [GitHub repository](https://github.com/infisical/infisical/issues). \ No newline at end of file diff --git a/docs/documentation/platform/external-migrations/vault.mdx b/docs/documentation/platform/external-migrations/vault.mdx new file mode 100644 index 000000000..ef139e8e4 --- /dev/null +++ b/docs/documentation/platform/external-migrations/vault.mdx @@ -0,0 +1,127 @@ +--- +title: "Migrating from Vault to Infisical" +sidebarTitle: "Vault" +description: "Learn how to migrate secrets from Vault to Infisical." +--- + +## Migrating from Vault + +Migrating from Vault Self-Hosted or Dedicated Vault is a straight forward process with our inbuilt migration option. In order to migrate from Vault, you'll need to provide Infisical an access token to your Vault instance. + +Currently the Vault migration only supports migrating secrets from the KV v2 secrets engine. If you're using a different secrets engine, please open an issue on our [GitHub repository](https://github.com/infisical/infisical/issues). + + +### Prerequisites + +- A Vault instance with the KV v2 secrets engine enabled. +- An access token to your Vault instance. + + +### Project Mapping + +When migrating from Vault, you'll need to choose how you want to map your Vault resources to Infisical projects. + +There are two options for project mapping: + +- `Namespace`: This will map your selected Vault namespace to a single Infisical project. When you select this option, each KV secret engine within the namespace will be mapped to a single Infisical project. Each KV secret engine will be mapped to a Infisical environment within the project. This means if you have 3 KV secret engines, you'll have 3 environments inside the same project, where the name of the environments correspond to the name of the KV secret engines. +- `Key Vault`: This will map all the KV secret engines within your Vault instance to a Infisical project. Each KV engine will be created as a Infisical project. This means if you have 3 KV secret engines, you'll have 3 Infisical projects. For each of the created projects, a single default environment will be created called `Production`, which will contain all your secrets from the corresponding KV secret engine. + + + + + + + In order to migrate from Vault, you'll need to create a Vault policy that allows Infisical to read the secrets and metadata from the KV v2 secrets engines within your Vault instance. + + + ```python + # Allow listing secret engines/mounts + path "sys/mounts" { + capabilities = ["read", "list"] + } + + # For KV v2 engines - access to both data and metadata + path "*/data/*" { + capabilities = ["read", "list"] + } + + path "*/metadata/*" { + capabilities = ["read", "list"] + } + + # If using Vault Enterprise - allow listing namespaces + path "sys/namespaces" { + capabilities = ["list", "read"] + } + + # Cross-namespace access (Enterprise only) + path "+/*" { + capabilities = ["read", "list"] + } + + path "+/sys/mounts" { + capabilities = ["read", "list"] + } + ``` + + Save this policy with the name `infisical-migration`. + + + + + You can use the Vault CLI to easily generate an access token for the new `infisical-migration` policy that you created in the previous step. + + ```bash + vault token create --policy="infisical-migration" + ``` + + After generating the token, you should see the following output: + + ```t + $ vault token create --policy="infisical-migration" + + Key Value + --- ----- + token + token_accessor p6kJDiBSzYYdabJUIpGCsCBm + token_duration 768h + token_renewable true + token_policies ["default" "infisical-migration"] + identity_policies [] + policies ["default" "infisical-migration"] + ``` + + Copy the `token` field and save it for later, as you'll need this when configuring the migration to Infisical. + + + + Open the Infisical dashboard and go to Organization Settings > External Migrations. + + ![Infisical Organization settings](/images/platform/external-migrations/infisical-external-migration-dashboard.png) + + + + Select the Vault platform and click on Next. + + ![Select Vault platform](/images/platform/external-migrations/infisical-import-vault-modal.png) + + + + Enter the Vault access token that you generated in the previous step and click Import data. + + ![Configure Vault migration](/images/platform/external-migrations/infisical-import-vault.png) + + - `Vault URL`: The URL of your Vault instance. + - `Vault Namespace`: The namespace of your Vault instance. This is optional, and can be left blank if you're not using namespaces for your Vault instance. + - `Vault Access Token`: The access token that you generated in the previous step. + + - `Project Mapping`: Choose how you want to map your Vault resources to Infisical projects. You can review the mapping options in the [Project Mapping](#project-mapping) section. + + Click on Import data to start the migration. + + + + + + It may take several minutes to complete the migration. You will receive an email when the migration is complete, or if there were any errors during the migration process. + \ No newline at end of file diff --git a/docs/images/guides/import-envkey/infisical-import-dashboard.png b/docs/images/guides/import-envkey/infisical-import-dashboard.png deleted file mode 100644 index ab197f126..000000000 Binary files a/docs/images/guides/import-envkey/infisical-import-dashboard.png and /dev/null differ diff --git a/docs/images/guides/import-envkey/infisical-import-envkey.png b/docs/images/guides/import-envkey/infisical-import-envkey.png deleted file mode 100644 index c504c9824..000000000 Binary files a/docs/images/guides/import-envkey/infisical-import-envkey.png and /dev/null differ diff --git a/docs/images/guides/import-envkey/copy-encryption-key.png b/docs/images/platform/external-migrations/envkey-copy-encryption-key.png similarity index 100% rename from docs/images/guides/import-envkey/copy-encryption-key.png rename to docs/images/platform/external-migrations/envkey-copy-encryption-key.png diff --git a/docs/images/guides/import-envkey/envkey-dashboard.png b/docs/images/platform/external-migrations/envkey-dashboard.png similarity index 100% rename from docs/images/guides/import-envkey/envkey-dashboard.png rename to docs/images/platform/external-migrations/envkey-dashboard.png diff --git a/docs/images/guides/import-envkey/envkey-export.png b/docs/images/platform/external-migrations/envkey-export.png similarity index 100% rename from docs/images/guides/import-envkey/envkey-export.png rename to docs/images/platform/external-migrations/envkey-export.png diff --git a/docs/images/platform/external-migrations/infisical-external-migration-dashboard.png b/docs/images/platform/external-migrations/infisical-external-migration-dashboard.png new file mode 100644 index 000000000..f978719f4 Binary files /dev/null and b/docs/images/platform/external-migrations/infisical-external-migration-dashboard.png differ diff --git a/docs/images/platform/external-migrations/infisical-import-envkey-modal.png b/docs/images/platform/external-migrations/infisical-import-envkey-modal.png new file mode 100644 index 000000000..9abe26fc8 Binary files /dev/null and b/docs/images/platform/external-migrations/infisical-import-envkey-modal.png differ diff --git a/docs/images/platform/external-migrations/infisical-import-envkey.png b/docs/images/platform/external-migrations/infisical-import-envkey.png new file mode 100644 index 000000000..0d73bc5a0 Binary files /dev/null and b/docs/images/platform/external-migrations/infisical-import-envkey.png differ diff --git a/docs/images/platform/external-migrations/infisical-import-vault-modal.png b/docs/images/platform/external-migrations/infisical-import-vault-modal.png new file mode 100644 index 000000000..c5c4a9063 Binary files /dev/null and b/docs/images/platform/external-migrations/infisical-import-vault-modal.png differ diff --git a/docs/images/platform/external-migrations/infisical-import-vault.png b/docs/images/platform/external-migrations/infisical-import-vault.png new file mode 100644 index 000000000..399e0100d Binary files /dev/null and b/docs/images/platform/external-migrations/infisical-import-vault.png differ diff --git a/frontend/src/hooks/api/migration/index.ts b/frontend/src/hooks/api/migration/index.ts new file mode 100644 index 000000000..f8dd99d03 --- /dev/null +++ b/frontend/src/hooks/api/migration/index.ts @@ -0,0 +1 @@ +export * from "./mutations"; diff --git a/frontend/src/hooks/api/migration/mutations.tsx b/frontend/src/hooks/api/migration/mutations.tsx index 87ba89eab..529d5c463 100644 --- a/frontend/src/hooks/api/migration/mutations.tsx +++ b/frontend/src/hooks/api/migration/mutations.tsx @@ -15,7 +15,7 @@ export const useImportEnvKey = () => { formData.append("file", file); try { - const response = await apiRequest.post("/api/v3/migrate/env-key/", formData, { + const response = await apiRequest.post("/api/v3/external-migration/env-key/", formData, { headers: { "Content-Type": "multipart/form-data" }, @@ -39,3 +39,26 @@ export const useImportEnvKey = () => { } }); }; + +export const useImportVault = () => { + return useMutation({ + mutationFn: async ({ + vaultAccessToken, + vaultNamespace, + vaultUrl, + mappingType + }: { + vaultAccessToken: string; + vaultNamespace?: string; + vaultUrl: string; + mappingType: string; + }) => { + await apiRequest.post("/api/v3/external-migration/vault/", { + vaultAccessToken, + vaultNamespace, + vaultUrl, + mappingType + }); + } + }); +}; diff --git a/frontend/src/pages/organization/SettingsPage/components/ImportTab/ImportTab.tsx b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/ExternalMigrationsTab.tsx similarity index 94% rename from frontend/src/pages/organization/SettingsPage/components/ImportTab/ImportTab.tsx rename to frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/ExternalMigrationsTab.tsx index 596cef9f4..6ca1a5c28 100644 --- a/frontend/src/pages/organization/SettingsPage/components/ImportTab/ImportTab.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/ExternalMigrationsTab.tsx @@ -9,7 +9,7 @@ import { ProjectMembershipRole } from "@app/hooks/api/roles/types"; import { SelectImportFromPlatformModal } from "./components/SelectImportFromPlatformModal"; -export const ImportTab = () => { +export const ExternalMigrationsTab = () => { const { membership } = useOrgPermission(); const { popUp, handlePopUpOpen, handlePopUpToggle } = usePopUp(["selectImportPlatform"] as const); @@ -30,7 +30,7 @@ export const ImportTab = () => {
diff --git a/frontend/src/pages/organization/SettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/EnvKeyPlatformModal.tsx similarity index 83% rename from frontend/src/pages/organization/SettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx rename to frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/EnvKeyPlatformModal.tsx index 55b360365..2c15cfb8a 100644 --- a/frontend/src/pages/organization/SettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/EnvKeyPlatformModal.tsx @@ -45,25 +45,21 @@ export const EnvKeyPlatformModal = ({ onClose }: Props) => { return; } - try { - await importEnvKey({ - file: data.file, - decryptionKey: data.encryptionKey - }); - createNotification({ - title: "Import started", - text: "Your data is being imported. You will receive an email when the import is complete or if the import fails. This may take up to 10 minutes.", - type: "info" - }); + await importEnvKey({ + file: data.file, + decryptionKey: data.encryptionKey + }); + createNotification({ + title: "Import started", + text: "Your data is being imported. You will receive an email when the import is complete or if the import fails. This may take up to 10 minutes.", + type: "info" + }); - onClose(); - reset(); + onClose(); + reset(); - if (fileUploadRef.current) { - fileUploadRef.current.value = ""; - } - } catch { - reset(); + if (fileUploadRef.current) { + fileUploadRef.current.value = ""; } }; diff --git a/frontend/src/pages/organization/SettingsPage/components/ImportTab/components/GenericDropzone.tsx b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/GenericDropzone.tsx similarity index 100% rename from frontend/src/pages/organization/SettingsPage/components/ImportTab/components/GenericDropzone.tsx rename to frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/GenericDropzone.tsx diff --git a/frontend/src/pages/organization/SettingsPage/components/ImportTab/components/SelectImportFromPlatformModal.tsx b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/SelectImportFromPlatformModal.tsx similarity index 70% rename from frontend/src/pages/organization/SettingsPage/components/ImportTab/components/SelectImportFromPlatformModal.tsx rename to frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/SelectImportFromPlatformModal.tsx index 06631ffe1..6da5a44c9 100644 --- a/frontend/src/pages/organization/SettingsPage/components/ImportTab/components/SelectImportFromPlatformModal.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/SelectImportFromPlatformModal.tsx @@ -1,11 +1,12 @@ import { useState } from "react"; -import { faKey } from "@fortawesome/free-solid-svg-icons"; +import { faKey, faVault } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { AnimatePresence, motion } from "framer-motion"; import { Modal, ModalContent } from "@app/components/v2"; import { EnvKeyPlatformModal } from "./EnvKeyPlatformModal"; +import { VaultPlatformModal } from "./VaultPlatformModal"; type Props = { isOpen?: boolean; @@ -22,6 +23,11 @@ const PLATFORM_LIST = [ icon: faKey, platform: "env-key", title: "Env Key" + }, + { + icon: faVault, + platform: "vault", + title: "Vault" } ] as const; @@ -82,18 +88,32 @@ export const SelectImportFromPlatformModal = ({ isOpen, onToggle }: Props) => {
)} - {wizardStep === WizardSteps.PlatformInputs && - selectedPlatform?.platform === "env-key" && ( - - handleFormReset(false)} /> - - )} + {wizardStep === WizardSteps.PlatformInputs && ( + <> + {selectedPlatform?.platform === "env-key" && ( + + handleFormReset(false)} /> + + )} + {selectedPlatform?.platform === "vault" && ( + + handleFormReset(false)} /> + + )} + + )} diff --git a/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx new file mode 100644 index 000000000..02ab5df1b --- /dev/null +++ b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx @@ -0,0 +1,222 @@ +import { Controller, useForm } from "react-hook-form"; +import { faQuestionCircle } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { twMerge } from "tailwind-merge"; +import { z } from "zod"; + +import { createNotification } from "@app/components/notifications"; +import { Button, FormControl, Input, Tooltip } from "@app/components/v2"; +import { NoticeBannerV2 } from "@app/components/v2/NoticeBannerV2/NoticeBannerV2"; +import { useImportVault } from "@app/hooks/api/migration/mutations"; + +type Props = { + id?: string; + onClose: () => void; +}; + +enum VaultMappingType { + KeyVault = "key-vault", + Namespace = "namespace" +} + +const MAPPING_TYPE_MENU_ITEMS = [ + { + value: VaultMappingType.KeyVault, + label: "Key Vaults", + tooltip: ( +
+ When using key vaults for mapping, each key vault within Vault will be created in Infisical + as a project. Each secret path inside the key vault, will be created as an environment + inside the corresponding project. When using Key Vaults as the project mapping type, a + default environment called "Production" will be created for each project, which + will contain the secrets from the key vault. +
+
Key Vault → Project
+
Default Environment (Production)
+
Secret Path → Secret Folder
+
Secret data → Secrets
+
+
+ ) + }, + { + value: VaultMappingType.Namespace, + label: "Namespaces", + tooltip: ( +
+ When using namespaces for mapping, each namespace within Vault will be created in Infisical + as a project. Each key vault (KV) inside the namespace, will be created as an environment + inside the corresponding project. +
+
Namespace → Project
+
Key Vault → Project Environment
+
Secret Path → Secret Folder
+
Secret data → Secrets
+
+
+ ) + } +]; + +export const VaultPlatformModal = ({ onClose }: Props) => { + const formSchema = z.object({ + vaultUrl: z.string().min(1), + vaultNamespace: z.string().trim().optional(), + vaultAccessToken: z.string().min(1), + mappingType: z.nativeEnum(VaultMappingType).default(VaultMappingType.KeyVault) + }); + type TFormData = z.infer; + + const { mutateAsync: importVault } = useImportVault(); + + const { + control, + handleSubmit, + reset, + formState: { isLoading, isDirty, isSubmitting, isValid, errors } + } = useForm({ + resolver: zodResolver(formSchema) + }); + + console.log({ + isSubmitting, + isLoading, + isValid, + errors + }); + + const onSubmit = async (data: TFormData) => { + await importVault({ + vaultAccessToken: data.vaultAccessToken, + vaultNamespace: data.vaultNamespace, + vaultUrl: data.vaultUrl, + mappingType: data.mappingType + }); + createNotification({ + title: "Import started", + text: "Your data is being imported. You will receive an email when the import is complete or if the import fails. This may take up to 10 minutes.", + type: "info" + }); + + onClose(); + reset(); + }; + + return ( +
+ +

+ The Vault migration currently supports importing static secrets from Vault + Dedicated/Self-Hosted. +

+ Currently only KV Secret Engine V2 is supported for Vault migrations. +
+

+ +
+ ( + + + + )} + /> + ( + + + + )} + /> + ( + + + + )} + /> + ( + +
+ {MAPPING_TYPE_MENU_ITEMS.map((el) => ( +
field.onChange(el.value)} + role="button" + tabIndex={0} + onKeyDown={(e) => { + if (e.key === "Enter") { + field.onChange(el.value); + } + }} + > +
+
{el.label}
+ {el.tooltip && ( +
+ + + +
+ )} +
+
+ ))} +
+ + )} + /> + +
+ + +
+ +
+ ); +}; diff --git a/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/index.tsx b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/index.tsx new file mode 100644 index 000000000..fea504547 --- /dev/null +++ b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/index.tsx @@ -0,0 +1 @@ +export { ExternalMigrationsTab } from "./ExternalMigrationsTab"; diff --git a/frontend/src/pages/organization/SettingsPage/components/ImportTab/index.tsx b/frontend/src/pages/organization/SettingsPage/components/ImportTab/index.tsx deleted file mode 100644 index ec2b6ead6..000000000 --- a/frontend/src/pages/organization/SettingsPage/components/ImportTab/index.tsx +++ /dev/null @@ -1 +0,0 @@ -export { ImportTab } from "./ImportTab"; diff --git a/frontend/src/pages/organization/SettingsPage/components/OrgTabGroup/OrgTabGroup.tsx b/frontend/src/pages/organization/SettingsPage/components/OrgTabGroup/OrgTabGroup.tsx index a71ee7692..42a8e5839 100644 --- a/frontend/src/pages/organization/SettingsPage/components/OrgTabGroup/OrgTabGroup.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/OrgTabGroup/OrgTabGroup.tsx @@ -5,7 +5,7 @@ import { Tab, TabList, TabPanel, Tabs } from "@app/components/v2"; import { ROUTE_PATHS } from "@app/const/routes"; import { AuditLogStreamsTab } from "../AuditLogStreamTab"; -import { ImportTab } from "../ImportTab"; +import { ExternalMigrationsTab } from "../ExternalMigrationsTab"; import { KmipTab } from "../KmipTab/OrgKmipTab"; import { OrgEncryptionTab } from "../OrgEncryptionTab"; import { OrgGeneralTab } from "../OrgGeneralTab"; @@ -39,7 +39,11 @@ export const OrgTabGroup = () => { component: OrgWorkflowIntegrationTab }, { name: "Audit Log Streams", key: "tag-audit-log-streams", component: AuditLogStreamsTab }, - { name: "Import", key: "tab-import", component: ImportTab }, + { + name: "External Migrations", + key: "tab-external-migrations", + component: ExternalMigrationsTab + }, { name: "Project Templates", key: "project-templates",