diff --git a/backend/src/server/routes/v3/external-migration-router.ts b/backend/src/server/routes/v3/external-migration-router.ts index 865287157..d7aca92e2 100644 --- a/backend/src/server/routes/v3/external-migration-router.ts +++ b/backend/src/server/routes/v3/external-migration-router.ts @@ -1,9 +1,11 @@ import fastifyMultipart from "@fastify/multipart"; +import { z } from "zod"; import { BadRequestError } from "@app/lib/errors"; import { readLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; +import { VaultMappingType } from "@app/services/external-migration/external-migration-types"; const MB25_IN_BYTES = 26214400; @@ -52,4 +54,30 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider }); } }); + + server.route({ + method: "POST", + url: "/vault", + config: { + rateLimit: readLimit + }, + schema: { + body: z.object({ + vaultAccessToken: z.string(), + vaultNamespace: z.string(), + vaultUrl: z.string(), + mappingType: z.nativeEnum(VaultMappingType) + }) + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + await server.services.migration.importVaultData({ + actorId: req.permission.id, + actor: req.permission.type, + actorOrgId: req.permission.orgId, + actorAuthMethod: req.permission.authMethod, + ...req.body + }); + } + }); }; diff --git a/backend/src/server/routes/v3/index.ts b/backend/src/server/routes/v3/index.ts index ed8401560..1c31741a1 100644 --- a/backend/src/server/routes/v3/index.ts +++ b/backend/src/server/routes/v3/index.ts @@ -11,5 +11,5 @@ export const registerV3Routes = async (server: FastifyZodProvider) => { await server.register(registerUserRouter, { prefix: "/users" }); await server.register(registerSecretRouter, { prefix: "/secrets" }); await server.register(registerSecretBlindIndexRouter, { prefix: "/workspaces" }); - await server.register(registerExternalMigrationRouter, { prefix: "/migrate" }); + await server.register(registerExternalMigrationRouter, { prefix: "/external-migration" }); }; diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns/envkey.ts similarity index 61% rename from backend/src/services/external-migration/external-migration-fns.ts rename to backend/src/services/external-migration/external-migration-fns/envkey.ts index 8af22d858..ffdf80e9c 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns/envkey.ts @@ -1,32 +1,26 @@ -import slugify from "@sindresorhus/slugify"; import sjcl from "sjcl"; import tweetnacl from "tweetnacl"; import tweetnaclUtil from "tweetnacl-util"; -import { SecretType, TSecretFolders } from "@app/db/schemas"; import { crypto } from "@app/lib/crypto/cryptography"; -import { BadRequestError, NotFoundError } from "@app/lib/errors"; -import { chunkArray } from "@app/lib/fn"; +import { BadRequestError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; -import { alphaNumericNanoId } from "@app/lib/nanoid"; -import { CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; -import { TKmsServiceFactory } from "../kms/kms-service"; -import { KmsDataKey } from "../kms/kms-types"; -import { TProjectDALFactory } from "../project/project-dal"; -import { TProjectServiceFactory } from "../project/project-service"; -import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; -import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; -import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; -import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; -import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; -import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; -import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; -import { fnSecretBulkInsert, getAllSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns"; -import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; -import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; -import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; -import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types"; +import { TFolderCommitServiceFactory } from "../../folder-commit/folder-commit-service"; +import { TKmsServiceFactory } from "../../kms/kms-service"; +import { TProjectDALFactory } from "../../project/project-dal"; +import { TProjectServiceFactory } from "../../project/project-service"; +import { TProjectEnvDALFactory } from "../../project-env/project-env-dal"; +import { TProjectEnvServiceFactory } from "../../project-env/project-env-service"; +import { TResourceMetadataDALFactory } from "../../resource-metadata/resource-metadata-dal"; +import { TSecretFolderDALFactory } from "../../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../../secret-folder/secret-folder-version-dal"; +import { TSecretTagDALFactory } from "../../secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "../../secret-v2-bridge/secret-v2-bridge-dal"; +import type { TSecretV2BridgeServiceFactory } from "../../secret-v2-bridge/secret-v2-bridge-service"; +import { TSecretVersionV2DALFactory } from "../../secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "../../secret-v2-bridge/secret-version-tag-dal"; +import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "../external-migration-types"; export type TImportDataIntoInfisicalDTO = { projectDAL: Pick; @@ -499,326 +493,3 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise { - // Import data to infisical - if (!data || !data.projects) { - throw new BadRequestError({ message: "No projects found in data" }); - } - - const originalToNewProjectId = new Map(); - const originalToNewEnvironmentId = new Map< - string, - { envId: string; envSlug: string; rootFolderId: string; projectId: string } - >(); - const originalToNewFolderId = new Map< - string, - { - folderId: string; - projectId: string; - } - >(); - const projectsNotImported: string[] = []; - - await projectDAL.transaction(async (tx) => { - for await (const project of data.projects) { - const newProject = await projectService - .createProject({ - actor, - actorId, - actorOrgId, - actorAuthMethod, - workspaceName: project.name, - createDefaultEnvs: false, - tx - }) - .catch((e) => { - logger.error(e, `Failed to import to project [name:${project.name}]`); - throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` }); - }); - originalToNewProjectId.set(project.id, newProject.id); - } - - // Import environments - if (data.environments) { - for await (const environment of data.environments) { - const projectId = originalToNewProjectId.get(environment.projectId); - const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`); - - if (!projectId) { - projectsNotImported.push(environment.projectId); - // eslint-disable-next-line no-continue - continue; - } - - const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx); - - if (existingEnv) { - throw new BadRequestError({ - message: `Environment with slug '${slug}' already exist`, - name: "CreateEnvironment" - }); - } - - const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); - const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx); - const folder = await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); - - originalToNewEnvironmentId.set(environment.id, { - envSlug: doc.slug, - envId: doc.id, - rootFolderId: folder.id, - projectId - }); - } - } - - if (data.folders) { - for await (const folder of data.folders) { - const parentEnv = originalToNewEnvironmentId.get(folder.parentFolderId as string); - - if (!parentEnv) { - // eslint-disable-next-line no-continue - continue; - } - - const newFolder = await folderDAL.create( - { - name: folder.name, - envId: parentEnv.envId, - parentId: parentEnv.rootFolderId - }, - tx - ); - - const newFolderVersion = await folderVersionDAL.create( - { - name: newFolder.name, - envId: newFolder.envId, - version: newFolder.version, - folderId: newFolder.id - }, - tx - ); - - await folderCommitService.createCommit( - { - actor: { - type: actor, - metadata: { - id: actorId - } - }, - message: "Changed by external migration", - folderId: parentEnv.rootFolderId, - changes: [ - { - type: CommitType.ADD, - folderVersionId: newFolderVersion.id - } - ] - }, - tx - ); - - originalToNewFolderId.set(folder.id, { - folderId: newFolder.id, - projectId: parentEnv.projectId - }); - } - } - - // Useful for debugging: - // console.log("data.secrets", data.secrets); - // console.log("data.folders", data.folders); - // console.log("data.environment", data.environments); - - if (data.secrets && data.secrets.length > 0) { - const mappedToEnvironmentId = new Map< - string, - { - secretKey: string; - secretValue: string; - folderId?: string; - isFromBlock?: boolean; - }[] - >(); - - for (const secret of data.secrets) { - const targetId = secret.folderId || secret.environmentId; - - // Skip if we can't find either an environment or folder mapping for this secret - if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(targetId)) { - logger.info({ secret }, "[importDataIntoInfisicalFn]: Could not find environment or folder for secret"); - - // eslint-disable-next-line no-continue - continue; - } - - if (!mappedToEnvironmentId.has(targetId)) { - mappedToEnvironmentId.set(targetId, []); - } - - const alreadyHasSecret = mappedToEnvironmentId - .get(targetId)! - .find((el) => el.secretKey === secret.name && el.folderId === secret.folderId); - - if (alreadyHasSecret && alreadyHasSecret.isFromBlock) { - // remove the existing secret if any - mappedToEnvironmentId - .get(targetId)! - .splice(mappedToEnvironmentId.get(targetId)!.indexOf(alreadyHasSecret), 1); - } - mappedToEnvironmentId.get(targetId)!.push({ - secretKey: secret.name, - secretValue: secret.value || "", - folderId: secret.folderId, - isFromBlock: secret.appBlockOrderIndex !== undefined - }); - } - - // for each of the mappedEnvironmentId - for await (const [targetId, secrets] of mappedToEnvironmentId) { - logger.info("[importDataIntoInfisicalFn]: Processing secrets for targetId", targetId); - - let selectedFolder: TSecretFolders | undefined; - let selectedProjectId: string | undefined; - - // Case 1: Secret belongs to a folder / branch / branch of a block - const foundFolder = originalToNewFolderId.get(targetId); - if (foundFolder) { - logger.info("[importDataIntoInfisicalFn]: Processing secrets for folder"); - selectedFolder = await folderDAL.findById(foundFolder.folderId, tx); - selectedProjectId = foundFolder.projectId; - } else { - logger.info("[importDataIntoInfisicalFn]: Processing secrets for normal environment"); - const environment = data.environments.find((env) => env.id === targetId); - if (!environment) { - logger.info( - { - targetId - }, - "[importDataIntoInfisicalFn]: Could not find environment for secret" - ); - // eslint-disable-next-line no-continue - continue; - } - - const projectId = originalToNewProjectId.get(environment.projectId)!; - - if (!projectId) { - throw new BadRequestError({ message: `Failed to import secret, project not found` }); - } - - const env = originalToNewEnvironmentId.get(targetId); - if (!env) { - logger.info( - { - targetId - }, - "[importDataIntoInfisicalFn]: Could not find environment for secret" - ); - - // eslint-disable-next-line no-continue - continue; - } - - const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx); - - if (!folder) { - throw new NotFoundError({ - message: `Folder not found for the given environment slug (${env.envSlug}) & secret path (/)`, - name: "Create secret" - }); - } - - selectedFolder = folder; - selectedProjectId = projectId; - } - - if (!selectedFolder) { - throw new NotFoundError({ - message: `Folder not found for the given environment slug & secret path`, - name: "CreateSecret" - }); - } - - if (!selectedProjectId) { - throw new NotFoundError({ - message: `Project not found for the given environment slug & secret path`, - name: "CreateSecret" - }); - } - - const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey( - { - type: KmsDataKey.SecretManager, - projectId: selectedProjectId - }, - tx - ); - - const secretBatches = chunkArray(secrets, 2500); - for await (const secretBatch of secretBatches) { - const secretsByKeys = await secretDAL.findBySecretKeys( - selectedFolder.id, - secretBatch.map((el) => ({ - key: el.secretKey, - type: SecretType.Shared - })), - tx - ); - if (secretsByKeys.length) { - throw new BadRequestError({ - message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}` - }); - } - await fnSecretBulkInsert({ - inputSecrets: secretBatch.map((el) => { - const references = getAllSecretReferences(el.secretValue).nestedReferences; - - return { - version: 1, - encryptedValue: el.secretValue - ? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob - : undefined, - key: el.secretKey, - references, - type: SecretType.Shared - }; - }), - folderId: selectedFolder.id, - orgId: actorOrgId, - resourceMetadataDAL, - secretDAL, - secretVersionDAL, - secretTagDAL, - secretVersionTagDAL, - folderCommitService, - actor: { - type: actor, - actorId - }, - tx - }); - } - } - } - }); - - return { projectsNotImported }; -}; diff --git a/backend/src/services/external-migration/external-migration-fns/import.ts b/backend/src/services/external-migration/external-migration-fns/import.ts new file mode 100644 index 000000000..5728bf1c0 --- /dev/null +++ b/backend/src/services/external-migration/external-migration-fns/import.ts @@ -0,0 +1,352 @@ +import slugify from "@sindresorhus/slugify"; + +import { SecretType, TSecretFolders } from "@app/db/schemas"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { chunkArray } from "@app/lib/fn"; +import { logger } from "@app/lib/logger"; +import { alphaNumericNanoId } from "@app/lib/nanoid"; +import { CommitType } from "@app/services/folder-commit/folder-commit-service"; +import { KmsDataKey } from "@app/services/kms/kms-types"; +import { fnSecretBulkInsert, getAllSecretReferences } from "@app/services/secret-v2-bridge/secret-v2-bridge-fns"; + +import { TImportDataIntoInfisicalDTO } from "./envkey"; + +export const importDataIntoInfisicalFn = async ({ + projectService, + projectEnvDAL, + projectDAL, + secretDAL, + kmsService, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL, + resourceMetadataDAL, + folderVersionDAL, + folderCommitService, + input: { data, actor, actorId, actorOrgId, actorAuthMethod } +}: TImportDataIntoInfisicalDTO) => { + // Import data to infisical + if (!data || !data.projects) { + throw new BadRequestError({ message: "No projects found in data" }); + } + + const originalToNewProjectId = new Map(); + const originalToNewEnvironmentId = new Map< + string, + { envId: string; envSlug: string; rootFolderId?: string; projectId: string } + >(); + const originalToNewFolderId = new Map< + string, + { + envId: string; + envSlug: string; + folderId: string; + projectId: string; + } + >(); + const projectsNotImported: string[] = []; + + await projectDAL.transaction(async (tx) => { + for await (const project of data.projects) { + const newProject = await projectService + .createProject({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + workspaceName: project.name, + createDefaultEnvs: false, + tx + }) + .catch((e) => { + logger.error(e, `Failed to import to project [name:${project.name}]`); + throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` }); + }); + originalToNewProjectId.set(project.id, newProject.id); + } + + // Import environments + if (data.environments) { + for await (const environment of data.environments) { + const projectId = originalToNewProjectId.get(environment.projectId); + const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`); + + if (!projectId) { + projectsNotImported.push(environment.projectId); + // eslint-disable-next-line no-continue + continue; + } + + const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx); + + if (existingEnv) { + throw new BadRequestError({ + message: `Environment with slug '${slug}' already exist`, + name: "CreateEnvironment" + }); + } + + const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); + const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx); + const folder = await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); + + originalToNewEnvironmentId.set(environment.id, { + envSlug: doc.slug, + envId: doc.id, + rootFolderId: folder.id, + projectId + }); + } + } + + if (data.folders) { + for await (const folder of data.folders) { + const parentEnv = originalToNewEnvironmentId.get(folder.parentFolderId as string); + const parentFolder = originalToNewFolderId.get(folder.parentFolderId as string); + + let newFolder: TSecretFolders; + + if (parentEnv?.rootFolderId) { + newFolder = await folderDAL.create( + { + name: folder.name, + envId: parentEnv.envId, + parentId: parentEnv.rootFolderId + }, + tx + ); + } else if (parentFolder) { + newFolder = await folderDAL.create( + { + name: folder.name, + envId: parentFolder.envId, + parentId: parentFolder.folderId + }, + tx + ); + } else { + logger.info({ folder }, "No parent environment found for folder"); + // eslint-disable-next-line no-continue + continue; + } + + const newFolderVersion = await folderVersionDAL.create( + { + name: newFolder.name, + envId: newFolder.envId, + version: newFolder.version, + folderId: newFolder.id + }, + tx + ); + + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Changed by external migration", + folderId: parentEnv?.rootFolderId || parentFolder?.folderId || "", + changes: [ + { + type: CommitType.ADD, + folderVersionId: newFolderVersion.id + } + ] + }, + tx + ); + + originalToNewFolderId.set(folder.id, { + folderId: newFolder.id, + envId: parentEnv?.envId || parentFolder?.envId || "", + envSlug: parentEnv?.envSlug || parentFolder?.envSlug || "", + projectId: parentEnv?.projectId || parentFolder?.projectId || "" + }); + } + } + + // Useful for debugging: + // console.log("data.secrets", data.secrets); + // console.log("data.folders", data.folders); + // console.log("data.environment", data.environments); + + if (data.secrets && data.secrets.length > 0) { + const mappedToEnvironmentId = new Map< + string, + { + secretKey: string; + secretValue: string; + folderId?: string; + isFromBlock?: boolean; + }[] + >(); + + for (const secret of data.secrets) { + const targetId = secret.folderId || secret.environmentId; + + // Skip if we can't find either an environment or folder mapping for this secret + if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(targetId)) { + logger.info({ secret }, "[importDataIntoInfisicalFn]: Could not find environment or folder for secret"); + + // eslint-disable-next-line no-continue + continue; + } + + if (!mappedToEnvironmentId.has(targetId)) { + mappedToEnvironmentId.set(targetId, []); + } + + const alreadyHasSecret = mappedToEnvironmentId + .get(targetId)! + .find((el) => el.secretKey === secret.name && el.folderId === secret.folderId); + + if (alreadyHasSecret && alreadyHasSecret.isFromBlock) { + // remove the existing secret if any + mappedToEnvironmentId + .get(targetId)! + .splice(mappedToEnvironmentId.get(targetId)!.indexOf(alreadyHasSecret), 1); + } + mappedToEnvironmentId.get(targetId)!.push({ + secretKey: secret.name, + secretValue: secret.value || "", + folderId: secret.folderId, + isFromBlock: secret.appBlockOrderIndex !== undefined + }); + } + + // for each of the mappedEnvironmentId + for await (const [targetId, secrets] of mappedToEnvironmentId) { + logger.info("[importDataIntoInfisicalFn]: Processing secrets for targetId", targetId); + + let selectedFolder: TSecretFolders | undefined; + let selectedProjectId: string | undefined; + + // Case 1: Secret belongs to a folder / branch / branch of a block + const foundFolder = originalToNewFolderId.get(targetId); + if (foundFolder) { + logger.info("[importDataIntoInfisicalFn]: Processing secrets for folder"); + selectedFolder = await folderDAL.findById(foundFolder.folderId, tx); + selectedProjectId = foundFolder.projectId; + } else { + logger.info("[importDataIntoInfisicalFn]: Processing secrets for normal environment"); + const environment = data.environments.find((env) => env.id === targetId); + if (!environment) { + logger.info( + { + targetId + }, + "[importDataIntoInfisicalFn]: Could not find environment for secret" + ); + // eslint-disable-next-line no-continue + continue; + } + + const projectId = originalToNewProjectId.get(environment.projectId)!; + + if (!projectId) { + throw new BadRequestError({ message: `Failed to import secret, project not found` }); + } + + const env = originalToNewEnvironmentId.get(targetId); + if (!env) { + logger.info( + { + targetId + }, + "[importDataIntoInfisicalFn]: Could not find environment for secret" + ); + + // eslint-disable-next-line no-continue + continue; + } + + const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx); + + if (!folder) { + throw new NotFoundError({ + message: `Folder not found for the given environment slug (${env.envSlug}) & secret path (/)`, + name: "Create secret" + }); + } + + selectedFolder = folder; + selectedProjectId = projectId; + } + + if (!selectedFolder) { + throw new NotFoundError({ + message: `Folder not found for the given environment slug & secret path`, + name: "CreateSecret" + }); + } + + if (!selectedProjectId) { + throw new NotFoundError({ + message: `Project not found for the given environment slug & secret path`, + name: "CreateSecret" + }); + } + + const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey( + { + type: KmsDataKey.SecretManager, + projectId: selectedProjectId + }, + tx + ); + + const secretBatches = chunkArray(secrets, 2500); + for await (const secretBatch of secretBatches) { + const secretsByKeys = await secretDAL.findBySecretKeys( + selectedFolder.id, + secretBatch.map((el) => ({ + key: el.secretKey, + type: SecretType.Shared + })), + tx + ); + if (secretsByKeys.length) { + throw new BadRequestError({ + message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}` + }); + } + await fnSecretBulkInsert({ + inputSecrets: secretBatch.map((el) => { + const references = getAllSecretReferences(el.secretValue).nestedReferences; + + return { + version: 1, + encryptedValue: el.secretValue + ? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob + : undefined, + key: el.secretKey, + references, + type: SecretType.Shared + }; + }), + folderId: selectedFolder.id, + orgId: actorOrgId, + resourceMetadataDAL, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + folderCommitService, + actor: { + type: actor, + actorId + }, + tx + }); + } + } + } + }); + + return { projectsNotImported }; +}; diff --git a/backend/src/services/external-migration/external-migration-fns/index.ts b/backend/src/services/external-migration/external-migration-fns/index.ts new file mode 100644 index 000000000..4af82bf22 --- /dev/null +++ b/backend/src/services/external-migration/external-migration-fns/index.ts @@ -0,0 +1,3 @@ +export * from "./envkey"; +export * from "./import"; +export * from "./vault"; diff --git a/backend/src/services/external-migration/external-migration-fns/vault.ts b/backend/src/services/external-migration/external-migration-fns/vault.ts new file mode 100644 index 000000000..a61b2e703 --- /dev/null +++ b/backend/src/services/external-migration/external-migration-fns/vault.ts @@ -0,0 +1,312 @@ +import axios, { AxiosInstance } from "axios"; +import { v4 as uuidv4 } from "uuid"; + +import { InfisicalImportData, VaultMappingType } from "../external-migration-types"; + +type VaultData = { + namespace: string; + mount: string; + path: string; + secretData: Record; +}; + +const vaultFactory = () => { + const getMounts = async (request: AxiosInstance) => { + const response = await request.get< + Record< + string, + { + accessor: string; + options: { + version?: string; + } | null; + type: string; + } + > + >("/v1/sys/mounts"); + return response.data; + }; + + const getPaths = async ( + request: AxiosInstance, + { mountPath, secretPath = "" }: { mountPath: string; secretPath?: string } + ) => { + try { + // For KV v2: /v1/{mount}/metadata/{path}?list=true + const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`; + const response = await request.get<{ + data: { + keys: string[]; + }; + }>(`/v1/${path}?list=true`); + + return response.data.data.keys; + } catch (err) { + if (axios.isAxiosError(err) && err.response?.status === 404) { + return null; + } + throw err; + } + }; + + const getSecrets = async ( + request: AxiosInstance, + { mountPath, secretPath }: { mountPath: string; secretPath: string } + ) => { + // For KV v2: /v1/{mount}/data/{path} + const response = await request.get<{ + data: { + data: Record; // KV v2 has nested data structure + metadata: { + created_time: string; + deletion_time: string; + destroyed: boolean; + version: number; + }; + }; + }>(`/v1/${mountPath}/data/${secretPath}`); + + return response.data.data.data; + }; + + // helper function to check if a mount is KV v2 (will be useful if we add support for Vault KV v1) + // const isKvV2Mount = (mountInfo: { type: string; options?: { version?: string } | null }) => { + // return mountInfo.type === "kv" && mountInfo.options?.version === "2"; + // }; + + const recursivelyGetAllPaths = async ( + request: AxiosInstance, + mountPath: string, + currentPath: string = "" + ): Promise => { + const paths = await getPaths(request, { mountPath, secretPath: currentPath }); + + if (paths === null || paths.length === 0) { + return []; + } + + const allSecrets: string[] = []; + + for await (const path of paths) { + const cleanPath = path.endsWith("/") ? path.slice(0, -1) : path; + const fullItemPath = currentPath ? `${currentPath}/${cleanPath}` : cleanPath; + + if (path.endsWith("/")) { + // it's a folder so we recurse into it + const subSecrets = await recursivelyGetAllPaths(request, mountPath, fullItemPath); + allSecrets.push(...subSecrets); + } else { + // it's a secret so we add it to our results + allSecrets.push(`${mountPath}/${fullItemPath}`); + } + } + + return allSecrets; + }; + + async function collectVaultData({ + baseUrl, + namespace, + accessToken + }: { + baseUrl: string; + namespace: string; + accessToken: string; + }): Promise { + const request = axios.create({ + baseURL: baseUrl, + headers: { + "X-Vault-Namespace": namespace, + "X-Vault-Token": accessToken + } + }); + + const allData: VaultData[] = []; + + // Get all mounts in this namespace + const mounts = await getMounts(request); + + for (const mount of Object.keys(mounts)) { + if (!mount.endsWith("/")) { + delete mounts[mount]; + } + } + + for await (const [mountPath, mountInfo] of Object.entries(mounts)) { + // skip non-KV mounts + if (!mountInfo.type.startsWith("kv")) { + // eslint-disable-next-line no-continue + continue; + } + + // get all paths in this mount + const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`); + + const cleanMountPath = mountPath.replace(/\/$/, ""); + + for await (const secretPath of paths) { + // get the actual secret data + const secretData = await getSecrets(request, { + mountPath: cleanMountPath, + secretPath: secretPath.replace(`${cleanMountPath}/`, "") + }); + + allData.push({ + namespace, + mount: mountPath.replace(/\/$/, ""), + path: secretPath.replace(`${cleanMountPath}/`, ""), + secretData + }); + } + } + + return allData; + } + + return { + collectVaultData, + getMounts, + getPaths, + getSecrets, + recursivelyGetAllPaths + }; +}; + +export const transformToInfisicalFormatNamespaceToProjects = ( + vaultData: VaultData[], + mappingType: VaultMappingType +): InfisicalImportData => { + const projects: Array<{ name: string; id: string }> = []; + const environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }> = []; + const folders: Array<{ id: string; name: string; environmentId: string; parentFolderId?: string }> = []; + const secrets: Array<{ id: string; name: string; environmentId: string; value: string; folderId?: string }> = []; + + // track created entities to avoid duplicates + const projectMap = new Map(); // namespace -> projectId + const environmentMap = new Map(); // namespace:mount -> environmentId + const folderMap = new Map(); // namespace:mount:folderPath -> folderId + + let environmentId: string = ""; + for (const data of vaultData) { + const { namespace, mount, path, secretData } = data; + + if (mappingType === "namespace") { + // create project (namespace) + if (!projectMap.has(namespace)) { + const projectId = uuidv4(); + projectMap.set(namespace, projectId); + projects.push({ + name: namespace, + id: projectId + }); + } + const projectId = projectMap.get(namespace)!; + + // create environment (mount) + const envKey = `${namespace}:${mount}`; + if (!environmentMap.has(envKey)) { + environmentId = uuidv4(); + environmentMap.set(envKey, environmentId); + environments.push({ + name: mount, + id: environmentId, + projectId + }); + } + environmentId = environmentMap.get(envKey)!; + } else if (mappingType === "key-vault") { + if (!projectMap.has(mount)) { + const projectId = uuidv4(); + projectMap.set(mount, projectId); + projects.push({ + name: mount, + id: projectId + }); + } + const projectId = projectMap.get(mount)!; + + // create single "Production" environment per project, because we have no good way of determining environments from vault + if (!environmentMap.has(mount)) { + environmentId = uuidv4(); + environmentMap.set(mount, environmentId); + environments.push({ + name: "Production", + id: environmentId, + projectId + }); + } + environmentId = environmentMap.get(mount)!; + } + + // create folder structure + let currentFolderId: string | undefined; + let currentPath = ""; + + if (path.includes("/")) { + const pathParts = path.split("/").filter(Boolean); + + const folderParts = pathParts; + + // create nested folder structure for the entire path + for (const folderName of folderParts) { + currentPath = currentPath ? `${currentPath}/${folderName}` : folderName; + const folderKey = `${namespace}:${mount}:${currentPath}`; + + if (!folderMap.has(folderKey)) { + const folderId = uuidv4(); + folderMap.set(folderKey, folderId); + folders.push({ + id: folderId, + name: folderName, + environmentId, + parentFolderId: currentFolderId || environmentId + }); + currentFolderId = folderId; + } else { + currentFolderId = folderMap.get(folderKey)!; + } + } + } + + for (const [key, value] of Object.entries(secretData)) { + secrets.push({ + id: uuidv4(), + name: key, + environmentId, + value: String(value), + folderId: currentFolderId + }); + } + } + + return { + projects, + environments, + folders, + secrets + }; +}; + +export const importVaultDataFn = async ({ + vaultAccessToken, + vaultNamespace, + vaultUrl, + mappingType +}: { + vaultAccessToken: string; + vaultNamespace: string; + vaultUrl: string; + mappingType: VaultMappingType; +}) => { + const vaultApi = vaultFactory(); + + const vaultData = await vaultApi.collectVaultData({ + accessToken: vaultAccessToken, + baseUrl: vaultUrl, + namespace: vaultNamespace + }); + + const infisicalData = transformToInfisicalFormatNamespaceToProjects(vaultData, mappingType); + + return infisicalData; +}; diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts index c4e6b43c5..8bde91aa5 100644 --- a/backend/src/services/external-migration/external-migration-queue.ts +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -19,7 +19,7 @@ import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-d import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service"; import { importDataIntoInfisicalFn } from "./external-migration-fns"; -import { ExternalPlatforms, TImportInfisicalDataCreate } from "./external-migration-types"; +import { ExternalPlatforms, ImportType, TImportInfisicalDataCreate } from "./external-migration-types"; export type TExternalMigrationQueueFactoryDep = { smtpService: TSmtpService; @@ -67,6 +67,7 @@ export const externalMigrationQueueFactory = ({ const startImport = async (dto: { actorEmail: string; data: { + importType: ImportType; iv: string; tag: string; ciphertext: string; diff --git a/backend/src/services/external-migration/external-migration-service.ts b/backend/src/services/external-migration/external-migration-service.ts index c310fd273..766b6cef4 100644 --- a/backend/src/services/external-migration/external-migration-service.ts +++ b/backend/src/services/external-migration/external-migration-service.ts @@ -4,9 +4,9 @@ import { crypto } from "@app/lib/crypto/cryptography"; import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors"; import { TUserDALFactory } from "../user/user-dal"; -import { decryptEnvKeyDataFn, parseEnvKeyDataFn } from "./external-migration-fns"; +import { decryptEnvKeyDataFn, importVaultDataFn, parseEnvKeyDataFn } from "./external-migration-fns"; import { TExternalMigrationQueueFactory } from "./external-migration-queue"; -import { TImportEnvKeyDataCreate } from "./external-migration-types"; +import { ImportType, TImportEnvKeyDataDTO, TImportVaultDataDTO } from "./external-migration-types"; type TExternalMigrationServiceFactoryDep = { permissionService: TPermissionServiceFactory; @@ -28,7 +28,7 @@ export const externalMigrationServiceFactory = ({ actorId, actorOrgId, actorAuthMethod - }: TImportEnvKeyDataCreate) => { + }: TImportEnvKeyDataDTO) => { if (crypto.isFipsModeEnabled()) { throw new BadRequestError({ message: "EnvKey migration is not supported when running in FIPS mode." }); } @@ -60,11 +60,65 @@ export const externalMigrationServiceFactory = ({ await externalMigrationQueue.startImport({ actorEmail: user.email!, - data: encrypted + data: { + importType: ImportType.EnvKey, + ...encrypted + } + }); + }; + + const importVaultData = async ({ + vaultAccessToken, + vaultNamespace, + mappingType, + vaultUrl, + actor, + actorId, + actorOrgId, + actorAuthMethod + }: TImportVaultDataDTO) => { + const { membership } = await permissionService.getOrgPermission( + actor, + actorId, + actorOrgId, + actorAuthMethod, + actorOrgId + ); + + if (membership.role !== OrgMembershipRole.Admin) { + throw new ForbiddenRequestError({ message: "Only admins can import data" }); + } + + const user = await userDAL.findById(actorId); + + const vaultData = await importVaultDataFn({ + vaultAccessToken, + vaultNamespace, + vaultUrl, + mappingType + }); + + const stringifiedJson = JSON.stringify({ + data: vaultData, + actor, + actorId, + actorOrgId, + actorAuthMethod + }); + + const encrypted = crypto.encryption().symmetric().encryptWithRootEncryptionKey(stringifiedJson); + + await externalMigrationQueue.startImport({ + actorEmail: user.email!, + data: { + importType: ImportType.Vault, + ...encrypted + } }); }; return { - importEnvKeyData + importEnvKeyData, + importVaultData }; }; diff --git a/backend/src/services/external-migration/external-migration-types.ts b/backend/src/services/external-migration/external-migration-types.ts index 32c70a688..f3431b5c9 100644 --- a/backend/src/services/external-migration/external-migration-types.ts +++ b/backend/src/services/external-migration/external-migration-types.ts @@ -1,5 +1,17 @@ +import { TOrgPermission } from "@app/lib/types"; + import { ActorAuthMethod, ActorType } from "../auth/auth-type"; +export enum ImportType { + EnvKey = "envkey", + Vault = "vault" +} + +export enum VaultMappingType { + Namespace = "namespace", + KeyVault = "key-vault" +} + export type InfisicalImportData = { projects: Array<{ name: string; id: string }>; environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }>; @@ -14,14 +26,17 @@ export type InfisicalImportData = { }>; }; -export type TImportEnvKeyDataCreate = { +export type TImportEnvKeyDataDTO = { decryptionKey: string; encryptedJson: { nonce: string; data: string }; - actor: ActorType; - actorId: string; - actorOrgId: string; - actorAuthMethod: ActorAuthMethod; -}; +} & Omit; + +export type TImportVaultDataDTO = { + vaultAccessToken: string; + vaultNamespace: string; + mappingType: VaultMappingType; + vaultUrl: string; +} & Omit; export type TImportInfisicalDataCreate = { data: InfisicalImportData; diff --git a/frontend/src/hooks/api/migration/index.ts b/frontend/src/hooks/api/migration/index.ts new file mode 100644 index 000000000..f8dd99d03 --- /dev/null +++ b/frontend/src/hooks/api/migration/index.ts @@ -0,0 +1 @@ +export * from "./mutations"; diff --git a/frontend/src/hooks/api/migration/mutations.tsx b/frontend/src/hooks/api/migration/mutations.tsx index 87ba89eab..51c0f4910 100644 --- a/frontend/src/hooks/api/migration/mutations.tsx +++ b/frontend/src/hooks/api/migration/mutations.tsx @@ -15,7 +15,7 @@ export const useImportEnvKey = () => { formData.append("file", file); try { - const response = await apiRequest.post("/api/v3/migrate/env-key/", formData, { + const response = await apiRequest.post("/api/v3/external-migration/env-key/", formData, { headers: { "Content-Type": "multipart/form-data" }, @@ -39,3 +39,23 @@ export const useImportEnvKey = () => { } }); }; + +export const useImportVault = () => { + return useMutation({ + mutationFn: async ({ + vaultAccessToken, + vaultNamespace, + vaultUrl + }: { + vaultAccessToken: string; + vaultNamespace: string; + vaultUrl: string; + }) => { + await apiRequest.post("/api/v3/external-migration/vault/", { + vaultAccessToken, + vaultNamespace, + vaultUrl + }); + } + }); +}; diff --git a/frontend/src/pages/organization/SettingsPage/components/ImportTab/components/SelectImportFromPlatformModal.tsx b/frontend/src/pages/organization/SettingsPage/components/ImportTab/components/SelectImportFromPlatformModal.tsx index 06631ffe1..f596a957d 100644 --- a/frontend/src/pages/organization/SettingsPage/components/ImportTab/components/SelectImportFromPlatformModal.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/ImportTab/components/SelectImportFromPlatformModal.tsx @@ -1,11 +1,12 @@ import { useState } from "react"; -import { faKey } from "@fortawesome/free-solid-svg-icons"; +import { faKey, faVault } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { AnimatePresence, motion } from "framer-motion"; import { Modal, ModalContent } from "@app/components/v2"; import { EnvKeyPlatformModal } from "./EnvKeyPlatformModal"; +import { VaultPlatformModal } from "./VaultPlatformModal copy"; type Props = { isOpen?: boolean; @@ -22,6 +23,11 @@ const PLATFORM_LIST = [ icon: faKey, platform: "env-key", title: "Env Key" + }, + { + icon: faVault, + platform: "vault", + title: "Vault" } ] as const; @@ -82,18 +88,32 @@ export const SelectImportFromPlatformModal = ({ isOpen, onToggle }: Props) => { )} - {wizardStep === WizardSteps.PlatformInputs && - selectedPlatform?.platform === "env-key" && ( - - handleFormReset(false)} /> - - )} + {wizardStep === WizardSteps.PlatformInputs && ( + <> + {selectedPlatform?.platform === "env-key" && ( + + handleFormReset(false)} /> + + )} + {selectedPlatform?.platform === "vault" && ( + + handleFormReset(false)} /> + + )} + + )} diff --git a/frontend/src/pages/organization/SettingsPage/components/ImportTab/components/VaultPlatformModal copy.tsx b/frontend/src/pages/organization/SettingsPage/components/ImportTab/components/VaultPlatformModal copy.tsx new file mode 100644 index 000000000..3d9755a28 --- /dev/null +++ b/frontend/src/pages/organization/SettingsPage/components/ImportTab/components/VaultPlatformModal copy.tsx @@ -0,0 +1,226 @@ +import { useRef } from "react"; +import { Controller, useForm } from "react-hook-form"; +import { faQuestionCircle } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { twMerge } from "tailwind-merge"; +import { z } from "zod"; + +import { createNotification } from "@app/components/notifications"; +import { Button, FormControl, Input, Tooltip } from "@app/components/v2"; +import { NoticeBannerV2 } from "@app/components/v2/NoticeBannerV2/NoticeBannerV2"; +import { useImportVault } from "@app/hooks/api/migration/mutations"; + +type Props = { + id?: string; + onClose: () => void; +}; + +enum VaultMappingType { + Namespace = "namespace", + KeyVault = "key-vault" +} + +const MAPPING_TYPE_MENU_ITEMS = [ + { + value: VaultMappingType.Namespace, + label: "Namespaces", + tooltip: ( +
+ When using namespaces for mapping, each namespace within Vault will be created in Infisical + as a project. Each key vault (KV) inside the namespace, will be created as an environment + inside the corresponding project. +
+
Namespace → Project
+
Key Vault → Project Environment
+
Secret Path → Secret Folder
+
Secret data → Secrets
+
+
+ ) + }, + { + value: VaultMappingType.KeyVault, + label: "Key Vaults", + tooltip: ( +
+ When using key vaults for mapping, each key vault within Vault will be created in Infisical + as a project. Each secret path inside the key vault, will be created as an environment + inside the corresponding project. When using Key Vaults as the mapping type, a default + environment called "Production" will be created for each project, which will + contain the secrets from the key vault. +
+
Key Vault → Project
+
Default Environment (Production)
+
Secret Path → Secret Folder
+
Secret data → Secrets
+
+
+ ) + } +]; + +export const VaultPlatformModal = ({ onClose }: Props) => { + const formSchema = z.object({ + vaultUrl: z.string().min(1), + vaultNamespace: z.string().min(1), + vaultAccessToken: z.string().min(1), + mappingType: z.nativeEnum(VaultMappingType) + }); + type TFormData = z.infer; + + const fileUploadRef = useRef(null); + + const { mutateAsync: importVault } = useImportVault(); + + const { + control, + handleSubmit, + reset, + formState: { isLoading, isDirty, isSubmitting, isValid } + } = useForm({ + resolver: zodResolver(formSchema) + }); + + const onSubmit = async (data: TFormData) => { + try { + await importVault({ + vaultAccessToken: data.vaultAccessToken, + vaultNamespace: data.vaultNamespace, + vaultUrl: data.vaultUrl + }); + createNotification({ + title: "Import started", + text: "Your data is being imported. You will receive an email when the import is complete or if the import fails. This may take up to 10 minutes.", + type: "info" + }); + + onClose(); + reset(); + + if (fileUploadRef.current) { + fileUploadRef.current.value = ""; + } + } catch { + reset(); + } + }; + + return ( +
+ +

+ The Vault migration currently supports importing static secrets from Vault + Dedicated/Self-Hosted. Namespaces are treated as projects, Secret Engines are treated as + environments, and secret paths are treated as folders. +

+ Currently only KV Secret Engine V2 is supported for Vault migrations. +
+

+ +
+ ( + + + + )} + /> + ( + + + + )} + /> + ( + + + + )} + /> + ( + +
+ {MAPPING_TYPE_MENU_ITEMS.map((el) => ( +
field.onChange(el.value)} + role="button" + tabIndex={0} + onKeyDown={(e) => { + if (e.key === "Enter") { + field.onChange(el.value); + } + }} + > +
+
{el.label}
+ {el.tooltip && ( +
+ + + +
+ )} +
+
+ ))} +
+ + )} + /> + +
+ + +
+ +
+ ); +};