From c99a4b7cc8d18108ff5c6f4c4deea7f521919cbc Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 29 Aug 2025 23:27:12 +0200 Subject: [PATCH 1/3] feat(vault-migration): gateway support & kv v1 support --- backend/src/server/routes/index.ts | 3 +- .../routes/v3/external-migration-router.ts | 3 +- .../external-migration-fns/vault.ts | 266 +++++++++++++----- .../external-migration-service.ts | 24 +- .../external-migration-types.ts | 1 + .../src/hooks/api/migration/mutations.tsx | 7 +- .../components/VaultPlatformModal.tsx | 84 +++++- 7 files changed, 292 insertions(+), 96 deletions(-) diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 3c1038ded..3d79836e5 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -1746,7 +1746,8 @@ export const registerRoutes = async ( const migrationService = externalMigrationServiceFactory({ externalMigrationQueue, userDAL, - permissionService + permissionService, + gatewayService }); const externalGroupOrgRoleMappingService = externalGroupOrgRoleMappingServiceFactory({ diff --git a/backend/src/server/routes/v3/external-migration-router.ts b/backend/src/server/routes/v3/external-migration-router.ts index 4325692da..259a97ddb 100644 --- a/backend/src/server/routes/v3/external-migration-router.ts +++ b/backend/src/server/routes/v3/external-migration-router.ts @@ -66,7 +66,8 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider vaultAccessToken: z.string(), vaultNamespace: z.string().trim().optional(), vaultUrl: z.string(), - mappingType: z.nativeEnum(VaultMappingType) + mappingType: z.nativeEnum(VaultMappingType), + gatewayId: z.string().optional() }) }, onRequest: verifyAuth([AuthMode.JWT]), diff --git a/backend/src/services/external-migration/external-migration-fns/vault.ts b/backend/src/services/external-migration/external-migration-fns/vault.ts index 1ebd56a80..f5f57aa1b 100644 --- a/backend/src/services/external-migration/external-migration-fns/vault.ts +++ b/backend/src/services/external-migration/external-migration-fns/vault.ts @@ -1,12 +1,21 @@ +import https from "node:https"; + import axios, { AxiosInstance } from "axios"; import { v4 as uuidv4 } from "uuid"; +import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service"; import { BadRequestError } from "@app/lib/errors"; +import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway"; import { logger } from "@app/lib/logger"; import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator"; import { InfisicalImportData, VaultMappingType } from "../external-migration-types"; +enum KvVersion { + V1 = "1", + V2 = "2" +} + type VaultData = { namespace: string; mount: string; @@ -14,7 +23,42 @@ type VaultData = { secretData: Record; }; -const vaultFactory = () => { +const vaultFactory = (gatewayService: Pick) => { + const $gatewayProxyWrapper = async ( + inputs: { + gatewayId: string; + targetHost?: string; + targetPort?: number; + }, + gatewayCallback: (host: string, port: number, httpsAgent?: https.Agent) => Promise + ): Promise => { + const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(inputs.gatewayId); + const [relayHost, relayPort] = relayDetails.relayAddress.split(":"); + + const callbackResult = await withGatewayProxy( + async (port, httpsAgent) => { + const res = await gatewayCallback("http://localhost", port, httpsAgent); + return res; + }, + { + protocol: GatewayProxyProtocol.Http, + targetHost: inputs.targetHost, + targetPort: inputs.targetPort, + relayHost, + relayPort: Number(relayPort), + identityId: relayDetails.identityId, + orgId: relayDetails.orgId, + tlsOptions: { + ca: relayDetails.certChain, + cert: relayDetails.certificate, + key: relayDetails.privateKey.toString() + } + } + ); + + return callbackResult; + }; + const getMounts = async (request: AxiosInstance) => { const response = await request .get<{ @@ -31,11 +75,24 @@ const vaultFactory = () => { const getPaths = async ( request: AxiosInstance, - { mountPath, secretPath = "" }: { mountPath: string; secretPath?: string } + { mountPath, secretPath = "" }: { mountPath: string; secretPath?: string }, + kvVersion: KvVersion ) => { try { - // For KV v2: /v1/{mount}/metadata/{path}?list=true - const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`; + if (kvVersion === KvVersion.V2) { + // For KV v2: /v1/{mount}/metadata/{path}?list=true + const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`; + const response = await request.get<{ + data: { + keys: string[]; + }; + }>(`/v1/${path}?list=true`); + + return response.data.data.keys; + } + + // kv version v1: /v1/{mount}?list=true + const path = secretPath ? `${mountPath}/${secretPath}` : mountPath; const response = await request.get<{ data: { keys: string[]; @@ -56,21 +113,42 @@ const vaultFactory = () => { const getSecrets = async ( request: AxiosInstance, - { mountPath, secretPath }: { mountPath: string; secretPath: string } + { mountPath, secretPath }: { mountPath: string; secretPath: string }, + kvVersion: KvVersion ) => { - // For KV v2: /v1/{mount}/data/{path} + if (kvVersion === KvVersion.V2) { + // For KV v2: /v1/{mount}/data/{path} + const response = await request + .get<{ + data: { + data: Record; // KV v2 has nested data structure + metadata: { + created_time: string; + deletion_time: string; + destroyed: boolean; + version: number; + }; + }; + }>(`/v1/${mountPath}/data/${secretPath}`) + .catch((err) => { + if (axios.isAxiosError(err)) { + logger.error(err.response?.data, "External migration: Failed to get Vault secret"); + } + throw err; + }); + + return response.data.data.data; + } + + // kv version v1 + const response = await request .get<{ - data: { - data: Record; // KV v2 has nested data structure - metadata: { - created_time: string; - deletion_time: string; - destroyed: boolean; - version: number; - }; - }; - }>(`/v1/${mountPath}/data/${secretPath}`) + data: Record; // KV v1 has flat data structure + lease_duration: number; + lease_id: string; + renewable: boolean; + }>(`/v1/${mountPath}/${secretPath}`) .catch((err) => { if (axios.isAxiosError(err)) { logger.error(err.response?.data, "External migration: Failed to get Vault secret"); @@ -78,7 +156,7 @@ const vaultFactory = () => { throw err; }); - return response.data.data.data; + return response.data.data; }; // helper function to check if a mount is KV v2 (will be useful if we add support for Vault KV v1) @@ -89,9 +167,10 @@ const vaultFactory = () => { const recursivelyGetAllPaths = async ( request: AxiosInstance, mountPath: string, + kvVersion: KvVersion, currentPath: string = "" ): Promise => { - const paths = await getPaths(request, { mountPath, secretPath: currentPath }); + const paths = await getPaths(request, { mountPath, secretPath: currentPath }, kvVersion); if (paths === null || paths.length === 0) { return []; @@ -105,7 +184,7 @@ const vaultFactory = () => { if (path.endsWith("/")) { // it's a folder so we recurse into it - const subSecrets = await recursivelyGetAllPaths(request, mountPath, fullItemPath); + const subSecrets = await recursivelyGetAllPaths(request, mountPath, kvVersion, fullItemPath); allSecrets.push(...subSecrets); } else { // it's a secret so we add it to our results @@ -119,60 +198,93 @@ const vaultFactory = () => { async function collectVaultData({ baseUrl, namespace, - accessToken + accessToken, + gatewayId }: { baseUrl: string; namespace?: string; accessToken: string; + gatewayId?: string; }): Promise { - const request = axios.create({ - baseURL: baseUrl, - headers: { - "X-Vault-Token": accessToken, - ...(namespace ? { "X-Vault-Namespace": namespace } : {}) + const getData = async (host: string, port?: number, httpsAgent?: https.Agent) => { + const allData: VaultData[] = []; + + const request = axios.create({ + baseURL: port ? `${host}:${port}` : host, + headers: { + "X-Vault-Token": accessToken, + ...(namespace ? { "X-Vault-Namespace": namespace } : {}) + }, + httpsAgent + }); + + // Get all mounts in this namespace + const mounts = await getMounts(request); + + for (const mount of Object.keys(mounts)) { + if (!mount.endsWith("/")) { + delete mounts[mount]; + } } - }); - const allData: VaultData[] = []; + for await (const [mountPath, mountInfo] of Object.entries(mounts)) { + // skip non-KV mounts + if (!mountInfo.type.startsWith("kv")) { + // eslint-disable-next-line no-continue + continue; + } - // Get all mounts in this namespace - const mounts = await getMounts(request); + const kvVersion = mountInfo.options?.version === "2" ? KvVersion.V2 : KvVersion.V1; - for (const mount of Object.keys(mounts)) { - if (!mount.endsWith("/")) { - delete mounts[mount]; + // get all paths in this mount + const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`, kvVersion); + + const cleanMountPath = mountPath.replace(/\/$/, ""); + + for await (const secretPath of paths) { + // get the actual secret data + const secretData = await getSecrets( + request, + { + mountPath: cleanMountPath, + secretPath: secretPath.replace(`${cleanMountPath}/`, "") + }, + kvVersion + ); + + allData.push({ + namespace: namespace || "", + mount: mountPath.replace(/\/$/, ""), + path: secretPath.replace(`${cleanMountPath}/`, ""), + secretData + }); + } } + + return allData; + }; + + let data; + + if (gatewayId) { + const url = new URL(baseUrl); + + const { port, protocol, hostname } = url; + const cleanedProtocol = protocol.slice(0, -1); + + data = await $gatewayProxyWrapper( + { + gatewayId, + targetHost: `${cleanedProtocol}://${hostname}`, + targetPort: port ? Number(port) : 8200 // 8200, default port for Vault self-hosted/dedicated + }, + getData + ); + } else { + data = await getData(baseUrl); } - for await (const [mountPath, mountInfo] of Object.entries(mounts)) { - // skip non-KV mounts - if (!mountInfo.type.startsWith("kv")) { - // eslint-disable-next-line no-continue - continue; - } - - // get all paths in this mount - const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`); - - const cleanMountPath = mountPath.replace(/\/$/, ""); - - for await (const secretPath of paths) { - // get the actual secret data - const secretData = await getSecrets(request, { - mountPath: cleanMountPath, - secretPath: secretPath.replace(`${cleanMountPath}/`, "") - }); - - allData.push({ - namespace: namespace || "", - mount: mountPath.replace(/\/$/, ""), - path: secretPath.replace(`${cleanMountPath}/`, ""), - secretData - }); - } - } - - return allData; + return data; } return { @@ -296,17 +408,22 @@ export const transformToInfisicalFormatNamespaceToProjects = ( }; }; -export const importVaultDataFn = async ({ - vaultAccessToken, - vaultNamespace, - vaultUrl, - mappingType -}: { - vaultAccessToken: string; - vaultNamespace?: string; - vaultUrl: string; - mappingType: VaultMappingType; -}) => { +export const importVaultDataFn = async ( + { + vaultAccessToken, + vaultNamespace, + vaultUrl, + mappingType, + gatewayId + }: { + vaultAccessToken: string; + vaultNamespace?: string; + vaultUrl: string; + mappingType: VaultMappingType; + gatewayId?: string; + }, + { gatewayService }: { gatewayService: Pick } +) => { await blockLocalAndPrivateIpAddresses(vaultUrl); if (mappingType === VaultMappingType.Namespace && !vaultNamespace) { @@ -315,12 +432,13 @@ export const importVaultDataFn = async ({ }); } - const vaultApi = vaultFactory(); + const vaultApi = vaultFactory(gatewayService); const vaultData = await vaultApi.collectVaultData({ accessToken: vaultAccessToken, baseUrl: vaultUrl, - namespace: vaultNamespace + namespace: vaultNamespace, + gatewayId }); const infisicalData = transformToInfisicalFormatNamespaceToProjects(vaultData, mappingType); diff --git a/backend/src/services/external-migration/external-migration-service.ts b/backend/src/services/external-migration/external-migration-service.ts index 3bbb88f5b..f1047d0ee 100644 --- a/backend/src/services/external-migration/external-migration-service.ts +++ b/backend/src/services/external-migration/external-migration-service.ts @@ -1,4 +1,5 @@ import { OrgMembershipRole } from "@app/db/schemas"; +import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types"; import { crypto } from "@app/lib/crypto/cryptography"; import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors"; @@ -12,6 +13,7 @@ type TExternalMigrationServiceFactoryDep = { permissionService: TPermissionServiceFactory; externalMigrationQueue: TExternalMigrationQueueFactory; userDAL: Pick; + gatewayService: Pick; }; export type TExternalMigrationServiceFactory = ReturnType; @@ -19,7 +21,8 @@ export type TExternalMigrationServiceFactory = ReturnType { const importEnvKeyData = async ({ decryptionKey, @@ -72,6 +75,7 @@ export const externalMigrationServiceFactory = ({ vaultNamespace, mappingType, vaultUrl, + gatewayId, actor, actorId, actorOrgId, @@ -91,12 +95,18 @@ export const externalMigrationServiceFactory = ({ const user = await userDAL.findById(actorId); - const vaultData = await importVaultDataFn({ - vaultAccessToken, - vaultNamespace, - vaultUrl, - mappingType - }); + const vaultData = await importVaultDataFn( + { + vaultAccessToken, + vaultNamespace, + vaultUrl, + mappingType, + gatewayId + }, + { + gatewayService + } + ); const stringifiedJson = JSON.stringify({ data: vaultData, diff --git a/backend/src/services/external-migration/external-migration-types.ts b/backend/src/services/external-migration/external-migration-types.ts index 7c0d4c9ea..ac8ff44e2 100644 --- a/backend/src/services/external-migration/external-migration-types.ts +++ b/backend/src/services/external-migration/external-migration-types.ts @@ -31,6 +31,7 @@ export type TImportVaultDataDTO = { vaultNamespace?: string; mappingType: VaultMappingType; vaultUrl: string; + gatewayId?: string; } & Omit; export type TImportInfisicalDataCreate = { diff --git a/frontend/src/hooks/api/migration/mutations.tsx b/frontend/src/hooks/api/migration/mutations.tsx index 529d5c463..182797954 100644 --- a/frontend/src/hooks/api/migration/mutations.tsx +++ b/frontend/src/hooks/api/migration/mutations.tsx @@ -46,18 +46,21 @@ export const useImportVault = () => { vaultAccessToken, vaultNamespace, vaultUrl, - mappingType + mappingType, + gatewayId }: { vaultAccessToken: string; vaultNamespace?: string; vaultUrl: string; mappingType: string; + gatewayId?: string; }) => { await apiRequest.post("/api/v3/external-migration/vault/", { vaultAccessToken, vaultNamespace, vaultUrl, - mappingType + mappingType, + gatewayId }); } }); diff --git a/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx index 02ab5df1b..74abe7fb8 100644 --- a/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx @@ -6,9 +6,16 @@ import { twMerge } from "tailwind-merge"; import { z } from "zod"; import { createNotification } from "@app/components/notifications"; -import { Button, FormControl, Input, Tooltip } from "@app/components/v2"; +import { Button, FormControl, Input, Select, SelectItem, Tooltip } from "@app/components/v2"; import { NoticeBannerV2 } from "@app/components/v2/NoticeBannerV2/NoticeBannerV2"; import { useImportVault } from "@app/hooks/api/migration/mutations"; +import { OrgPermissionCan } from "@app/components/permissions"; +import { + OrgGatewayPermissionActions, + OrgPermissionSubjects +} from "@app/context/OrgPermissionContext/types"; +import { useQuery } from "@tanstack/react-query"; +import { gatewaysQueryKeys } from "@app/hooks/api"; type Props = { id?: string; @@ -62,36 +69,32 @@ const MAPPING_TYPE_MENU_ITEMS = [ export const VaultPlatformModal = ({ onClose }: Props) => { const formSchema = z.object({ vaultUrl: z.string().min(1), + gatewayId: z.string().optional(), vaultNamespace: z.string().trim().optional(), vaultAccessToken: z.string().min(1), mappingType: z.nativeEnum(VaultMappingType).default(VaultMappingType.KeyVault) }); type TFormData = z.infer; + const { data: gateways, isPending: isGatewayLoading } = useQuery(gatewaysQueryKeys.list()); const { mutateAsync: importVault } = useImportVault(); const { control, handleSubmit, reset, - formState: { isLoading, isDirty, isSubmitting, isValid, errors } + formState: { isLoading, isDirty, isSubmitting, isValid } } = useForm({ resolver: zodResolver(formSchema) }); - console.log({ - isSubmitting, - isLoading, - isValid, - errors - }); - const onSubmit = async (data: TFormData) => { await importVault({ vaultAccessToken: data.vaultAccessToken, vaultNamespace: data.vaultNamespace, vaultUrl: data.vaultUrl, - mappingType: data.mappingType + mappingType: data.mappingType, + ...(data.gatewayId && { gatewayId: data.gatewayId }) }); createNotification({ title: "Import started", @@ -110,11 +113,70 @@ export const VaultPlatformModal = ({ onClose }: Props) => { The Vault migration currently supports importing static secrets from Vault Dedicated/Self-Hosted.
- Currently only KV Secret Engine V2 is supported for Vault migrations. + Currently only KV Secret Engine is supported for Vault migrations.

+
+ + {(isAllowed) => ( + ( + + +
+ +
+ + + )} + /> + )} + +
+ Date: Fri, 29 Aug 2025 23:32:58 +0200 Subject: [PATCH 2/3] Update frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../ExternalMigrationsTab/components/VaultPlatformModal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx index 74abe7fb8..355f8fe9a 100644 --- a/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx @@ -155,7 +155,7 @@ export const VaultPlatformModal = ({ onClose }: Props) => { position="popper" > { onChange(undefined); }} From 6bccb1e5eb8828baac4921a088d3b6f56b77ba7c Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 29 Aug 2025 23:57:36 +0200 Subject: [PATCH 3/3] Update vault.mdx --- docs/documentation/platform/external-migrations/vault.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/documentation/platform/external-migrations/vault.mdx b/docs/documentation/platform/external-migrations/vault.mdx index ef139e8e4..8254e104f 100644 --- a/docs/documentation/platform/external-migrations/vault.mdx +++ b/docs/documentation/platform/external-migrations/vault.mdx @@ -8,12 +8,12 @@ description: "Learn how to migrate secrets from Vault to Infisical." Migrating from Vault Self-Hosted or Dedicated Vault is a straight forward process with our inbuilt migration option. In order to migrate from Vault, you'll need to provide Infisical an access token to your Vault instance. -Currently the Vault migration only supports migrating secrets from the KV v2 secrets engine. If you're using a different secrets engine, please open an issue on our [GitHub repository](https://github.com/infisical/infisical/issues). +Currently the Vault migration only supports migrating secrets from the KV V2 and V1 secrets engine. If you're using a different secrets engine, please open an issue on our [GitHub repository](https://github.com/infisical/infisical/issues). ### Prerequisites -- A Vault instance with the KV v2 secrets engine enabled. +- A Vault instance with the KV secret engine enabled. - An access token to your Vault instance.