Merge pull request #4440 from Infisical/daniel/vault-migration

feat(vault-migration): gateway support & kv v1 support

backend/src/server/routes/index.ts

@@ -1746,7 +1746,8 @@ export const registerRoutes = async (
   const migrationService = externalMigrationServiceFactory({
     externalMigrationQueue,
     userDAL,
-    permissionService
+    permissionService,
+    gatewayService
   });
 
   const externalGroupOrgRoleMappingService = externalGroupOrgRoleMappingServiceFactory({

backend/src/server/routes/v3/external-migration-router.ts

@@ -66,7 +66,8 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider
         vaultAccessToken: z.string(),
         vaultNamespace: z.string().trim().optional(),
         vaultUrl: z.string(),
-        mappingType: z.nativeEnum(VaultMappingType)
+        mappingType: z.nativeEnum(VaultMappingType),
+        gatewayId: z.string().optional()
       })
     },
     onRequest: verifyAuth([AuthMode.JWT]),

backend/src/services/external-migration/external-migration-fns/vault.ts

@@ -1,12 +1,21 @@
+import https from "node:https";
+
 import axios, { AxiosInstance } from "axios";
 import { v4 as uuidv4 } from "uuid";
 
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { BadRequestError } from "@app/lib/errors";
+import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
 import { logger } from "@app/lib/logger";
 import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
 
 import { InfisicalImportData, VaultMappingType } from "../external-migration-types";
 
+enum KvVersion {
+  V1 = "1",
+  V2 = "2"
+}
+
 type VaultData = {
   namespace: string;
   mount: string;
@@ -14,7 +23,42 @@ type VaultData = {
   secretData: Record<string, string>;
 };
 
-const vaultFactory = () => {
+const vaultFactory = (gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">) => {
+  const $gatewayProxyWrapper = async <T>(
+    inputs: {
+      gatewayId: string;
+      targetHost?: string;
+      targetPort?: number;
+    },
+    gatewayCallback: (host: string, port: number, httpsAgent?: https.Agent) => Promise<T>
+  ): Promise<T> => {
+    const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(inputs.gatewayId);
+    const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
+
+    const callbackResult = await withGatewayProxy(
+      async (port, httpsAgent) => {
+        const res = await gatewayCallback("http://localhost", port, httpsAgent);
+        return res;
+      },
+      {
+        protocol: GatewayProxyProtocol.Http,
+        targetHost: inputs.targetHost,
+        targetPort: inputs.targetPort,
+        relayHost,
+        relayPort: Number(relayPort),
+        identityId: relayDetails.identityId,
+        orgId: relayDetails.orgId,
+        tlsOptions: {
+          ca: relayDetails.certChain,
+          cert: relayDetails.certificate,
+          key: relayDetails.privateKey.toString()
+        }
+      }
+    );
+
+    return callbackResult;
+  };
+
   const getMounts = async (request: AxiosInstance) => {
     const response = await request
       .get<{
@@ -31,11 +75,24 @@ const vaultFactory = () => {
 
   const getPaths = async (
     request: AxiosInstance,
-    { mountPath, secretPath = "" }: { mountPath: string; secretPath?: string }
+    { mountPath, secretPath = "" }: { mountPath: string; secretPath?: string },
+    kvVersion: KvVersion
   ) => {
     try {
-      // For KV v2: /v1/{mount}/metadata/{path}?list=true
+      if (kvVersion === KvVersion.V2) {
-      const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`;
+        // For KV v2: /v1/{mount}/metadata/{path}?list=true
+        const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`;
+        const response = await request.get<{
+          data: {
+            keys: string[];
+          };
+        }>(`/v1/${path}?list=true`);
+
+        return response.data.data.keys;
+      }
+
+      // kv version v1: /v1/{mount}?list=true
+      const path = secretPath ? `${mountPath}/${secretPath}` : mountPath;
       const response = await request.get<{
         data: {
           keys: string[];
@@ -56,21 +113,42 @@ const vaultFactory = () => {
 
   const getSecrets = async (
     request: AxiosInstance,
-    { mountPath, secretPath }: { mountPath: string; secretPath: string }
+    { mountPath, secretPath }: { mountPath: string; secretPath: string },
+    kvVersion: KvVersion
   ) => {
-    // For KV v2: /v1/{mount}/data/{path}
+    if (kvVersion === KvVersion.V2) {
+      // For KV v2: /v1/{mount}/data/{path}
+      const response = await request
+        .get<{
+          data: {
+            data: Record<string, string>; // KV v2 has nested data structure
+            metadata: {
+              created_time: string;
+              deletion_time: string;
+              destroyed: boolean;
+              version: number;
+            };
+          };
+        }>(`/v1/${mountPath}/data/${secretPath}`)
+        .catch((err) => {
+          if (axios.isAxiosError(err)) {
+            logger.error(err.response?.data, "External migration: Failed to get Vault secret");
+          }
+          throw err;
+        });
+
+      return response.data.data.data;
+    }
+
+    // kv version v1
+
     const response = await request
       .get<{
-        data: {
+        data: Record<string, string>; // KV v1 has flat data structure
-          data: Record<string, string>; // KV v2 has nested data structure
+        lease_duration: number;
-          metadata: {
+        lease_id: string;
-            created_time: string;
+        renewable: boolean;
-            deletion_time: string;
+      }>(`/v1/${mountPath}/${secretPath}`)
-            destroyed: boolean;
-            version: number;
-          };
-        };
-      }>(`/v1/${mountPath}/data/${secretPath}`)
       .catch((err) => {
         if (axios.isAxiosError(err)) {
           logger.error(err.response?.data, "External migration: Failed to get Vault secret");
@@ -78,7 +156,7 @@ const vaultFactory = () => {
         throw err;
       });
 
-    return response.data.data.data;
+    return response.data.data;
   };
 
   // helper function to check if a mount is KV v2 (will be useful if we add support for Vault KV v1)
@@ -89,9 +167,10 @@ const vaultFactory = () => {
   const recursivelyGetAllPaths = async (
     request: AxiosInstance,
     mountPath: string,
+    kvVersion: KvVersion,
     currentPath: string = ""
   ): Promise<string[]> => {
-    const paths = await getPaths(request, { mountPath, secretPath: currentPath });
+    const paths = await getPaths(request, { mountPath, secretPath: currentPath }, kvVersion);
 
     if (paths === null || paths.length === 0) {
       return [];
@@ -105,7 +184,7 @@ const vaultFactory = () => {
 
       if (path.endsWith("/")) {
         // it's a folder so we recurse into it
-        const subSecrets = await recursivelyGetAllPaths(request, mountPath, fullItemPath);
+        const subSecrets = await recursivelyGetAllPaths(request, mountPath, kvVersion, fullItemPath);
         allSecrets.push(...subSecrets);
       } else {
         // it's a secret so we add it to our results
@@ -119,60 +198,93 @@ const vaultFactory = () => {
   async function collectVaultData({
     baseUrl,
     namespace,
-    accessToken
+    accessToken,
+    gatewayId
   }: {
     baseUrl: string;
     namespace?: string;
     accessToken: string;
+    gatewayId?: string;
   }): Promise<VaultData[]> {
-    const request = axios.create({
+    const getData = async (host: string, port?: number, httpsAgent?: https.Agent) => {
-      baseURL: baseUrl,
+      const allData: VaultData[] = [];
-      headers: {
+
-        "X-Vault-Token": accessToken,
+      const request = axios.create({
-        ...(namespace ? { "X-Vault-Namespace": namespace } : {})
+        baseURL: port ? `${host}:${port}` : host,
+        headers: {
+          "X-Vault-Token": accessToken,
+          ...(namespace ? { "X-Vault-Namespace": namespace } : {})
+        },
+        httpsAgent
+      });
+
+      // Get all mounts in this namespace
+      const mounts = await getMounts(request);
+
+      for (const mount of Object.keys(mounts)) {
+        if (!mount.endsWith("/")) {
+          delete mounts[mount];
+        }
       }
-    });
 
-    const allData: VaultData[] = [];
+      for await (const [mountPath, mountInfo] of Object.entries(mounts)) {
+        // skip non-KV mounts
+        if (!mountInfo.type.startsWith("kv")) {
+          // eslint-disable-next-line no-continue
+          continue;
+        }
 
-    // Get all mounts in this namespace
+        const kvVersion = mountInfo.options?.version === "2" ? KvVersion.V2 : KvVersion.V1;
-    const mounts = await getMounts(request);
 
-    for (const mount of Object.keys(mounts)) {
+        // get all paths in this mount
-      if (!mount.endsWith("/")) {
+        const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`, kvVersion);
-        delete mounts[mount];
+
+        const cleanMountPath = mountPath.replace(/\/$/, "");
+
+        for await (const secretPath of paths) {
+          // get the actual secret data
+          const secretData = await getSecrets(
+            request,
+            {
+              mountPath: cleanMountPath,
+              secretPath: secretPath.replace(`${cleanMountPath}/`, "")
+            },
+            kvVersion
+          );
+
+          allData.push({
+            namespace: namespace || "",
+            mount: mountPath.replace(/\/$/, ""),
+            path: secretPath.replace(`${cleanMountPath}/`, ""),
+            secretData
+          });
+        }
       }
+
+      return allData;
+    };
+
+    let data;
+
+    if (gatewayId) {
+      const url = new URL(baseUrl);
+
+      const { port, protocol, hostname } = url;
+      const cleanedProtocol = protocol.slice(0, -1);
+
+      data = await $gatewayProxyWrapper(
+        {
+          gatewayId,
+          targetHost: `${cleanedProtocol}://${hostname}`,
+          targetPort: port ? Number(port) : 8200 // 8200, default port for Vault self-hosted/dedicated
+        },
+        getData
+      );
+    } else {
+      data = await getData(baseUrl);
     }
 
-    for await (const [mountPath, mountInfo] of Object.entries(mounts)) {
+    return data;
-      // skip non-KV mounts
-      if (!mountInfo.type.startsWith("kv")) {
-        // eslint-disable-next-line no-continue
-        continue;
-      }
-
-      // get all paths in this mount
-      const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`);
-
-      const cleanMountPath = mountPath.replace(/\/$/, "");
-
-      for await (const secretPath of paths) {
-        // get the actual secret data
-        const secretData = await getSecrets(request, {
-          mountPath: cleanMountPath,
-          secretPath: secretPath.replace(`${cleanMountPath}/`, "")
-        });
-
-        allData.push({
-          namespace: namespace || "",
-          mount: mountPath.replace(/\/$/, ""),
-          path: secretPath.replace(`${cleanMountPath}/`, ""),
-          secretData
-        });
-      }
-    }
-
-    return allData;
   }
 
   return {
@@ -296,17 +408,22 @@ export const transformToInfisicalFormatNamespaceToProjects = (
   };
 };
 
-export const importVaultDataFn = async ({
+export const importVaultDataFn = async (
-  vaultAccessToken,
+  {
-  vaultNamespace,
+    vaultAccessToken,
-  vaultUrl,
+    vaultNamespace,
-  mappingType
+    vaultUrl,
-}: {
+    mappingType,
-  vaultAccessToken: string;
+    gatewayId
-  vaultNamespace?: string;
+  }: {
-  vaultUrl: string;
+    vaultAccessToken: string;
-  mappingType: VaultMappingType;
+    vaultNamespace?: string;
-}) => {
+    vaultUrl: string;
+    mappingType: VaultMappingType;
+    gatewayId?: string;
+  },
+  { gatewayService }: { gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId"> }
+) => {
   await blockLocalAndPrivateIpAddresses(vaultUrl);
 
   if (mappingType === VaultMappingType.Namespace && !vaultNamespace) {
@@ -315,12 +432,13 @@ export const importVaultDataFn = async ({
     });
   }
 
-  const vaultApi = vaultFactory();
+  const vaultApi = vaultFactory(gatewayService);
 
   const vaultData = await vaultApi.collectVaultData({
     accessToken: vaultAccessToken,
     baseUrl: vaultUrl,
-    namespace: vaultNamespace
+    namespace: vaultNamespace,
+    gatewayId
   });
 
   const infisicalData = transformToInfisicalFormatNamespaceToProjects(vaultData, mappingType);

backend/src/services/external-migration/external-migration-service.ts

@@ -1,4 +1,5 @@
 import { OrgMembershipRole } from "@app/db/schemas";
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
 import { crypto } from "@app/lib/crypto/cryptography";
 import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors";
@@ -12,6 +13,7 @@ type TExternalMigrationServiceFactoryDep = {
   permissionService: TPermissionServiceFactory;
   externalMigrationQueue: TExternalMigrationQueueFactory;
   userDAL: Pick<TUserDALFactory, "findById">;
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
 };
 
 export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrationServiceFactory>;
@@ -19,7 +21,8 @@ export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrati
 export const externalMigrationServiceFactory = ({
   permissionService,
   externalMigrationQueue,
-  userDAL
+  userDAL,
+  gatewayService
 }: TExternalMigrationServiceFactoryDep) => {
   const importEnvKeyData = async ({
     decryptionKey,
@@ -72,6 +75,7 @@ export const externalMigrationServiceFactory = ({
     vaultNamespace,
     mappingType,
     vaultUrl,
+    gatewayId,
     actor,
     actorId,
     actorOrgId,
@@ -91,12 +95,18 @@ export const externalMigrationServiceFactory = ({
 
     const user = await userDAL.findById(actorId);
 
-    const vaultData = await importVaultDataFn({
+    const vaultData = await importVaultDataFn(
-      vaultAccessToken,
+      {
-      vaultNamespace,
+        vaultAccessToken,
-      vaultUrl,
+        vaultNamespace,
-      mappingType
+        vaultUrl,
-    });
+        mappingType,
+        gatewayId
+      },
+      {
+        gatewayService
+      }
+    );
 
     const stringifiedJson = JSON.stringify({
       data: vaultData,

backend/src/services/external-migration/external-migration-types.ts

@@ -31,6 +31,7 @@ export type TImportVaultDataDTO = {
   vaultNamespace?: string;
   mappingType: VaultMappingType;
   vaultUrl: string;
+  gatewayId?: string;
 } & Omit<TOrgPermission, "orgId">;
 
 export type TImportInfisicalDataCreate = {

docs/documentation/platform/external-migrations/vault.mdx

@@ -8,12 +8,12 @@ description: "Learn how to migrate secrets from Vault to Infisical."
 
 Migrating from Vault Self-Hosted or Dedicated Vault is a straight forward process with our inbuilt migration option. In order to migrate from Vault, you'll need to provide Infisical an access token to your Vault instance.
 
-Currently the Vault migration only supports migrating secrets from the KV v2 secrets engine. If you're using a different secrets engine, please open an issue on our [GitHub repository](https://github.com/infisical/infisical/issues).
+Currently the Vault migration only supports migrating secrets from the KV V2 and V1 secrets engine. If you're using a different secrets engine, please open an issue on our [GitHub repository](https://github.com/infisical/infisical/issues).
 
 
 ### Prerequisites
 
-- A Vault instance with the KV v2 secrets engine enabled.
+- A Vault instance with the KV secret engine enabled.
 - An access token to your Vault instance.
 
 

frontend/src/hooks/api/migration/mutations.tsx

@@ -46,18 +46,21 @@ export const useImportVault = () => {
       vaultAccessToken,
       vaultNamespace,
       vaultUrl,
-      mappingType
+      mappingType,
+      gatewayId
     }: {
       vaultAccessToken: string;
       vaultNamespace?: string;
       vaultUrl: string;
       mappingType: string;
+      gatewayId?: string;
     }) => {
       await apiRequest.post("/api/v3/external-migration/vault/", {
         vaultAccessToken,
         vaultNamespace,
         vaultUrl,
-        mappingType
+        mappingType,
+        gatewayId
       });
     }
   });

frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx

@@ -6,9 +6,16 @@ import { twMerge } from "tailwind-merge";
 import { z } from "zod";
 
 import { createNotification } from "@app/components/notifications";
-import { Button, FormControl, Input, Tooltip } from "@app/components/v2";
+import { Button, FormControl, Input, Select, SelectItem, Tooltip } from "@app/components/v2";
 import { NoticeBannerV2 } from "@app/components/v2/NoticeBannerV2/NoticeBannerV2";
 import { useImportVault } from "@app/hooks/api/migration/mutations";
+import { OrgPermissionCan } from "@app/components/permissions";
+import {
+  OrgGatewayPermissionActions,
+  OrgPermissionSubjects
+} from "@app/context/OrgPermissionContext/types";
+import { useQuery } from "@tanstack/react-query";
+import { gatewaysQueryKeys } from "@app/hooks/api";
 
 type Props = {
   id?: string;
@@ -62,36 +69,32 @@ const MAPPING_TYPE_MENU_ITEMS = [
 export const VaultPlatformModal = ({ onClose }: Props) => {
   const formSchema = z.object({
     vaultUrl: z.string().min(1),
+    gatewayId: z.string().optional(),
     vaultNamespace: z.string().trim().optional(),
     vaultAccessToken: z.string().min(1),
     mappingType: z.nativeEnum(VaultMappingType).default(VaultMappingType.KeyVault)
   });
   type TFormData = z.infer<typeof formSchema>;
 
+  const { data: gateways, isPending: isGatewayLoading } = useQuery(gatewaysQueryKeys.list());
   const { mutateAsync: importVault } = useImportVault();
 
   const {
     control,
     handleSubmit,
     reset,
-    formState: { isLoading, isDirty, isSubmitting, isValid, errors }
+    formState: { isLoading, isDirty, isSubmitting, isValid }
   } = useForm<TFormData>({
     resolver: zodResolver(formSchema)
   });
 
-  console.log({
-    isSubmitting,
-    isLoading,
-    isValid,
-    errors
-  });
-
   const onSubmit = async (data: TFormData) => {
     await importVault({
       vaultAccessToken: data.vaultAccessToken,
       vaultNamespace: data.vaultNamespace,
       vaultUrl: data.vaultUrl,
-      mappingType: data.mappingType
+      mappingType: data.mappingType,
+      ...(data.gatewayId && { gatewayId: data.gatewayId })
     });
     createNotification({
       title: "Import started",
@@ -110,11 +113,70 @@ export const VaultPlatformModal = ({ onClose }: Props) => {
           The Vault migration currently supports importing static secrets from Vault
           Dedicated/Self-Hosted.
           <div className="mt-2 text-xs opacity-80">
-            Currently only KV Secret Engine V2 is supported for Vault migrations.
+            Currently only KV Secret Engine is supported for Vault migrations.
           </div>
         </p>
       </NoticeBannerV2>
       <form onSubmit={handleSubmit(onSubmit)} autoComplete="off">
+        <div className="w-full flex-1">
+          <OrgPermissionCan
+            I={OrgGatewayPermissionActions.AttachGateways}
+            a={OrgPermissionSubjects.Gateway}
+          >
+            {(isAllowed) => (
+              <Controller
+                control={control}
+                name="gatewayId"
+                defaultValue=""
+                render={({ field: { value, onChange }, fieldState: { error } }) => (
+                  <FormControl
+                    isError={Boolean(error?.message)}
+                    errorText={error?.message}
+                    label="Gateway"
+                    isOptional
+                  >
+                    <Tooltip
+                      isDisabled={isAllowed}
+                      content="Restricted access. You don't have permission to attach gateways to resources."
+                    >
+                      <div>
+                        <Select
+                          isDisabled={!isAllowed}
+                          value={value as string}
+                          onValueChange={(v) => {
+                            if (v !== "") {
+                              onChange(v);
+                            }
+                          }}
+                          className="w-full border border-mineshaft-500"
+                          dropdownContainerClassName="max-w-none"
+                          isLoading={isGatewayLoading}
+                          placeholder="Default: Internet Gateway"
+                          position="popper"
+                        >
+                          <SelectItem
+                            value=""
+                            onClick={() => {
+                              onChange(undefined);
+                            }}
+                          >
+                            Internet Gateway
+                          </SelectItem>
+                          {gateways?.map((el) => (
+                            <SelectItem value={el.id} key={el.id}>
+                              {el.name}
+                            </SelectItem>
+                          ))}
+                        </Select>
+                      </div>
+                    </Tooltip>
+                  </FormControl>
+                )}
+              />
+            )}
+          </OrgPermissionCan>
+        </div>
+
         <Controller
           control={control}
           name="vaultUrl"