feat: add in-platform support for secret imports

backend/src/server/routes/index.ts

@@ -2199,7 +2199,9 @@ export const registerRoutes = async (
     gatewayService,
     kmsService,
     appConnectionService,
-    externalMigrationConfigDAL
+    externalMigrationConfigDAL,
+    secretService,
+    auditLogService
   });
 
   // setup the communication with license key server

backend/src/server/routes/v3/external-migration-router.ts

@@ -139,7 +139,7 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider
         })
       }
     },
-    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
+    onRequest: verifyAuth([AuthMode.JWT]),
     handler: async (req) => {
       const config = await server.services.migration.getExternalMigrationConfig({
         platform: req.query.platform,
@@ -260,4 +260,63 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider
       return { mounts };
     }
   });
+
+  server.route({
+    method: "POST",
+    url: "/vault/import-secrets",
+    config: {
+      rateLimit: writeLimit
+    },
+    schema: {
+      body: z.object({
+        projectId: z.string(),
+        environment: z.string(),
+        secretPath: z.string(),
+        vaultNamespace: z.string(),
+        vaultSecretPath: z.string()
+      }),
+      response: {
+        200: z.object({
+          message: z.string()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
+    handler: async (req) => {
+      await server.services.migration.importVaultSecrets({
+        actor: req.permission,
+        auditLogInfo: req.auditLogInfo,
+        ...req.body
+      });
+
+      return { message: "Successfully imported vault secrets" };
+    }
+  });
+
+  server.route({
+    method: "GET",
+    url: "/vault/secret-paths",
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      querystring: z.object({
+        namespace: z.string().optional()
+      }),
+      response: {
+        200: z.object({
+          secretPaths: z.string().array()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
+    handler: async (req) => {
+      const secretPaths = await server.services.migration.getVaultSecretPaths({
+        actor: req.permission,
+        namespace: req.query.namespace
+      });
+
+      return { secretPaths };
+    }
+  });
 };

backend/src/services/app-connection/hc-vault/hc-vault-connection-fns.ts

@@ -363,3 +363,210 @@ export const listHCVaultMounts = async (
 
   return mounts;
 };
+
+export const listHCVaultSecretPaths = async (
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
+  namespace?: string
+) => {
+  const instanceUrl = await getHCVaultInstanceUrl(connection);
+  const accessToken = await getHCVaultAccessToken(connection, gatewayService);
+
+  if (namespace && connection.credentials.namespace) {
+    throw new BadRequestError({
+      message: "Namespace cannot be specified when namespace is already set in the connection credentials"
+    });
+  }
+
+  const targetNamespace = namespace || connection.credentials.namespace;
+
+  const getPaths = async (mountPath: string, secretPath: string, kvVersion: "1" | "2"): Promise<string[] | null> => {
+    try {
+      let path: string;
+      if (kvVersion === "2") {
+        // For KV v2: /v1/{mount}/metadata/{path}?list=true
+        path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`;
+      } else {
+        // For KV v1: /v1/{mount}/{path}?list=true
+        path = secretPath ? `${mountPath}/${secretPath}` : mountPath;
+      }
+
+      const { data } = await requestWithHCVaultGateway<{
+        data: {
+          keys: string[];
+        };
+      }>(connection, gatewayService, {
+        url: `${instanceUrl}/v1/${path}?list=true`,
+        method: "GET",
+        headers: {
+          "X-Vault-Token": accessToken,
+          ...(targetNamespace ? { "X-Vault-Namespace": targetNamespace } : {})
+        }
+      });
+
+      return data.data.keys;
+    } catch (error) {
+      if (error instanceof AxiosError && error.response?.status === 404) {
+        return null;
+      }
+
+      throw error;
+    }
+  };
+
+  // Recursive function to get all secret paths in a mount
+  const recursivelyGetAllPaths = async (
+    mountPath: string,
+    kvVersion: "1" | "2",
+    currentPath: string = ""
+  ): Promise<string[]> => {
+    const paths = await getPaths(mountPath, currentPath, kvVersion);
+
+    if (paths === null || paths.length === 0) {
+      return [];
+    }
+
+    const allSecrets: string[] = [];
+
+    // Process paths sequentially to maintain tree traversal order
+    // eslint-disable-next-line no-restricted-syntax
+    for (const path of paths) {
+      const cleanPath = path.endsWith("/") ? path.slice(0, -1) : path;
+      const fullItemPath = currentPath ? `${currentPath}/${cleanPath}` : cleanPath;
+
+      if (path.endsWith("/")) {
+        // it's a folder so we recurse into it
+        // eslint-disable-next-line no-await-in-loop
+        const subSecrets = await recursivelyGetAllPaths(mountPath, kvVersion, fullItemPath);
+        allSecrets.push(...subSecrets);
+      } else {
+        // it's a secret so we add it to our results
+        allSecrets.push(`${mountPath}/${fullItemPath}`);
+      }
+    }
+
+    return allSecrets;
+  };
+
+  // Get all mounts
+  const mounts = await listHCVaultMounts(connection, gatewayService, namespace);
+
+  // Filter for KV mounts (kv, kv-v1, kv-v2)
+  const kvMounts = mounts.filter((mount) => mount.type === "kv" || mount.type.startsWith("kv"));
+
+  // Collect all secret paths from all KV mounts in parallel
+  const allSecretPathsArrays = await Promise.all(
+    kvMounts.map(async (mount) => {
+      const kvVersion = mount.version === "2" ? "2" : "1";
+      const cleanMountPath = mount.path.replace(/\/$/, ""); // Remove trailing slash
+      return recursivelyGetAllPaths(cleanMountPath, kvVersion);
+    })
+  );
+
+  // Flatten the arrays into a single array
+  const allSecretPaths = allSecretPathsArrays.flat();
+
+  return allSecretPaths;
+};
+
+export const getHCVaultSecretsForPath = async (
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
+  namespace: string,
+  secretPath: string
+) => {
+  const instanceUrl = await getHCVaultInstanceUrl(connection);
+  const accessToken = await getHCVaultAccessToken(connection, gatewayService);
+
+  if (connection.credentials.namespace && connection.credentials.namespace !== namespace) {
+    throw new BadRequestError({
+      message: "Specified namespace does not match the namespace in the connection credentials"
+    });
+  }
+
+  const targetNamespace = namespace || connection.credentials.namespace;
+
+  try {
+    // Extract mount and path from the secretPath
+    // secretPath format: {mount}/{path}
+    const pathParts = secretPath.split("/");
+    const mountPath = pathParts[0];
+    const actualPath = pathParts.slice(1).join("/");
+
+    if (!mountPath || !actualPath) {
+      throw new BadRequestError({
+        message: "Invalid secret path format. Expected format: {mount}/{path}"
+      });
+    }
+
+    // Get mounts to determine KV version
+    const mounts = await listHCVaultMounts(connection, gatewayService, namespace);
+    const mount = mounts.find((m) => m.path.replace(/\/$/, "") === mountPath);
+
+    if (!mount) {
+      throw new BadRequestError({
+        message: `Mount '${mountPath}' not found in HashiCorp Vault`
+      });
+    }
+
+    const kvVersion = mount.version === "2" ? "2" : "1";
+
+    // Fetch secrets based on KV version
+    if (kvVersion === "2") {
+      // For KV v2: /v1/{mount}/data/{path}
+      const { data } = await requestWithHCVaultGateway<{
+        data: {
+          data: Record<string, string>; // KV v2 has nested data structure
+          metadata: {
+            created_time: string;
+            deletion_time: string;
+            destroyed: boolean;
+            version: number;
+          };
+        };
+      }>(connection, gatewayService, {
+        url: `${instanceUrl}/v1/${mountPath}/data/${actualPath}`,
+        method: "GET",
+        headers: {
+          "X-Vault-Token": accessToken,
+          ...(targetNamespace ? { "X-Vault-Namespace": targetNamespace } : {})
+        }
+      });
+
+      return data.data.data;
+    }
+
+    // For KV v1: /v1/{mount}/{path}
+    const { data } = await requestWithHCVaultGateway<{
+      data: Record<string, string>; // KV v1 has flat data structure
+      lease_duration: number;
+      lease_id: string;
+      renewable: boolean;
+    }>(connection, gatewayService, {
+      url: `${instanceUrl}/v1/${mountPath}/${actualPath}`,
+      method: "GET",
+      headers: {
+        "X-Vault-Token": accessToken,
+        ...(targetNamespace ? { "X-Vault-Namespace": targetNamespace } : {})
+      }
+    });
+
+    return data.data;
+  } catch (error: unknown) {
+    logger.error(error, "Unable to fetch secrets from HC Vault path");
+
+    if (error instanceof AxiosError) {
+      throw new BadRequestError({
+        message: `Failed to fetch secrets: ${error.message || "Unknown error"}`
+      });
+    }
+
+    if (error instanceof BadRequestError) {
+      throw error;
+    }
+
+    throw new BadRequestError({
+      message: "Unable to fetch secrets from HashiCorp Vault"
+    });
+  }
+};

backend/src/services/external-migration/external-migration-service.ts

@@ -1,4 +1,10 @@
 import { OrgMembershipRole } from "@app/db/schemas";
+import {
+  AuditLogInfo,
+  EventType,
+  SecretApprovalEvent,
+  TAuditLogServiceFactory
+} from "@app/ee/services/audit-log/audit-log-types";
 import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
 import { crypto } from "@app/lib/crypto/cryptography";
@@ -9,12 +15,16 @@ import { AppConnection } from "../app-connection/app-connection-enums";
 import { decryptAppConnectionCredentials } from "../app-connection/app-connection-fns";
 import { TAppConnectionServiceFactory } from "../app-connection/app-connection-service";
 import {
+  getHCVaultSecretsForPath,
   listHCVaultMounts,
   listHCVaultNamespaces,
   listHCVaultPolicies,
+  listHCVaultSecretPaths,
   THCVaultConnection
 } from "../app-connection/hc-vault";
 import { TKmsServiceFactory } from "../kms/kms-service";
+import { TSecretServiceFactory } from "../secret/secret-service";
+import { SecretProtectionType } from "../secret/secret-types";
 import { TUserDALFactory } from "../user/user-dal";
 import { TExternalMigrationConfigDALFactory } from "./external-migration-config-dal";
 import {
@@ -35,6 +45,8 @@ import {
 
 type TExternalMigrationServiceFactoryDep = {
   permissionService: TPermissionServiceFactory;
+  secretService: TSecretServiceFactory;
+  auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
   externalMigrationQueue: TExternalMigrationQueueFactory;
   appConnectionService: Pick<TAppConnectionServiceFactory, "connectAppConnectionById">;
   externalMigrationConfigDAL: Pick<TExternalMigrationConfigDALFactory, "create" | "upsert" | "findOne" | "transaction">;
@@ -50,6 +62,8 @@ export const externalMigrationServiceFactory = ({
   externalMigrationQueue,
   userDAL,
   gatewayService,
+  secretService,
+  auditLogService,
   appConnectionService,
   externalMigrationConfigDAL,
   kmsService
@@ -371,6 +385,143 @@ export const externalMigrationServiceFactory = ({
     return mounts;
   };
 
+  const getVaultSecretPaths = async ({ actor, namespace }: { actor: OrgServiceActor; namespace?: string }) => {
+    const { hasRole } = await permissionService.getOrgPermission(
+      actor.type,
+      actor.id,
+      actor.orgId,
+      actor.authMethod,
+      actor.orgId
+    );
+
+    if (!hasRole(OrgMembershipRole.Admin)) {
+      throw new ForbiddenRequestError({ message: "Only admins can view vault secret paths" });
+    }
+
+    const vaultConfig = await externalMigrationConfigDAL.findOne({
+      orgId: actor.orgId,
+      platform: ExternalMigrationProviders.Vault
+    });
+
+    if (!vaultConfig) {
+      throw new NotFoundError({ message: "Vault migration config not found" });
+    }
+
+    if (!vaultConfig.connection) {
+      throw new BadRequestError({ message: "Vault migration connection is not configured" });
+    }
+
+    const credentials = await decryptAppConnectionCredentials({
+      orgId: vaultConfig.orgId,
+      encryptedCredentials: vaultConfig.connection.encryptedCredentials,
+      kmsService,
+      projectId: null
+    });
+
+    const connection = {
+      ...vaultConfig.connection,
+      credentials
+    } as THCVaultConnection;
+
+    const secretPaths = await listHCVaultSecretPaths(connection, gatewayService, namespace);
+
+    return secretPaths;
+  };
+
+  const importVaultSecrets = async ({
+    actor,
+    projectId,
+    environment,
+    secretPath,
+    vaultNamespace,
+    vaultSecretPath,
+    auditLogInfo
+  }: {
+    actor: OrgServiceActor;
+    projectId: string;
+    environment: string;
+    secretPath: string;
+    vaultNamespace: string;
+    vaultSecretPath: string;
+    auditLogInfo: AuditLogInfo;
+  }) => {
+    const { hasRole } = await permissionService.getOrgPermission(
+      actor.type,
+      actor.id,
+      actor.orgId,
+      actor.authMethod,
+      actor.orgId
+    );
+
+    if (!hasRole(OrgMembershipRole.Admin)) {
+      throw new ForbiddenRequestError({ message: "Only admins can import vault secrets" });
+    }
+
+    const vaultConfig = await externalMigrationConfigDAL.findOne({
+      orgId: actor.orgId,
+      platform: ExternalMigrationProviders.Vault
+    });
+
+    if (!vaultConfig) {
+      throw new NotFoundError({ message: "Vault migration config not found" });
+    }
+
+    if (!vaultConfig.connection) {
+      throw new BadRequestError({ message: "Vault migration connection is not configured" });
+    }
+
+    const credentials = await decryptAppConnectionCredentials({
+      orgId: vaultConfig.orgId,
+      encryptedCredentials: vaultConfig.connection.encryptedCredentials,
+      kmsService,
+      projectId: null
+    });
+
+    const connection = {
+      ...vaultConfig.connection,
+      credentials
+    } as THCVaultConnection;
+
+    const vaultSecrets = await getHCVaultSecretsForPath(connection, gatewayService, vaultNamespace, vaultSecretPath);
+
+    const secretOperation = await secretService.createManySecretsRaw({
+      actorId: actor.id,
+      actor: actor.type,
+      actorAuthMethod: actor.authMethod,
+      actorOrgId: actor.orgId,
+      secretPath,
+      environment,
+      projectId,
+      secrets: Object.entries(vaultSecrets).map(([secretKey, secretValue]) => ({
+        secretKey,
+        secretValue
+      }))
+    });
+
+    if (secretOperation.type === SecretProtectionType.Approval) {
+      await auditLogService.createAuditLog({
+        projectId,
+        ...auditLogInfo,
+        event: {
+          type: EventType.SECRET_APPROVAL_REQUEST,
+          metadata: {
+            committedBy: secretOperation.approval.committerUserId,
+            secretApprovalRequestId: secretOperation.approval.id,
+            secretApprovalRequestSlug: secretOperation.approval.slug,
+            secretPath,
+            environment,
+            secrets: Object.entries(vaultSecrets).map(([secretKey]) => ({
+              secretKey
+            })),
+            eventType: SecretApprovalEvent.CreateMany
+          }
+        }
+      });
+
+      return { approval: secretOperation.approval };
+    }
+  };
+
   return {
     importEnvKeyData,
     importVaultData,
@@ -379,6 +530,8 @@ export const externalMigrationServiceFactory = ({
     getExternalMigrationConfig,
     getVaultNamespaces,
     getVaultPolicies,
-    getVaultMounts
+    getVaultMounts,
+    getVaultSecretPaths,
+    importVaultSecrets
   };
 };