feat: in-platform migration tooling for Vault policies + scaffolding

backend/src/services/app-connection/hc-vault/hc-vault-connection-fns.ts

@@ -13,7 +13,12 @@ import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
 import { AppConnection } from "@app/services/app-connection/app-connection-enums";
 
 import { HCVaultConnectionMethod } from "./hc-vault-connection-enums";
-import { THCVaultConnection, THCVaultConnectionConfig, THCVaultMountResponse } from "./hc-vault-connection-types";
+import {
+  THCVaultConnection,
+  THCVaultConnectionConfig,
+  THCVaultMount,
+  THCVaultMountResponse
+} from "./hc-vault-connection-types";
 
 export const getHCVaultInstanceUrl = async (config: THCVaultConnectionConfig) => {
   const instanceUrl = removeTrailingSlash(config.credentials.instanceUrl);
@@ -181,29 +186,179 @@ export const validateHCVaultConnectionCredentials = async (
   }
 };
 
-export const listHCVaultMounts = async (
+export const listHCVaultPolicies = async (
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
+  namespace?: string
+) => {
+  const instanceUrl = await getHCVaultInstanceUrl(connection);
+  const accessToken = await getHCVaultAccessToken(connection, gatewayService);
+
+  if (namespace && connection.credentials.namespace) {
+    throw new BadRequestError({
+      message: "Namespace cannot be specified when namespace is already set in the connection credentials"
+    });
+  }
+
+  const targetNamespace = namespace || connection.credentials.namespace;
+
+  try {
+    const { data: listData } = await requestWithHCVaultGateway<{
+      policies: string[];
+    }>(connection, gatewayService, {
+      url: `${instanceUrl}/v1/sys/policy`,
+      method: "GET",
+      headers: {
+        "X-Vault-Token": accessToken,
+        ...(targetNamespace ? { "X-Vault-Namespace": targetNamespace } : {})
+      }
+    });
+
+    const policyNames = listData.policies || [];
+
+    const policies = await Promise.all(
+      policyNames.map(async (policyName) => {
+        try {
+          const { data: policyData } = await requestWithHCVaultGateway<{
+            name: string;
+            rules: string;
+          }>(connection, gatewayService, {
+            url: `${instanceUrl}/v1/sys/policy/${policyName}`,
+            method: "GET",
+            headers: {
+              "X-Vault-Token": accessToken,
+              ...(targetNamespace ? { "X-Vault-Namespace": targetNamespace } : {})
+            }
+          });
+
+          return {
+            name: policyData.name,
+            rules: policyData.rules
+          };
+        } catch (error: unknown) {
+          logger.error(error, `Unable to fetch policy details for ${policyName}`);
+          return {
+            name: policyName,
+            rules: ""
+          };
+        }
+      })
+    );
+
+    return policies;
+  } catch (error: unknown) {
+    logger.error(error, "Unable to list HC Vault policies");
+
+    if (error instanceof AxiosError) {
+      throw new BadRequestError({
+        message: `Failed to list policies: ${error.message || "Unknown error"}`
+      });
+    }
+
+    throw new BadRequestError({
+      message: "Unable to list policies from HashiCorp Vault"
+    });
+  }
+};
+
+export const listHCVaultNamespaces = async (
   connection: THCVaultConnection,
   gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
 ) => {
   const instanceUrl = await getHCVaultInstanceUrl(connection);
   const accessToken = await getHCVaultAccessToken(connection, gatewayService);
 
+  try {
+    const { data } = await requestWithHCVaultGateway<{
+      data: {
+        keys: string[];
+        key_info?: {
+          [key: string]: {
+            id: string;
+            path: string;
+            custom_metadata?: Record<string, unknown>;
+          };
+        };
+      };
+    }>(connection, gatewayService, {
+      url: `${instanceUrl}/v1/sys/namespaces`,
+      method: "LIST",
+      headers: {
+        "X-Vault-Token": accessToken,
+        ...(connection.credentials.namespace ? { "X-Vault-Namespace": connection.credentials.namespace } : {})
+      }
+    });
+
+    // Transform using key_info if available, otherwise fall back to keys array
+    const namespaces = (data.data.keys || []).map((namespaceKey) => {
+      const keyInfo = data.data.key_info?.[namespaceKey];
+      return {
+        id: keyInfo?.id || namespaceKey.replace(/\/$/, ""), // Use Vault's ID if available, otherwise use the key
+        name: namespaceKey.replace(/\/$/, "") // Remove trailing slash for display
+      };
+    });
+
+    return namespaces;
+  } catch (error: unknown) {
+    // 404 means namespaces endpoint doesn't exist (Vault Community Edition)
+    // Return empty array to gracefully degrade
+    if (error instanceof AxiosError && error.response?.status === 404) {
+      logger.info("Namespaces endpoint not available (likely Vault Community Edition). Returning empty list.");
+      return [
+        {
+          id: "default",
+          name: "default"
+        }
+      ];
+    }
+
+    logger.error(error, "Unable to list HC Vault namespaces");
+
+    if (error instanceof AxiosError) {
+      throw new BadRequestError({
+        message: `Failed to list namespaces: ${error.message || "Unknown error"}`
+      });
+    }
+
+    throw new BadRequestError({
+      message: "Unable to list namespaces from HashiCorp Vault"
+    });
+  }
+};
+
+export const listHCVaultMounts = async (
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
+  namespace?: string
+) => {
+  const instanceUrl = await getHCVaultInstanceUrl(connection);
+  const accessToken = await getHCVaultAccessToken(connection, gatewayService);
+
+  if (namespace && connection.credentials.namespace) {
+    throw new BadRequestError({
+      message: "Namespace cannot be specified when namespace is already set in the connection credentials"
+    });
+  }
+
+  const targetNamespace = namespace || connection.credentials.namespace;
+
   const { data } = await requestWithHCVaultGateway<THCVaultMountResponse>(connection, gatewayService, {
     url: `${instanceUrl}/v1/sys/mounts`,
     method: "GET",
     headers: {
       "X-Vault-Token": accessToken,
-      ...(connection.credentials.namespace ? { "X-Vault-Namespace": connection.credentials.namespace } : {})
+      ...(targetNamespace ? { "X-Vault-Namespace": targetNamespace } : {})
     }
   });
 
-  const mounts: string[] = [];
+  const mounts: THCVaultMount[] = [];
 
-  // Filter for "kv" version 2 type only
   Object.entries(data.data).forEach(([path, mount]) => {
-    if (mount.type === "kv" && mount.options?.version === "2") {
-      mounts.push(path);
-    }
+    mounts.push({
+      path,
+      type: mount.type,
+      version: mount.options?.version
+    });
   });
 
   return mounts;

backend/src/services/app-connection/hc-vault/hc-vault-connection-service.ts

@@ -21,7 +21,8 @@ export const hcVaultConnectionService = (
 
     try {
       const mounts = await listHCVaultMounts(appConnection, gatewayService);
-      return mounts;
+      // Filter for KV version 2 mounts only and extract just the paths
+      return mounts.filter((mount) => mount.type === "kv" && mount.version === "2").map((mount) => mount.path);
     } catch (error) {
       logger.error(error, "Failed to establish connection with Hashicorp Vault");
       return [];

backend/src/services/app-connection/hc-vault/hc-vault-connection-types.ts

@@ -33,3 +33,9 @@ export type THCVaultMountResponse = {
     };
   };
 };
+
+export type THCVaultMount = {
+  path: string;
+  type: string;
+  version?: string | null;
+};

backend/src/services/external-migration/external-migration-config-dal.ts

@@ -0,0 +1,80 @@
+import { Knex } from "knex";
+
+import { TDbClient } from "@app/db";
+import { TableName, TExternalMigrationConfigsInsert } from "@app/db/schemas";
+import { DatabaseError } from "@app/lib/errors";
+import { buildFindFilter, ormify, prependTableNameToFindFilter, selectAllTableCols } from "@app/lib/knex";
+
+export type TExternalMigrationConfigDALFactory = ReturnType<typeof externalMigrationConfigDALFactory>;
+
+export const externalMigrationConfigDALFactory = (db: TDbClient) => {
+  const orm = ormify(db, TableName.ExternalMigrationConfig);
+
+  const upsert = async (data: TExternalMigrationConfigsInsert, tx?: Knex) => {
+    try {
+      const [doc] = await (tx || db)(TableName.ExternalMigrationConfig)
+        .insert(data)
+        .onConflict(["orgId", "platform"])
+        .merge()
+        .returning("*");
+      return doc;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "UpsertExternalMigrationConfig" });
+    }
+  };
+
+  const findOne = async (filter: { orgId: string; platform: string }, tx?: Knex) => {
+    try {
+      const result = await (tx || db?.replicaNode?.() || db)(TableName.ExternalMigrationConfig)
+        .leftJoin(
+          TableName.AppConnection,
+          `${TableName.AppConnection}.id`,
+          `${TableName.ExternalMigrationConfig}.connectionId`
+        )
+        /* eslint-disable @typescript-eslint/no-misused-promises */
+        .where(buildFindFilter(prependTableNameToFindFilter(TableName.ExternalMigrationConfig, filter)))
+        .select(selectAllTableCols(TableName.ExternalMigrationConfig))
+        .select(
+          db.ref("id").withSchema(TableName.AppConnection).as("appConnectionId"),
+          db.ref("name").withSchema(TableName.AppConnection).as("appConnectionName"),
+          db.ref("app").withSchema(TableName.AppConnection).as("appConnectionApp"),
+          db.ref("encryptedCredentials").withSchema(TableName.AppConnection).as("appConnectionEncryptedCredentials"),
+          db.ref("orgId").withSchema(TableName.AppConnection).as("appConnectionOrgId"),
+          db.ref("method").withSchema(TableName.AppConnection).as("appConnectionMethod"),
+          db.ref("description").withSchema(TableName.AppConnection).as("appConnectionDescription"),
+          db.ref("version").withSchema(TableName.AppConnection).as("appConnectionVersion"),
+          db.ref("gatewayId").withSchema(TableName.AppConnection).as("appConnectionGatewayId"),
+          db.ref("projectId").withSchema(TableName.AppConnection).as("appConnectionProjectId"),
+          db.ref("createdAt").withSchema(TableName.AppConnection).as("appConnectionCreatedAt"),
+          db.ref("updatedAt").withSchema(TableName.AppConnection).as("appConnectionUpdatedAt")
+        )
+        .first();
+
+      if (!result) return undefined;
+
+      return {
+        ...result,
+        connection: result.appConnectionId
+          ? {
+              id: result.appConnectionId,
+              name: result.appConnectionName,
+              app: result.appConnectionApp,
+              encryptedCredentials: result.appConnectionEncryptedCredentials,
+              orgId: result.appConnectionOrgId,
+              method: result.appConnectionMethod,
+              description: result.appConnectionDescription,
+              version: result.appConnectionVersion,
+              gatewayId: result.appConnectionGatewayId,
+              projectId: result.appConnectionProjectId,
+              createdAt: result.appConnectionCreatedAt,
+              updatedAt: result.appConnectionUpdatedAt
+            }
+          : undefined
+      };
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find one" });
+    }
+  };
+
+  return { ...orm, upsert, findOne };
+};

backend/src/services/external-migration/external-migration-service.ts

@@ -2,9 +2,21 @@ import { OrgMembershipRole } from "@app/db/schemas";
 import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
 import { crypto } from "@app/lib/crypto/cryptography";
-import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors";
+import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors";
+import { OrgServiceActor } from "@app/lib/types";
 
+import { AppConnection } from "../app-connection/app-connection-enums";
+import { decryptAppConnectionCredentials } from "../app-connection/app-connection-fns";
+import { TAppConnectionServiceFactory } from "../app-connection/app-connection-service";
+import {
+  listHCVaultMounts,
+  listHCVaultNamespaces,
+  listHCVaultPolicies,
+  THCVaultConnection
+} from "../app-connection/hc-vault";
+import { TKmsServiceFactory } from "../kms/kms-service";
 import { TUserDALFactory } from "../user/user-dal";
+import { TExternalMigrationConfigDALFactory } from "./external-migration-config-dal";
 import {
   decryptEnvKeyDataFn,
   importVaultDataFn,
@@ -15,6 +27,7 @@ import { TExternalMigrationQueueFactory } from "./external-migration-queue";
 import {
   ExternalMigrationProviders,
   ExternalPlatforms,
+  TConfigureExternalMigrationDTO,
   THasCustomVaultMigrationDTO,
   TImportEnvKeyDataDTO,
   TImportVaultDataDTO
@@ -23,8 +36,11 @@ import {
 type TExternalMigrationServiceFactoryDep = {
   permissionService: TPermissionServiceFactory;
   externalMigrationQueue: TExternalMigrationQueueFactory;
+  appConnectionService: Pick<TAppConnectionServiceFactory, "connectAppConnectionById">;
+  externalMigrationConfigDAL: Pick<TExternalMigrationConfigDALFactory, "create" | "upsert" | "findOne" | "transaction">;
   userDAL: Pick<TUserDALFactory, "findById">;
   gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
+  kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
 };
 
 export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrationServiceFactory>;
@@ -33,7 +49,10 @@ export const externalMigrationServiceFactory = ({
   permissionService,
   externalMigrationQueue,
   userDAL,
-  gatewayService
+  gatewayService,
+  appConnectionService,
+  externalMigrationConfigDAL,
+  kmsService
 }: TExternalMigrationServiceFactoryDep) => {
   const importEnvKeyData = async ({
     decryptionKey,
@@ -171,9 +190,195 @@ export const externalMigrationServiceFactory = ({
     return actorOrgId in vaultMigrationTransformMappings;
   };
 
+  const configureExternalMigration = async ({ platform, connectionId, actor }: TConfigureExternalMigrationDTO) => {
+    const { hasRole } = await permissionService.getOrgPermission(
+      actor.type,
+      actor.id,
+      actor.orgId,
+      actor.authMethod,
+      actor.orgId
+    );
+
+    if (!hasRole(OrgMembershipRole.Admin)) {
+      throw new ForbiddenRequestError({ message: "Only admins can configure external migration" });
+    }
+
+    if (connectionId) {
+      if (platform === ExternalMigrationProviders.Vault) {
+        await appConnectionService.connectAppConnectionById(AppConnection.HCVault, connectionId, actor);
+      } else {
+        throw new BadRequestError({ message: "Invalid platform" });
+      }
+    }
+
+    const config = await externalMigrationConfigDAL.upsert({
+      platform,
+      connectionId,
+      orgId: actor.orgId
+    });
+
+    return config;
+  };
+
+  const getExternalMigrationConfig = async ({ platform, actor }: { platform: string; actor: OrgServiceActor }) => {
+    const { hasRole } = await permissionService.getOrgPermission(
+      actor.type,
+      actor.id,
+      actor.orgId,
+      actor.authMethod,
+      actor.orgId
+    );
+
+    if (!hasRole(OrgMembershipRole.Admin)) {
+      throw new ForbiddenRequestError({ message: "Only admins can view external migration config" });
+    }
+
+    const config = await externalMigrationConfigDAL.findOne({
+      orgId: actor.orgId,
+      platform
+    });
+
+    if (!config) {
+      throw new NotFoundError({ message: "External migration config not found" });
+    }
+
+    return config;
+  };
+
+  const getVaultNamespaces = async ({ actor }: { actor: OrgServiceActor }) => {
+    const { hasRole } = await permissionService.getOrgPermission(
+      actor.type,
+      actor.id,
+      actor.orgId,
+      actor.authMethod,
+      actor.orgId
+    );
+
+    if (!hasRole(OrgMembershipRole.Admin)) {
+      throw new ForbiddenRequestError({ message: "Only admins can view vault namespaces" });
+    }
+
+    const vaultConfig = await externalMigrationConfigDAL.findOne({
+      orgId: actor.orgId,
+      platform: ExternalMigrationProviders.Vault
+    });
+
+    if (!vaultConfig) {
+      throw new BadRequestError({ message: "Vault migration config not found" });
+    }
+
+    if (!vaultConfig.connection) {
+      throw new BadRequestError({ message: "Vault migration connection is not configured" });
+    }
+
+    const credentials = await decryptAppConnectionCredentials({
+      orgId: vaultConfig.orgId,
+      encryptedCredentials: vaultConfig.connection.encryptedCredentials,
+      kmsService,
+      projectId: null
+    });
+
+    const connection = {
+      ...vaultConfig.connection,
+      credentials
+    } as THCVaultConnection;
+
+    const namespaces = await listHCVaultNamespaces(connection, gatewayService);
+    return namespaces;
+  };
+
+  const getVaultPolicies = async ({ actor, namespace }: { actor: OrgServiceActor; namespace?: string }) => {
+    const { hasRole } = await permissionService.getOrgPermission(
+      actor.type,
+      actor.id,
+      actor.orgId,
+      actor.authMethod,
+      actor.orgId
+    );
+
+    if (!hasRole(OrgMembershipRole.Admin)) {
+      throw new ForbiddenRequestError({ message: "Only admins can view vault policies" });
+    }
+
+    const vaultConfig = await externalMigrationConfigDAL.findOne({
+      orgId: actor.orgId,
+      platform: ExternalMigrationProviders.Vault
+    });
+
+    if (!vaultConfig) {
+      throw new NotFoundError({ message: "Vault migration config not found" });
+    }
+
+    if (!vaultConfig.connection) {
+      throw new BadRequestError({ message: "Vault migration connection is not configured" });
+    }
+
+    const credentials = await decryptAppConnectionCredentials({
+      orgId: vaultConfig.orgId,
+      encryptedCredentials: vaultConfig.connection.encryptedCredentials,
+      kmsService,
+      projectId: null
+    });
+
+    const connection = {
+      ...vaultConfig.connection,
+      credentials
+    } as THCVaultConnection;
+
+    const policies = await listHCVaultPolicies(connection, gatewayService, namespace);
+    return policies;
+  };
+
+  const getVaultMounts = async ({ actor, namespace }: { actor: OrgServiceActor; namespace?: string }) => {
+    const { hasRole } = await permissionService.getOrgPermission(
+      actor.type,
+      actor.id,
+      actor.orgId,
+      actor.authMethod,
+      actor.orgId
+    );
+
+    if (!hasRole(OrgMembershipRole.Admin)) {
+      throw new ForbiddenRequestError({ message: "Only admins can view vault mounts" });
+    }
+
+    const vaultConfig = await externalMigrationConfigDAL.findOne({
+      orgId: actor.orgId,
+      platform: ExternalMigrationProviders.Vault
+    });
+
+    if (!vaultConfig) {
+      throw new NotFoundError({ message: "Vault migration config not found" });
+    }
+
+    if (!vaultConfig.connection) {
+      throw new BadRequestError({ message: "Vault migration connection is not configured" });
+    }
+
+    const credentials = await decryptAppConnectionCredentials({
+      orgId: vaultConfig.orgId,
+      encryptedCredentials: vaultConfig.connection.encryptedCredentials,
+      kmsService,
+      projectId: null
+    });
+
+    const connection = {
+      ...vaultConfig.connection,
+      credentials
+    } as THCVaultConnection;
+
+    const mounts = await listHCVaultMounts(connection, gatewayService, namespace);
+    return mounts;
+  };
+
   return {
     importEnvKeyData,
     importVaultData,
-    hasCustomVaultMigration
+    hasCustomVaultMigration,
+    configureExternalMigration,
+    getExternalMigrationConfig,
+    getVaultNamespaces,
+    getVaultPolicies,
+    getVaultMounts
   };
 };

backend/src/services/external-migration/external-migration-types.ts

@@ -1,4 +1,4 @@
-import { TOrgPermission } from "@app/lib/types";
+import { OrgServiceActor, TOrgPermission } from "@app/lib/types";
 
 import { ActorAuthMethod, ActorType } from "../auth/auth-type";
 
@@ -121,3 +121,9 @@ export enum ExternalMigrationProviders {
   Vault = "vault",
   EnvKey = "env-key"
 }
+
+export type TConfigureExternalMigrationDTO = {
+  platform: ExternalMigrationProviders;
+  connectionId: string | null;
+  actor: OrgServiceActor;
+};