From dffc204ec2b521a0aace3fddc3a7e009150f495a Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 14 Oct 2025 20:25:08 +0800 Subject: [PATCH] feat: add in-platform support for secret imports --- backend/src/server/routes/index.ts | 4 +- .../routes/v3/external-migration-router.ts | 61 +++++- .../hc-vault/hc-vault-connection-fns.ts | 207 ++++++++++++++++++ .../external-migration-service.ts | 155 ++++++++++++- .../src/hooks/api/migration/mutations.tsx | 32 ++- frontend/src/hooks/api/migration/queries.tsx | 21 +- frontend/src/hooks/api/migration/types.ts | 8 + .../components/ActionBar/ActionBar.tsx | 88 +++++++- .../ActionBar/VaultSecretImportModal.tsx | 199 +++++++++++++++++ 9 files changed, 763 insertions(+), 12 deletions(-) create mode 100644 frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/VaultSecretImportModal.tsx diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index c22024380..66ff0f395 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -2199,7 +2199,9 @@ export const registerRoutes = async ( gatewayService, kmsService, appConnectionService, - externalMigrationConfigDAL + externalMigrationConfigDAL, + secretService, + auditLogService }); // setup the communication with license key server diff --git a/backend/src/server/routes/v3/external-migration-router.ts b/backend/src/server/routes/v3/external-migration-router.ts index 7b436f43f..008e20902 100644 --- a/backend/src/server/routes/v3/external-migration-router.ts +++ b/backend/src/server/routes/v3/external-migration-router.ts @@ -139,7 +139,7 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider }) } }, - onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { const config = await server.services.migration.getExternalMigrationConfig({ platform: req.query.platform, @@ -260,4 +260,63 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider return { mounts }; } }); + + server.route({ + method: "POST", + url: "/vault/import-secrets", + config: { + rateLimit: writeLimit + }, + schema: { + body: z.object({ + projectId: z.string(), + environment: z.string(), + secretPath: z.string(), + vaultNamespace: z.string(), + vaultSecretPath: z.string() + }), + response: { + 200: z.object({ + message: z.string() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + await server.services.migration.importVaultSecrets({ + actor: req.permission, + auditLogInfo: req.auditLogInfo, + ...req.body + }); + + return { message: "Successfully imported vault secrets" }; + } + }); + + server.route({ + method: "GET", + url: "/vault/secret-paths", + config: { + rateLimit: readLimit + }, + schema: { + querystring: z.object({ + namespace: z.string().optional() + }), + response: { + 200: z.object({ + secretPaths: z.string().array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const secretPaths = await server.services.migration.getVaultSecretPaths({ + actor: req.permission, + namespace: req.query.namespace + }); + + return { secretPaths }; + } + }); }; diff --git a/backend/src/services/app-connection/hc-vault/hc-vault-connection-fns.ts b/backend/src/services/app-connection/hc-vault/hc-vault-connection-fns.ts index cda8c4166..6f7016ac1 100644 --- a/backend/src/services/app-connection/hc-vault/hc-vault-connection-fns.ts +++ b/backend/src/services/app-connection/hc-vault/hc-vault-connection-fns.ts @@ -363,3 +363,210 @@ export const listHCVaultMounts = async ( return mounts; }; + +export const listHCVaultSecretPaths = async ( + connection: THCVaultConnection, + gatewayService: Pick, + namespace?: string +) => { + const instanceUrl = await getHCVaultInstanceUrl(connection); + const accessToken = await getHCVaultAccessToken(connection, gatewayService); + + if (namespace && connection.credentials.namespace) { + throw new BadRequestError({ + message: "Namespace cannot be specified when namespace is already set in the connection credentials" + }); + } + + const targetNamespace = namespace || connection.credentials.namespace; + + const getPaths = async (mountPath: string, secretPath: string, kvVersion: "1" | "2"): Promise => { + try { + let path: string; + if (kvVersion === "2") { + // For KV v2: /v1/{mount}/metadata/{path}?list=true + path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`; + } else { + // For KV v1: /v1/{mount}/{path}?list=true + path = secretPath ? `${mountPath}/${secretPath}` : mountPath; + } + + const { data } = await requestWithHCVaultGateway<{ + data: { + keys: string[]; + }; + }>(connection, gatewayService, { + url: `${instanceUrl}/v1/${path}?list=true`, + method: "GET", + headers: { + "X-Vault-Token": accessToken, + ...(targetNamespace ? { "X-Vault-Namespace": targetNamespace } : {}) + } + }); + + return data.data.keys; + } catch (error) { + if (error instanceof AxiosError && error.response?.status === 404) { + return null; + } + + throw error; + } + }; + + // Recursive function to get all secret paths in a mount + const recursivelyGetAllPaths = async ( + mountPath: string, + kvVersion: "1" | "2", + currentPath: string = "" + ): Promise => { + const paths = await getPaths(mountPath, currentPath, kvVersion); + + if (paths === null || paths.length === 0) { + return []; + } + + const allSecrets: string[] = []; + + // Process paths sequentially to maintain tree traversal order + // eslint-disable-next-line no-restricted-syntax + for (const path of paths) { + const cleanPath = path.endsWith("/") ? path.slice(0, -1) : path; + const fullItemPath = currentPath ? `${currentPath}/${cleanPath}` : cleanPath; + + if (path.endsWith("/")) { + // it's a folder so we recurse into it + // eslint-disable-next-line no-await-in-loop + const subSecrets = await recursivelyGetAllPaths(mountPath, kvVersion, fullItemPath); + allSecrets.push(...subSecrets); + } else { + // it's a secret so we add it to our results + allSecrets.push(`${mountPath}/${fullItemPath}`); + } + } + + return allSecrets; + }; + + // Get all mounts + const mounts = await listHCVaultMounts(connection, gatewayService, namespace); + + // Filter for KV mounts (kv, kv-v1, kv-v2) + const kvMounts = mounts.filter((mount) => mount.type === "kv" || mount.type.startsWith("kv")); + + // Collect all secret paths from all KV mounts in parallel + const allSecretPathsArrays = await Promise.all( + kvMounts.map(async (mount) => { + const kvVersion = mount.version === "2" ? "2" : "1"; + const cleanMountPath = mount.path.replace(/\/$/, ""); // Remove trailing slash + return recursivelyGetAllPaths(cleanMountPath, kvVersion); + }) + ); + + // Flatten the arrays into a single array + const allSecretPaths = allSecretPathsArrays.flat(); + + return allSecretPaths; +}; + +export const getHCVaultSecretsForPath = async ( + connection: THCVaultConnection, + gatewayService: Pick, + namespace: string, + secretPath: string +) => { + const instanceUrl = await getHCVaultInstanceUrl(connection); + const accessToken = await getHCVaultAccessToken(connection, gatewayService); + + if (connection.credentials.namespace && connection.credentials.namespace !== namespace) { + throw new BadRequestError({ + message: "Specified namespace does not match the namespace in the connection credentials" + }); + } + + const targetNamespace = namespace || connection.credentials.namespace; + + try { + // Extract mount and path from the secretPath + // secretPath format: {mount}/{path} + const pathParts = secretPath.split("/"); + const mountPath = pathParts[0]; + const actualPath = pathParts.slice(1).join("/"); + + if (!mountPath || !actualPath) { + throw new BadRequestError({ + message: "Invalid secret path format. Expected format: {mount}/{path}" + }); + } + + // Get mounts to determine KV version + const mounts = await listHCVaultMounts(connection, gatewayService, namespace); + const mount = mounts.find((m) => m.path.replace(/\/$/, "") === mountPath); + + if (!mount) { + throw new BadRequestError({ + message: `Mount '${mountPath}' not found in HashiCorp Vault` + }); + } + + const kvVersion = mount.version === "2" ? "2" : "1"; + + // Fetch secrets based on KV version + if (kvVersion === "2") { + // For KV v2: /v1/{mount}/data/{path} + const { data } = await requestWithHCVaultGateway<{ + data: { + data: Record; // KV v2 has nested data structure + metadata: { + created_time: string; + deletion_time: string; + destroyed: boolean; + version: number; + }; + }; + }>(connection, gatewayService, { + url: `${instanceUrl}/v1/${mountPath}/data/${actualPath}`, + method: "GET", + headers: { + "X-Vault-Token": accessToken, + ...(targetNamespace ? { "X-Vault-Namespace": targetNamespace } : {}) + } + }); + + return data.data.data; + } + + // For KV v1: /v1/{mount}/{path} + const { data } = await requestWithHCVaultGateway<{ + data: Record; // KV v1 has flat data structure + lease_duration: number; + lease_id: string; + renewable: boolean; + }>(connection, gatewayService, { + url: `${instanceUrl}/v1/${mountPath}/${actualPath}`, + method: "GET", + headers: { + "X-Vault-Token": accessToken, + ...(targetNamespace ? { "X-Vault-Namespace": targetNamespace } : {}) + } + }); + + return data.data; + } catch (error: unknown) { + logger.error(error, "Unable to fetch secrets from HC Vault path"); + + if (error instanceof AxiosError) { + throw new BadRequestError({ + message: `Failed to fetch secrets: ${error.message || "Unknown error"}` + }); + } + + if (error instanceof BadRequestError) { + throw error; + } + + throw new BadRequestError({ + message: "Unable to fetch secrets from HashiCorp Vault" + }); + } +}; diff --git a/backend/src/services/external-migration/external-migration-service.ts b/backend/src/services/external-migration/external-migration-service.ts index e70d36694..917886ec6 100644 --- a/backend/src/services/external-migration/external-migration-service.ts +++ b/backend/src/services/external-migration/external-migration-service.ts @@ -1,4 +1,10 @@ import { OrgMembershipRole } from "@app/db/schemas"; +import { + AuditLogInfo, + EventType, + SecretApprovalEvent, + TAuditLogServiceFactory +} from "@app/ee/services/audit-log/audit-log-types"; import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types"; import { crypto } from "@app/lib/crypto/cryptography"; @@ -9,12 +15,16 @@ import { AppConnection } from "../app-connection/app-connection-enums"; import { decryptAppConnectionCredentials } from "../app-connection/app-connection-fns"; import { TAppConnectionServiceFactory } from "../app-connection/app-connection-service"; import { + getHCVaultSecretsForPath, listHCVaultMounts, listHCVaultNamespaces, listHCVaultPolicies, + listHCVaultSecretPaths, THCVaultConnection } from "../app-connection/hc-vault"; import { TKmsServiceFactory } from "../kms/kms-service"; +import { TSecretServiceFactory } from "../secret/secret-service"; +import { SecretProtectionType } from "../secret/secret-types"; import { TUserDALFactory } from "../user/user-dal"; import { TExternalMigrationConfigDALFactory } from "./external-migration-config-dal"; import { @@ -35,6 +45,8 @@ import { type TExternalMigrationServiceFactoryDep = { permissionService: TPermissionServiceFactory; + secretService: TSecretServiceFactory; + auditLogService: Pick; externalMigrationQueue: TExternalMigrationQueueFactory; appConnectionService: Pick; externalMigrationConfigDAL: Pick; @@ -50,6 +62,8 @@ export const externalMigrationServiceFactory = ({ externalMigrationQueue, userDAL, gatewayService, + secretService, + auditLogService, appConnectionService, externalMigrationConfigDAL, kmsService @@ -371,6 +385,143 @@ export const externalMigrationServiceFactory = ({ return mounts; }; + const getVaultSecretPaths = async ({ actor, namespace }: { actor: OrgServiceActor; namespace?: string }) => { + const { hasRole } = await permissionService.getOrgPermission( + actor.type, + actor.id, + actor.orgId, + actor.authMethod, + actor.orgId + ); + + if (!hasRole(OrgMembershipRole.Admin)) { + throw new ForbiddenRequestError({ message: "Only admins can view vault secret paths" }); + } + + const vaultConfig = await externalMigrationConfigDAL.findOne({ + orgId: actor.orgId, + platform: ExternalMigrationProviders.Vault + }); + + if (!vaultConfig) { + throw new NotFoundError({ message: "Vault migration config not found" }); + } + + if (!vaultConfig.connection) { + throw new BadRequestError({ message: "Vault migration connection is not configured" }); + } + + const credentials = await decryptAppConnectionCredentials({ + orgId: vaultConfig.orgId, + encryptedCredentials: vaultConfig.connection.encryptedCredentials, + kmsService, + projectId: null + }); + + const connection = { + ...vaultConfig.connection, + credentials + } as THCVaultConnection; + + const secretPaths = await listHCVaultSecretPaths(connection, gatewayService, namespace); + + return secretPaths; + }; + + const importVaultSecrets = async ({ + actor, + projectId, + environment, + secretPath, + vaultNamespace, + vaultSecretPath, + auditLogInfo + }: { + actor: OrgServiceActor; + projectId: string; + environment: string; + secretPath: string; + vaultNamespace: string; + vaultSecretPath: string; + auditLogInfo: AuditLogInfo; + }) => { + const { hasRole } = await permissionService.getOrgPermission( + actor.type, + actor.id, + actor.orgId, + actor.authMethod, + actor.orgId + ); + + if (!hasRole(OrgMembershipRole.Admin)) { + throw new ForbiddenRequestError({ message: "Only admins can import vault secrets" }); + } + + const vaultConfig = await externalMigrationConfigDAL.findOne({ + orgId: actor.orgId, + platform: ExternalMigrationProviders.Vault + }); + + if (!vaultConfig) { + throw new NotFoundError({ message: "Vault migration config not found" }); + } + + if (!vaultConfig.connection) { + throw new BadRequestError({ message: "Vault migration connection is not configured" }); + } + + const credentials = await decryptAppConnectionCredentials({ + orgId: vaultConfig.orgId, + encryptedCredentials: vaultConfig.connection.encryptedCredentials, + kmsService, + projectId: null + }); + + const connection = { + ...vaultConfig.connection, + credentials + } as THCVaultConnection; + + const vaultSecrets = await getHCVaultSecretsForPath(connection, gatewayService, vaultNamespace, vaultSecretPath); + + const secretOperation = await secretService.createManySecretsRaw({ + actorId: actor.id, + actor: actor.type, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + secretPath, + environment, + projectId, + secrets: Object.entries(vaultSecrets).map(([secretKey, secretValue]) => ({ + secretKey, + secretValue + })) + }); + + if (secretOperation.type === SecretProtectionType.Approval) { + await auditLogService.createAuditLog({ + projectId, + ...auditLogInfo, + event: { + type: EventType.SECRET_APPROVAL_REQUEST, + metadata: { + committedBy: secretOperation.approval.committerUserId, + secretApprovalRequestId: secretOperation.approval.id, + secretApprovalRequestSlug: secretOperation.approval.slug, + secretPath, + environment, + secrets: Object.entries(vaultSecrets).map(([secretKey]) => ({ + secretKey + })), + eventType: SecretApprovalEvent.CreateMany + } + } + }); + + return { approval: secretOperation.approval }; + } + }; + return { importEnvKeyData, importVaultData, @@ -379,6 +530,8 @@ export const externalMigrationServiceFactory = ({ getExternalMigrationConfig, getVaultNamespaces, getVaultPolicies, - getVaultMounts + getVaultMounts, + getVaultSecretPaths, + importVaultSecrets }; }; diff --git a/frontend/src/hooks/api/migration/mutations.tsx b/frontend/src/hooks/api/migration/mutations.tsx index f43ffeb2f..c3c55ad5a 100644 --- a/frontend/src/hooks/api/migration/mutations.tsx +++ b/frontend/src/hooks/api/migration/mutations.tsx @@ -1,10 +1,16 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { apiRequest } from "@app/config/request"; +import { dashboardKeys } from "@app/hooks/api/dashboard/queries"; +import { secretKeys } from "@app/hooks/api/secrets/queries"; import { projectKeys } from "../projects"; import { externalMigrationQueryKeys } from "./queries"; -import { ExternalMigrationProviders, TExternalMigrationConfig } from "./types"; +import { + ExternalMigrationProviders, + TExternalMigrationConfig, + TImportVaultSecretsDTO +} from "./types"; export const useImportEnvKey = () => { const queryClient = useQueryClient(); @@ -90,3 +96,27 @@ export const useUpdateExternalMigrationConfig = (platform: ExternalMigrationProv } }); }; + +export const useImportVaultSecrets = () => { + const queryClient = useQueryClient(); + + return useMutation<{ message: string }, object, TImportVaultSecretsDTO>({ + mutationFn: async (dto) => { + const { data } = await apiRequest.post<{ message: string }>( + "/api/v3/external-migration/vault/import-secrets", + dto + ); + return data; + }, + onSuccess: (_, { projectId, environment, secretPath }) => { + queryClient.invalidateQueries({ queryKey: dashboardKeys.all() }); + queryClient.invalidateQueries({ + queryKey: secretKeys.getProjectSecret({ + projectId, + environment, + secretPath + }) + }); + } + }); +}; diff --git a/frontend/src/hooks/api/migration/queries.tsx b/frontend/src/hooks/api/migration/queries.tsx index ff08eeca3..4b4d94904 100644 --- a/frontend/src/hooks/api/migration/queries.tsx +++ b/frontend/src/hooks/api/migration/queries.tsx @@ -12,7 +12,8 @@ export const externalMigrationQueryKeys = { config: (platform: string) => ["external-migration-config", { platform }], vaultNamespaces: () => ["vault-namespaces"], vaultPolicies: () => ["vault-policies"], - vaultMounts: () => ["vault-mounts"] + vaultMounts: () => ["vault-mounts"], + vaultSecretPaths: () => ["vault-secret-paths"] }; export const useHasCustomMigrationAvailable = (provider: ExternalMigrationProviders) => { @@ -88,3 +89,21 @@ export const useGetVaultMounts = (enabled = true, namespace?: string) => { enabled }); }; + +export const useGetVaultSecretPaths = (enabled = true, namespace?: string) => { + return useQuery({ + queryKey: externalMigrationQueryKeys.vaultSecretPaths(), + queryFn: async () => { + const { data } = await apiRequest.get<{ + secretPaths: string[]; + }>("/api/v3/external-migration/vault/secret-paths", { + params: { + namespace + } + }); + + return data.secretPaths; + }, + enabled + }); +}; diff --git a/frontend/src/hooks/api/migration/types.ts b/frontend/src/hooks/api/migration/types.ts index c8e00fb71..230b11da9 100644 --- a/frontend/src/hooks/api/migration/types.ts +++ b/frontend/src/hooks/api/migration/types.ts @@ -11,3 +11,11 @@ export type TExternalMigrationConfig = { createdAt: string; updatedAt: string; }; + +export type TImportVaultSecretsDTO = { + projectId: string; + environment: string; + secretPath: string; + vaultNamespace: string; + vaultSecretPath: string; +}; diff --git a/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/ActionBar.tsx b/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/ActionBar.tsx index e48a3c468..41c0af267 100644 --- a/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/ActionBar.tsx +++ b/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/ActionBar.tsx @@ -77,6 +77,8 @@ import { fetchDashboardProjectSecretsByKeys } from "@app/hooks/api/dashboard/queries"; import { UsedBySecretSyncs } from "@app/hooks/api/dashboard/types"; +import { useGetExternalMigrationConfig, useImportVaultSecrets } from "@app/hooks/api/migration"; +import { ExternalMigrationProviders } from "@app/hooks/api/migration/types"; import { secretApprovalRequestKeys } from "@app/hooks/api/secretApprovalRequest/queries"; import { PendingAction } from "@app/hooks/api/secretFolders/types"; import { fetchProjectSecrets, secretKeys } from "@app/hooks/api/secrets/queries"; @@ -98,6 +100,7 @@ import { CreateDynamicSecretForm } from "./CreateDynamicSecretForm"; import { CreateSecretImportForm } from "./CreateSecretImportForm"; import { FolderForm } from "./FolderForm"; import { MoveSecretsModal } from "./MoveSecretsModal"; +import { VaultSecretImportModal } from "./VaultSecretImportModal"; type TParsedEnv = { value: string; comments: string[]; secretPath?: string; secretKey: string }[]; type TParsedFolderEnv = Record< @@ -171,7 +174,8 @@ export const ActionBar = ({ "upgradePlan", "replicateFolder", "confirmUpload", - "requestAccess" + "requestAccess", + "importFromVault" ] as const); const isProtectedBranch = Boolean(protectedBranchPolicyName); const { subscription } = useSubscription(); @@ -185,6 +189,7 @@ export const ActionBar = ({ const { mutateAsync: createSecretBatch, isPending: isCreatingSecrets } = useCreateSecretBatch({ options: { onSuccess: undefined } }); + const { mutateAsync: importVaultSecrets } = useImportVaultSecrets(); const queryClient = useQueryClient(); const { addPendingChange } = useBatchModeActions(); @@ -193,6 +198,8 @@ export const ActionBar = ({ const isMultiSelectActive = Boolean(Object.keys(selectedSecrets).length); const { permission } = useProjectPermission(); + const { data: vaultConfig } = useGetExternalMigrationConfig(ExternalMigrationProviders.Vault); + const hasVaultConnection = Boolean(vaultConfig?.connectionId); const handleFolderCreate = async (folderName: string, description: string | null) => { try { @@ -663,6 +670,33 @@ export const ActionBar = ({ } }; + const handleVaultImport = async (vaultPath: string, namespace: string) => { + try { + await importVaultSecrets({ + projectId, + environment, + secretPath, + vaultNamespace: namespace, + vaultSecretPath: vaultPath + }); + + createNotification({ + type: "success", + text: "Successfully imported secrets from HashiCorp Vault" + }); + } catch (err) { + console.error("Vault import error:", err); + const error = err as AxiosError<{ message?: string }>; + const errorMessage = + error.response?.data?.message || "Failed to import secrets from Vault. Please try again."; + + createNotification({ + type: "error", + text: errorMessage + }); + } + }; + const isTableFiltered = Object.values(filter.tags).some(Boolean) || Object.values(filter.include).some(Boolean); @@ -784,8 +818,8 @@ export const ActionBar = ({ {Boolean(filteredTags) && {filteredTags} Applied} - - + +
Filter by Secret Tags @@ -926,7 +960,7 @@ export const ActionBar = ({ @@ -1059,6 +1093,39 @@ export const ActionBar = ({ )} + {hasVaultConnection && ( + + {(isAllowed) => ( + + )} + + )}
@@ -1070,11 +1137,11 @@ export const ActionBar = ({ isMultiSelectActive && "h-16" )} > -
+
{Object.keys(selectedSecrets).length} Selected
+ + + +
+ + ); +}; + +export const VaultSecretImportModal = ({ + isOpen, + onOpenChange, + environment, + secretPath, + onImport +}: Props) => { + return ( + + + onOpenChange(false)} + environment={environment} + secretPath={secretPath} + onImport={onImport} + /> + + + ); +};