feat(secrey-sync): Hashicorp Vault Secret Sync (and minor app connection fixes)

backend/src/lib/api-docs/constants.ts

@@ -2002,6 +2002,10 @@ export const SecretSyncs = {
     WINDMILL: {
       workspace: "The Windmill workspace to sync secrets to.",
       path: "The Windmill workspace path to sync secrets to."
+    },
+    HC_VAULT: {
+      mount: "The Hashicorp Vault Secrets Engine Mount to sync secrets to.",
+      path: "The Hashicorp Vault path to sync secrets to."
     }
   }
 };

backend/src/server/routes/v1/secret-sync-routers/hc-vault-sync-router.ts

@@ -0,0 +1,17 @@
+import {
+  CreateHCVaultSyncSchema,
+  HCVaultSyncSchema,
+  UpdateHCVaultSyncSchema
+} from "@app/services/secret-sync/hc-vault";
+import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
+
+import { registerSyncSecretsEndpoints } from "./secret-sync-endpoints";
+
+export const registerHCVaultSyncRouter = async (server: FastifyZodProvider) =>
+  registerSyncSecretsEndpoints({
+    destination: SecretSync.HCVault,
+    server,
+    responseSchema: HCVaultSyncSchema,
+    createSchema: CreateHCVaultSyncSchema,
+    updateSchema: UpdateHCVaultSyncSchema
+  });

backend/src/server/routes/v1/secret-sync-routers/index.ts

@@ -8,6 +8,7 @@ import { registerCamundaSyncRouter } from "./camunda-sync-router";
 import { registerDatabricksSyncRouter } from "./databricks-sync-router";
 import { registerGcpSyncRouter } from "./gcp-sync-router";
 import { registerGitHubSyncRouter } from "./github-sync-router";
+import { registerHCVaultSyncRouter } from "./hc-vault-sync-router";
 import { registerHumanitecSyncRouter } from "./humanitec-sync-router";
 import { registerTerraformCloudSyncRouter } from "./terraform-cloud-sync-router";
 import { registerVercelSyncRouter } from "./vercel-sync-router";
@@ -27,5 +28,6 @@ export const SECRET_SYNC_REGISTER_ROUTER_MAP: Record<SecretSync, (server: Fastif
   [SecretSync.TerraformCloud]: registerTerraformCloudSyncRouter,
   [SecretSync.Camunda]: registerCamundaSyncRouter,
   [SecretSync.Vercel]: registerVercelSyncRouter,
-  [SecretSync.Windmill]: registerWindmillSyncRouter
+  [SecretSync.Windmill]: registerWindmillSyncRouter,
+  [SecretSync.HCVault]: registerHCVaultSyncRouter
 };

backend/src/server/routes/v1/secret-sync-routers/secret-sync-router.ts

@@ -22,6 +22,7 @@ import { CamundaSyncListItemSchema, CamundaSyncSchema } from "@app/services/secr
 import { DatabricksSyncListItemSchema, DatabricksSyncSchema } from "@app/services/secret-sync/databricks";
 import { GcpSyncListItemSchema, GcpSyncSchema } from "@app/services/secret-sync/gcp";
 import { GitHubSyncListItemSchema, GitHubSyncSchema } from "@app/services/secret-sync/github";
+import { HCVaultSyncListItemSchema, HCVaultSyncSchema } from "@app/services/secret-sync/hc-vault";
 import { HumanitecSyncListItemSchema, HumanitecSyncSchema } from "@app/services/secret-sync/humanitec";
 import { TerraformCloudSyncListItemSchema, TerraformCloudSyncSchema } from "@app/services/secret-sync/terraform-cloud";
 import { VercelSyncListItemSchema, VercelSyncSchema } from "@app/services/secret-sync/vercel";
@@ -39,7 +40,8 @@ const SecretSyncSchema = z.discriminatedUnion("destination", [
   TerraformCloudSyncSchema,
   CamundaSyncSchema,
   VercelSyncSchema,
-  WindmillSyncSchema
+  WindmillSyncSchema,
+  HCVaultSyncSchema
 ]);
 
 const SecretSyncOptionsSchema = z.discriminatedUnion("destination", [
@@ -54,7 +56,8 @@ const SecretSyncOptionsSchema = z.discriminatedUnion("destination", [
   TerraformCloudSyncListItemSchema,
   CamundaSyncListItemSchema,
   VercelSyncListItemSchema,
-  WindmillSyncListItemSchema
+  WindmillSyncListItemSchema,
+  HCVaultSyncListItemSchema
 ]);
 
 export const registerSecretSyncRouter = async (server: FastifyZodProvider) => {

backend/src/services/app-connection/app-connection-maps.ts

@@ -14,5 +14,6 @@ export const APP_CONNECTION_NAME_MAP: Record<AppConnection, string> = {
   [AppConnection.MsSql]: "Microsoft SQL Server",
   [AppConnection.Camunda]: "Camunda",
   [AppConnection.Windmill]: "Windmill",
-  [AppConnection.Auth0]: "Auth0"
+  [AppConnection.Auth0]: "Auth0",
+  [AppConnection.HCVault]: "Hashicorp Vault"
 };

backend/src/services/app-connection/hc-vault/hc-vault-connection-fns.ts

@@ -91,9 +91,9 @@ export const listHCVaultMounts = async (appConnection: THCVaultConnection) => {
 
   const mounts: string[] = [];
 
-  // Filter for "kv" type only
+  // Filter for "kv" version 2 type only
   Object.entries(data.data).forEach(([path, mount]) => {
-    if (mount.type === "kv") {
+    if (mount.type === "kv" && mount.options?.version === "2") {
       mounts.push(path);
     }
   });

backend/src/services/app-connection/hc-vault/hc-vault-connection-types.ts

@@ -26,6 +26,9 @@ export type THCVaultConnectionConfig = DiscriminativePick<THCVaultConnectionInpu
 export type THCVaultMountResponse = {
   data: {
     [key: string]: {
+      options: {
+        version?: string | null;
+      } | null;
       type: string; // We're only interested in "kv" types
     };
   };

backend/src/services/secret-sync/hc-vault/hc-vault-sync-constants.ts

@@ -0,0 +1,10 @@
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
+import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
+
+export const HC_VAULT_SYNC_LIST_OPTION: TSecretSyncListItem = {
+  name: "Hashicorp Vault",
+  destination: SecretSync.HCVault,
+  connection: AppConnection.HCVault,
+  canImportSecrets: true
+};

backend/src/services/secret-sync/hc-vault/hc-vault-sync-fns.ts

@@ -0,0 +1,126 @@
+import { request } from "@app/lib/config/request";
+import { removeTrailingSlash } from "@app/lib/fn";
+import { getHCVaultAccessToken } from "@app/services/app-connection/hc-vault";
+import {
+  THCVaultListVariables,
+  THCVaultListVariablesResponse,
+  THCVaultSyncWithCredentials,
+  TPostHCVaultVariable
+} from "@app/services/secret-sync/hc-vault/hc-vault-sync-types";
+import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
+import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
+
+const listHCVaultVariables = async ({ instanceUrl, mount, accessToken, path }: THCVaultListVariables) => {
+  const { data } = await request.get<THCVaultListVariablesResponse>(
+    `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
+    {
+      headers: {
+        "X-Vault-Token": accessToken
+      }
+    }
+  );
+
+  return data.data.data;
+};
+
+// Hashicorp Vault updates all variables in one batch. This is to respect their versioning
+const updateHCVaultVariables = async ({ path, instanceUrl, accessToken, mount, data }: TPostHCVaultVariable) =>
+  request.post(
+    `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
+    {
+      data
+    },
+    {
+      headers: {
+        "X-Vault-Token": accessToken,
+        "Content-Type": "application/json"
+      }
+    }
+  );
+
+export const HCVaultSyncFns = {
+  syncSecrets: async (secretSync: THCVaultSyncWithCredentials, secretMap: TSecretMap) => {
+    const {
+      connection,
+      destinationConfig: { mount, path },
+      syncOptions: { disableSecretDeletion }
+    } = secretSync;
+
+    const accessToken = await getHCVaultAccessToken(connection);
+    const { instanceUrl } = connection.credentials;
+
+    const variables = await listHCVaultVariables({ instanceUrl, accessToken, mount, path });
+    let tainted = false;
+
+    for await (const entry of Object.entries(secretMap)) {
+      const [key, { value }] = entry;
+      if (value !== variables[key]) {
+        variables[key] = value;
+        tainted = true;
+      }
+    }
+
+    if (disableSecretDeletion) return;
+
+    for await (const [key] of Object.entries(variables)) {
+      if (!(key in secretMap)) {
+        delete variables[key];
+        tainted = true;
+      }
+    }
+
+    // Only update variables if there was a change detected
+    if (!tainted) return;
+
+    try {
+      await updateHCVaultVariables({ accessToken, instanceUrl, mount, path, data: variables });
+    } catch (error) {
+      throw new SecretSyncError({
+        error
+      });
+    }
+  },
+  removeSecrets: async (secretSync: THCVaultSyncWithCredentials, secretMap: TSecretMap) => {
+    const {
+      connection,
+      destinationConfig: { mount, path }
+    } = secretSync;
+
+    const accessToken = await getHCVaultAccessToken(connection);
+    const { instanceUrl } = connection.credentials;
+
+    const variables = await listHCVaultVariables({ instanceUrl, accessToken, mount, path });
+
+    for await (const [key] of Object.entries(variables)) {
+      if (key in secretMap) {
+        delete variables[key];
+      }
+    }
+
+    try {
+      await updateHCVaultVariables({ accessToken, instanceUrl, mount, path, data: variables });
+    } catch (error) {
+      throw new SecretSyncError({
+        error
+      });
+    }
+  },
+  getSecrets: async (secretSync: THCVaultSyncWithCredentials) => {
+    const {
+      connection,
+      destinationConfig: { mount, path }
+    } = secretSync;
+
+    const accessToken = await getHCVaultAccessToken(connection);
+    const { instanceUrl } = connection.credentials;
+
+    const variables = await listHCVaultVariables({
+      instanceUrl,
+      accessToken,
+      mount,
+      path
+    });
+
+    return Object.fromEntries(Object.entries(variables).map(([key, value]) => [key, { value }]));
+  }
+};

backend/src/services/secret-sync/hc-vault/hc-vault-sync-schemas.ts

@@ -0,0 +1,57 @@
+import { z } from "zod";
+
+import { SecretSyncs } from "@app/lib/api-docs";
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
+import {
+  BaseSecretSyncSchema,
+  GenericCreateSecretSyncFieldsSchema,
+  GenericUpdateSecretSyncFieldsSchema
+} from "@app/services/secret-sync/secret-sync-schemas";
+import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";
+
+const HCVaultSyncDestinationConfigSchema = z.object({
+  mount: z
+    .string()
+    .trim()
+    .min(1, "Secrets Engine Mount required")
+    .describe(SecretSyncs.DESTINATION_CONFIG.HC_VAULT.mount),
+  path: z
+    .string()
+    .trim()
+    .min(1, "Path required")
+    .transform((val) => val.trim().replace(/^\/+|\/+$/g, "")) // removes leading/trailing slashes
+    .refine((val) => /^([a-zA-Z0-9._-]+\/)*[a-zA-Z0-9._-]+$/.test(val), {
+      message:
+        "Invalid Vault path format. Use alphanumerics, dots, dashes, underscores, and single slashes between segments."
+    })
+    .describe(SecretSyncs.DESTINATION_CONFIG.HC_VAULT.path)
+});
+
+const HCVaultSyncOptionsConfig: TSyncOptionsConfig = { canImportSecrets: true };
+
+export const HCVaultSyncSchema = BaseSecretSyncSchema(SecretSync.HCVault, HCVaultSyncOptionsConfig).extend({
+  destination: z.literal(SecretSync.HCVault),
+  destinationConfig: HCVaultSyncDestinationConfigSchema
+});
+
+export const CreateHCVaultSyncSchema = GenericCreateSecretSyncFieldsSchema(
+  SecretSync.HCVault,
+  HCVaultSyncOptionsConfig
+).extend({
+  destinationConfig: HCVaultSyncDestinationConfigSchema
+});
+
+export const UpdateHCVaultSyncSchema = GenericUpdateSecretSyncFieldsSchema(
+  SecretSync.HCVault,
+  HCVaultSyncOptionsConfig
+).extend({
+  destinationConfig: HCVaultSyncDestinationConfigSchema.optional()
+});
+
+export const HCVaultSyncListItemSchema = z.object({
+  name: z.literal("Hashicorp Vault"),
+  connection: z.literal(AppConnection.HCVault),
+  destination: z.literal(SecretSync.HCVault),
+  canImportSecrets: z.literal(true)
+});

backend/src/services/secret-sync/hc-vault/hc-vault-sync-types.ts

@@ -0,0 +1,38 @@
+import { z } from "zod";
+
+import { THCVaultConnection } from "@app/services/app-connection/hc-vault";
+
+import { CreateHCVaultSyncSchema, HCVaultSyncListItemSchema, HCVaultSyncSchema } from "./hc-vault-sync-schemas";
+
+export type THCVaultSync = z.infer<typeof HCVaultSyncSchema>;
+
+export type THCVaultSyncInput = z.infer<typeof CreateHCVaultSyncSchema>;
+
+export type THCVaultSyncListItem = z.infer<typeof HCVaultSyncListItemSchema>;
+
+export type THCVaultSyncWithCredentials = THCVaultSync & {
+  connection: THCVaultConnection;
+};
+
+export type THCVaultListVariablesResponse = {
+  data: {
+    data: {
+      [key: string]: string;
+    };
+  };
+};
+
+export type THCVaultListVariables = {
+  accessToken: string;
+  instanceUrl: string;
+  mount: string;
+  path: string;
+};
+
+export type TPostHCVaultVariable = THCVaultListVariables & {
+  data: {
+    [key: string]: string;
+  };
+};
+
+export type TDeleteHCVaultVariable = THCVaultListVariables;

backend/src/services/secret-sync/hc-vault/index.ts

@@ -0,0 +1,4 @@
+export * from "./hc-vault-sync-constants";
+export * from "./hc-vault-sync-fns";
+export * from "./hc-vault-sync-schemas";
+export * from "./hc-vault-sync-types";

backend/src/services/secret-sync/secret-sync-enums.ts

@@ -10,7 +10,8 @@ export enum SecretSync {
   TerraformCloud = "terraform-cloud",
   Camunda = "camunda",
   Vercel = "vercel",
-  Windmill = "windmill"
+  Windmill = "windmill",
+  HCVault = "hashicorp-vault"
 }
 
 export enum SecretSyncInitialSyncBehavior {

backend/src/services/secret-sync/secret-sync-fns.ts

@@ -25,6 +25,7 @@ import { AZURE_KEY_VAULT_SYNC_LIST_OPTION, azureKeyVaultSyncFactory } from "./az
 import { CAMUNDA_SYNC_LIST_OPTION, camundaSyncFactory } from "./camunda";
 import { GCP_SYNC_LIST_OPTION } from "./gcp";
 import { GcpSyncFns } from "./gcp/gcp-sync-fns";
+import { HC_VAULT_SYNC_LIST_OPTION, HCVaultSyncFns } from "./hc-vault";
 import { HUMANITEC_SYNC_LIST_OPTION } from "./humanitec";
 import { HumanitecSyncFns } from "./humanitec/humanitec-sync-fns";
 import { TERRAFORM_CLOUD_SYNC_LIST_OPTION, TerraformCloudSyncFns } from "./terraform-cloud";
@@ -43,7 +44,8 @@ const SECRET_SYNC_LIST_OPTIONS: Record<SecretSync, TSecretSyncListItem> = {
   [SecretSync.TerraformCloud]: TERRAFORM_CLOUD_SYNC_LIST_OPTION,
   [SecretSync.Camunda]: CAMUNDA_SYNC_LIST_OPTION,
   [SecretSync.Vercel]: VERCEL_SYNC_LIST_OPTION,
-  [SecretSync.Windmill]: WINDMILL_SYNC_LIST_OPTION
+  [SecretSync.Windmill]: WINDMILL_SYNC_LIST_OPTION,
+  [SecretSync.HCVault]: HC_VAULT_SYNC_LIST_OPTION
 };
 
 export const listSecretSyncOptions = () => {
@@ -140,6 +142,8 @@ export const SecretSyncFns = {
         return VercelSyncFns.syncSecrets(secretSync, secretMap);
       case SecretSync.Windmill:
         return WindmillSyncFns.syncSecrets(secretSync, secretMap);
+      case SecretSync.HCVault:
+        return HCVaultSyncFns.syncSecrets(secretSync, secretMap);
       default:
         throw new Error(
           `Unhandled sync destination for sync secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`
@@ -199,6 +203,9 @@ export const SecretSyncFns = {
       case SecretSync.Windmill:
         secretMap = await WindmillSyncFns.getSecrets(secretSync);
         break;
+      case SecretSync.HCVault:
+        secretMap = await HCVaultSyncFns.getSecrets(secretSync);
+        break;
       default:
         throw new Error(
           `Unhandled sync destination for get secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`
@@ -252,6 +259,8 @@ export const SecretSyncFns = {
         return VercelSyncFns.removeSecrets(secretSync, secretMap);
       case SecretSync.Windmill:
         return WindmillSyncFns.removeSecrets(secretSync, secretMap);
+      case SecretSync.HCVault:
+        return HCVaultSyncFns.removeSecrets(secretSync, secretMap);
       default:
         throw new Error(
           `Unhandled sync destination for remove secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`

backend/src/services/secret-sync/secret-sync-maps.ts

@@ -13,7 +13,8 @@ export const SECRET_SYNC_NAME_MAP: Record<SecretSync, string> = {
   [SecretSync.TerraformCloud]: "Terraform Cloud",
   [SecretSync.Camunda]: "Camunda",
   [SecretSync.Vercel]: "Vercel",
-  [SecretSync.Windmill]: "Windmill"
+  [SecretSync.Windmill]: "Windmill",
+  [SecretSync.HCVault]: "Hashicorp Vault"
 };
 
 export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
@@ -28,5 +29,6 @@ export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
   [SecretSync.TerraformCloud]: AppConnection.TerraformCloud,
   [SecretSync.Camunda]: AppConnection.Camunda,
   [SecretSync.Vercel]: AppConnection.Vercel,
-  [SecretSync.Windmill]: AppConnection.Windmill
+  [SecretSync.Windmill]: AppConnection.Windmill,
+  [SecretSync.HCVault]: AppConnection.HCVault
 };

backend/src/services/secret-sync/secret-sync-types.ts

@@ -55,6 +55,12 @@ import {
   TAzureKeyVaultSyncWithCredentials
 } from "./azure-key-vault";
 import { TGcpSync, TGcpSyncInput, TGcpSyncListItem, TGcpSyncWithCredentials } from "./gcp";
+import {
+  THCVaultSync,
+  THCVaultSyncInput,
+  THCVaultSyncListItem,
+  THCVaultSyncWithCredentials
+} from "./hc-vault/hc-vault-sync-types";
 import {
   THumanitecSync,
   THumanitecSyncInput,
@@ -81,7 +87,8 @@ export type TSecretSync =
   | TTerraformCloudSync
   | TCamundaSync
   | TVercelSync
-  | TWindmillSync;
+  | TWindmillSync
+  | THCVaultSync;
 
 export type TSecretSyncWithCredentials =
   | TAwsParameterStoreSyncWithCredentials
@@ -95,7 +102,8 @@ export type TSecretSyncWithCredentials =
   | TTerraformCloudSyncWithCredentials
   | TCamundaSyncWithCredentials
   | TVercelSyncWithCredentials
-  | TWindmillSyncWithCredentials;
+  | TWindmillSyncWithCredentials
+  | THCVaultSyncWithCredentials;
 
 export type TSecretSyncInput =
   | TAwsParameterStoreSyncInput
@@ -109,7 +117,8 @@ export type TSecretSyncInput =
   | TTerraformCloudSyncInput
   | TCamundaSyncInput
   | TVercelSyncInput
-  | TWindmillSyncInput;
+  | TWindmillSyncInput
+  | THCVaultSyncInput;
 
 export type TSecretSyncListItem =
   | TAwsParameterStoreSyncListItem
@@ -123,7 +132,8 @@ export type TSecretSyncListItem =
   | TTerraformCloudSyncListItem
   | TCamundaSyncListItem
   | TVercelSyncListItem
-  | TWindmillSyncListItem;
+  | TWindmillSyncListItem
+  | THCVaultSyncListItem;
 
 export type TSyncOptionsConfig = {
   canImportSecrets: boolean;