feature: windmill connection and sync

backend/src/services/app-connection/app-connection-enums.ts

@@ -10,7 +10,8 @@ export enum AppConnection {
   Vercel = "vercel",
   Postgres = "postgres",
   MsSql = "mssql",
-  Camunda = "camunda"
+  Camunda = "camunda",
+  Windmill = "windmill"
 }
 
 export enum AWSRegion {

backend/src/services/app-connection/app-connection-fns.ts

@@ -49,6 +49,11 @@ import {
 } from "./terraform-cloud";
 import { VercelConnectionMethod } from "./vercel";
 import { getVercelConnectionListItem, validateVercelConnectionCredentials } from "./vercel/vercel-connection-fns";
+import {
+  getWindmillConnectionListItem,
+  validateWindmillConnectionCredentials,
+  WindmillConnectionMethod
+} from "./windmill";
 
 export const listAppConnectionOptions = () => {
   return [
@@ -63,7 +68,8 @@ export const listAppConnectionOptions = () => {
     getVercelConnectionListItem(),
     getPostgresConnectionListItem(),
     getMsSqlConnectionListItem(),
-    getCamundaConnectionListItem()
+    getCamundaConnectionListItem(),
+    getWindmillConnectionListItem()
   ].sort((a, b) => a.name.localeCompare(b.name));
 };
 
@@ -122,7 +128,8 @@ const VALIDATE_APP_CONNECTION_CREDENTIALS_MAP: Record<AppConnection, TAppConnect
   [AppConnection.MsSql]: validateSqlConnectionCredentials as TAppConnectionCredentialsValidator,
   [AppConnection.TerraformCloud]: validateTerraformCloudConnectionCredentials as TAppConnectionCredentialsValidator,
   [AppConnection.Camunda]: validateCamundaConnectionCredentials as TAppConnectionCredentialsValidator,
-  [AppConnection.Vercel]: validateVercelConnectionCredentials as TAppConnectionCredentialsValidator
+  [AppConnection.Vercel]: validateVercelConnectionCredentials as TAppConnectionCredentialsValidator,
+  [AppConnection.Windmill]: validateWindmillConnectionCredentials as TAppConnectionCredentialsValidator
 };
 
 export const validateAppConnectionCredentials = async (
@@ -154,6 +161,8 @@ export const getAppConnectionMethodName = (method: TAppConnection["method"]) =>
     case PostgresConnectionMethod.UsernameAndPassword:
     case MsSqlConnectionMethod.UsernameAndPassword:
       return "Username & Password";
+    case WindmillConnectionMethod.AccessToken:
+      return "Access Token";
     default:
       // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
       throw new Error(`Unhandled App Connection Method: ${method}`);
@@ -196,5 +205,6 @@ export const TRANSITION_CONNECTION_CREDENTIALS_TO_PLATFORM: Record<
   [AppConnection.MsSql]: transferSqlConnectionCredentialsToPlatform as TAppConnectionTransitionCredentialsToPlatform,
   [AppConnection.TerraformCloud]: platformManagedCredentialsNotSupported,
   [AppConnection.Camunda]: platformManagedCredentialsNotSupported,
-  [AppConnection.Vercel]: platformManagedCredentialsNotSupported
+  [AppConnection.Vercel]: platformManagedCredentialsNotSupported,
+  [AppConnection.Windmill]: platformManagedCredentialsNotSupported
 };

backend/src/services/app-connection/app-connection-maps.ts

@@ -12,5 +12,6 @@ export const APP_CONNECTION_NAME_MAP: Record<AppConnection, string> = {
   [AppConnection.Vercel]: "Vercel",
   [AppConnection.Postgres]: "PostgreSQL",
   [AppConnection.MsSql]: "Microsoft SQL Server",
-  [AppConnection.Camunda]: "Camunda"
+  [AppConnection.Camunda]: "Camunda",
+  [AppConnection.Windmill]: "Windmill"
 };

backend/src/services/app-connection/app-connection-service.ts

@@ -47,6 +47,8 @@ import { ValidateTerraformCloudConnectionCredentialsSchema } from "./terraform-c
 import { terraformCloudConnectionService } from "./terraform-cloud/terraform-cloud-connection-service";
 import { ValidateVercelConnectionCredentialsSchema } from "./vercel";
 import { vercelConnectionService } from "./vercel/vercel-connection-service";
+import { ValidateWindmillConnectionCredentialsSchema } from "./windmill";
+import { windmillConnectionService } from "./windmill/windmill-connection-service";
 
 export type TAppConnectionServiceFactoryDep = {
   appConnectionDAL: TAppConnectionDALFactory;
@@ -68,7 +70,8 @@ const VALIDATE_APP_CONNECTION_CREDENTIALS_MAP: Record<AppConnection, TValidateAp
   [AppConnection.Vercel]: ValidateVercelConnectionCredentialsSchema,
   [AppConnection.Postgres]: ValidatePostgresConnectionCredentialsSchema,
   [AppConnection.MsSql]: ValidateMsSqlConnectionCredentialsSchema,
-  [AppConnection.Camunda]: ValidateCamundaConnectionCredentialsSchema
+  [AppConnection.Camunda]: ValidateCamundaConnectionCredentialsSchema,
+  [AppConnection.Windmill]: ValidateWindmillConnectionCredentialsSchema
 };
 
 export const appConnectionServiceFactory = ({
@@ -442,6 +445,7 @@ export const appConnectionServiceFactory = ({
     humanitec: humanitecConnectionService(connectAppConnectionById),
     terraformCloud: terraformCloudConnectionService(connectAppConnectionById),
     camunda: camundaConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
-    vercel: vercelConnectionService(connectAppConnectionById)
+    vercel: vercelConnectionService(connectAppConnectionById),
+    windmill: windmillConnectionService(connectAppConnectionById)
   };
 };

backend/src/services/app-connection/app-connection-types.ts

@@ -69,6 +69,12 @@ import {
   TVercelConnectionConfig,
   TVercelConnectionInput
 } from "./vercel";
+import {
+  TValidateWindmillConnectionCredentialsSchema,
+  TWindmillConnection,
+  TWindmillConnectionConfig,
+  TWindmillConnectionInput
+} from "./windmill";
 
 export type TAppConnection = { id: string } & (
   | TAwsConnection
@@ -83,6 +89,7 @@ export type TAppConnection = { id: string } & (
   | TPostgresConnection
   | TMsSqlConnection
   | TCamundaConnection
+  | TWindmillConnection
 );
 
 export type TAppConnectionRaw = NonNullable<Awaited<ReturnType<TAppConnectionDALFactory["findById"]>>>;
@@ -102,6 +109,7 @@ export type TAppConnectionInput = { id: string } & (
   | TPostgresConnectionInput
   | TMsSqlConnectionInput
   | TCamundaConnectionInput
+  | TWindmillConnectionInput
 );
 
 export type TSqlConnectionInput = TPostgresConnectionInput | TMsSqlConnectionInput;
@@ -126,7 +134,8 @@ export type TAppConnectionConfig =
   | TTerraformCloudConnectionConfig
   | TVercelConnectionConfig
   | TSqlConnectionConfig
-  | TCamundaConnectionConfig;
+  | TCamundaConnectionConfig
+  | TWindmillConnectionConfig;
 
 export type TValidateAppConnectionCredentialsSchema =
   | TValidateAwsConnectionCredentialsSchema
@@ -140,7 +149,8 @@ export type TValidateAppConnectionCredentialsSchema =
   | TValidateMsSqlConnectionCredentialsSchema
   | TValidateCamundaConnectionCredentialsSchema
   | TValidateTerraformCloudConnectionCredentialsSchema
-  | TValidateVercelConnectionCredentialsSchema;
+  | TValidateVercelConnectionCredentialsSchema
+  | TValidateWindmillConnectionCredentialsSchema;
 
 export type TListAwsConnectionKmsKeys = {
   connectionId: string;

backend/src/services/app-connection/terraform-cloud/terraform-cloud-connection-fns.ts

@@ -44,7 +44,7 @@ export const validateTerraformCloudConnectionCredentials = async (config: TTerra
       });
     }
     throw new BadRequestError({
-      message: "Unable to validate connection - verify credentials"
+      message: "Unable to validate connection: verify credentials"
     });
   }
 

backend/src/services/app-connection/vercel/vercel-connection-fns.ts

@@ -42,7 +42,7 @@ export const validateVercelConnectionCredentials = async (config: TVercelConnect
       });
     }
     throw new BadRequestError({
-      message: "Unable to validate connection - verify credentials"
+      message: "Unable to validate connection: verify credentials"
     });
   }
 

backend/src/services/app-connection/windmill/index.ts

@@ -0,0 +1,4 @@
+export * from "./windmill-connection-enums";
+export * from "./windmill-connection-fns";
+export * from "./windmill-connection-schemas";
+export * from "./windmill-connection-types";

backend/src/services/app-connection/windmill/windmill-connection-enums.ts

@@ -0,0 +1,3 @@
+export enum WindmillConnectionMethod {
+  AccessToken = "access-token"
+}

backend/src/services/app-connection/windmill/windmill-connection-fns.ts

@@ -0,0 +1,65 @@
+import { AxiosError } from "axios";
+
+import { request } from "@app/lib/config/request";
+import { BadRequestError } from "@app/lib/errors";
+import { removeTrailingSlash } from "@app/lib/fn";
+import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+
+import { WindmillConnectionMethod } from "./windmill-connection-enums";
+import { TWindmillConnection, TWindmillConnectionConfig, TWindmillWorkspace } from "./windmill-connection-types";
+
+export const getWindmillInstanceUrl = async (config: TWindmillConnectionConfig) => {
+  const instanceUrl = config.credentials.instanceUrl
+    ? removeTrailingSlash(config.credentials.instanceUrl)
+    : "https://app.windmill.dev";
+
+  await blockLocalAndPrivateIpAddresses(instanceUrl);
+
+  return instanceUrl;
+};
+
+export const getWindmillConnectionListItem = () => {
+  return {
+    name: "Windmill" as const,
+    app: AppConnection.Windmill as const,
+    methods: Object.values(WindmillConnectionMethod) as [WindmillConnectionMethod.AccessToken]
+  };
+};
+
+export const validateWindmillConnectionCredentials = async (config: TWindmillConnectionConfig) => {
+  const instanceUrl = await getWindmillInstanceUrl(config);
+  const { accessToken } = config.credentials;
+
+  try {
+    await request.get(`${instanceUrl}/api/workspaces/list`, {
+      headers: {
+        Authorization: `Bearer ${accessToken}`
+      }
+    });
+  } catch (error: unknown) {
+    if (error instanceof AxiosError) {
+      throw new BadRequestError({
+        message: `Failed to validate credentials: ${error.message || "Unknown error"}`
+      });
+    }
+    throw new BadRequestError({
+      message: "Unable to validate connection: verify credentials"
+    });
+  }
+
+  return config.credentials;
+};
+
+export const listWindmillWorkspaces = async (appConnection: TWindmillConnection) => {
+  const instanceUrl = await getWindmillInstanceUrl(appConnection);
+  const { accessToken } = appConnection.credentials;
+
+  const resp = await request.get<TWindmillWorkspace[]>(`${instanceUrl}/api/workspaces/list`, {
+    headers: {
+      Authorization: `Bearer ${accessToken}`
+    }
+  });
+
+  return resp.data.filter((workspace) => !workspace.deleted);
+};

backend/src/services/app-connection/windmill/windmill-connection-schemas.ts

@@ -0,0 +1,70 @@
+import z from "zod";
+
+import { AppConnections } from "@app/lib/api-docs";
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+import {
+  BaseAppConnectionSchema,
+  GenericCreateAppConnectionFieldsSchema,
+  GenericUpdateAppConnectionFieldsSchema
+} from "@app/services/app-connection/app-connection-schemas";
+
+import { WindmillConnectionMethod } from "./windmill-connection-enums";
+
+export const WindmillConnectionAccessTokenCredentialsSchema = z.object({
+  accessToken: z
+    .string()
+    .trim()
+    .min(1, "Access Token required")
+    .describe(AppConnections.CREDENTIALS.WINDMILL.accessToken),
+  instanceUrl: z
+    .string()
+    .trim()
+    .url("Invalid Instance URL")
+    .optional()
+    .describe(AppConnections.CREDENTIALS.WINDMILL.instanceUrl)
+});
+
+const BaseWindmillConnectionSchema = BaseAppConnectionSchema.extend({ app: z.literal(AppConnection.Windmill) });
+
+export const WindmillConnectionSchema = BaseWindmillConnectionSchema.extend({
+  method: z.literal(WindmillConnectionMethod.AccessToken),
+  credentials: WindmillConnectionAccessTokenCredentialsSchema
+});
+
+export const SanitizedWindmillConnectionSchema = z.discriminatedUnion("method", [
+  BaseWindmillConnectionSchema.extend({
+    method: z.literal(WindmillConnectionMethod.AccessToken),
+    credentials: WindmillConnectionAccessTokenCredentialsSchema.pick({
+      instanceUrl: true
+    })
+  })
+]);
+
+export const ValidateWindmillConnectionCredentialsSchema = z.discriminatedUnion("method", [
+  z.object({
+    method: z
+      .literal(WindmillConnectionMethod.AccessToken)
+      .describe(AppConnections?.CREATE(AppConnection.Windmill).method),
+    credentials: WindmillConnectionAccessTokenCredentialsSchema.describe(
+      AppConnections.CREATE(AppConnection.Windmill).credentials
+    )
+  })
+]);
+
+export const CreateWindmillConnectionSchema = ValidateWindmillConnectionCredentialsSchema.and(
+  GenericCreateAppConnectionFieldsSchema(AppConnection.Windmill)
+);
+
+export const UpdateWindmillConnectionSchema = z
+  .object({
+    credentials: WindmillConnectionAccessTokenCredentialsSchema.optional().describe(
+      AppConnections.UPDATE(AppConnection.Windmill).credentials
+    )
+  })
+  .and(GenericUpdateAppConnectionFieldsSchema(AppConnection.Windmill));
+
+export const WindmillConnectionListItemSchema = z.object({
+  name: z.literal("Windmill"),
+  app: z.literal(AppConnection.Windmill),
+  methods: z.nativeEnum(WindmillConnectionMethod).array()
+});

backend/src/services/app-connection/windmill/windmill-connection-service.ts

@@ -0,0 +1,28 @@
+import { OrgServiceActor } from "@app/lib/types";
+
+import { AppConnection } from "../app-connection-enums";
+import { listWindmillWorkspaces } from "./windmill-connection-fns";
+import { TWindmillConnection } from "./windmill-connection-types";
+
+type TGetAppConnectionFunc = (
+  app: AppConnection,
+  connectionId: string,
+  actor: OrgServiceActor
+) => Promise<TWindmillConnection>;
+
+export const windmillConnectionService = (getAppConnection: TGetAppConnectionFunc) => {
+  const listWorkspaces = async (connectionId: string, actor: OrgServiceActor) => {
+    const appConnection = await getAppConnection(AppConnection.Windmill, connectionId, actor);
+
+    try {
+      const workspaces = await listWindmillWorkspaces(appConnection);
+      return workspaces;
+    } catch (error) {
+      return [];
+    }
+  };
+
+  return {
+    listWorkspaces
+  };
+};

backend/src/services/app-connection/windmill/windmill-connection-types.ts

@@ -0,0 +1,27 @@
+import z from "zod";
+
+import { DiscriminativePick } from "@app/lib/types";
+
+import { AppConnection } from "../app-connection-enums";
+import {
+  CreateWindmillConnectionSchema,
+  ValidateWindmillConnectionCredentialsSchema,
+  WindmillConnectionSchema
+} from "./windmill-connection-schemas";
+
+export type TWindmillConnection = z.infer<typeof WindmillConnectionSchema>;
+
+export type TWindmillConnectionInput = z.infer<typeof CreateWindmillConnectionSchema> & {
+  app: AppConnection.Windmill;
+};
+
+export type TValidateWindmillConnectionCredentialsSchema = typeof ValidateWindmillConnectionCredentialsSchema;
+
+export type TWindmillConnectionConfig = DiscriminativePick<
+  TWindmillConnectionInput,
+  "method" | "app" | "credentials"
+> & {
+  orgId: string;
+};
+
+export type TWindmillWorkspace = { id: string; name: string; deleted: boolean };

backend/src/services/secret-sync/secret-sync-enums.ts

@@ -9,7 +9,8 @@ export enum SecretSync {
   Humanitec = "humanitec",
   TerraformCloud = "terraform-cloud",
   Camunda = "camunda",
-  Vercel = "vercel"
+  Vercel = "vercel",
+  Windmill = "windmill"
 }
 
 export enum SecretSyncInitialSyncBehavior {

backend/src/services/secret-sync/secret-sync-fns.ts

@@ -29,6 +29,7 @@ import { HUMANITEC_SYNC_LIST_OPTION } from "./humanitec";
 import { HumanitecSyncFns } from "./humanitec/humanitec-sync-fns";
 import { TERRAFORM_CLOUD_SYNC_LIST_OPTION, TerraformCloudSyncFns } from "./terraform-cloud";
 import { VERCEL_SYNC_LIST_OPTION, VercelSyncFns } from "./vercel";
+import { WINDMILL_SYNC_LIST_OPTION, WindmillSyncFns } from "./windmill";
 
 const SECRET_SYNC_LIST_OPTIONS: Record<SecretSync, TSecretSyncListItem> = {
   [SecretSync.AWSParameterStore]: AWS_PARAMETER_STORE_SYNC_LIST_OPTION,
@@ -41,7 +42,8 @@ const SECRET_SYNC_LIST_OPTIONS: Record<SecretSync, TSecretSyncListItem> = {
   [SecretSync.Humanitec]: HUMANITEC_SYNC_LIST_OPTION,
   [SecretSync.TerraformCloud]: TERRAFORM_CLOUD_SYNC_LIST_OPTION,
   [SecretSync.Camunda]: CAMUNDA_SYNC_LIST_OPTION,
-  [SecretSync.Vercel]: VERCEL_SYNC_LIST_OPTION
+  [SecretSync.Vercel]: VERCEL_SYNC_LIST_OPTION,
+  [SecretSync.Windmill]: WINDMILL_SYNC_LIST_OPTION
 };
 
 export const listSecretSyncOptions = () => {
@@ -136,6 +138,8 @@ export const SecretSyncFns = {
         }).syncSecrets(secretSync, secretMap);
       case SecretSync.Vercel:
         return VercelSyncFns.syncSecrets(secretSync, secretMap);
+      case SecretSync.Windmill:
+        return WindmillSyncFns.syncSecrets(secretSync, secretMap);
       default:
         throw new Error(
           `Unhandled sync destination for sync secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`
@@ -192,6 +196,9 @@ export const SecretSyncFns = {
       case SecretSync.Vercel:
         secretMap = await VercelSyncFns.getSecrets(secretSync);
         break;
+      case SecretSync.Windmill:
+        secretMap = await WindmillSyncFns.getSecrets(secretSync);
+        break;
       default:
         throw new Error(
           `Unhandled sync destination for get secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`
@@ -243,6 +250,8 @@ export const SecretSyncFns = {
         }).removeSecrets(secretSync, secretMap);
       case SecretSync.Vercel:
         return VercelSyncFns.removeSecrets(secretSync, secretMap);
+      case SecretSync.Windmill:
+        return WindmillSyncFns.removeSecrets(secretSync, secretMap);
       default:
         throw new Error(
           `Unhandled sync destination for remove secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`

backend/src/services/secret-sync/secret-sync-maps.ts

@@ -12,7 +12,8 @@ export const SECRET_SYNC_NAME_MAP: Record<SecretSync, string> = {
   [SecretSync.Humanitec]: "Humanitec",
   [SecretSync.TerraformCloud]: "Terraform Cloud",
   [SecretSync.Camunda]: "Camunda",
-  [SecretSync.Vercel]: "Vercel"
+  [SecretSync.Vercel]: "Vercel",
+  [SecretSync.Windmill]: "Windmill"
 };
 
 export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
@@ -26,5 +27,6 @@ export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
   [SecretSync.Humanitec]: AppConnection.Humanitec,
   [SecretSync.TerraformCloud]: AppConnection.TerraformCloud,
   [SecretSync.Camunda]: AppConnection.Camunda,
-  [SecretSync.Vercel]: AppConnection.Vercel
+  [SecretSync.Vercel]: AppConnection.Vercel,
+  [SecretSync.Windmill]: AppConnection.Windmill
 };

backend/src/services/secret-sync/secret-sync-queue.ts

@@ -76,6 +76,7 @@ type TSecretSyncQueueFactoryDep = {
     | "findBySecretKeys"
     | "bulkUpdate"
     | "deleteMany"
+    | "invalidateSecretCacheByProjectId"
   >;
   secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
   secretSyncDAL: Pick<TSecretSyncDALFactory, "findById" | "find" | "updateById" | "deleteById">;
@@ -382,6 +383,9 @@ export const secretSyncQueueFactory = ({
       });
     }
 
+    if (secretsToUpdate.length || secretsToCreate.length)
+      await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
+
     return importedSecretMap;
   };
 

backend/src/services/secret-sync/secret-sync-types.ts

@@ -29,6 +29,12 @@ import {
 } from "@app/services/secret-sync/github";
 import { TSecretSyncDALFactory } from "@app/services/secret-sync/secret-sync-dal";
 import { SecretSync, SecretSyncImportBehavior } from "@app/services/secret-sync/secret-sync-enums";
+import {
+  TWindmillSync,
+  TWindmillSyncInput,
+  TWindmillSyncListItem,
+  TWindmillSyncWithCredentials
+} from "@app/services/secret-sync/windmill";
 
 import {
   TAwsParameterStoreSync,
@@ -74,7 +80,8 @@ export type TSecretSync =
   | THumanitecSync
   | TTerraformCloudSync
   | TCamundaSync
-  | TVercelSync;
+  | TVercelSync
+  | TWindmillSync;
 
 export type TSecretSyncWithCredentials =
   | TAwsParameterStoreSyncWithCredentials
@@ -87,7 +94,8 @@ export type TSecretSyncWithCredentials =
   | THumanitecSyncWithCredentials
   | TTerraformCloudSyncWithCredentials
   | TCamundaSyncWithCredentials
-  | TVercelSyncWithCredentials;
+  | TVercelSyncWithCredentials
+  | TWindmillSyncWithCredentials;
 
 export type TSecretSyncInput =
   | TAwsParameterStoreSyncInput
@@ -100,7 +108,8 @@ export type TSecretSyncInput =
   | THumanitecSyncInput
   | TTerraformCloudSyncInput
   | TCamundaSyncInput
-  | TVercelSyncInput;
+  | TVercelSyncInput
+  | TWindmillSyncInput;
 
 export type TSecretSyncListItem =
   | TAwsParameterStoreSyncListItem
@@ -113,7 +122,8 @@ export type TSecretSyncListItem =
   | THumanitecSyncListItem
   | TTerraformCloudSyncListItem
   | TCamundaSyncListItem
-  | TVercelSyncListItem;
+  | TVercelSyncListItem
+  | TWindmillSyncListItem;
 
 export type TSyncOptionsConfig = {
   canImportSecrets: boolean;

backend/src/services/secret-sync/windmill/index.ts

@@ -0,0 +1,4 @@
+export * from "./windmill-sync-constants";
+export * from "./windmill-sync-fns";
+export * from "./windmill-sync-schemas";
+export * from "./windmill-sync-types";

backend/src/services/secret-sync/windmill/windmill-sync-constants.ts

@@ -0,0 +1,10 @@
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
+import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
+
+export const WINDMILL_SYNC_LIST_OPTION: TSecretSyncListItem = {
+  name: "Windmill",
+  destination: SecretSync.Windmill,
+  connection: AppConnection.Windmill,
+  canImportSecrets: true
+};

backend/src/services/secret-sync/windmill/windmill-sync-fns.ts

@@ -0,0 +1,232 @@
+import { request } from "@app/lib/config/request";
+import { getWindmillInstanceUrl } from "@app/services/app-connection/windmill";
+import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
+import {
+  TDeleteWindmillVariable,
+  TPostWindmillVariable,
+  TWindmillListVariables,
+  TWindmillListVariablesResponse,
+  TWindmillSyncWithCredentials,
+  TWindmillVariable
+} from "@app/services/secret-sync/windmill/windmill-sync-types";
+
+import { TSecretMap } from "../secret-sync-types";
+
+const PAGE_LIMIT = 100;
+
+const listWindmillVariables = async ({ instanceUrl, workspace, accessToken, path }: TWindmillListVariables) => {
+  const variables: Record<string, TWindmillVariable> = {};
+
+  // windmill paginates but doesn't return if there's more pages so we need to check if page size full
+  let page: number | null = 1;
+
+  while (page) {
+    // eslint-disable-next-line no-await-in-loop
+    const { data: variablesPage } = await request.get<TWindmillListVariablesResponse>(
+      `${instanceUrl}/api/w/${workspace}/variables/list`,
+      {
+        headers: {
+          Authorization: `Bearer ${accessToken}`
+        },
+        params: {
+          page,
+          limit: PAGE_LIMIT,
+          path_start: path
+        }
+      }
+    );
+
+    // eslint-disable-next-line no-await-in-loop
+    for await (const variable of variablesPage) {
+      const variableName = variable.path.replace(path, "");
+
+      if (variable.is_secret) {
+        const { data: variableValue } = await request.get<string>(
+          `${instanceUrl}/api/w/${workspace}/variables/get_value/${variable.path}`,
+          {
+            headers: {
+              Authorization: `Bearer ${accessToken}`
+            }
+          }
+        );
+
+        variables[variableName] = {
+          ...variable,
+          value: variableValue
+        };
+      } else {
+        variables[variableName] = variable;
+      }
+    }
+
+    if (variablesPage.length >= PAGE_LIMIT) {
+      page += 1;
+    } else {
+      page = null;
+    }
+  }
+
+  return variables;
+};
+
+const createWindmillVariable = async ({
+  path,
+  value,
+  instanceUrl,
+  accessToken,
+  workspace,
+  description
+}: TPostWindmillVariable) =>
+  request.post(
+    `${instanceUrl}/api/w/${workspace}/variables/create`,
+    {
+      path,
+      value,
+      is_secret: true,
+      description
+    },
+    {
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+        "Content-Type": "application/json"
+      }
+    }
+  );
+
+const updateWindmillVariable = async ({
+  path,
+  value,
+  instanceUrl,
+  accessToken,
+  workspace,
+  description
+}: TPostWindmillVariable) =>
+  request.post(
+    `${instanceUrl}/api/w/${workspace}/variables/update/${path}`,
+    {
+      value,
+      is_secret: true,
+      description
+    },
+    {
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+        "Content-Type": "application/json"
+      }
+    }
+  );
+
+const deleteWindmillVariable = async ({ path, instanceUrl, accessToken, workspace }: TDeleteWindmillVariable) =>
+  request.delete(`${instanceUrl}/api/w/${workspace}/variables/delete/${path}`, {
+    headers: {
+      Authorization: `Bearer ${accessToken}`
+    }
+  });
+
+export const WindmillSyncFns = {
+  syncSecrets: async (secretSync: TWindmillSyncWithCredentials, secretMap: TSecretMap) => {
+    const {
+      connection,
+      destinationConfig: { workspace, path },
+      syncOptions: { disableSecretDeletion }
+    } = secretSync;
+
+    const instanceUrl = await getWindmillInstanceUrl(connection);
+
+    const { accessToken } = connection.credentials;
+
+    const variables = await listWindmillVariables({ instanceUrl, accessToken, workspace, path });
+
+    for await (const entry of Object.entries(secretMap)) {
+      const [key, { value, comment = "" }] = entry;
+
+      try {
+        const payload = {
+          instanceUrl,
+          workspace,
+          path: path + key,
+          value,
+          accessToken,
+          description: comment
+        };
+        if (key in variables) {
+          if (variables[key].value !== value || variables[key].description !== comment)
+            await updateWindmillVariable(payload);
+        } else {
+          await createWindmillVariable(payload);
+        }
+      } catch (error) {
+        throw new SecretSyncError({
+          error,
+          secretKey: key
+        });
+      }
+    }
+
+    if (disableSecretDeletion) return;
+
+    for await (const [key, variable] of Object.entries(variables)) {
+      if (!(key in secretMap)) {
+        try {
+          await deleteWindmillVariable({
+            instanceUrl,
+            workspace,
+            path: variable.path,
+            accessToken
+          });
+        } catch (error) {
+          throw new SecretSyncError({
+            error,
+            secretKey: key
+          });
+        }
+      }
+    }
+  },
+  removeSecrets: async (secretSync: TWindmillSyncWithCredentials, secretMap: TSecretMap) => {
+    const {
+      connection,
+      destinationConfig: { workspace, path }
+    } = secretSync;
+
+    const instanceUrl = await getWindmillInstanceUrl(connection);
+
+    const { accessToken } = connection.credentials;
+
+    const variables = await listWindmillVariables({ instanceUrl, accessToken, workspace, path });
+
+    for await (const [key, variable] of Object.entries(variables)) {
+      if (key in secretMap) {
+        try {
+          await deleteWindmillVariable({
+            path: variable.path,
+            instanceUrl,
+            workspace,
+            accessToken
+          });
+        } catch (error) {
+          throw new SecretSyncError({
+            error,
+            secretKey: key
+          });
+        }
+      }
+    }
+  },
+  getSecrets: async (secretSync: TWindmillSyncWithCredentials) => {
+    const {
+      connection,
+      destinationConfig: { workspace, path }
+    } = secretSync;
+
+    const instanceUrl = await getWindmillInstanceUrl(connection);
+
+    const { accessToken } = connection.credentials;
+
+    const variables = await listWindmillVariables({ instanceUrl, accessToken, workspace, path });
+
+    return Object.fromEntries(
+      Object.entries(variables).map(([key, variable]) => [key, { value: variable.value ?? "" }])
+    );
+  }
+};

backend/src/services/secret-sync/windmill/windmill-sync-schemas.ts

@@ -0,0 +1,66 @@
+import { z } from "zod";
+
+import { SecretSyncs } from "@app/lib/api-docs";
+import { CharacterType, characterValidator } from "@app/lib/validator/validate-string";
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
+import {
+  BaseSecretSyncSchema,
+  GenericCreateSecretSyncFieldsSchema,
+  GenericUpdateSecretSyncFieldsSchema
+} from "@app/services/secret-sync/secret-sync-schemas";
+import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";
+
+const pathCharacterValidator = characterValidator([
+  CharacterType.AlphaNumeric,
+  CharacterType.Underscore,
+  CharacterType.Hyphen
+]);
+
+const WindmillSyncDestinationConfigSchema = z.object({
+  workspace: z.string().trim().min(1, "Workspace required").describe(SecretSyncs.DESTINATION_CONFIG.WINDMILL.workspace),
+  path: z
+    .string()
+    .trim()
+    .min(1, "Path required")
+    .refine(
+      (val) =>
+        (val.startsWith("u/") || val.startsWith("f/")) &&
+        val.endsWith("/") &&
+        val.split("/").length >= 3 &&
+        val
+          .split("/")
+          .filter(Boolean)
+          .every((segment) => pathCharacterValidator(segment)),
+      'Invalid path - must follow Windmill path format. ex: "/f/folder/path/"'
+    )
+    .describe(SecretSyncs.DESTINATION_CONFIG.WINDMILL.path)
+});
+
+const WindmillSyncOptionsConfig: TSyncOptionsConfig = { canImportSecrets: true };
+
+export const WindmillSyncSchema = BaseSecretSyncSchema(SecretSync.Windmill, WindmillSyncOptionsConfig).extend({
+  destination: z.literal(SecretSync.Windmill),
+  destinationConfig: WindmillSyncDestinationConfigSchema
+});
+
+export const CreateWindmillSyncSchema = GenericCreateSecretSyncFieldsSchema(
+  SecretSync.Windmill,
+  WindmillSyncOptionsConfig
+).extend({
+  destinationConfig: WindmillSyncDestinationConfigSchema
+});
+
+export const UpdateWindmillSyncSchema = GenericUpdateSecretSyncFieldsSchema(
+  SecretSync.Windmill,
+  WindmillSyncOptionsConfig
+).extend({
+  destinationConfig: WindmillSyncDestinationConfigSchema.optional()
+});
+
+export const WindmillSyncListItemSchema = z.object({
+  name: z.literal("Windmill"),
+  connection: z.literal(AppConnection.Windmill),
+  destination: z.literal(SecretSync.Windmill),
+  canImportSecrets: z.literal(true)
+});

backend/src/services/secret-sync/windmill/windmill-sync-types.ts

@@ -0,0 +1,39 @@
+import { z } from "zod";
+
+import { TWindmillConnection } from "@app/services/app-connection/windmill";
+
+import { CreateWindmillSyncSchema, WindmillSyncListItemSchema, WindmillSyncSchema } from "./windmill-sync-schemas";
+
+export type TWindmillSync = z.infer<typeof WindmillSyncSchema>;
+
+export type TWindmillSyncInput = z.infer<typeof CreateWindmillSyncSchema>;
+
+export type TWindmillSyncListItem = z.infer<typeof WindmillSyncListItemSchema>;
+
+export type TWindmillSyncWithCredentials = TWindmillSync & {
+  connection: TWindmillConnection;
+};
+
+export type TWindmillVariable = {
+  path: string;
+  value: string;
+  is_secret: boolean;
+  is_oauth: boolean;
+  description: string;
+};
+
+export type TWindmillListVariablesResponse = TWindmillVariable[];
+
+export type TWindmillListVariables = {
+  accessToken: string;
+  instanceUrl: string;
+  path: string;
+  workspace: string;
+  description?: string;
+};
+
+export type TPostWindmillVariable = TWindmillListVariables & {
+  value: string;
+};
+
+export type TDeleteWindmillVariable = TWindmillListVariables;