Merge pull request #765 from sunilk4u/feat/windmill-integration

Feature: Windmill.dev cloud Integeration
This commit is contained in:
BlackMagiq
2023-07-28 02:11:27 +07:00
committed by GitHub
13 changed files with 587 additions and 16 deletions

View File

@@ -275,3 +275,70 @@ export const decryptSymmetricHelper = async ({
return plaintext;
};
/**
* Return decrypted comments for workspace secrets with id [workspaceId]
* and [envionment] using bot
* @param {Object} obj
* @param {String} obj.workspaceId - id of workspace
* @param {String} obj.environment - environment
*/
export const getSecretsCommentBotHelper = async ({
workspaceId,
environment,
secretPath
} : {
workspaceId: Types.ObjectId;
environment: string;
secretPath: string;
}) => {
const content = {} as any;
const key = await getKey({ workspaceId: workspaceId });
let folderId = "root";
const folders = await Folder.findOne({
workspace: workspaceId,
environment,
});
if (!folders && secretPath !== "/") {
throw InternalServerError({ message: "Folder not found" });
}
if (folders) {
const folder = getFolderByPath(folders.nodes, secretPath);
if (!folder) {
throw InternalServerError({ message: "Folder not found" });
}
folderId = folder.id;
}
const secrets = await Secret.find({
workspace: workspaceId,
environment,
type: SECRET_SHARED,
folder: folderId,
});
secrets.forEach((secret: ISecret) => {
if(secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) {
const secretKey = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretKeyCiphertext,
iv: secret.secretKeyIV,
tag: secret.secretKeyTag,
key,
});
const commentValue = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretCommentCiphertext,
iv: secret.secretCommentIV,
tag: secret.secretCommentTag,
key,
});
content[secretKey] = commentValue;
}
});
return content;
}

View File

@@ -123,7 +123,7 @@ export const syncIntegrationsHelper = async ({
? {
environment,
}
: {}),
: {}),
isActive: true,
app: { $ne: null },
});
@@ -133,17 +133,24 @@ export const syncIntegrationsHelper = async ({
for await (const integration of integrations) {
// get workspace, environment (shared) secrets
const secrets = await BotService.getSecrets({
// issue here?
workspaceId: integration.workspace,
environment: integration.environment,
secretPath: integration.secretPath,
});
// get workspace, environment (shared) secrets comments
const secretComments = await BotService.getSecretComments({
workspaceId: integration.workspace,
environment: integration.environment,
secretPath: integration.secretPath,
})
const integrationAuth = await IntegrationAuth.findById(
integration.integrationAuth
);
if (!integrationAuth) throw new Error("Failed to find integration auth");
if (!integrationAuth) throw new Error("Failed to find integration auth");
// get integration auth access token
const access = await getIntegrationAuthAccessHelper({
integrationAuthId: integration.integrationAuth,
@@ -156,6 +163,7 @@ export const syncIntegrationsHelper = async ({
secrets,
accessId: access.accessId === undefined ? null : access.accessId,
accessToken: access.accessToken,
secretComments
});
}
} catch (err) {

View File

@@ -40,7 +40,9 @@ import {
INTEGRATION_TRAVISCI,
INTEGRATION_TRAVISCI_API_URL,
INTEGRATION_VERCEL,
INTEGRATION_VERCEL_API_URL
INTEGRATION_VERCEL_API_URL,
INTEGRATION_WINDMILL,
INTEGRATION_WINDMILL_API_URL,
} from "../variables";
import { IIntegrationAuth } from "../models";
import { Octokit } from "@octokit/rest";
@@ -181,6 +183,11 @@ const getApps = async ({
accessToken,
});
break;
case INTEGRATION_WINDMILL:
apps = await getAppsWindmill({
accessToken
});
break;
case INTEGRATION_DIGITAL_OCEAN_APP_PLATFORM:
apps = await getAppsDigitalOceanAppPlatform({
accessToken
@@ -941,6 +948,106 @@ const getAppsCodefresh = async ({
};
/**
* Return list of projects for Windmill integration
* @param {Object} obj
* @param {String} obj.accessToken - access token for Windmill API
* @returns {Object[]} apps - names of Windmill workspaces
* @returns {String} apps.name - name of Windmill workspace
*/
const getAppsWindmill = async ({ accessToken }: { accessToken: string }) => {
const { data } = await standardRequest.get(
`${INTEGRATION_WINDMILL_API_URL}/workspaces/list`,
{
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
);
// check for write access of secrets in windmill workspaces
const writeAccessCheck = data.map(async (app: any) => {
try {
const userPath = "u/user/variable";
const folderPath = "f/folder/variable";
const { data: writeUser } = await standardRequest.post(
`${INTEGRATION_WINDMILL_API_URL}/w/${app.id}/variables/create`,
{
path: userPath,
value: "variable",
is_secret: true,
description: "variable description"
},
{
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
);
const { data: writeFolder } = await standardRequest.post(
`${INTEGRATION_WINDMILL_API_URL}/w/${app.id}/variables/create`,
{
path: folderPath,
value: "variable",
is_secret: true,
description: "variable description"
},
{
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
);
// is write access is allowed then delete the created secrets from workspace
if (writeUser && writeFolder) {
await standardRequest.delete(
`${INTEGRATION_WINDMILL_API_URL}/w/${app.id}/variables/delete/${userPath}`,
{
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
);
await standardRequest.delete(
`${INTEGRATION_WINDMILL_API_URL}/w/${app.id}/variables/delete/${folderPath}`,
{
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
);
return app;
} else {
return { error: "cannot write secret" };
}
} catch (err: any) {
return { error: err.message };
}
});
const appsWriteResponses = await Promise.all(writeAccessCheck);
const appsWithWriteAccess = appsWriteResponses.filter((appRes: any) => !appRes.error);
const apps = appsWithWriteAccess.map((a: any) => {
return {
name: a.name,
appId: a.id,
};
});
return apps;
}
/**
* Return list of applications for DigitalOcean App Platform integration
* @param {Object} obj

View File

@@ -49,7 +49,9 @@ import {
INTEGRATION_TRAVISCI,
INTEGRATION_TRAVISCI_API_URL,
INTEGRATION_VERCEL,
INTEGRATION_VERCEL_API_URL
INTEGRATION_VERCEL_API_URL,
INTEGRATION_WINDMILL,
INTEGRATION_WINDMILL_API_URL,
} from "../variables";
import AWS from "aws-sdk";
import { Octokit } from "@octokit/rest";
@@ -65,19 +67,22 @@ import { standardRequest } from "../config/request";
* @param {Object} obj.secrets - secrets to push to integration (object where keys are secret keys and values are secret values)
* @param {String} obj.accessId - access id for integration
* @param {String} obj.accessToken - access token for integration
* @param {Object} obj.secretComments - secret comments to push to integration (object where keys are secret keys and values are comment values)
*/
const syncSecrets = async ({
integration,
integrationAuth,
secrets,
accessId,
accessToken
accessToken,
secretComments
}: {
integration: IIntegration;
integrationAuth: IIntegrationAuth;
secrets: any;
accessId: string | null;
accessToken: string;
secretComments: any;
}) => {
switch (integration.integration) {
case INTEGRATION_AZURE_KEY_VAULT:
@@ -256,7 +261,15 @@ const syncSecrets = async ({
accessToken
});
break;
}
case INTEGRATION_WINDMILL:
await syncSecretsWindmill({
integration,
secrets,
accessToken,
secretComments
});
break;
}
};
/**
@@ -2244,7 +2257,7 @@ const syncSecretsCodefresh = async ({
* @param {IIntegration} obj.integration - integration details
* @param {IIntegrationAuth} obj.integrationAuth - integration auth details
* @param {Object} obj.secrets - secrets to push to integration (object where keys are secret keys and values are secret values)
* @param {String} obj.accessToken - personal access token for DigitalOcean
* @param {String} obj.accessToken - access token for integration
*/
const syncSecretsDigitalOceanAppPlatform = async ({
integration,
@@ -2272,6 +2285,114 @@ const syncSecretsDigitalOceanAppPlatform = async ({
);
}
/**
* Sync/push [secrets] to Windmill with name [integration.app]
* @param {Object} obj
* @param {IIntegration} obj.integration - integration details
* @param {IIntegrationAuth} obj.integrationAuth - integration auth details
* @param {Object} obj.secrets - secrets to push to integration (object where keys are secret keys and values are secret values)
* @param {String} obj.accessToken - access token for windmill integration
* @param {Object} obj.secretComments - secret comments to push to integration (object where keys are secret keys and values are comment values)
*/
const syncSecretsWindmill = async ({
integration,
secrets,
accessToken,
secretComments
}: {
integration: IIntegration;
secrets: any;
accessToken: string;
secretComments: any;
}) => {
interface WindmillSecret {
path: string;
value: string;
is_secret: boolean;
description?: string;
}
// get secrets stored in windmill workspace
const res = (await standardRequest.get(
`${INTEGRATION_WINDMILL_API_URL}/w/${integration.appId}/variables/list`,
{
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
))
.data
.reduce(
(obj: any, secret: WindmillSecret) => ({
...obj,
[secret.path]: secret
}),
{}
);
// eslint-disable-next-line no-useless-escape
const pattern = new RegExp("^(u\/|f\/)[a-zA-Z0-9_-]+\/([a-zA-Z0-9_-]+\/)*[a-zA-Z0-9_-]*[^\/]$");
for await (const key of Object.keys(secrets)) {
if((key.startsWith("u/") || key.startsWith("f/")) && pattern.test(key)) {
if(!(key in res)) {
// case: secret does not exist in windmill
// -> create secret
await standardRequest.post(
`${INTEGRATION_WINDMILL_API_URL}/w/${integration.appId}/variables/create`,
{
path: key,
value: secrets[key],
is_secret: true,
description: secretComments[key] || ""
},
{
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
);
} else {
// -> update secret
await standardRequest.post(
`${INTEGRATION_WINDMILL_API_URL}/w/${integration.appId}/variables/update/${res[key].path}`,
{
path: key,
value: secrets[key],
is_secret: true,
description: secretComments[key] || ""
},
{
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
);
}
}
}
for await (const key of Object.keys(res)) {
if (!(key in secrets)) {
// -> delete secret
await standardRequest.delete(
`${INTEGRATION_WINDMILL_API_URL}/w/${integration.appId}/variables/delete/${res[key].path}`,
{
headers: {
Authorization: `Bearer ${accessToken}`,
"Content-Type": "application/json",
"Accept-Encoding": "application/json",
}
}
);
}
}
}
/**
* Sync/push [secrets] to Cloud66 application with name [integration.app]
* @param {Object} obj

View File

@@ -22,7 +22,8 @@ import {
INTEGRATION_SUPABASE,
INTEGRATION_TERRAFORM_CLOUD,
INTEGRATION_TRAVISCI,
INTEGRATION_VERCEL
INTEGRATION_VERCEL,
INTEGRATION_WINDMILL
} from "../variables";
import { Schema, Types, model } from "mongoose";
@@ -67,6 +68,7 @@ export interface IIntegration {
| "digital-ocean-app-platform"
| "cloud-66"
| "northflank"
| "windmill";
integrationAuth: Types.ObjectId;
}
@@ -157,9 +159,10 @@ const integrationSchema = new Schema<IIntegration>(
INTEGRATION_TERRAFORM_CLOUD,
INTEGRATION_HASHICORP_VAULT,
INTEGRATION_CLOUDFLARE_PAGES,
INTEGRATION_CODEFRESH,
INTEGRATION_WINDMILL,
INTEGRATION_BITBUCKET,
INTEGRATION_DIGITAL_OCEAN_APP_PLATFORM,
INTEGRATION_CODEFRESH,
INTEGRATION_CLOUD_66,
INTEGRATION_NORTHFLANK
],

View File

@@ -24,7 +24,8 @@ import {
INTEGRATION_SUPABASE,
INTEGRATION_TERRAFORM_CLOUD,
INTEGRATION_TRAVISCI,
INTEGRATION_VERCEL
INTEGRATION_VERCEL,
INTEGRATION_WINDMILL
} from "../variables";
import { Document, Schema, Types, model } from "mongoose";
@@ -54,7 +55,8 @@ export interface IIntegrationAuth extends Document {
| "bitbucket"
| "cloud-66"
| "terraform-cloud"
| "northflank";
| "northflank"
| "windmill";
teamId: string;
accountId: string;
url: string;
@@ -101,9 +103,10 @@ const integrationAuthSchema = new Schema<IIntegrationAuth>(
INTEGRATION_TERRAFORM_CLOUD,
INTEGRATION_HASHICORP_VAULT,
INTEGRATION_CLOUDFLARE_PAGES,
INTEGRATION_CODEFRESH,
INTEGRATION_WINDMILL,
INTEGRATION_BITBUCKET,
INTEGRATION_DIGITAL_OCEAN_APP_PLATFORM,
INTEGRATION_CODEFRESH,
INTEGRATION_CLOUD_66,
INTEGRATION_NORTHFLANK
],

View File

@@ -5,6 +5,7 @@ import {
getIsWorkspaceE2EEHelper,
getKey,
getSecretsBotHelper,
getSecretsCommentBotHelper,
} from "../helpers/bot";
/**
@@ -107,6 +108,30 @@ class BotService {
tag,
});
}
/**
* Return decrypted secret comments for workspace with id [worskpaceId] and
* environment [environment] shared to bot.
* @param {Object} obj
* @param {String} obj.workspaceId - id of workspace of secrets
* @param {String} obj.environment - environment for secrets
* @returns {Object} secretObj - object where keys are secret keys and values are comments
*/
static async getSecretComments({
workspaceId,
environment,
secretPath
}: {
workspaceId: Types.ObjectId;
environment: string;
secretPath: string;
}) {
return await getSecretsCommentBotHelper({
workspaceId,
environment,
secretPath
});
}
}
export default BotService;

View File

@@ -30,6 +30,7 @@ export const INTEGRATION_HASHICORP_VAULT = "hashicorp-vault";
export const INTEGRATION_CLOUDFLARE_PAGES = "cloudflare-pages";
export const INTEGRATION_BITBUCKET = "bitbucket";
export const INTEGRATION_CODEFRESH = "codefresh";
export const INTEGRATION_WINDMILL = "windmill";
export const INTEGRATION_DIGITAL_OCEAN_APP_PLATFORM = "digital-ocean-app-platform";
export const INTEGRATION_CLOUD_66 = "cloud-66";
export const INTEGRATION_NORTHFLANK = "northflank";
@@ -50,9 +51,10 @@ export const INTEGRATION_SET = new Set([
INTEGRATION_TERRAFORM_CLOUD,
INTEGRATION_HASHICORP_VAULT,
INTEGRATION_CLOUDFLARE_PAGES,
INTEGRATION_CODEFRESH,
INTEGRATION_WINDMILL,
INTEGRATION_BITBUCKET,
INTEGRATION_DIGITAL_OCEAN_APP_PLATFORM,
INTEGRATION_CODEFRESH,
INTEGRATION_CLOUD_66,
INTEGRATION_NORTHFLANK
]);
@@ -88,6 +90,7 @@ export const INTEGRATION_TERRAFORM_CLOUD_API_URL = "https://app.terraform.io";
export const INTEGRATION_CLOUDFLARE_PAGES_API_URL = "https://api.cloudflare.com";
export const INTEGRATION_BITBUCKET_API_URL = "https://api.bitbucket.org";
export const INTEGRATION_CODEFRESH_API_URL = "https://g.codefresh.io/api";
export const INTEGRATION_WINDMILL_API_URL = "https://app.windmill.dev/api";
export const INTEGRATION_DIGITAL_OCEAN_API_URL = "https://api.digitalocean.com";
export const INTEGRATION_CLOUD_66_API_URL = "https://app.cloud66.com/api";
export const INTEGRATION_NORTHFLANK_API_URL = "https://api.northflank.com";
@@ -293,6 +296,15 @@ export const getIntegrationOptions = async () => {
clientId: "",
docsLink: "",
},
{
name: "Windmill",
slug: "windmill",
image: "Windmill.png",
isAvailable: true,
type: "pat",
clientId: "",
docsLink: "",
},
{
name: "Digital Ocean App Platform",
slug: "digital-ocean-app-platform",

View File

@@ -26,8 +26,9 @@ const integrationSlugNameMapping: Mapping = {
"digital-ocean-app-platform": "Digital Ocean App Platform",
bitbucket: "BitBucket",
"cloud-66": "Cloud 66",
northflank: "Northflank"
};
northflank: "Northflank",
'windmill': 'Windmill'
}
const envMapping: Mapping = {
Development: "dev",

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.9 KiB

View File

@@ -0,0 +1,65 @@
import { useState } from "react";
import { useRouter } from "next/router";
import { Button, Card, CardTitle, FormControl, Input } from "../../../components/v2";
import saveIntegrationAccessToken from "../../api/integrations/saveIntegrationAccessToken";
export default function WindmillCreateIntegrationPage() {
const router = useRouter();
const [apiKey, setApiKey] = useState("");
const [apiKeyErrorText, setApiKeyErrorText] = useState("");
const [isLoading, setIsLoading] = useState(false);
const handleButtonClick = async () => {
try {
setApiKeyErrorText("");
if (apiKey.length === 0) {
setApiKeyErrorText("API Key cannot be blank");
return;
}
setIsLoading(true);
const integrationAuth = await saveIntegrationAccessToken({
workspaceId: localStorage.getItem("projectData.id"),
integration: "windmill",
accessToken: apiKey,
accessId: null,
url: null,
namespace: null
});
setIsLoading(false);
router.push(`/integrations/windmill/create?integrationAuthId=${integrationAuth._id}`);
} catch (err) {
console.error(err);
}
};
return (
<div className="flex h-full w-full items-center justify-center">
<Card className="max-w-md rounded-md p-8">
<CardTitle className="text-center">Windmill Integration</CardTitle>
<FormControl
label="Windmill Access Token"
errorText={apiKeyErrorText}
isError={apiKeyErrorText !== "" ?? false}
>
<Input placeholder="" value={apiKey} onChange={(e) => setApiKey(e.target.value)} />
</FormControl>
<Button
onClick={handleButtonClick}
color="mineshaft"
className="mt-4"
isLoading={isLoading}
>
Connect to Windmill
</Button>
</Card>
</div>
);
}
WindmillCreateIntegrationPage.requireAuth = true;

View File

@@ -0,0 +1,156 @@
import { useEffect, useState } from "react";
import { useRouter } from "next/router";
import queryString from "query-string";
import {
Button,
Card,
CardTitle,
FormControl,
Input,
Select,
SelectItem
} from "../../../components/v2";
import {
useGetIntegrationAuthApps,
useGetIntegrationAuthById
} from "../../../hooks/api/integrationAuth";
import { useGetWorkspaceById } from "../../../hooks/api/workspace";
import createIntegration from "../../api/integrations/createIntegration";
export default function WindmillCreateIntegrationPage() {
const router = useRouter();
const { integrationAuthId } = queryString.parse(router.asPath.split("?")[1]);
const { data: workspace } = useGetWorkspaceById(localStorage.getItem("projectData.id") ?? "");
const { data: integrationAuth } = useGetIntegrationAuthById((integrationAuthId as string) ?? "");
const { data: integrationAuthApps } = useGetIntegrationAuthApps({
integrationAuthId: (integrationAuthId as string) ?? ""
});
const [selectedSourceEnvironment, setSelectedSourceEnvironment] = useState("");
const [secretPath, setSecretPath] = useState("/");
const [targetApp, setTargetApp] = useState("");
const [isLoading, setIsLoading] = useState(false);
useEffect(() => {
if (workspace) {
setSelectedSourceEnvironment(workspace.environments[0].slug);
}
}, [workspace]);
useEffect(() => {
if (integrationAuthApps) {
if (integrationAuthApps.length > 0) {
setTargetApp(integrationAuthApps[0].name);
} else {
setTargetApp("none");
}
}
}, [integrationAuthApps]);
const handleButtonClick = async () => {
try {
if (!integrationAuth?._id) return;
setIsLoading(true);
await createIntegration({
integrationAuthId: integrationAuth?._id,
isActive: true,
app: targetApp,
appId:
integrationAuthApps?.find((integrationAuthApp) => integrationAuthApp.name === targetApp)
?.appId ?? null,
sourceEnvironment: selectedSourceEnvironment,
targetEnvironment: null,
targetEnvironmentId: null,
targetService: null,
targetServiceId: null,
owner: null,
path: null,
region: null,
secretPath
});
setIsLoading(false);
router.push(`/integrations/${localStorage.getItem("projectData.id")}`);
} catch (err) {
console.error(err);
}
};
return integrationAuth &&
workspace &&
selectedSourceEnvironment &&
integrationAuthApps &&
targetApp ? (
<div className="flex h-full w-full items-center justify-center">
<Card className="max-w-md rounded-md p-8">
<CardTitle className="text-center">Windmill Integration</CardTitle>
<FormControl label="Project Environment" className="mt-4">
<Select
value={selectedSourceEnvironment}
onValueChange={(val) => setSelectedSourceEnvironment(val)}
className="w-full border border-mineshaft-500"
>
{workspace?.environments.map((sourceEnvironment) => (
<SelectItem
value={sourceEnvironment.slug}
key={`source-environment-${sourceEnvironment.slug}`}
>
{sourceEnvironment.name}
</SelectItem>
))}
</Select>
</FormControl>
<FormControl label="Secrets Path">
<Input
value={secretPath}
onChange={(evt) => setSecretPath(evt.target.value)}
placeholder="Provide a path, default is /"
/>
</FormControl>
<FormControl label="Windmill Workspace" className="mt-4">
<Select
value={targetApp}
onValueChange={(val) => setTargetApp(val)}
className="w-full border border-mineshaft-500"
isDisabled={integrationAuthApps.length === 0}
>
{integrationAuthApps.length > 0 ? (
integrationAuthApps.map((integrationAuthApp) => (
<SelectItem
value={integrationAuthApp.name}
key={`target-environment-${integrationAuthApp.name}`}
>
{integrationAuthApp.name}
</SelectItem>
))
) : (
<SelectItem value="none" key="target-app-none">
No projects found
</SelectItem>
)}
</Select>
</FormControl>
<Button
onClick={handleButtonClick}
color="mineshaft"
className="mt-4"
isLoading={isLoading}
isDisabled={integrationAuthApps.length === 0}
>
Create Integration
</Button>
</Card>
</div>
) : (
<div />
);
}
WindmillCreateIntegrationPage.requireAuth = true;

View File

@@ -110,6 +110,9 @@ export const redirectForProviderAuth = (integrationOption: TCloudIntegration) =>
case "northflank":
link = `${window.location.origin}/integrations/northflank/authorize`;
break;
case "windmill":
link = `${window.location.origin}/integrations/windmill/authorize`;
break;
default:
break;
}