From 00dfcfcf4e53ec60d536919cbc2818c81d605e99 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Fri, 9 Jun 2023 15:36:37 +0100 Subject: [PATCH] Finish preliminary Vault integration, made docs for Vault and Checkly --- .../v1/integrationAuthController.ts | 78 +++++---- .../controllers/v1/integrationController.ts | 1 + backend/src/integrations/sync.ts | 72 +++++++- backend/src/middleware/requestErrorHandler.ts | 6 + backend/src/models/integration.ts | 9 +- backend/src/models/integrationAuth.ts | 14 +- backend/src/routes/v1/integration.ts | 2 +- backend/src/routes/v1/integrationAuth.ts | 2 + backend/src/variables/integration.ts | 13 +- docs/images/integrations-checkly-auth.png | Bin 0 -> 1184375 bytes docs/images/integrations-checkly-create.png | Bin 0 -> 2442788 bytes .../images/integrations-checkly-dashboard.png | Bin 0 -> 1167890 bytes docs/images/integrations-checkly-token.png | Bin 0 -> 1028746 bytes docs/images/integrations-checkly.png | Bin 0 -> 1514015 bytes .../integrations-hashicorp-vault-access-1.png | Bin 0 -> 143803 bytes .../integrations-hashicorp-vault-access-2.png | Bin 0 -> 623066 bytes .../integrations-hashicorp-vault-access-3.png | Bin 0 -> 753721 bytes .../integrations-hashicorp-vault-auth.png | Bin 0 -> 1204206 bytes .../integrations-hashicorp-vault-create.png | Bin 0 -> 1180803 bytes .../integrations-hashicorp-vault-engine-1.png | Bin 0 -> 144333 bytes .../integrations-hashicorp-vault-engine-2.png | Bin 0 -> 788293 bytes .../integrations-hashicorp-vault-engine-3.png | Bin 0 -> 888482 bytes .../integrations-hashicorp-vault-policy-1.png | Bin 0 -> 143773 bytes .../integrations-hashicorp-vault-policy-2.png | Bin 0 -> 656785 bytes .../integrations-hashicorp-vault-policy-3.png | Bin 0 -> 770623 bytes .../integrations-hashicorp-vault-shell.png | Bin 0 -> 558020 bytes docs/images/integrations-hashicorp-vault.png | Bin 0 -> 613631 bytes docs/images/integrations.png | Bin 1153570 -> 1657991 bytes docs/integrations/cloud/checkly.mdx | 37 ++++ docs/integrations/cloud/hashicorp-vault.mdx | 159 ++++++++++++++++++ docs/integrations/overview.mdx | 2 + docs/mint.json | 2 + frontend/public/data/frequentConstants.ts | 3 +- frontend/public/images/integrations/Vault.png | Bin 0 -> 1557 bytes .../components/integrations/Integration.tsx | 10 +- .../saveIntegrationAccessToken.ts | 20 ++- frontend/src/pages/integrations/[id].tsx | 6 + .../aws-parameter-store/authorize.tsx | 10 +- .../aws-secret-manager/authorize.tsx | 4 +- .../pages/integrations/checkly/authorize.tsx | 4 +- .../pages/integrations/circleci/authorize.tsx | 4 +- .../pages/integrations/flyio/authorize.tsx | 4 +- .../hashicorp-vault/authorize.tsx | 129 ++++++++++++++ .../integrations/hashicorp-vault/create.tsx | 136 +++++++++++++++ .../pages/integrations/railway/authorize.tsx | 4 +- .../pages/integrations/render/authorize.tsx | 4 +- .../pages/integrations/supabase/authorize.tsx | 4 +- .../pages/integrations/travisci/authorize.tsx | 4 +- 48 files changed, 680 insertions(+), 63 deletions(-) create mode 100644 docs/images/integrations-checkly-auth.png create mode 100644 docs/images/integrations-checkly-create.png create mode 100644 docs/images/integrations-checkly-dashboard.png create mode 100644 docs/images/integrations-checkly-token.png create mode 100644 docs/images/integrations-checkly.png create mode 100644 docs/images/integrations-hashicorp-vault-access-1.png create mode 100644 docs/images/integrations-hashicorp-vault-access-2.png create mode 100644 docs/images/integrations-hashicorp-vault-access-3.png create mode 100644 docs/images/integrations-hashicorp-vault-auth.png create mode 100644 docs/images/integrations-hashicorp-vault-create.png create mode 100644 docs/images/integrations-hashicorp-vault-engine-1.png create mode 100644 docs/images/integrations-hashicorp-vault-engine-2.png create mode 100644 docs/images/integrations-hashicorp-vault-engine-3.png create mode 100644 docs/images/integrations-hashicorp-vault-policy-1.png create mode 100644 docs/images/integrations-hashicorp-vault-policy-2.png create mode 100644 docs/images/integrations-hashicorp-vault-policy-3.png create mode 100644 docs/images/integrations-hashicorp-vault-shell.png create mode 100644 docs/images/integrations-hashicorp-vault.png create mode 100644 docs/integrations/cloud/checkly.mdx create mode 100644 docs/integrations/cloud/hashicorp-vault.mdx create mode 100644 frontend/public/images/integrations/Vault.png create mode 100644 frontend/src/pages/integrations/hashicorp-vault/authorize.tsx create mode 100644 frontend/src/pages/integrations/hashicorp-vault/create.tsx diff --git a/backend/src/controllers/v1/integrationAuthController.ts b/backend/src/controllers/v1/integrationAuthController.ts index a002187c8..fc79e22cf 100644 --- a/backend/src/controllers/v1/integrationAuthController.ts +++ b/backend/src/controllers/v1/integrationAuthController.ts @@ -86,47 +86,53 @@ export const saveIntegrationAccessToken = async ( // TODO: check if access token is valid for each integration let integrationAuth; - const { - workspaceId, - accessId, - accessToken, - integration - }: { - workspaceId: string; - accessId: string | null; - accessToken: string; - integration: string; - } = req.body; + const { + workspaceId, + accessId, + accessToken, + url, + namespace, + integration + }: { + workspaceId: string; + accessId: string | null; + accessToken: string; + url: string; + namespace: string; + integration: string; + } = req.body; - const bot = await Bot.findOne({ - workspace: new Types.ObjectId(workspaceId), - isActive: true - }); + const bot = await Bot.findOne({ + workspace: new Types.ObjectId(workspaceId), + isActive: true + }); - if (!bot) throw new Error('Bot must be enabled to save integration access token'); + if (!bot) throw new Error('Bot must be enabled to save integration access token'); - integrationAuth = await IntegrationAuth.findOneAndUpdate({ - workspace: new Types.ObjectId(workspaceId), - integration - }, { - workspace: new Types.ObjectId(workspaceId), - integration, - algorithm: ALGORITHM_AES_256_GCM, - keyEncoding: ENCODING_SCHEME_UTF8 - }, { - new: true, - upsert: true - }); + integrationAuth = await IntegrationAuth.findOneAndUpdate({ + workspace: new Types.ObjectId(workspaceId), + integration + }, { + workspace: new Types.ObjectId(workspaceId), + integration, + url, + namespace, + algorithm: ALGORITHM_AES_256_GCM, + keyEncoding: ENCODING_SCHEME_UTF8 + }, { + new: true, + upsert: true + }); - // encrypt and save integration access details - integrationAuth = await IntegrationService.setIntegrationAuthAccess({ - integrationAuthId: integrationAuth._id.toString(), - accessId, - accessToken, - accessExpiresAt: undefined - }); + // encrypt and save integration access details + integrationAuth = await IntegrationService.setIntegrationAuthAccess({ + integrationAuthId: integrationAuth._id.toString(), + accessId, + accessToken, + accessExpiresAt: undefined + }); - if (!integrationAuth) throw new Error('Failed to save integration access token'); + if (!integrationAuth) throw new Error('Failed to save integration access token'); return res.status(200).send({ integrationAuth diff --git a/backend/src/controllers/v1/integrationController.ts b/backend/src/controllers/v1/integrationController.ts index 83f7c784a..5119761d0 100644 --- a/backend/src/controllers/v1/integrationController.ts +++ b/backend/src/controllers/v1/integrationController.ts @@ -57,6 +57,7 @@ export const createIntegration = async (req: Request, res: Response) => { }) }); } + return res.status(200).send({ integration, }); diff --git a/backend/src/integrations/sync.ts b/backend/src/integrations/sync.ts index 3b655e2fd..76d009944 100644 --- a/backend/src/integrations/sync.ts +++ b/backend/src/integrations/sync.ts @@ -36,7 +36,8 @@ import { INTEGRATION_TRAVISCI_API_URL, INTEGRATION_SUPABASE_API_URL, INTEGRATION_CHECKLY, - INTEGRATION_CHECKLY_API_URL + INTEGRATION_CHECKLY_API_URL, + INTEGRATION_HASHICORP_VAULT } from "../variables"; import { standardRequest} from '../config/request'; @@ -200,6 +201,15 @@ const syncSecrets = async ({ accessToken, }); break; + case INTEGRATION_HASHICORP_VAULT: + await syncSecretsHashiCorpVault({ + integration, + integrationAuth, + secrets, + accessId, + accessToken + }); + break; } }; @@ -1762,5 +1772,65 @@ const syncSecretsCheckly = async ({ } }; +/** + * Sync/push [secrets] to HashiCorp Vault path + * @param {Object} obj + * @param {IIntegration} obj.integration - integration details + * @param {Object} obj.secrets - secrets to push to integration (object where keys are secret keys and values are secret values) + * @param {String} obj.accessToken - access token for HashiCorp Vault integration + */ +const syncSecretsHashiCorpVault = async ({ + integration, + integrationAuth, + secrets, + accessId, + accessToken, +}: { + integration: IIntegration; + integrationAuth: IIntegrationAuth; + secrets: any; + accessId: string | null; + accessToken: string; +}) => { + if (!accessId) return; + + interface LoginAppRoleRes { + auth: { + client_token: string; + } + } + + // get Vault client token (could be optimized) + const { data }: { data: LoginAppRoleRes } = await standardRequest.post( + `${integrationAuth.url}/v1/auth/approle/login`, + { + "role_id": accessId, + "secret_id": accessToken + }, + { + headers: { + "X-Vault-Namespace": integrationAuth.namespace + } + } + ); + + const clientToken = data.auth.client_token; + + await standardRequest.post( + `${integrationAuth.url}/v1/${integration.app}/data/${integration.path}`, + { + data: secrets + }, + { + headers: { + "Authorization": `Bearer ${accessToken}`, + "Accept": "application/json", + "Content-Type": "application/json", + "X-Vault-Token": clientToken, + "X-Vault-Namespace": integrationAuth.namespace + }, + } + ); +}; export { syncSecrets }; diff --git a/backend/src/middleware/requestErrorHandler.ts b/backend/src/middleware/requestErrorHandler.ts index 08aa0d5fd..6aa73954b 100644 --- a/backend/src/middleware/requestErrorHandler.ts +++ b/backend/src/middleware/requestErrorHandler.ts @@ -3,6 +3,7 @@ import { ErrorRequestHandler } from 'express'; import { InternalServerError } from '../utils/errors'; import { getLogger } from '../utils/logger'; import RequestError, { LogLevel } from '../utils/requestError'; +import { getNodeEnv } from '../config'; export const requestErrorHandler: ErrorRequestHandler = async ( error: RequestError | Error, @@ -12,6 +13,11 @@ export const requestErrorHandler: ErrorRequestHandler = async ( ) => { if (res.headersSent) return next(); + if (await getNodeEnv() !== "production") { + /* eslint-disable no-console */ + console.error(error); + } + //TODO: Find better way to type check for error. In current setting you need to cast type to get the functions and variables from RequestError if (!(error instanceof RequestError)) { error = InternalServerError({ diff --git a/backend/src/models/integration.ts b/backend/src/models/integration.ts index 504d4a2be..d4fbae807 100644 --- a/backend/src/models/integration.ts +++ b/backend/src/models/integration.ts @@ -14,7 +14,8 @@ import { INTEGRATION_CIRCLECI, INTEGRATION_TRAVISCI, INTEGRATION_SUPABASE, - INTEGRATION_CHECKLY + INTEGRATION_CHECKLY, + INTEGRATION_HASHICORP_VAULT } from "../variables"; export interface IIntegration { @@ -47,7 +48,8 @@ export interface IIntegration { | 'circleci' | 'travisci' | 'supabase' - | 'checkly'; + | 'checkly' + | 'hashicorp-vault'; integrationAuth: Types.ObjectId; } @@ -133,7 +135,8 @@ const integrationSchema = new Schema( INTEGRATION_CIRCLECI, INTEGRATION_TRAVISCI, INTEGRATION_SUPABASE, - INTEGRATION_CHECKLY + INTEGRATION_CHECKLY, + INTEGRATION_HASHICORP_VAULT ], required: true, }, diff --git a/backend/src/models/integrationAuth.ts b/backend/src/models/integrationAuth.ts index 4f2209f2d..cc28c9fd6 100644 --- a/backend/src/models/integrationAuth.ts +++ b/backend/src/models/integrationAuth.ts @@ -14,6 +14,7 @@ import { INTEGRATION_CIRCLECI, INTEGRATION_TRAVISCI, INTEGRATION_SUPABASE, + INTEGRATION_HASHICORP_VAULT, ALGORITHM_AES_256_GCM, ENCODING_SCHEME_UTF8, ENCODING_SCHEME_BASE64 @@ -25,6 +26,8 @@ export interface IIntegrationAuth extends Document { integration: 'heroku' | 'vercel' | 'netlify' | 'github' | 'gitlab' | 'render' | 'railway' | 'flyio' | 'azure-key-vault' | 'circleci' | 'travisci' | 'supabase' | 'aws-parameter-store' | 'aws-secret-manager' | 'checkly'; teamId: string; accountId: string; + url: string; + namespace: string; refreshCiphertext?: string; refreshIV?: string; refreshTag?: string; @@ -62,7 +65,8 @@ const integrationAuthSchema = new Schema( INTEGRATION_FLYIO, INTEGRATION_CIRCLECI, INTEGRATION_TRAVISCI, - INTEGRATION_SUPABASE + INTEGRATION_SUPABASE, + INTEGRATION_HASHICORP_VAULT ], required: true, }, @@ -70,6 +74,14 @@ const integrationAuthSchema = new Schema( // vercel-specific integration param type: String, }, + url: { + // for any self-hosted integrations (e.g. self-hosted hashicorp-vault) + type: String + }, + namespace: { + // hashicorp-vault-specific integration param + type: String + }, accountId: { // netlify-specific integration param type: String, diff --git a/backend/src/routes/v1/integration.ts b/backend/src/routes/v1/integration.ts index 52930ce0c..b8e0b38bd 100644 --- a/backend/src/routes/v1/integration.ts +++ b/backend/src/routes/v1/integration.ts @@ -15,7 +15,7 @@ import { import { body, param } from 'express-validator'; import { integrationController } from '../../controllers/v1'; -router.post( // new: add new integration for integration auth +router.post( '/', requireAuth({ acceptedAuthModes: [AUTH_MODE_JWT, AUTH_MODE_API_KEY] diff --git a/backend/src/routes/v1/integrationAuth.ts b/backend/src/routes/v1/integrationAuth.ts index fa5863029..f8fe15a4b 100644 --- a/backend/src/routes/v1/integrationAuth.ts +++ b/backend/src/routes/v1/integrationAuth.ts @@ -57,6 +57,8 @@ router.post( body('workspaceId').exists().trim().notEmpty(), body('accessId').trim(), body('accessToken').exists().trim().notEmpty(), + body('url').trim(), + body('namespace').trim(), body('integration').exists().trim().notEmpty(), validateRequest, requireAuth({ diff --git a/backend/src/variables/integration.ts b/backend/src/variables/integration.ts index 7fa8f6a23..26e9f2544 100644 --- a/backend/src/variables/integration.ts +++ b/backend/src/variables/integration.ts @@ -23,6 +23,7 @@ export const INTEGRATION_CIRCLECI = "circleci"; export const INTEGRATION_TRAVISCI = "travisci"; export const INTEGRATION_SUPABASE = 'supabase'; export const INTEGRATION_CHECKLY = 'checkly'; +export const INTEGRATION_HASHICORP_VAULT = 'hashicorp-vault'; export const INTEGRATION_SET = new Set([ INTEGRATION_AZURE_KEY_VAULT, INTEGRATION_HEROKU, @@ -35,7 +36,8 @@ export const INTEGRATION_SET = new Set([ INTEGRATION_CIRCLECI, INTEGRATION_TRAVISCI, INTEGRATION_SUPABASE, - INTEGRATION_CHECKLY + INTEGRATION_CHECKLY, + INTEGRATION_HASHICORP_VAULT ]); // integration types @@ -202,6 +204,15 @@ export const getIntegrationOptions = async () => { clientId: '', docsLink: '' }, + { + name: 'HashiCorp Vault', + slug: 'hashicorp-vault', + image: 'Vault.png', + isAvailable: true, + type: 'pat', + clientId: '', + docsLink: '' + }, { name: 'Google Cloud Platform', slug: 'gcp', diff --git a/docs/images/integrations-checkly-auth.png b/docs/images/integrations-checkly-auth.png new file mode 100644 index 0000000000000000000000000000000000000000..b6299aefe63fd282873b4194131313dd0d9a93eb GIT binary patch literal 1184375 zcmZU)1z1$g|2|BINC?ugbSM(ixpWGMpooC0G}0jl80?P$FJUmKW9gSOfco!7$ z@JM*cNT4&OB$D!Yco*HA)YWh5s;hI}^m4a%a!MPn~i5MMGbTe}i%ZGm59W5&?+gOTD&-^VwxlxZ?t z_*Tbh+qv7>fnbi@e!iE3*EYV&KA{I z-MgNi#&~`6*zLV7lG8aC`Jk{kQoQR_i#I;SWl-;W;_WdiTs){e&|Vm9x>S7AJVgHKuUK}u@tvy}cnLx6u`z$mc z9wgmJ?#?|zrmrCNL-K;YL{YMtzLHc_O@fy#l+WrWUAXbI!6Jv5uJnymCrl&A?mNDU zkXnF$^(cqbO7GPBke%m*<4=bv6C$UW7H@jqzmck!pT&3Kr^-b?3i;cnt-9e;c#E0@ z<~jv^_9;HM!uuG%CGy$Tn4WoCu7@I?R+8?`+NQ^V@i$wIcWqcS+lx%2g=j43V{Tb< zPKHjIr)M+0t!B33we7z6P3I?h%t<+?Ax(5gcq-=Z!*oeWRXaqoDC+>@4DV%6Yg2zj zn3#~IIc)PXN`VGGK-?ruQ@KvXSm9MQN#+{V>6TeV8y=~WQ38d1b4sM^R-qO-SX zdercgQ}o8G`p_w>JDG3p#g<%d9elgoUBgT4soN*JTfpX~k#n6RNG_Vdh2|oY3Y9j0 zCXrX@oARbAR^Zl9B!Ng`xC{M*|DF6r|Ljvq__@fw>LR4|9^jZ=TGu)s0z zAWE=Mm2{c<_5}C-`Mr7R#4$qZb}XlY?dL;Jp83cob-1u1-jk!ydv#abG;cG(J-}-b z+j-CW5;oPedOLc;Y@1+ak7OPFFIEXY!UhMYR@qZcWGhMT7b1`RCRRHFmr?0c+%9lm z28FQPZYrDj&@lFn7w_+EMiWr~t`Hz>a{gk`uA{)4SDJx!#U;N(ohAXC3foQ}X`#Kv`zzgX< zJ3prm9@Atad{D`#2#})JH#6}ZO9?%;IE_TEnf~(VdWr|~NiOJ05WQ}tooWlR3=`NP z4&x-!2)&y~G}=mZ*N$s;^_Cd%0G_w1xFB(V+m-hWE<_e8lvBhqMfe)c&MrG(=onl935a-1gB@i_s0-H#%YDss)Keyp+8v)tAJ%we30glWrAV zG>>)BdX&`Cqq0keyr(TADE<7a%Kjv$ z(whCQ%+V!*MW387g*JA!1c4>RpzF#Mi15lT=N%Hv2~~fC;xPL^6hAi0_BnPHR54w7V#+A+=j@8yoPN5oGHVg8=HJdI?s5u zlVL*XoqY6(F}lt=3cB4oW;(e=)Q^5WvVTL|&2kMmmY9Tf|`(&F4ArI*29 zUOLM3i!P$&&$r5OT+$Yh#>cxyuf3QG4YRlxdy8vt7G+}Mq>(( zpMR|T$o8G>`>pR*A6-8xJ&}L1^+f22$>F<$gtflFLy@$_NgLNL#4Pn+Nm$Cwe4!ZEGy z9gm~Vsa7dPnLoLou-lAJwaK>mf81qQy0dhj⋘Q?s+duN(%q?*JrQ8Udz4y`C8kc zX<*GBu;;Gvsb>uDQ1dwN%?j(0=95~QYII9;o~?;1RL))HTC@NQ5oTPu6Dj--xlhI>YSCeT;1 z39qTwcf$AU=JTT)=a%P(M;YulNvTO+si$@DX^)SL880{ds0t=}7_~8L#+15_5@;Bew2M3cQD6BB1AtW0sk6{Lw zfozNd!2n?c{scaX;2Pm+*bG5_)uy>zS#Ft`s)*|7OG4G{Y30qV&84=x9(O%-JuIRP zb-a4MC6c~kcs2d%@hhc=91q7j5~AqZc{J^>~6fPxkr-LF#0xp`MZMT>1|M zY1_?rRk=%wpXpIe;y-*bax}8km;*6O_BKR7arC}U%PkpNiJPpjreM?o6FYG zEYLjO{QdONDf205m;9pT?pxj0DX$G5WytEq@Uf=^^H;u@<%v&qOW~^Ct$s~lbz(es zr9K7ZMc!RQ+`WDBP_Hi)Yx1Earlii$JG0LJTTQ{20>ta3v4AlP`^@1?g{6fXlLLb_ ziRM`~_v|f(?u#{f8&2iCXiRNv^hRc{Uaxfdf!H@$l@S#)7jtM^%5M*OcXd)Qong*$ zt;u7FJ1EJ~c*eTE%b5XrZ6U)_@Uc+2AhtQOez|?PFr)(0h;*79t+UJPmlhTfVH8DI zaX-*B$nrHeo?c2Dxystd{FCn|viwEEub`)aU7sgXC-x@T(~Q#$oXaMK5&Hq<9={LY zy?pn5Bkt?9`^O9z(*;&jcG(C zdfZLo+ntZU`+awh5?(~UF!^x*!v}>pKjBjKjpfHy3f;e(o3HwtnY5T-SH7P$=^uwG z?J0>K*>AQ~iFzUWLsoz@L^;yRFEr$35x!IAc=H^UkI8=hys<666AYn_}M9jU|-~=aMUV?+I^g zR(Xf4Qsp1ZA$JJ&)s{4eV=2oy%1J}^PkyUpJ6Lq^Rq7jLw1~>oquk(6_~!ebm$sP= zTP|5P`75nFIgejvt#?akn)ZxEZk+yZo~hiv{A0aIVU=Uyes6I3IX!h8%oKD3&$p$6|kab6K>+3<+g1>|Iy%rtyGoTk(S}ap>+}4Eg3*OdOW(s+8+Jo^wGWk{F z5Nx8HqHGpY1&(5*_BL7o3=l$Y*2jKwxpiKprW((`h8^!73!cwZz47NOfo=Gzsn`uX z0+9Ubc^CdJ(MygHbhlqeWp*~+=JbUb(Vj~3zzJCvJl!97kG)x0t|ZwtHeP9b^WKO{ zacvt(5hy*XhxbW%dwaWdYa6U4CZP!IUb42Men-Q-$t>L!+&eH#1Zt&X*Zt%Gf6^GT z**4TM693LrV&lM-3)+wXiLtw@Yj0qHcLgew;Su4p;1NS5eCVZ&&-!0k3tteA@XvVy zJiI6;JfeT@F@)Z6e~HiwSLg3LVe&IPQs_Tg=;i;C;NQD1D83~8w@ktdUBgo~QP