Merge pull request #423 from Infisical/check-integrations

Patch create integration page on no integration projects and add support for groups in GitLab integration
This commit is contained in:
BlackMagiq
2023-03-10 21:50:10 +07:00
committed by GitHub
27 changed files with 800 additions and 239 deletions
@@ -2,13 +2,16 @@ import { Request, Response } from 'express';
import { Types } from 'mongoose';
import * as Sentry from '@sentry/node';
import {
Integration,
IntegrationAuth,
Bot
} from '../../models';
import { INTEGRATION_SET, INTEGRATION_OPTIONS } from '../../variables';
import { IntegrationService } from '../../services';
import { getApps, revokeAccess } from '../../integrations';
import {
getApps,
getTeams,
revokeAccess
} from '../../integrations';
/***
* Return integration authorization with id [integrationAuthId]
@@ -154,25 +157,54 @@ export const saveIntegrationAccessToken = async (
* @returns
*/
export const getIntegrationAuthApps = async (req: Request, res: Response) => {
let apps;
try {
apps = await getApps({
integrationAuth: req.integrationAuth,
accessToken: req.accessToken,
});
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: "Failed to get integration authorization applications",
});
}
let apps;
try {
const teamId = req.query.teamId as string;
apps = await getApps({
integrationAuth: req.integrationAuth,
accessToken: req.accessToken,
...teamId && { teamId }
});
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: "Failed to get integration authorization applications",
});
}
return res.status(200).send({
apps,
});
return res.status(200).send({
apps
});
};
/**
* Return list of teams allowed for integration with integration authorization id [integrationAuthId]
* @param req
* @param res
* @returns
*/
export const getIntegrationAuthTeams = async (req: Request, res: Response) => {
let teams;
try {
teams = await getTeams({
integrationAuth: req.integrationAuth,
accessToken: req.accessToken
});
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: "Failed to get integration authorization teams"
});
}
return res.status(200).send({
teams
});
}
/**
* Delete integration authorization with id [integrationAuthId]
* @param req
@@ -2,10 +2,7 @@ import { Request, Response } from 'express';
import { Types } from 'mongoose';
import * as Sentry from '@sentry/node';
import {
Integration,
Workspace,
Bot,
BotKey
Integration
} from '../../models';
import { EventService } from '../../services';
import { eventPushSecrets } from '../../events';
@@ -18,6 +15,7 @@ import { eventPushSecrets } from '../../events';
*/
export const createIntegration = async (req: Request, res: Response) => {
let integration;
try {
const {
integrationAuthId,
@@ -34,19 +32,19 @@ export const createIntegration = async (req: Request, res: Response) => {
// TODO: validate [sourceEnvironment] and [targetEnvironment]
// initialize new integration after saving integration access token
integration = await new Integration({
workspace: req.integrationAuth.workspace._id,
environment: sourceEnvironment,
isActive,
app,
integration = await new Integration({
workspace: req.integrationAuth.workspace._id,
environment: sourceEnvironment,
isActive,
app,
appId,
targetEnvironment,
owner,
path,
region,
integration: req.integrationAuth.integration,
integrationAuth: new Types.ObjectId(integrationAuthId)
}).save();
integration: req.integrationAuth.integration,
integrationAuth: new Types.ObjectId(integrationAuthId)
}).save();
if (integration) {
// trigger event - push secrets
+1 -1
View File
@@ -229,7 +229,7 @@ const getIntegrationAuthAccessHelper = async ({ integrationAuthId }: { integrati
// access token is expired
const refreshToken = await getIntegrationAuthRefreshHelper({ integrationAuthId });
accessToken = await exchangeRefresh({
integration: integrationAuth.integration,
integrationAuth,
refreshToken
});
}
+104 -36
View File
@@ -25,26 +25,30 @@ import {
INTEGRATION_TRAVISCI_API_URL,
} from "../variables";
interface App {
name: string;
appId?: string;
owner?: string;
}
/**
* Return list of names of apps for integration named [integration]
* @param {Object} obj
* @param {String} obj.integration - name of integration
* @param {String} obj.accessToken - access token for integration
* @param {String} obj.teamId - (optional) id of team for getting integration apps (used for integrations like GitLab)
* @returns {Object[]} apps - names of integration apps
* @returns {String} apps.name - name of integration app
*/
const getApps = async ({
integrationAuth,
accessToken,
teamId
}: {
integrationAuth: IIntegrationAuth;
accessToken: string;
teamId?: string;
}) => {
interface App {
name: string;
appId?: string;
owner?: string;
}
let apps: App[] = [];
try {
@@ -82,6 +86,7 @@ const getApps = async ({
case INTEGRATION_GITLAB:
apps = await getAppsGitlab({
accessToken,
teamId
});
break;
case INTEGRATION_RENDER:
@@ -434,44 +439,107 @@ const getAppsTravisCI = async ({ accessToken }: { accessToken: string }) => {
* @returns {Object[]} apps - names of GitLab sites
* @returns {String} apps.name - name of GitLab site
*/
const getAppsGitlab = async ({ accessToken }: {accessToken: string}) => {
let apps;
const getAppsGitlab = async ({
accessToken,
teamId
}: {
accessToken: string;
teamId?: string;
}) => {
const apps: App[] = [];
let page = 1;
const perPage = 10;
let hasMorePages = true;
try {
const { id } = (
await request.get(
`${INTEGRATION_GITLAB_API_URL}/v4/user`,
{
headers: {
"Authorization": `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
)
).data;
const res = (
await request.get(
`${INTEGRATION_GITLAB_API_URL}/v4/users/${id}/projects`,
{
headers: {
"Authorization": `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
)
).data;
if (teamId) {
// case: fetch projects for group with id [teamId] in GitLab
while (hasMorePages) {
const params = new URLSearchParams({
page: String(page),
per_page: String(perPage)
});
apps = res?.map((a: any) => {
return {
name: a?.name,
appId: `${a?.id}`,
const { data } = (
await request.get(
`${INTEGRATION_GITLAB_API_URL}/v4/groups/${teamId}/projects`,
{
params,
headers: {
"Authorization": `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
)
);
data.map((a: any) => {
apps.push({
name: a.name,
appId: a.id
});
});
if (data.length < perPage) {
hasMorePages = false;
}
page++;
}
});
}catch (err) {
} else {
// case: fetch projects for individual in GitLab
const { id } = (
await request.get(
`${INTEGRATION_GITLAB_API_URL}/v4/user`,
{
headers: {
"Authorization": `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
)
).data;
while (hasMorePages) {
const params = new URLSearchParams({
page: String(page),
per_page: String(perPage)
});
const { data } = (
await request.get(
`${INTEGRATION_GITLAB_API_URL}/v4/users/${id}/projects`,
{
params,
headers: {
"Authorization": `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
)
);
data.map((a: any) => {
apps.push({
name: a.name,
appId: a.id
});
});
if (data.length < perPage) {
hasMorePages = false;
}
page++;
}
}
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error("Failed to get GitLab repos");
throw new Error("Failed to get GitLab projects");
}
return apps;
+11 -6
View File
@@ -12,7 +12,7 @@ import {
INTEGRATION_VERCEL_TOKEN_URL,
INTEGRATION_NETLIFY_TOKEN_URL,
INTEGRATION_GITHUB_TOKEN_URL,
INTEGRATION_GITLAB_TOKEN_URL,
INTEGRATION_GITLAB_TOKEN_URL
} from '../variables';
import {
SITE_URL,
@@ -73,7 +73,7 @@ interface ExchangeCodeGithubResponse {
interface ExchangeCodeGitlabResponse {
access_token: string;
token_type: string;
expires_in: string;
expires_in: number;
refresh_token: string;
scope: string;
created_at: number;
@@ -168,7 +168,7 @@ const exchangeCodeAzure = async ({
accessExpiresAt.setSeconds(
accessExpiresAt.getSeconds() + res.expires_in
);
} catch (err: any) {
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed OAuth2 code-token exchange with Azure');
@@ -370,6 +370,7 @@ const exchangeCodeGithub = async ({ code }: { code: string }) => {
*/
const exchangeCodeGitlab = async ({ code }: { code: string }) => {
let res: ExchangeCodeGitlabResponse;
const accessExpiresAt = new Date();
try {
res = (
@@ -389,7 +390,11 @@ const exchangeCodeGitlab = async ({ code }: { code: string }) => {
}
)
).data;
}catch (err) {
accessExpiresAt.setSeconds(
accessExpiresAt.getSeconds() + res.expires_in
);
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed OAuth2 code-token exchange with Gitlab');
@@ -397,8 +402,8 @@ const exchangeCodeGitlab = async ({ code }: { code: string }) => {
return {
accessToken: res.access_token,
refreshToken: null,
accessExpiresAt: null
refreshToken: res.refresh_token,
accessExpiresAt
};
}
+2
View File
@@ -1,6 +1,7 @@
import { exchangeCode } from './exchange';
import { exchangeRefresh } from './refresh';
import { getApps } from './apps';
import { getTeams } from './teams';
import { syncSecrets } from './sync';
import { revokeAccess } from './revoke';
@@ -8,6 +9,7 @@ export {
exchangeCode,
exchangeRefresh,
getApps,
getTeams,
syncSecrets,
revokeAccess
}
+144 -19
View File
@@ -1,16 +1,29 @@
import request from '../config/request';
import * as Sentry from '@sentry/node';
import { INTEGRATION_AZURE_KEY_VAULT, INTEGRATION_HEROKU } from '../variables';
import {
IIntegrationAuth
} from '../models';
import {
INTEGRATION_AZURE_KEY_VAULT,
INTEGRATION_HEROKU,
INTEGRATION_GITLAB,
} from '../variables';
import {
SITE_URL,
CLIENT_ID_AZURE,
CLIENT_ID_GITLAB,
CLIENT_SECRET_AZURE,
CLIENT_SECRET_HEROKU
CLIENT_SECRET_HEROKU,
CLIENT_SECRET_GITLAB
} from '../config';
import {
INTEGRATION_AZURE_TOKEN_URL,
INTEGRATION_HEROKU_TOKEN_URL
INTEGRATION_HEROKU_TOKEN_URL,
INTEGRATION_GITLAB_TOKEN_URL
} from '../variables';
import {
IntegrationService
} from '../services';
interface RefreshTokenAzureResponse {
token_type: string;
@@ -21,6 +34,23 @@ interface RefreshTokenAzureResponse {
refresh_token: string;
}
interface RefreshTokenHerokuResponse {
access_token: string;
expires_in: number;
refresh_token: string;
token_type: string;
user_id: string;
}
interface RefreshTokenGitLabResponse {
token_type: string;
scope: string;
expires_in: number;
access_token: string;
refresh_token: string;
created_at: number;
}
/**
* Return new access token by exchanging refresh token [refreshToken] for integration
* named [integration]
@@ -29,33 +59,61 @@ interface RefreshTokenAzureResponse {
* @param {String} obj.refreshToken - refresh token to use to get new access token for Heroku
*/
const exchangeRefresh = async ({
integration,
integrationAuth,
refreshToken
}: {
integration: string;
integrationAuth: IIntegrationAuth;
refreshToken: string;
}) => {
let accessToken;
interface TokenDetails {
accessToken: string;
refreshToken: string;
accessExpiresAt: Date;
}
let tokenDetails: TokenDetails;
try {
switch (integration) {
switch (integrationAuth.integration) {
case INTEGRATION_AZURE_KEY_VAULT:
accessToken = await exchangeRefreshAzure({
tokenDetails = await exchangeRefreshAzure({
refreshToken
});
break;
case INTEGRATION_HEROKU:
accessToken = await exchangeRefreshHeroku({
tokenDetails = await exchangeRefreshHeroku({
refreshToken
});
break;
case INTEGRATION_GITLAB:
tokenDetails = await exchangeRefreshGitLab({
refreshToken
});
break;
default:
throw new Error('Failed to exchange token for incompatible integration');
}
if (tokenDetails?.accessToken && tokenDetails?.refreshToken && tokenDetails?.accessExpiresAt) {
await IntegrationService.setIntegrationAuthAccess({
integrationAuthId: integrationAuth._id.toString(),
accessId: null,
accessToken: tokenDetails.accessToken,
accessExpiresAt: tokenDetails.accessExpiresAt
});
await IntegrationService.setIntegrationAuthRefresh({
integrationAuthId: integrationAuth._id.toString(),
refreshToken: tokenDetails.refreshToken
});
}
return tokenDetails.accessToken;
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to get new OAuth2 access token');
}
return accessToken;
};
/**
@@ -71,7 +129,8 @@ const exchangeRefreshAzure = async ({
refreshToken: string;
}) => {
try {
const res: RefreshTokenAzureResponse = (await request.post(
const accessExpiresAt = new Date();
const { data }: { data: RefreshTokenAzureResponse } = await request.post(
INTEGRATION_AZURE_TOKEN_URL,
new URLSearchParams({
client_id: CLIENT_ID_AZURE,
@@ -80,9 +139,17 @@ const exchangeRefreshAzure = async ({
grant_type: 'refresh_token',
client_secret: CLIENT_SECRET_AZURE
} as any)
)).data;
);
return res.access_token;
accessExpiresAt.setSeconds(
accessExpiresAt.getSeconds() + data.expires_in
);
return ({
accessToken: data.access_token,
refreshToken: data.refresh_token,
accessExpiresAt
});
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
@@ -102,10 +169,13 @@ const exchangeRefreshHeroku = async ({
}: {
refreshToken: string;
}) => {
let accessToken;
try {
const res = await request.post(
const accessExpiresAt = new Date();
const {
data
}: {
data: RefreshTokenHerokuResponse
} = await request.post(
INTEGRATION_HEROKU_TOKEN_URL,
new URLSearchParams({
grant_type: 'refresh_token',
@@ -114,14 +184,69 @@ const exchangeRefreshHeroku = async ({
} as any)
);
accessToken = res.data.access_token;
accessExpiresAt.setSeconds(
accessExpiresAt.getSeconds() + data.expires_in
);
return ({
accessToken: data.access_token,
refreshToken: data.refresh_token,
accessExpiresAt
});
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to refresh OAuth2 access token for Heroku');
}
};
return accessToken;
/**
* Return new access token by exchanging refresh token [refreshToken] for the
* GitLab integration
* @param {Object} obj
* @param {String} obj.refreshToken - refresh token to use to get new access token for GitLab
* @returns
*/
const exchangeRefreshGitLab = async ({
refreshToken
}: {
refreshToken: string;
}) => {
try {
const accessExpiresAt = new Date();
const {
data
}: {
data: RefreshTokenGitLabResponse
} = await request.post(
INTEGRATION_GITLAB_TOKEN_URL,
new URLSearchParams({
grant_type: 'refresh_token',
refresh_token: refreshToken,
client_id: CLIENT_ID_GITLAB,
client_secret: CLIENT_SECRET_GITLAB,
redirect_uri: `${SITE_URL}/integrations/gitlab/oauth2/callback`
} as any),
{
headers: {
"Accept-Encoding": "application/json",
}
});
accessExpiresAt.setSeconds(
accessExpiresAt.getSeconds() + data.expires_in
);
return ({
accessToken: data.access_token,
refreshToken: data.refresh_token,
accessExpiresAt
});
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to refresh OAuth2 access token for GitLab');
}
};
export { exchangeRefresh };
+3 -4
View File
@@ -172,7 +172,6 @@ const syncSecretsAzureKeyVault = async ({
accessToken: string;
}) => {
try {
interface GetAzureKeyVaultSecret {
id: string; // secret URI
attributes: {
@@ -1512,7 +1511,7 @@ const syncSecretsGitLab = async ({
)
).data;
for (const key of Object.keys(secrets)) {
for await (const key of Object.keys(secrets)) {
const existingSecret = getSecretsRes.find((s: any) => s.key == key);
if (!existingSecret) {
await request.post(
@@ -1533,7 +1532,7 @@ const syncSecretsGitLab = async ({
},
}
)
}else {
} else {
// udpate secret
await request.put(
`${INTEGRATION_GITLAB_API_URL}/v4/projects/${integration?.appId}/variables/${existingSecret.key}`,
@@ -1553,7 +1552,7 @@ const syncSecretsGitLab = async ({
}
// delete secrets
for (const sec of getSecretsRes) {
for await (const sec of getSecretsRes) {
if (!(sec.key in secrets)) {
await request.delete(
`${INTEGRATION_GITLAB_API_URL}/v4/projects/${integration?.appId}/variables/${sec.key}`,
+92
View File
@@ -0,0 +1,92 @@
import * as Sentry from "@sentry/node";
import {
IIntegrationAuth
} from '../models';
import {
INTEGRATION_GITLAB,
INTEGRATION_GITLAB_API_URL
} from '../variables';
import request from '../config/request';
interface Team {
name: string;
teamId: string;
}
/**
* Return list of teams for integration authorization [integrationAuth]
* @param {Object} obj
* @param {String} obj.integrationAuth - integration authorization to get teams for
* @param {String} obj.accessToken - access token for integration authorization
* @returns {Object[]} teams - teams of integration authorization
* @returns {String} teams.name - name of team
* @returns {String} teams.teamId - id of team
*/
const getTeams = async ({
integrationAuth,
accessToken
}: {
integrationAuth: IIntegrationAuth;
accessToken: string;
}) => {
let teams: Team[] = [];
try {
switch (integrationAuth.integration) {
case INTEGRATION_GITLAB:
teams = await getTeamsGitLab({
accessToken
});
break;
}
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to get integration teams');
}
return teams;
}
/**
* Return list of teams for GitLab integration
* @param {Object} obj
* @param {String} obj.accessToken - access token for GitLab API
* @returns {Object[]} teams - teams that user is part of in GitLab
* @returns {String} teams.name - name of team
* @returns {String} teams.teamId - id of team
*/
const getTeamsGitLab = async ({
accessToken
}: {
accessToken: string;
}) => {
let teams: Team[] = [];
try {
const res = (await request.get(
`${INTEGRATION_GITLAB_API_URL}/v4/groups`,
{
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json"
}
}
)).data;
teams = res.map((t: any) => ({
name: t.name,
teamId: t.id
}));
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error("Failed to get GitLab integration teams");
}
return teams;
}
export {
getTeams
}
-2
View File
@@ -62,13 +62,11 @@ const integrationSchema = new Schema<IIntegration>(
default: null,
},
appId: {
// (new)
// id of app in provider
type: String,
default: null,
},
targetEnvironment: {
// (new)
// target environment
type: String,
default: null,
+3 -3
View File
@@ -23,9 +23,9 @@ export interface IIntegrationAuth {
refreshCiphertext?: string;
refreshIV?: string;
refreshTag?: string;
accessIdCiphertext?: string; // new
accessIdIV?: string; // new
accessIdTag?: string; // new
accessIdCiphertext?: string;
accessIdIV?: string;
accessIdTag?: string;
accessCiphertext?: string;
accessIV?: string;
accessTag?: string;
+15 -1
View File
@@ -1,6 +1,6 @@
import express from 'express';
const router = express.Router();
import { body, param } from 'express-validator';
import { body, param, query } from 'express-validator';
import {
requireAuth,
requireWorkspaceAuth,
@@ -73,10 +73,24 @@ router.get(
acceptedRoles: [ADMIN, MEMBER]
}),
param('integrationAuthId'),
query('teamId'),
validateRequest,
integrationAuthController.getIntegrationAuthApps
);
router.get(
'/:integrationAuthId/teams',
requireAuth({
acceptedAuthModes: ['jwt']
}),
requireIntegrationAuthorizationAuth({
acceptedRoles: [ADMIN, MEMBER]
}),
param('integrationAuthId'),
validateRequest,
integrationAuthController.getIntegrationAuthTeams
);
router.delete(
'/:integrationAuthId',
requireAuth({
@@ -7,9 +7,6 @@ import {
setIntegrationAuthAccessHelper,
} from '../helpers/integration';
// should sync stuff be here too? Probably.
// TODO: move bot functions to IntegrationService.
/**
* Class to handle integrations
*/
+2 -1
View File
@@ -59,6 +59,7 @@ const INTEGRATION_FLYIO_API_URL = "https://api.fly.io/graphql";
const INTEGRATION_CIRCLECI_API_URL = "https://circleci.com/api";
const INTEGRATION_TRAVISCI_API_URL = "https://api.travis-ci.com";
// TODO: deprecate types?
const INTEGRATION_OPTIONS = [
{
name: 'Heroku',
@@ -156,7 +157,7 @@ const INTEGRATION_OPTIONS = [
slug: 'gitlab',
image: 'GitLab.png',
isAvailable: true,
type: 'oauth',
type: 'custom',
clientId: CLIENT_ID_GITLAB,
docsLink: ''
},