Finish support for GitLab groups integration

This commit is contained in:
Tuan Dang
2023-03-10 21:25:04 +07:00
parent 3f3516b7ba
commit 0269b58a3c
13 changed files with 319 additions and 181 deletions

View File

@@ -2,7 +2,6 @@ import { Request, Response } from 'express';
import { Types } from 'mongoose';
import * as Sentry from '@sentry/node';
import {
Integration,
IntegrationAuth,
Bot
} from '../../models';
@@ -161,7 +160,7 @@ export const getIntegrationAuthApps = async (req: Request, res: Response) => {
let apps;
try {
const teamId = req.query.teamId as string;
apps = await getApps({
integrationAuth: req.integrationAuth,
accessToken: req.accessToken,

View File

@@ -2,10 +2,7 @@ import { Request, Response } from 'express';
import { Types } from 'mongoose';
import * as Sentry from '@sentry/node';
import {
Integration,
Workspace,
Bot,
BotKey
Integration
} from '../../models';
import { EventService } from '../../services';
import { eventPushSecrets } from '../../events';
@@ -18,6 +15,7 @@ import { eventPushSecrets } from '../../events';
*/
export const createIntegration = async (req: Request, res: Response) => {
let integration;
try {
const {
integrationAuthId,
@@ -34,19 +32,19 @@ export const createIntegration = async (req: Request, res: Response) => {
// TODO: validate [sourceEnvironment] and [targetEnvironment]
// initialize new integration after saving integration access token
integration = await new Integration({
workspace: req.integrationAuth.workspace._id,
environment: sourceEnvironment,
isActive,
app,
integration = await new Integration({
workspace: req.integrationAuth.workspace._id,
environment: sourceEnvironment,
isActive,
app,
appId,
targetEnvironment,
owner,
path,
region,
integration: req.integrationAuth.integration,
integrationAuth: new Types.ObjectId(integrationAuthId)
}).save();
integration: req.integrationAuth.integration,
integrationAuth: new Types.ObjectId(integrationAuthId)
}).save();
if (integration) {
// trigger event - push secrets

View File

@@ -229,7 +229,7 @@ const getIntegrationAuthAccessHelper = async ({ integrationAuthId }: { integrati
// access token is expired
const refreshToken = await getIntegrationAuthRefreshHelper({ integrationAuthId });
accessToken = await exchangeRefresh({
integration: integrationAuth.integration,
integrationAuth,
refreshToken
});
}

View File

@@ -24,7 +24,12 @@ import {
INTEGRATION_CIRCLECI_API_URL,
INTEGRATION_TRAVISCI_API_URL,
} from "../variables";
import { requireIntegrationAuthorizationAuth } from "../middleware";
interface App {
name: string;
appId?: string;
owner?: string;
}
/**
* Return list of names of apps for integration named [integration]
@@ -45,12 +50,6 @@ const getApps = async ({
teamId?: string;
}) => {
interface App {
name: string;
appId?: string;
owner?: string;
}
let apps: App[] = [];
try {
switch (integrationAuth.integration) {
@@ -87,6 +86,7 @@ const getApps = async ({
case INTEGRATION_GITLAB:
apps = await getAppsGitlab({
accessToken,
teamId
});
break;
case INTEGRATION_RENDER:
@@ -439,44 +439,107 @@ const getAppsTravisCI = async ({ accessToken }: { accessToken: string }) => {
* @returns {Object[]} apps - names of GitLab sites
* @returns {String} apps.name - name of GitLab site
*/
const getAppsGitlab = async ({ accessToken }: {accessToken: string}) => {
let apps;
const getAppsGitlab = async ({
accessToken,
teamId
}: {
accessToken: string;
teamId?: string;
}) => {
const apps: App[] = [];
let page = 1;
const perPage = 10;
let hasMorePages = true;
try {
const { id } = (
await request.get(
`${INTEGRATION_GITLAB_API_URL}/v4/user`,
{
headers: {
"Authorization": `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
)
).data;
const res = (
await request.get(
`${INTEGRATION_GITLAB_API_URL}/v4/users/${id}/projects`,
{
headers: {
"Authorization": `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
)
).data;
if (teamId) {
// case: fetch projects for group with id [teamId] in GitLab
while (hasMorePages) {
const params = new URLSearchParams({
page: String(page),
per_page: String(perPage)
});
apps = res?.map((a: any) => {
return {
name: a?.name,
appId: `${a?.id}`,
const { data } = (
await request.get(
`${INTEGRATION_GITLAB_API_URL}/v4/groups/${teamId}/projects`,
{
params,
headers: {
"Authorization": `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
)
);
data.map((a: any) => {
apps.push({
name: a.name,
appId: a.id
});
});
if (data.length < perPage) {
hasMorePages = false;
}
page++;
}
});
}catch (err) {
} else {
// case: fetch projects for individual in GitLab
const { id } = (
await request.get(
`${INTEGRATION_GITLAB_API_URL}/v4/user`,
{
headers: {
"Authorization": `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
)
).data;
while (hasMorePages) {
const params = new URLSearchParams({
page: String(page),
per_page: String(perPage)
});
const { data } = (
await request.get(
`${INTEGRATION_GITLAB_API_URL}/v4/users/${id}/projects`,
{
params,
headers: {
"Authorization": `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
},
}
)
);
data.map((a: any) => {
apps.push({
name: a.name,
appId: a.id
});
});
if (data.length < perPage) {
hasMorePages = false;
}
page++;
}
}
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error("Failed to get GitLab repos");
throw new Error("Failed to get GitLab projects");
}
return apps;

View File

@@ -12,8 +12,7 @@ import {
INTEGRATION_VERCEL_TOKEN_URL,
INTEGRATION_NETLIFY_TOKEN_URL,
INTEGRATION_GITHUB_TOKEN_URL,
INTEGRATION_GITLAB_TOKEN_URL,
INTEGRATION_GITLAB_API_URL
INTEGRATION_GITLAB_TOKEN_URL
} from '../variables';
import {
SITE_URL,
@@ -169,7 +168,7 @@ const exchangeCodeAzure = async ({
accessExpiresAt.setSeconds(
accessExpiresAt.getSeconds() + res.expires_in
);
} catch (err: any) {
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed OAuth2 code-token exchange with Azure');
@@ -392,46 +391,10 @@ const exchangeCodeGitlab = async ({ code }: { code: string }) => {
)
).data;
// 1. try getting groups
// https://gitlab.com/api/v4/groups
// const res2 = (await request.get(
// `${INTEGRATION_GITLAB_API_URL}/v4/groups`,
// {
// headers: {
// Authorization: `Bearer ${res.access_token}`,
// "Accept-Encoding": "application/json"
// }
// }
// )).data;
// 2. try getting projects of that group
// const res3 = (await request.get(
// `${INTEGRATION_GITLAB_API_URL}/v4/groups/${res2[0].id}`,
// {
// headers: {
// Authorization: `Bearer ${res.access_token}`,
// "Accept-Encoding": "application/json"
// }
// }
// )).data;
// const res = (
// await request.get(
// `${INTEGRATION_GITLAB_API_URL}/v4/users/${id}/projects`,
// {
// headers: {
// "Authorization": `Bearer ${accessToken}`,
// "Accept-Encoding": "application/json",
// },
// }
// )
// ).data;
accessExpiresAt.setSeconds(
accessExpiresAt.getSeconds() + res.expires_in
);
}catch (err) {
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed OAuth2 code-token exchange with Gitlab');

View File

@@ -1,16 +1,29 @@
import request from '../config/request';
import * as Sentry from '@sentry/node';
import { INTEGRATION_AZURE_KEY_VAULT, INTEGRATION_HEROKU } from '../variables';
import {
IIntegrationAuth
} from '../models';
import {
INTEGRATION_AZURE_KEY_VAULT,
INTEGRATION_HEROKU,
INTEGRATION_GITLAB,
} from '../variables';
import {
SITE_URL,
CLIENT_ID_AZURE,
CLIENT_ID_GITLAB,
CLIENT_SECRET_AZURE,
CLIENT_SECRET_HEROKU
CLIENT_SECRET_HEROKU,
CLIENT_SECRET_GITLAB
} from '../config';
import {
INTEGRATION_AZURE_TOKEN_URL,
INTEGRATION_HEROKU_TOKEN_URL
INTEGRATION_HEROKU_TOKEN_URL,
INTEGRATION_GITLAB_TOKEN_URL
} from '../variables';
import {
IntegrationService
} from '../services';
interface RefreshTokenAzureResponse {
token_type: string;
@@ -21,6 +34,23 @@ interface RefreshTokenAzureResponse {
refresh_token: string;
}
interface RefreshTokenHerokuResponse {
access_token: string;
expires_in: number;
refresh_token: string;
token_type: string;
user_id: string;
}
interface RefreshTokenGitLabResponse {
token_type: string;
scope: string;
expires_in: number;
access_token: string;
refresh_token: string;
created_at: number;
}
/**
* Return new access token by exchanging refresh token [refreshToken] for integration
* named [integration]
@@ -29,33 +59,61 @@ interface RefreshTokenAzureResponse {
* @param {String} obj.refreshToken - refresh token to use to get new access token for Heroku
*/
const exchangeRefresh = async ({
integration,
integrationAuth,
refreshToken
}: {
integration: string;
integrationAuth: IIntegrationAuth;
refreshToken: string;
}) => {
let accessToken;
interface TokenDetails {
accessToken: string;
refreshToken: string;
accessExpiresAt: Date;
}
let tokenDetails: TokenDetails;
try {
switch (integration) {
switch (integrationAuth.integration) {
case INTEGRATION_AZURE_KEY_VAULT:
accessToken = await exchangeRefreshAzure({
tokenDetails = await exchangeRefreshAzure({
refreshToken
});
break;
case INTEGRATION_HEROKU:
accessToken = await exchangeRefreshHeroku({
tokenDetails = await exchangeRefreshHeroku({
refreshToken
});
break;
case INTEGRATION_GITLAB:
tokenDetails = await exchangeRefreshGitLab({
refreshToken
});
break;
default:
throw new Error('Failed to exchange token for incompatible integration');
}
if (tokenDetails?.accessToken && tokenDetails?.refreshToken && tokenDetails?.accessExpiresAt) {
await IntegrationService.setIntegrationAuthAccess({
integrationAuthId: integrationAuth._id.toString(),
accessId: null,
accessToken: tokenDetails.accessToken,
accessExpiresAt: tokenDetails.accessExpiresAt
});
await IntegrationService.setIntegrationAuthRefresh({
integrationAuthId: integrationAuth._id.toString(),
refreshToken: tokenDetails.refreshToken
});
}
return tokenDetails.accessToken;
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to get new OAuth2 access token');
}
return accessToken;
};
/**
@@ -71,7 +129,8 @@ const exchangeRefreshAzure = async ({
refreshToken: string;
}) => {
try {
const res: RefreshTokenAzureResponse = (await request.post(
const accessExpiresAt = new Date();
const { data }: { data: RefreshTokenAzureResponse } = await request.post(
INTEGRATION_AZURE_TOKEN_URL,
new URLSearchParams({
client_id: CLIENT_ID_AZURE,
@@ -80,9 +139,17 @@ const exchangeRefreshAzure = async ({
grant_type: 'refresh_token',
client_secret: CLIENT_SECRET_AZURE
} as any)
)).data;
);
return res.access_token;
accessExpiresAt.setSeconds(
accessExpiresAt.getSeconds() + data.expires_in
);
return ({
accessToken: data.access_token,
refreshToken: data.refresh_token,
accessExpiresAt
});
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
@@ -102,10 +169,13 @@ const exchangeRefreshHeroku = async ({
}: {
refreshToken: string;
}) => {
let accessToken;
try {
const res = await request.post(
const accessExpiresAt = new Date();
const {
data
}: {
data: RefreshTokenHerokuResponse
} = await request.post(
INTEGRATION_HEROKU_TOKEN_URL,
new URLSearchParams({
grant_type: 'refresh_token',
@@ -114,14 +184,69 @@ const exchangeRefreshHeroku = async ({
} as any)
);
accessToken = res.data.access_token;
accessExpiresAt.setSeconds(
accessExpiresAt.getSeconds() + data.expires_in
);
return ({
accessToken: data.access_token,
refreshToken: data.refresh_token,
accessExpiresAt
});
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to refresh OAuth2 access token for Heroku');
}
};
return accessToken;
/**
* Return new access token by exchanging refresh token [refreshToken] for the
* GitLab integration
* @param {Object} obj
* @param {String} obj.refreshToken - refresh token to use to get new access token for GitLab
* @returns
*/
const exchangeRefreshGitLab = async ({
refreshToken
}: {
refreshToken: string;
}) => {
try {
const accessExpiresAt = new Date();
const {
data
}: {
data: RefreshTokenGitLabResponse
} = await request.post(
INTEGRATION_GITLAB_TOKEN_URL,
new URLSearchParams({
grant_type: 'refresh_token',
refresh_token: refreshToken,
client_id: CLIENT_ID_GITLAB,
client_secret: CLIENT_SECRET_GITLAB,
redirect_uri: `${SITE_URL}/integrations/gitlab/oauth2/callback`
} as any),
{
headers: {
"Accept-Encoding": "application/json",
}
});
accessExpiresAt.setSeconds(
accessExpiresAt.getSeconds() + data.expires_in
);
return ({
accessToken: data.access_token,
refreshToken: data.refresh_token,
accessExpiresAt
});
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to refresh OAuth2 access token for GitLab');
}
};
export { exchangeRefresh };

View File

@@ -172,7 +172,6 @@ const syncSecretsAzureKeyVault = async ({
accessToken: string;
}) => {
try {
interface GetAzureKeyVaultSecret {
id: string; // secret URI
attributes: {
@@ -1512,7 +1511,7 @@ const syncSecretsGitLab = async ({
)
).data;
for (const key of Object.keys(secrets)) {
for await (const key of Object.keys(secrets)) {
const existingSecret = getSecretsRes.find((s: any) => s.key == key);
if (!existingSecret) {
await request.post(
@@ -1533,7 +1532,7 @@ const syncSecretsGitLab = async ({
},
}
)
}else {
} else {
// udpate secret
await request.put(
`${INTEGRATION_GITLAB_API_URL}/v4/projects/${integration?.appId}/variables/${existingSecret.key}`,
@@ -1553,7 +1552,7 @@ const syncSecretsGitLab = async ({
}
// delete secrets
for (const sec of getSecretsRes) {
for await (const sec of getSecretsRes) {
if (!(sec.key in secrets)) {
await request.delete(
`${INTEGRATION_GITLAB_API_URL}/v4/projects/${integration?.appId}/variables/${sec.key}`,

View File

@@ -8,6 +8,20 @@ import {
} from '../variables';
import request from '../config/request';
interface Team {
name: string;
teamId: string;
}
/**
* Return list of teams for integration authorization [integrationAuth]
* @param {Object} obj
* @param {String} obj.integrationAuth - integration authorization to get teams for
* @param {String} obj.accessToken - access token for integration authorization
* @returns {Object[]} teams - teams of integration authorization
* @returns {String} teams.name - name of team
* @returns {String} teams.teamId - id of team
*/
const getTeams = async ({
integrationAuth,
accessToken
@@ -15,10 +29,6 @@ const getTeams = async ({
integrationAuth: IIntegrationAuth;
accessToken: string;
}) => {
interface Team {
name: string;
teamId: string;
}
let teams: Team[] = [];
try {
@@ -39,12 +49,20 @@ const getTeams = async ({
return teams;
}
/**
* Return list of teams for GitLab integration
* @param {Object} obj
* @param {String} obj.accessToken - access token for GitLab API
* @returns {Object[]} teams - teams that user is part of in GitLab
* @returns {String} teams.name - name of team
* @returns {String} teams.teamId - id of team
*/
const getTeamsGitLab = async ({
accessToken
}: {
accessToken: string;
}) => {
let teams = [];
let teams: Team[] = [];
try {
const res = (await request.get(
`${INTEGRATION_GITLAB_API_URL}/v4/groups`,

View File

@@ -62,13 +62,11 @@ const integrationSchema = new Schema<IIntegration>(
default: null,
},
appId: {
// (new)
// id of app in provider
type: String,
default: null,
},
targetEnvironment: {
// (new)
// target environment
type: String,
default: null,

View File

@@ -64,9 +64,6 @@ router.post(
integrationAuthController.saveIntegrationAccessToken
);
// this can optionally accept a teamId?
// IF teamId is passed in then it probably means that we want to get
// the apps for that team
router.get(
'/:integrationAuthId/apps',
requireAuth({
@@ -76,7 +73,7 @@ router.get(
acceptedRoles: [ADMIN, MEMBER]
}),
param('integrationAuthId'),
query('entity'),
query('teamId'),
validateRequest,
integrationAuthController.getIntegrationAuthApps
);

View File

@@ -7,9 +7,6 @@ import {
setIntegrationAuthAccessHelper,
} from '../helpers/integration';
// should sync stuff be here too? Probably.
// TODO: move bot functions to IntegrationService.
/**
* Class to handle integrations
*/

View File

@@ -25,7 +25,6 @@ const fetchIntegrationAuthApps = async ({
integrationAuthId: string;
teamId?: string;
}) => {
console.log('fetchIntegrationAuthApps: ', integrationAuthId);
const searchParams = new URLSearchParams(teamId ? { teamId } : undefined);
const { data } = await apiRequest.get<{ apps: App[] }>(
`/api/v1/integration-auth/${integrationAuthId}/apps`,

View File

@@ -23,8 +23,6 @@ const gitLabEntities = [
{ name: 'Group', value: 'group' }
]
// TODO: flesh out the data flow...
export default function GitLabCreateIntegrationPage() {
const router = useRouter();
@@ -33,59 +31,51 @@ export default function GitLabCreateIntegrationPage() {
const { data: workspace } = useGetWorkspaceById(localStorage.getItem('projectData.id') ?? '');
const { data: integrationAuth } = useGetIntegrationAuthById(integrationAuthId as string ?? '');
const [targetEntity, setTargetEntity] = useState(gitLabEntities[0].value);
const [targetTeam, setTargetTeam] = useState('aa'); // ?
const [targetTeamId] = useState(undefined);
const [targetTeamId, setTargetTeamId] = useState<string | null>(null);
const { data: integrationAuthApps } = useGetIntegrationAuthApps({
integrationAuthId: integrationAuthId as string ?? '',
...(targetTeamId ? { teamId: targetTeamId } : {})
});
const { data: integrationAuthTeams } = useGetIntegrationAuthTeams(integrationAuthId as string ?? '');
console.log('integrationAuthTeams: ', integrationAuthTeams);
const [targetEntity, setTargetEntity] = useState(gitLabEntities[0].value);
const [selectedSourceEnvironment, setSelectedSourceEnvironment] = useState('');
const [targetApp, setTargetApp] = useState('');
const [targetAppId, setTargetAppId] = useState('');
const [isLoading, setIsLoading] = useState(false);
useEffect(() => {
if (workspace) {
setSelectedSourceEnvironment(workspace.environments[0].slug);
}
if (workspace) {
setSelectedSourceEnvironment(workspace.environments[0].slug);
}
}, [workspace]);
useEffect(() => {
if (integrationAuthApps) {
if (integrationAuthApps.length > 0) {
console.log('AA');
setTargetApp(integrationAuthApps[0].name);
setTargetAppId(integrationAuthApps[0].appId as string);
} else {
console.log('BB');
setTargetApp('none');
setTargetAppId('none');
}
}
}, [integrationAuthApps]);
useEffect(() => {
// if (targetEntity === 'group' && integrationAuthTeams) {
// if (integrationAuthTeams.length > 0) {
// setTargetTeam(integrationAuthTeams[0].name);
// } else {
// setTargetTeam('none');
// }
// }
// if (targetEntity === 'group') {
// if (integrationAuthTeams && integrationAuthTeams.length > 0) {
// setTargetTeamId(integrationAuthTeams[0].teamId);
// } else {
// setTargetTeamId('');
// }
// } else {
// }
}, [integrationAuthTeams, integrationAuthApps, targetEntity]);
if (targetEntity === 'group' && integrationAuthTeams && integrationAuthTeams.length > 0) {
if (integrationAuthTeams) {
if (integrationAuthTeams.length > 0) {
// case: user is part of at least 1 group in GitLab
setTargetTeamId(integrationAuthTeams[0].teamId);
} else {
// case: user is not part of any groups in GitLab
setTargetTeamId('none');
}
}
} else if (targetEntity === 'individual') {
setTargetTeamId(null);
}
}, [targetEntity, integrationAuthTeams]);
const handleButtonClick = async () => {
try {
@@ -95,8 +85,8 @@ export default function GitLabCreateIntegrationPage() {
await createIntegration({
integrationAuthId: integrationAuth?._id,
isActive: true,
app: targetApp,
appId: (integrationAuthApps?.find((integrationAuthApp) => integrationAuthApp.name === targetApp))?.appId ?? null,
app: (integrationAuthApps?.find((integrationAuthApp) => integrationAuthApp.appId === targetAppId))?.name ?? null,
appId: targetAppId,
sourceEnvironment: selectedSourceEnvironment,
targetEnvironment: null,
owner: null,
@@ -113,15 +103,7 @@ export default function GitLabCreateIntegrationPage() {
}
}
console.log('A', (integrationAuth && workspace && selectedSourceEnvironment && integrationAuthApps && integrationAuthTeams && targetApp && targetTeam));
console.log('B', integrationAuth);
console.log('C', workspace);
console.log('D', selectedSourceEnvironment);
console.log('E', integrationAuthApps);
console.log('F', integrationAuthTeams);
console.log('G', targetApp);
console.log('H', targetTeam);
return (integrationAuth && workspace && selectedSourceEnvironment && integrationAuthApps && integrationAuthTeams && targetApp && targetTeam) ? (
return (integrationAuth && workspace && selectedSourceEnvironment && integrationAuthApps && integrationAuthTeams && targetAppId) ? (
<div className="h-full w-full flex justify-center items-center">
<Card className="max-w-md p-8 rounded-md">
<CardTitle className='text-center'>GitLab Integration</CardTitle>
@@ -165,13 +147,13 @@ export default function GitLabCreateIntegrationPage() {
className='mt-4'
>
<Select
value={targetTeam}
onValueChange={(val) => setTargetTeam(val)}
value={targetTeamId}
onValueChange={(val) => setTargetTeamId(val)}
className='w-full border border-mineshaft-500'
>
{integrationAuthTeams.length > 0 ? (
integrationAuthTeams.map((integrationAuthTeam) => (
<SelectItem value={integrationAuthTeam.name} key={`target-team-${integrationAuthTeam.name}`}>
<SelectItem value={integrationAuthTeam.teamId} key={`target-team-${integrationAuthTeam.teamId}`}>
{integrationAuthTeam.name}
</SelectItem>
))
@@ -188,14 +170,14 @@ export default function GitLabCreateIntegrationPage() {
className='mt-4'
>
<Select
value={targetApp}
onValueChange={(val) => setTargetApp(val)}
value={targetAppId}
onValueChange={(val) => setTargetAppId(val)}
className='w-full border border-mineshaft-500'
isDisabled={integrationAuthApps.length === 0}
>
{integrationAuthApps.length > 0 ? (
integrationAuthApps.map((integrationAuthApp) => (
<SelectItem value={integrationAuthApp.name} key={`target-app-${integrationAuthApp.name}`}>
<SelectItem value={integrationAuthApp.appId as string} key={`target-app-${integrationAuthApp.appId}`}>
{integrationAuthApp.name}
</SelectItem>
))