diff --git a/backend/package-lock.json b/backend/package-lock.json index 49df5a596..b90e448cb 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -30,6 +30,7 @@ "@fastify/static": "^7.0.4", "@fastify/swagger": "^8.14.0", "@fastify/swagger-ui": "^2.1.0", + "@gitbeaker/rest": "^42.5.0", "@google-cloud/kms": "^4.5.0", "@infisical/quic": "^1.0.8", "@node-saml/passport-saml": "^5.0.1", @@ -7807,6 +7808,48 @@ "p-limit": "^3.1.0" } }, + "node_modules/@gitbeaker/core": { + "version": "42.5.0", + "resolved": "https://registry.npmjs.org/@gitbeaker/core/-/core-42.5.0.tgz", + "integrity": "sha512-rMWpOPaZi1iLiifnOIoVO57p2EmQQdfIwP4txqNyMvG4WjYP5Ez0U7jRD9Nra41x6K5kTPBZkuQcAdxVWRJcEQ==", + "license": "MIT", + "dependencies": { + "@gitbeaker/requester-utils": "^42.5.0", + "qs": "^6.12.2", + "xcase": "^2.0.1" + }, + "engines": { + "node": ">=18.20.0" + } + }, + "node_modules/@gitbeaker/requester-utils": { + "version": "42.5.0", + "resolved": "https://registry.npmjs.org/@gitbeaker/requester-utils/-/requester-utils-42.5.0.tgz", + "integrity": "sha512-HLdLS9LPBMVQumvroQg/4qkphLDtwDB+ygEsrD2u4oYCMUtXV4V1xaVqU4yTXjbTJ5sItOtdB43vYRkBcgueBw==", + "license": "MIT", + "dependencies": { + "picomatch-browser": "^2.2.6", + "qs": "^6.12.2", + "rate-limiter-flexible": "^4.0.1", + "xcase": "^2.0.1" + }, + "engines": { + "node": ">=18.20.0" + } + }, + "node_modules/@gitbeaker/rest": { + "version": "42.5.0", + "resolved": "https://registry.npmjs.org/@gitbeaker/rest/-/rest-42.5.0.tgz", + "integrity": "sha512-oC5cM6jS7aFOp0luTw5mWSRuMgdxwHRLZQ/aWkI+ETMfsprR/HyxsXfljlMY/XJ/fRxTbRJiodR5Axf66WjO3w==", + "license": "MIT", + "dependencies": { + "@gitbeaker/core": "^42.5.0", + "@gitbeaker/requester-utils": "^42.5.0" + }, + "engines": { + "node": ">=18.20.0" + } + }, "node_modules/@google-cloud/kms": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/@google-cloud/kms/-/kms-4.5.0.tgz", @@ -24628,6 +24671,18 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/picomatch-browser": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/picomatch-browser/-/picomatch-browser-2.2.6.tgz", + "integrity": "sha512-0ypsOQt9D4e3hziV8O4elD9uN0z/jtUEfxVRtNaAAtXIyUx9m/SzlO020i8YNL2aL/E6blOvvHQcin6HZlFy/w==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/pify": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", @@ -25562,6 +25617,12 @@ "node": ">= 0.6" } }, + "node_modules/rate-limiter-flexible": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/rate-limiter-flexible/-/rate-limiter-flexible-4.0.1.tgz", + "integrity": "sha512-2/dGHpDFpeA0+755oUkW+EKyklqLS9lu0go9pDsbhqQjZcxfRyJ6LA4JI0+HAdZ2bemD/oOjUeZQB2lCZqXQfQ==", + "license": "ISC" + }, "node_modules/raw-body": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", @@ -31039,6 +31100,12 @@ } } }, + "node_modules/xcase": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/xcase/-/xcase-2.0.1.tgz", + "integrity": "sha512-UmFXIPU+9Eg3E9m/728Bii0lAIuoc+6nbrNUKaRPJOFp91ih44qqGlWtxMB6kXFrRD6po+86ksHM5XHCfk6iPw==", + "license": "MIT" + }, "node_modules/xml-crypto": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/xml-crypto/-/xml-crypto-6.0.1.tgz", diff --git a/backend/package.json b/backend/package.json index c2bfc29d9..128de7bd6 100644 --- a/backend/package.json +++ b/backend/package.json @@ -149,6 +149,7 @@ "@fastify/static": "^7.0.4", "@fastify/swagger": "^8.14.0", "@fastify/swagger-ui": "^2.1.0", + "@gitbeaker/rest": "^42.5.0", "@google-cloud/kms": "^4.5.0", "@infisical/quic": "^1.0.8", "@node-saml/passport-saml": "^5.0.1", diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index cb234e98b..5d4a91eec 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -2409,10 +2409,11 @@ export const SecretSyncs = { appId: "The ID of the Fly.io app to sync secrets to." }, GITLAB: { - projectId: "The GitLab project to sync secrets to.", - projectName: "The GitLab project name to sync secrets to.", - groupId: "The GitLab group to sync secrets to.", - scope: "The GitLab project scope that secrets should be synced to. (default: individual)", + projectId: "The GitLab Project ID to sync secrets to.", + projectName: "The GitLab Project Name to sync secrets to.", + groupId: "The GitLab Group ID to sync secrets to.", + groupName: "The GitLab Group Name to sync secrets to.", + scope: "The GitLab scope that secrets should be synced to. (default: project)", targetEnvironment: "The GitLab environment scope that secrets should be synced to. (default: *)", shouldProtectSecrets: "Whether variables should be protected", shouldMaskSecrets: "Whether variables should be masked in logs", diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 547de9898..9166e7569 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -247,6 +247,10 @@ const envSchema = z INF_APP_CONNECTION_GITHUB_RADAR_APP_ID: zpStr(z.string().optional()), INF_APP_CONNECTION_GITHUB_RADAR_APP_WEBHOOK_SECRET: zpStr(z.string().optional()), + // gitlab oauth + INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID: zpStr(z.string().optional()), + INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET: zpStr(z.string().optional()), + // gcp app INF_APP_CONNECTION_GCP_SERVICE_ACCOUNT_CREDENTIAL: zpStr(z.string().optional()), diff --git a/backend/src/server/routes/v1/app-connection-routers/gitlab-connection-router.ts b/backend/src/server/routes/v1/app-connection-routers/gitlab-connection-router.ts index 59bb7d3a4..b168cc97f 100644 --- a/backend/src/server/routes/v1/app-connection-routers/gitlab-connection-router.ts +++ b/backend/src/server/routes/v1/app-connection-routers/gitlab-connection-router.ts @@ -34,9 +34,6 @@ export const registerGitLabConnectionRouter = async (server: FastifyZodProvider) params: z.object({ connectionId: z.string().uuid() }), - querystring: z.object({ - group: z.string().optional() - }), response: { 200: z .object({ @@ -52,8 +49,7 @@ export const registerGitLabConnectionRouter = async (server: FastifyZodProvider) const projects: TGitLabProject[] = await server.services.appConnection.gitlab.listProjects( connectionId, - req.permission, - req.query.group + req.permission ); return projects; diff --git a/backend/src/services/app-connection/gitlab/gitlab-connection-fns.ts b/backend/src/services/app-connection/gitlab/gitlab-connection-fns.ts index 3b643ac3d..d874d1447 100644 --- a/backend/src/services/app-connection/gitlab/gitlab-connection-fns.ts +++ b/backend/src/services/app-connection/gitlab/gitlab-connection-fns.ts @@ -1,5 +1,6 @@ /* eslint-disable no-await-in-loop */ -import { AxiosError, AxiosResponse } from "axios"; +import { GitbeakerRequestError, Gitlab } from "@gitbeaker/rest"; +import { AxiosError } from "axios"; import { getConfig } from "@app/lib/config/env"; import { request } from "@app/lib/config/request"; @@ -13,7 +14,7 @@ import { IntegrationUrls } from "@app/services/integration-auth/integration-list import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { TAppConnectionDALFactory } from "../app-connection-dal"; -import { GitLabConnectionMethod } from "./gitlab-connection-enums"; +import { GitLabAccessTokenType, GitLabConnectionMethod } from "./gitlab-connection-enums"; import { TGitLabConnection, TGitLabConnectionConfig, TGitLabGroup, TGitLabProject } from "./gitlab-connection-types"; interface GitLabOAuthTokenResponse { @@ -26,7 +27,7 @@ interface GitLabOAuthTokenResponse { } export const getGitLabConnectionListItem = () => { - const { CLIENT_ID_GITLAB_LOGIN } = getConfig(); + const { INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID } = getConfig(); return { name: "GitLab" as const, @@ -35,7 +36,7 @@ export const getGitLabConnectionListItem = () => { GitLabConnectionMethod.AccessToken, GitLabConnectionMethod.OAuth ], - oauthClientId: CLIENT_ID_GITLAB_LOGIN + oauthClientId: INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID }; }; @@ -47,6 +48,18 @@ export const getGitLabInstanceUrl = async (instanceUrl?: string) => { return gitLabInstanceUrl; }; +export const getGitLabClient = async (accessToken: string, instanceUrl?: string, isOAuth = false) => { + const host = await getGitLabInstanceUrl(instanceUrl); + + const client = new Gitlab({ + host, + ...(isOAuth ? { oauthToken: accessToken } : { token: accessToken }), + camelize: true + }); + + return client; +}; + export const refreshGitLabToken = async ( refreshToken: string, appId: string, @@ -55,8 +68,9 @@ export const refreshGitLabToken = async ( kmsService: Pick, instanceUrl?: string ): Promise => { - const { CLIENT_ID_GITLAB_LOGIN, CLIENT_SECRET_GITLAB_LOGIN, SITE_URL } = getConfig(); - if (!CLIENT_SECRET_GITLAB_LOGIN || !CLIENT_ID_GITLAB_LOGIN || !SITE_URL) { + const { INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID, INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET, SITE_URL } = + getConfig(); + if (!INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET || !INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID || !SITE_URL) { throw new InternalServerError({ message: `GitLab environment variables have not been configured` }); @@ -65,9 +79,9 @@ export const refreshGitLabToken = async ( const payload = new URLSearchParams({ grant_type: "refresh_token", refresh_token: refreshToken, - client_id: CLIENT_ID_GITLAB_LOGIN, - client_secret: CLIENT_SECRET_GITLAB_LOGIN, - redirect_uri: `${SITE_URL}/integrations/gitlab/oauth2/callback` + client_id: INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID, + client_secret: INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET, + redirect_uri: `${SITE_URL}/organization/app-connections/gitlab/oauth/callback` }); try { @@ -95,7 +109,6 @@ export const refreshGitLabToken = async ( }); await appConnectionDAL.updateById(appId, { encryptedCredentials }); - return data.access_token; } catch (error: unknown) { if (error instanceof AxiosError) { @@ -113,8 +126,9 @@ export const exchangeGitLabOAuthCode = async ( code: string, instanceUrl?: string ): Promise => { - const { CLIENT_ID_GITLAB_LOGIN, CLIENT_SECRET_GITLAB_LOGIN, SITE_URL } = getConfig(); - if (!CLIENT_SECRET_GITLAB_LOGIN || !CLIENT_ID_GITLAB_LOGIN || !SITE_URL) { + const { INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID, INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET, SITE_URL } = + getConfig(); + if (!INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET || !INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID || !SITE_URL) { throw new InternalServerError({ message: `GitLab environment variables have not been configured` }); @@ -124,9 +138,9 @@ export const exchangeGitLabOAuthCode = async ( const payload = new URLSearchParams({ grant_type: "authorization_code", code, - client_id: CLIENT_ID_GITLAB_LOGIN, - client_secret: CLIENT_SECRET_GITLAB_LOGIN, - redirect_uri: `${SITE_URL}/integrations/gitlab/oauth2/callback` + client_id: INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID, + client_secret: INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET, + redirect_uri: `${SITE_URL}/organization/app-connections/gitlab/oauth/callback` }); const url = await getGitLabInstanceUrl(instanceUrl); @@ -173,30 +187,24 @@ export const validateGitLabConnectionCredentials = async (config: TGitLabConnect }); } - let response: AxiosResponse | null = null; - try { - const url = await getGitLabInstanceUrl(inputCredentials.instanceUrl); - response = await request.get(`${url}/api/v4/user`, { - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: "application/json" - } - }); + const client = await getGitLabClient( + accessToken, + inputCredentials.instanceUrl, + method === GitLabConnectionMethod.OAuth + ); + await client.Users.showCurrentUser(); } catch (error: unknown) { - if (error instanceof AxiosError) { + logger.error(error, "Error validating GitLab connection credentials"); + + if (error instanceof GitbeakerRequestError) { throw new BadRequestError({ - message: `Failed to validate credentials: ${error.message}` + message: `Failed to validate credentials: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}` }); } - throw new BadRequestError({ - message: "Unable to validate connection: verify credentials" - }); - } - if (!response?.data) { - throw new InternalServerError({ - message: "Failed to validate credentials: Response was empty" + throw new BadRequestError({ + message: `Failed to validate credentials: ${(error as Error)?.message || "verify credentials"}` }); } @@ -217,20 +225,18 @@ export const validateGitLabConnectionCredentials = async (config: TGitLabConnect export const listGitLabProjects = async ({ appConnection, appConnectionDAL, - kmsService, - teamId + kmsService }: { appConnection: TGitLabConnection; appConnectionDAL: Pick; kmsService: Pick; - teamId?: string; }): Promise => { let { accessToken } = appConnection.credentials; if ( appConnection.method === GitLabConnectionMethod.OAuth && appConnection.credentials.refreshToken && - appConnection.credentials.expiresAt < new Date() + new Date(appConnection.credentials.expiresAt) < new Date() ) { accessToken = await refreshGitLabToken( appConnection.credentials.refreshToken, @@ -242,160 +248,29 @@ export const listGitLabProjects = async ({ ); } - const url = await getGitLabInstanceUrl(appConnection.credentials.instanceUrl); - const gitLabApiUrl = `${url}/api/v4`; - - const projects: TGitLabProject[] = []; - let page = 1; - const perPage = 100; - let hasMorePages = true; - try { - if (teamId) { - while (hasMorePages) { - const { data } = await request.get(`${gitLabApiUrl}/groups/${teamId}/projects`, { - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: "application/json" - }, - params: { - page: page.toString(), - per_page: perPage.toString(), - order_by: "updated_at", - sort: "desc", - include_subgroups: "true" - } - }); + const client = await getGitLabClient( + accessToken, + appConnection.credentials.instanceUrl, + appConnection.method === GitLabConnectionMethod.OAuth + ); + const projects = await client.Projects.all({ + archived: false, + includePendingDelete: false, + membership: true, + includeHidden: false, + imported: false + }); - if (!data) { - throw new InternalServerError({ - message: "Failed to get group projects: Response was empty" - }); - } - - data.forEach((project) => { - projects.push({ - name: project.name, - id: project.id.toString() - }); - }); - - hasMorePages = data.length === perPage; - page += 1; - } - } else { - const { data: userData } = await request.get<{ id: string }>(`${gitLabApiUrl}/user`, { - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: "application/json" - } - }); - - if (!userData?.id) { - throw new InternalServerError({ - message: "Failed to get current user information" - }); - } - - while (hasMorePages) { - const { data } = await request.get(`${gitLabApiUrl}/users/${userData.id}/projects`, { - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: "application/json" - }, - params: { - page: page.toString(), - per_page: perPage.toString(), - order_by: "updated_at", - sort: "desc" - } - }); - - if (!data) { - throw new InternalServerError({ - message: "Failed to get user projects: Response was empty" - }); - } - - data.forEach((project) => { - projects.push({ - name: project.name, - id: project.id.toString() - }); - }); - - hasMorePages = data.length === perPage; - page += 1; - } - - if (projects.length === 0 && appConnection.method === GitLabConnectionMethod.AccessToken) { - try { - const { data: tokenAssociations } = await request.get<{ - projects?: TGitLabProject[]; - groups?: Array<{ projects?: TGitLabProject[] }>; - }>(`${gitLabApiUrl}/personal_access_tokens/self/associations`, { - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: "application/json" - }, - params: { - min_access_level: "50" - } - }); - - if (tokenAssociations?.projects) { - tokenAssociations.projects.forEach((project) => { - projects.push({ - name: project.name, - id: project.id.toString() - }); - }); - } - - if (tokenAssociations?.groups) { - tokenAssociations.groups.forEach((group) => { - if (group.projects) { - group.projects.forEach((project) => { - const existingProject = projects.find((p) => p.id === project.id.toString()); - if (!existingProject) { - projects.push({ - name: project.name, - id: project.id.toString() - }); - } - }); - } - }); - } - } catch (error) { - logger.warn(error, "Failed to fetch projects via personal access token associations:"); - } - } - } - - return projects; + return projects.map((project) => ({ + name: project.pathWithNamespace, + id: project.id.toString() + })); } catch (error: unknown) { - if (error instanceof AxiosError) { - const status = error.response?.status; - const { message } = error; - - if (status === 401) { - throw new BadRequestError({ - message: `GitLab authentication failed: ${message}` - }); - } else if (status === 403) { - throw new BadRequestError({ - message: `GitLab access forbidden: ${message}` - }); - } else if (status === 404) { - throw new BadRequestError({ - message: teamId ? `GitLab group not found or access denied: ${message}` : `GitLab user not found: ${message}` - }); - } else { - throw new BadRequestError({ - message: `Failed to fetch GitLab projects: ${message}` - }); - } + if (error instanceof GitbeakerRequestError) { + throw new BadRequestError({ + message: `Failed to fetch GitLab projects: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}` + }); } if (error instanceof InternalServerError) { @@ -411,22 +286,25 @@ export const listGitLabProjects = async ({ export const listGitLabGroups = async ({ appConnection, appConnectionDAL, - kmsService, - includeSubgroups = true, - owned = false + kmsService }: { appConnection: TGitLabConnection; appConnectionDAL: Pick; kmsService: Pick; - includeSubgroups?: boolean; - owned?: boolean; }): Promise => { let { accessToken } = appConnection.credentials; + if ( + appConnection.method === GitLabConnectionMethod.AccessToken && + appConnection.credentials.accessTokenType === GitLabAccessTokenType.Project + ) { + return []; + } + if ( appConnection.method === GitLabConnectionMethod.OAuth && appConnection.credentials.refreshToken && - appConnection.credentials.expiresAt < new Date() + new Date(appConnection.credentials.expiresAt) < new Date() ) { accessToken = await refreshGitLabToken( appConnection.credentials.refreshToken, @@ -438,69 +316,28 @@ export const listGitLabGroups = async ({ ); } - const url = await getGitLabInstanceUrl(appConnection.credentials.instanceUrl); - const gitLabApiUrl = `${url}/api/v4`; - - const groups: TGitLabGroup[] = []; - let page = 1; - const perPage = 100; - let hasMorePages = true; - try { - while (hasMorePages) { - const { data } = await request.get(`${gitLabApiUrl}/groups`, { - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: "application/json" - }, - params: { - page: page.toString(), - per_page: perPage.toString(), - order_by: "name", - sort: "asc", - all_available: (!owned).toString(), - owned: owned.toString(), - min_access_level: "10", - ...(includeSubgroups && { with_custom_attributes: "true" }) - } - }); + const client = await getGitLabClient( + accessToken, + appConnection.credentials.instanceUrl, + appConnection.method === GitLabConnectionMethod.OAuth + ); - if (!data) { - throw new InternalServerError({ - message: "Failed to get groups: Response was empty" - }); - } + const groups = await client.Groups.all({ + orderBy: "name", + sort: "asc", + minAccessLevel: 50 + }); - data.forEach((group) => { - groups.push({ - id: group.id.toString(), - name: group.name - }); - }); - - hasMorePages = data.length === perPage; - page += 1; - } - - return groups; + return groups.map((group) => ({ + id: group.id.toString(), + name: group.name + })); } catch (error: unknown) { - if (error instanceof AxiosError) { - const status = error.response?.status; - const { message } = error; - - if (status === 401) { - throw new BadRequestError({ - message: `GitLab authentication failed: ${message}` - }); - } else if (status === 403) { - throw new BadRequestError({ - message: `GitLab access forbidden: ${message}` - }); - } else { - throw new BadRequestError({ - message: `Failed to fetch GitLab groups: ${message}` - }); - } + if (error instanceof GitbeakerRequestError) { + throw new BadRequestError({ + message: `Failed to fetch GitLab groups: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}` + }); } if (error instanceof InternalServerError) { diff --git a/backend/src/services/app-connection/gitlab/gitlab-connection-service.ts b/backend/src/services/app-connection/gitlab/gitlab-connection-service.ts index 863ef1659..818697be3 100644 --- a/backend/src/services/app-connection/gitlab/gitlab-connection-service.ts +++ b/backend/src/services/app-connection/gitlab/gitlab-connection-service.ts @@ -18,10 +18,10 @@ export const gitlabConnectionService = ( appConnectionDAL: Pick, kmsService: Pick ) => { - const listProjects = async (connectionId: string, actor: OrgServiceActor, teamId?: string) => { + const listProjects = async (connectionId: string, actor: OrgServiceActor) => { try { const appConnection = await getAppConnection(AppConnection.GitLab, connectionId, actor); - const projects = await listGitLabProjects({ appConnection, appConnectionDAL, kmsService, teamId }); + const projects = await listGitLabProjects({ appConnection, appConnectionDAL, kmsService }); return projects; } catch (error) { logger.error(error, `Failed to establish connection with GitLab for app ${connectionId}`); diff --git a/backend/src/services/secret-sync/gitlab/gitlab-sync-enums.ts b/backend/src/services/secret-sync/gitlab/gitlab-sync-enums.ts index c93e51f5f..9f735f563 100644 --- a/backend/src/services/secret-sync/gitlab/gitlab-sync-enums.ts +++ b/backend/src/services/secret-sync/gitlab/gitlab-sync-enums.ts @@ -1,4 +1,4 @@ export enum GitLabSyncScope { - Individual = "individual", + Project = "project", Group = "group" } diff --git a/backend/src/services/secret-sync/gitlab/gitlab-sync-fns.ts b/backend/src/services/secret-sync/gitlab/gitlab-sync-fns.ts index cbdce57a2..37a6d7cf7 100644 --- a/backend/src/services/secret-sync/gitlab/gitlab-sync-fns.ts +++ b/backend/src/services/secret-sync/gitlab/gitlab-sync-fns.ts @@ -1,9 +1,13 @@ /* eslint-disable no-await-in-loop */ -import { request } from "@app/lib/config/request"; -import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator"; +import { GitbeakerRequestError } from "@gitbeaker/rest"; + import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal"; -import { GitLabConnectionMethod, refreshGitLabToken, TGitLabConnection } from "@app/services/app-connection/gitlab"; -import { IntegrationUrls } from "@app/services/integration-auth/integration-list"; +import { + getGitLabClient, + GitLabConnectionMethod, + refreshGitLabToken, + TGitLabConnection +} from "@app/services/app-connection/gitlab"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { TGitLabSyncWithCredentials, TGitLabVariable } from "@app/services/secret-sync/gitlab/gitlab-sync-types"; import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors"; @@ -11,6 +15,7 @@ import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns"; import { TSecretMap } from "@app/services/secret-sync/secret-sync-types"; import { SECRET_SYNC_NAME_MAP } from "../secret-sync-maps"; +import { GitLabSyncScope } from "./gitlab-sync-enums"; interface TGitLabVariablePayload { key?: string; @@ -42,81 +47,64 @@ const getValidAccessToken = async ( if ( connection.method === GitLabConnectionMethod.OAuth && connection.credentials.refreshToken && - connection.credentials.expiresAt < new Date() + new Date(connection.credentials.expiresAt) < new Date() ) { const accessToken = await refreshGitLabToken( connection.credentials.refreshToken, connection.id, connection.orgId, appConnectionDAL, - kmsService + kmsService, + connection.credentials.instanceUrl ); return accessToken; } return connection.credentials.accessToken; }; -const getGitLabApiUrl = async (connection: TGitLabConnection): Promise => { - const baseUrl = connection.credentials.instanceUrl || IntegrationUrls.GITLAB_API_URL; - await blockLocalAndPrivateIpAddresses(baseUrl); - return baseUrl.includes("/api") ? baseUrl : `${baseUrl}/api`; -}; - -const buildVariablesEndpoint = (apiUrl: string, projectId: string): string => { - return `${apiUrl}/v4/projects/${encodeURIComponent(projectId)}/variables`; -}; - const getGitLabVariables = async ({ accessToken, connection, - projectId, + scope, + resourceId, targetEnvironment }: { accessToken: string; connection: TGitLabConnection; - projectId: string; + scope: GitLabSyncScope; + resourceId: string; targetEnvironment?: string; }): Promise => { try { - const apiUrl = await getGitLabApiUrl(connection); - const baseEndpoint = buildVariablesEndpoint(apiUrl, projectId); + const client = await getGitLabClient( + accessToken, + connection.credentials.instanceUrl, + connection.method === GitLabConnectionMethod.OAuth + ); - const headers = { - Authorization: `Bearer ${accessToken}`, - "Accept-Encoding": "application/json", - "Content-Type": "application/json" - }; + let variables: TGitLabVariable[] = []; - let allVariables: TGitLabVariable[] = []; - let url: string | null = `${baseEndpoint}?per_page=100`; - - if (targetEnvironment) { - url += `&filter[environment_scope]=${encodeURIComponent(targetEnvironment)}`; - } - - while (url) { - const response = await request.get(url, { headers }); - allVariables = [...allVariables, ...(response.data || [])]; - - const linkHeader = response.headers.link as string; - const nextLink = linkHeader?.split(",").find((part: string) => part.includes('rel="next"')); - - if (nextLink) { - url = nextLink.trim().split(";")[0].slice(1, -1); - } else { - url = null; - } + if (scope === GitLabSyncScope.Project) { + variables = await client.ProjectVariables.all(resourceId); + } else { + variables = await client.GroupVariables.all(resourceId); } if (targetEnvironment) { - return allVariables.filter((variable) => variable.environment_scope === targetEnvironment); + variables = variables.filter((v) => v.environmentScope === targetEnvironment); } - return allVariables; + return variables; } catch (error) { + if (error instanceof GitbeakerRequestError) { + throw new SecretSyncError({ + error: new Error( + `Failed to fetch variables: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}` + ) + }); + } throw new SecretSyncError({ - error, - secretKey: "list_variables" + error }); } }; @@ -124,38 +112,61 @@ const getGitLabVariables = async ({ const createGitLabVariable = async ({ accessToken, connection, - projectId, + scope, + resourceId, variable }: { accessToken: string; connection: TGitLabConnection; - projectId: string; + scope: GitLabSyncScope; + resourceId: string; variable: TGitLabVariableCreate; }): Promise => { try { - const apiUrl = await getGitLabApiUrl(connection); - const endpoint = buildVariablesEndpoint(apiUrl, projectId); + const client = await getGitLabClient( + accessToken, + connection.credentials.instanceUrl, + connection.method === GitLabConnectionMethod.OAuth + ); const payload = { key: variable.key, value: variable.value, - variable_type: variable.variable_type || "env_var", - environment_scope: variable.environment_scope || "*", + variableType: "env_var", + environmentScope: variable.environment_scope || "*", protected: variable.protected || false, masked: variable.masked || false, masked_and_hidden: variable.masked_and_hidden || false, - raw: false, - ...(variable.description && { description: variable.description }) + raw: false }; - await request.post(endpoint, payload, { - headers: { - Authorization: `Bearer ${accessToken}`, - "Accept-Encoding": "application/json", - "Content-Type": "application/json" - } - }); + if (scope === GitLabSyncScope.Project) { + await client.ProjectVariables.create(resourceId, payload.key, payload.value, { + variableType: "env_var", + environmentScope: payload.environmentScope, + protected: payload.protected, + masked: payload.masked, + masked_and_hidden: payload.masked_and_hidden, + raw: false + }); + } else { + await client.GroupVariables.create(resourceId, payload.key, payload.value, { + variableType: "env_var", + environmentScope: payload.environmentScope, + protected: payload.protected, + masked: payload.masked, + raw: false + }); + } } catch (error) { + if (error instanceof GitbeakerRequestError) { + throw new SecretSyncError({ + error: new Error( + `Failed to create variable: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}` + ), + secretKey: variable.key + }); + } throw new SecretSyncError({ error, secretKey: variable.key @@ -166,45 +177,57 @@ const createGitLabVariable = async ({ const updateGitLabVariable = async ({ accessToken, connection, - projectId, + scope, + resourceId, key, variable, targetEnvironment }: { accessToken: string; connection: TGitLabConnection; - projectId: string; + scope: GitLabSyncScope; + resourceId: string; key: string; variable: TGitLabVariableUpdate; targetEnvironment?: string; }): Promise => { try { - const apiUrl = await getGitLabApiUrl(connection); - const baseEndpoint = buildVariablesEndpoint(apiUrl, projectId); - let url = `${baseEndpoint}/${encodeURIComponent(key)}`; + const client = await getGitLabClient( + accessToken, + connection.credentials.instanceUrl, + connection.method === GitLabConnectionMethod.OAuth + ); - if (targetEnvironment) { - url += `?filter[environment_scope]=${encodeURIComponent(targetEnvironment)}`; - } - - const payload = { - value: variable.value, - ...(variable.variable_type && { variable_type: variable.variable_type }), - ...(variable.environment_scope && { environment_scope: variable.environment_scope }), + const options = { + ...(variable.environment_scope && { environmentScope: variable.environment_scope }), ...(variable.protected !== undefined && { protected: variable.protected }), - ...(variable.masked !== undefined && { masked: variable.masked }), - ...(variable.masked_and_hidden !== undefined && { masked_and_hidden: variable.masked_and_hidden }), - ...(variable.description !== undefined && { description: variable.description || "" }) + ...(variable.masked !== undefined && { masked: variable.masked }) }; - await request.put(url, payload, { - headers: { - Authorization: `Bearer ${accessToken}`, - "Accept-Encoding": "application/json", - "Content-Type": "application/json" - } - }); + if (targetEnvironment) { + options.environmentScope = targetEnvironment; + } + + if (scope === GitLabSyncScope.Project) { + await client.ProjectVariables.edit(resourceId, key, variable.value, { + ...options, + filter: { environment_scope: targetEnvironment || "*" } + }); + } else { + await client.GroupVariables.edit(resourceId, key, variable.value, { + ...options, + filter: { environment_scope: targetEnvironment || "*" } + }); + } } catch (error) { + if (error instanceof GitbeakerRequestError) { + throw new SecretSyncError({ + error: new Error( + `Failed to update variable: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}` + ), + secretKey: key + }); + } throw new SecretSyncError({ error, secretKey: key @@ -215,35 +238,52 @@ const updateGitLabVariable = async ({ const deleteGitLabVariable = async ({ accessToken, connection, - projectId, + scope, + resourceId, key, - targetEnvironment + targetEnvironment, + allVariables }: { accessToken: string; connection: TGitLabConnection; - projectId: string; + scope: GitLabSyncScope; + resourceId: string; key: string; targetEnvironment?: string; + allVariables?: TGitLabVariable[]; }): Promise => { + if (allVariables && !allVariables.find((v) => v.key === key)) { + return; + } try { - const apiUrl = await getGitLabApiUrl(connection); - const baseEndpoint = buildVariablesEndpoint(apiUrl, projectId); - let url = `${baseEndpoint}/${encodeURIComponent(key)}`; + const client = await getGitLabClient( + accessToken, + connection.credentials.instanceUrl, + connection.method === GitLabConnectionMethod.OAuth + ); + const options: { filter?: { environment_scope: string } } = {}; if (targetEnvironment) { - url += `?filter[environment_scope]=${encodeURIComponent(targetEnvironment)}`; + options.filter = { environment_scope: targetEnvironment || "*" }; } - await request.delete(url, { - headers: { - Authorization: `Bearer ${accessToken}`, - "Accept-Encoding": "application/json", - "Content-Type": "application/json" - } - }); + if (scope === GitLabSyncScope.Project) { + await client.ProjectVariables.remove(resourceId, key, options); + } else { + await client.GroupVariables.remove(resourceId, key); + } } catch (error: unknown) { + if (error instanceof GitbeakerRequestError) { + throw new SecretSyncError({ + error: new Error( + `Failed to delete variable: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}` + ), + secretKey: key + }); + } throw new SecretSyncError({ - error + error, + secretKey: key }); } }; @@ -255,8 +295,9 @@ export const GitLabSyncFns = { { appConnectionDAL, kmsService }: TGitLabSyncFactoryDeps ): Promise => { const { connection, environment, destinationConfig } = secretSync; + const { scope, targetEnvironment } = destinationConfig; - const { projectId, targetEnvironment } = destinationConfig; + const resourceId = scope === GitLabSyncScope.Project ? destinationConfig.projectId : destinationConfig.groupId; const accessToken = await getValidAccessToken(connection, appConnectionDAL, kmsService); @@ -264,13 +305,20 @@ export const GitLabSyncFns = { const currentVariables = await getGitLabVariables({ accessToken, connection, - projectId, + scope, + resourceId, targetEnvironment }); const currentVariableMap = new Map(currentVariables.map((v) => [v.key, v])); for (const [key, { value }] of Object.entries(secretMap)) { + if (value?.length < 8 && destinationConfig.shouldMaskSecrets) { + throw new SecretSyncError({ + message: `Secret ${key} is too short to be masked. GitLab requires a minimum of 8 characters for masked secrets.`, + secretKey: key + }); + } try { const existingVariable = currentVariableMap.get(key); @@ -279,17 +327,14 @@ export const GitLabSyncFns = { await updateGitLabVariable({ accessToken, connection, - projectId, + scope, + resourceId, key, variable: { value, - variable_type: existingVariable.variable_type, - environment_scope: targetEnvironment || existingVariable.environment_scope, + environment_scope: targetEnvironment, protected: destinationConfig.shouldProtectSecrets ?? existingVariable.protected, - ...(!existingVariable.masked && destinationConfig.shouldMaskSecrets && { masked: value?.length > 8 }), - ...(!existingVariable.hidden && - destinationConfig.shouldHideSecrets && { masked_and_hidden: value?.length > 8 }), - description: existingVariable.description ?? undefined + masked: existingVariable.masked || destinationConfig.shouldMaskSecrets }, targetEnvironment }); @@ -298,15 +343,16 @@ export const GitLabSyncFns = { await createGitLabVariable({ accessToken, connection, - projectId, + scope, + resourceId, variable: { key, value, variable_type: "env_var", environment_scope: targetEnvironment || "*", protected: destinationConfig.shouldProtectSecrets || false, - masked: value?.length > 8 ? destinationConfig.shouldMaskSecrets || false : false, - masked_and_hidden: value?.length > 8 ? destinationConfig.shouldHideSecrets || false : false + masked: destinationConfig.shouldMaskSecrets || false, + masked_and_hidden: destinationConfig.shouldHideSecrets || false } }); } @@ -323,13 +369,14 @@ export const GitLabSyncFns = { try { const shouldDelete = matchesSchema(variable.key, environment?.slug || "", secretSync.syncOptions.keySchema) && - !(variable.key in secretMap); + variable.key in secretMap; if (shouldDelete) { await deleteGitLabVariable({ accessToken, connection, - projectId, + scope, + resourceId, key: variable.key, targetEnvironment }); @@ -359,19 +406,30 @@ export const GitLabSyncFns = { { appConnectionDAL, kmsService }: TGitLabSyncFactoryDeps ): Promise => { const { connection, destinationConfig } = secretSync; + const { scope, targetEnvironment } = destinationConfig; - const { projectId, targetEnvironment } = destinationConfig; + const resourceId = scope === GitLabSyncScope.Project ? destinationConfig.projectId : destinationConfig.groupId; const accessToken = await getValidAccessToken(connection, appConnectionDAL, kmsService); + const allVariables = await getGitLabVariables({ + accessToken, + connection, + scope, + resourceId, + targetEnvironment + }); + for (const key of Object.keys(secretMap)) { try { await deleteGitLabVariable({ accessToken, connection, - projectId, + scope, + resourceId, key, - targetEnvironment + targetEnvironment, + allVariables }); } catch (error) { throw new SecretSyncError({ diff --git a/backend/src/services/secret-sync/gitlab/gitlab-sync-schemas.ts b/backend/src/services/secret-sync/gitlab/gitlab-sync-schemas.ts index 1715c9799..8797cc98d 100644 --- a/backend/src/services/secret-sync/gitlab/gitlab-sync-schemas.ts +++ b/backend/src/services/secret-sync/gitlab/gitlab-sync-schemas.ts @@ -14,7 +14,7 @@ import { GitLabSyncScope } from "./gitlab-sync-enums"; const GitLabSyncDestinationConfigSchema = z.discriminatedUnion("scope", [ z.object({ - scope: z.literal(GitLabSyncScope.Individual).describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.scope), + scope: z.literal(GitLabSyncScope.Project).describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.scope), projectId: z.string().min(1, "Project ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.projectId), projectName: z .string() @@ -44,11 +44,7 @@ const GitLabSyncDestinationConfigSchema = z.discriminatedUnion("scope", [ z.object({ scope: z.literal(GitLabSyncScope.Group).describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.scope), groupId: z.string().min(1, "Group ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.groupId), - projectId: z.string().min(1, "Project ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.projectId), - projectName: z - .string() - .min(1, "Project name is required") - .describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.projectName), + groupName: z.string().min(1, "Group name is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.groupName), targetEnvironment: z .string() .optional() diff --git a/backend/src/services/secret-sync/gitlab/gitlab-sync-types.ts b/backend/src/services/secret-sync/gitlab/gitlab-sync-types.ts index 3713f7b59..1f11e8a80 100644 --- a/backend/src/services/secret-sync/gitlab/gitlab-sync-types.ts +++ b/backend/src/services/secret-sync/gitlab/gitlab-sync-types.ts @@ -15,13 +15,9 @@ export type TGitLabSyncWithCredentials = TGitLabSync & { export type TGitLabVariable = { key: string; value: string; - variable_type: "env_var" | "file"; protected: boolean; masked: boolean; - hidden: boolean; - raw: boolean; - environment_scope: string; - description: string | null; + environmentScope?: string; }; export type TGitLabVariableCreate = { diff --git a/docs/self-hosting/configuration/envars.mdx b/docs/self-hosting/configuration/envars.mdx index 42756e593..6f3dffa77 100644 --- a/docs/self-hosting/configuration/envars.mdx +++ b/docs/self-hosting/configuration/envars.mdx @@ -489,10 +489,10 @@ When set, all visits to the Infisical login page will automatically redirect use Follow detailed guide to configure [GitLab SSO](/documentation/platform/sso/gitlab) - + OAuth2 client ID for GitLab login - + OAuth2 client secret for GitLab login diff --git a/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/GitLabSyncFields.tsx b/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/GitLabSyncFields.tsx index eab2120e6..98ced2cd6 100644 --- a/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/GitLabSyncFields.tsx +++ b/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/GitLabSyncFields.tsx @@ -68,7 +68,6 @@ export const GitLabSyncFields = () => { const connectionId = useWatch({ name: "connection.id", control }); const scope = useWatch({ name: "destinationConfig.scope", control }); - const selectedGroup = useWatch({ name: "destinationConfig.groupId", control }); const shouldMaskSecrets = useWatch({ name: "destinationConfig.shouldMaskSecrets", control }); const { data: groups, isLoading: isGroupsLoading } = useGitlabConnectionListGroups(connectionId, { @@ -77,7 +76,6 @@ export const GitLabSyncFields = () => { const { data: projects, isLoading: isProjectsLoading } = useGitlabConnectionListProjects( connectionId, - selectedGroup, { enabled: Boolean(connectionId) } @@ -90,14 +88,15 @@ export const GitLabSyncFields = () => { setValue("destinationConfig.projectId", ""); setValue("destinationConfig.projectName", ""); setValue("destinationConfig.groupId", ""); - setValue("destinationConfig.scope", GitlabSyncScope.Individual); + setValue("destinationConfig.groupName", ""); + setValue("destinationConfig.scope", GitlabSyncScope.Project); }} /> (