feat(secret-sync): rework GitLab secret-sync to add group variables

backend/src/lib/api-docs/constants.ts

@@ -2409,10 +2409,11 @@ export const SecretSyncs = {
       appId: "The ID of the Fly.io app to sync secrets to."
     },
     GITLAB: {
-      projectId: "The GitLab project to sync secrets to.",
-      projectName: "The GitLab project name to sync secrets to.",
-      groupId: "The GitLab group to sync secrets to.",
-      scope: "The GitLab project scope that secrets should be synced to. (default: individual)",
+      projectId: "The GitLab Project ID to sync secrets to.",
+      projectName: "The GitLab Project Name to sync secrets to.",
+      groupId: "The GitLab Group ID to sync secrets to.",
+      groupName: "The GitLab Group Name to sync secrets to.",
+      scope: "The GitLab scope that secrets should be synced to. (default: project)",
       targetEnvironment: "The GitLab environment scope that secrets should be synced to. (default: *)",
       shouldProtectSecrets: "Whether variables should be protected",
       shouldMaskSecrets: "Whether variables should be masked in logs",

backend/src/lib/config/env.ts

@@ -247,6 +247,10 @@ const envSchema = z
     INF_APP_CONNECTION_GITHUB_RADAR_APP_ID: zpStr(z.string().optional()),
     INF_APP_CONNECTION_GITHUB_RADAR_APP_WEBHOOK_SECRET: zpStr(z.string().optional()),
 
+    // gitlab oauth
+    INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID: zpStr(z.string().optional()),
+    INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET: zpStr(z.string().optional()),
+
     // gcp app
     INF_APP_CONNECTION_GCP_SERVICE_ACCOUNT_CREDENTIAL: zpStr(z.string().optional()),
 

backend/src/server/routes/v1/app-connection-routers/gitlab-connection-router.ts

@@ -34,9 +34,6 @@ export const registerGitLabConnectionRouter = async (server: FastifyZodProvider)
       params: z.object({
         connectionId: z.string().uuid()
       }),
-      querystring: z.object({
-        group: z.string().optional()
-      }),
       response: {
         200: z
           .object({
@@ -52,8 +49,7 @@ export const registerGitLabConnectionRouter = async (server: FastifyZodProvider)
 
       const projects: TGitLabProject[] = await server.services.appConnection.gitlab.listProjects(
         connectionId,
-        req.permission,
-        req.query.group
+        req.permission
       );
 
       return projects;

backend/src/services/app-connection/gitlab/gitlab-connection-fns.ts

@@ -1,5 +1,6 @@
 /* eslint-disable no-await-in-loop */
-import { AxiosError, AxiosResponse } from "axios";
+import { GitbeakerRequestError, Gitlab } from "@gitbeaker/rest";
+import { AxiosError } from "axios";
 
 import { getConfig } from "@app/lib/config/env";
 import { request } from "@app/lib/config/request";
@@ -13,7 +14,7 @@ import { IntegrationUrls } from "@app/services/integration-auth/integration-list
 import { TKmsServiceFactory } from "@app/services/kms/kms-service";
 
 import { TAppConnectionDALFactory } from "../app-connection-dal";
-import { GitLabConnectionMethod } from "./gitlab-connection-enums";
+import { GitLabAccessTokenType, GitLabConnectionMethod } from "./gitlab-connection-enums";
 import { TGitLabConnection, TGitLabConnectionConfig, TGitLabGroup, TGitLabProject } from "./gitlab-connection-types";
 
 interface GitLabOAuthTokenResponse {
@@ -26,7 +27,7 @@ interface GitLabOAuthTokenResponse {
 }
 
 export const getGitLabConnectionListItem = () => {
-  const { CLIENT_ID_GITLAB_LOGIN } = getConfig();
+  const { INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID } = getConfig();
 
   return {
     name: "GitLab" as const,
@@ -35,7 +36,7 @@ export const getGitLabConnectionListItem = () => {
       GitLabConnectionMethod.AccessToken,
       GitLabConnectionMethod.OAuth
     ],
-    oauthClientId: CLIENT_ID_GITLAB_LOGIN
+    oauthClientId: INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID
   };
 };
 
@@ -47,6 +48,18 @@ export const getGitLabInstanceUrl = async (instanceUrl?: string) => {
   return gitLabInstanceUrl;
 };
 
+export const getGitLabClient = async (accessToken: string, instanceUrl?: string, isOAuth = false) => {
+  const host = await getGitLabInstanceUrl(instanceUrl);
+
+  const client = new Gitlab<true>({
+    host,
+    ...(isOAuth ? { oauthToken: accessToken } : { token: accessToken }),
+    camelize: true
+  });
+
+  return client;
+};
+
 export const refreshGitLabToken = async (
   refreshToken: string,
   appId: string,
@@ -55,8 +68,9 @@ export const refreshGitLabToken = async (
   kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">,
   instanceUrl?: string
 ): Promise<string> => {
-  const { CLIENT_ID_GITLAB_LOGIN, CLIENT_SECRET_GITLAB_LOGIN, SITE_URL } = getConfig();
-  if (!CLIENT_SECRET_GITLAB_LOGIN || !CLIENT_ID_GITLAB_LOGIN || !SITE_URL) {
+  const { INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID, INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET, SITE_URL } =
+    getConfig();
+  if (!INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET || !INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID || !SITE_URL) {
     throw new InternalServerError({
       message: `GitLab environment variables have not been configured`
     });
@@ -65,9 +79,9 @@ export const refreshGitLabToken = async (
   const payload = new URLSearchParams({
     grant_type: "refresh_token",
     refresh_token: refreshToken,
-    client_id: CLIENT_ID_GITLAB_LOGIN,
-    client_secret: CLIENT_SECRET_GITLAB_LOGIN,
-    redirect_uri: `${SITE_URL}/integrations/gitlab/oauth2/callback`
+    client_id: INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID,
+    client_secret: INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET,
+    redirect_uri: `${SITE_URL}/organization/app-connections/gitlab/oauth/callback`
   });
 
   try {
@@ -95,7 +109,6 @@ export const refreshGitLabToken = async (
     });
 
     await appConnectionDAL.updateById(appId, { encryptedCredentials });
-
     return data.access_token;
   } catch (error: unknown) {
     if (error instanceof AxiosError) {
@@ -113,8 +126,9 @@ export const exchangeGitLabOAuthCode = async (
   code: string,
   instanceUrl?: string
 ): Promise<GitLabOAuthTokenResponse> => {
-  const { CLIENT_ID_GITLAB_LOGIN, CLIENT_SECRET_GITLAB_LOGIN, SITE_URL } = getConfig();
-  if (!CLIENT_SECRET_GITLAB_LOGIN || !CLIENT_ID_GITLAB_LOGIN || !SITE_URL) {
+  const { INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID, INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET, SITE_URL } =
+    getConfig();
+  if (!INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET || !INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID || !SITE_URL) {
     throw new InternalServerError({
       message: `GitLab environment variables have not been configured`
     });
@@ -124,9 +138,9 @@ export const exchangeGitLabOAuthCode = async (
     const payload = new URLSearchParams({
       grant_type: "authorization_code",
       code,
-      client_id: CLIENT_ID_GITLAB_LOGIN,
-      client_secret: CLIENT_SECRET_GITLAB_LOGIN,
-      redirect_uri: `${SITE_URL}/integrations/gitlab/oauth2/callback`
+      client_id: INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_ID,
+      client_secret: INF_APP_CONNECTION_GITLAB_OAUTH_CLIENT_SECRET,
+      redirect_uri: `${SITE_URL}/organization/app-connections/gitlab/oauth/callback`
     });
     const url = await getGitLabInstanceUrl(instanceUrl);
 
@@ -173,30 +187,24 @@ export const validateGitLabConnectionCredentials = async (config: TGitLabConnect
     });
   }
 
-  let response: AxiosResponse<TGitLabProject[]> | null = null;
-
   try {
-    const url = await getGitLabInstanceUrl(inputCredentials.instanceUrl);
-    response = await request.get<TGitLabProject[]>(`${url}/api/v4/user`, {
-      headers: {
-        Authorization: `Bearer ${accessToken}`,
-        Accept: "application/json"
-      }
-    });
+    const client = await getGitLabClient(
+      accessToken,
+      inputCredentials.instanceUrl,
+      method === GitLabConnectionMethod.OAuth
+    );
+    await client.Users.showCurrentUser();
   } catch (error: unknown) {
-    if (error instanceof AxiosError) {
+    logger.error(error, "Error validating GitLab connection credentials");
+
+    if (error instanceof GitbeakerRequestError) {
       throw new BadRequestError({
-        message: `Failed to validate credentials: ${error.message}`
+        message: `Failed to validate credentials: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}`
       });
     }
-    throw new BadRequestError({
-      message: "Unable to validate connection: verify credentials"
-    });
-  }
 
-  if (!response?.data) {
-    throw new InternalServerError({
-      message: "Failed to validate credentials: Response was empty"
+    throw new BadRequestError({
+      message: `Failed to validate credentials: ${(error as Error)?.message || "verify credentials"}`
     });
   }
 
@@ -217,20 +225,18 @@ export const validateGitLabConnectionCredentials = async (config: TGitLabConnect
 export const listGitLabProjects = async ({
   appConnection,
   appConnectionDAL,
-  kmsService,
-  teamId
+  kmsService
 }: {
   appConnection: TGitLabConnection;
   appConnectionDAL: Pick<TAppConnectionDALFactory, "updateById">;
   kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
-  teamId?: string;
 }): Promise<TGitLabProject[]> => {
   let { accessToken } = appConnection.credentials;
 
   if (
     appConnection.method === GitLabConnectionMethod.OAuth &&
     appConnection.credentials.refreshToken &&
-    appConnection.credentials.expiresAt < new Date()
+    new Date(appConnection.credentials.expiresAt) < new Date()
   ) {
     accessToken = await refreshGitLabToken(
       appConnection.credentials.refreshToken,
@@ -242,160 +248,29 @@ export const listGitLabProjects = async ({
     );
   }
 
-  const url = await getGitLabInstanceUrl(appConnection.credentials.instanceUrl);
-  const gitLabApiUrl = `${url}/api/v4`;
-
-  const projects: TGitLabProject[] = [];
-  let page = 1;
-  const perPage = 100;
-  let hasMorePages = true;
-
   try {
-    if (teamId) {
-      while (hasMorePages) {
-        const { data } = await request.get<TGitLabProject[]>(`${gitLabApiUrl}/groups/${teamId}/projects`, {
-          headers: {
-            Authorization: `Bearer ${accessToken}`,
-            Accept: "application/json"
-          },
-          params: {
-            page: page.toString(),
-            per_page: perPage.toString(),
-            order_by: "updated_at",
-            sort: "desc",
-            include_subgroups: "true"
-          }
-        });
+    const client = await getGitLabClient(
+      accessToken,
+      appConnection.credentials.instanceUrl,
+      appConnection.method === GitLabConnectionMethod.OAuth
+    );
+    const projects = await client.Projects.all({
+      archived: false,
+      includePendingDelete: false,
+      membership: true,
+      includeHidden: false,
+      imported: false
+    });
 
-        if (!data) {
-          throw new InternalServerError({
-            message: "Failed to get group projects: Response was empty"
-          });
-        }
-
-        data.forEach((project) => {
-          projects.push({
-            name: project.name,
-            id: project.id.toString()
-          });
-        });
-
-        hasMorePages = data.length === perPage;
-        page += 1;
-      }
-    } else {
-      const { data: userData } = await request.get<{ id: string }>(`${gitLabApiUrl}/user`, {
-        headers: {
-          Authorization: `Bearer ${accessToken}`,
-          Accept: "application/json"
-        }
-      });
-
-      if (!userData?.id) {
-        throw new InternalServerError({
-          message: "Failed to get current user information"
-        });
-      }
-
-      while (hasMorePages) {
-        const { data } = await request.get<TGitLabProject[]>(`${gitLabApiUrl}/users/${userData.id}/projects`, {
-          headers: {
-            Authorization: `Bearer ${accessToken}`,
-            Accept: "application/json"
-          },
-          params: {
-            page: page.toString(),
-            per_page: perPage.toString(),
-            order_by: "updated_at",
-            sort: "desc"
-          }
-        });
-
-        if (!data) {
-          throw new InternalServerError({
-            message: "Failed to get user projects: Response was empty"
-          });
-        }
-
-        data.forEach((project) => {
-          projects.push({
-            name: project.name,
-            id: project.id.toString()
-          });
-        });
-
-        hasMorePages = data.length === perPage;
-        page += 1;
-      }
-
-      if (projects.length === 0 && appConnection.method === GitLabConnectionMethod.AccessToken) {
-        try {
-          const { data: tokenAssociations } = await request.get<{
-            projects?: TGitLabProject[];
-            groups?: Array<{ projects?: TGitLabProject[] }>;
-          }>(`${gitLabApiUrl}/personal_access_tokens/self/associations`, {
-            headers: {
-              Authorization: `Bearer ${accessToken}`,
-              Accept: "application/json"
-            },
-            params: {
-              min_access_level: "50"
-            }
-          });
-
-          if (tokenAssociations?.projects) {
-            tokenAssociations.projects.forEach((project) => {
-              projects.push({
-                name: project.name,
-                id: project.id.toString()
-              });
-            });
-          }
-
-          if (tokenAssociations?.groups) {
-            tokenAssociations.groups.forEach((group) => {
-              if (group.projects) {
-                group.projects.forEach((project) => {
-                  const existingProject = projects.find((p) => p.id === project.id.toString());
-                  if (!existingProject) {
-                    projects.push({
-                      name: project.name,
-                      id: project.id.toString()
-                    });
-                  }
-                });
-              }
-            });
-          }
-        } catch (error) {
-          logger.warn(error, "Failed to fetch projects via personal access token associations:");
-        }
-      }
-    }
-
-    return projects;
+    return projects.map((project) => ({
+      name: project.pathWithNamespace,
+      id: project.id.toString()
+    }));
   } catch (error: unknown) {
-    if (error instanceof AxiosError) {
-      const status = error.response?.status;
-      const { message } = error;
-
-      if (status === 401) {
-        throw new BadRequestError({
-          message: `GitLab authentication failed: ${message}`
-        });
-      } else if (status === 403) {
-        throw new BadRequestError({
-          message: `GitLab access forbidden: ${message}`
-        });
-      } else if (status === 404) {
-        throw new BadRequestError({
-          message: teamId ? `GitLab group not found or access denied: ${message}` : `GitLab user not found: ${message}`
-        });
-      } else {
-        throw new BadRequestError({
-          message: `Failed to fetch GitLab projects: ${message}`
-        });
-      }
+    if (error instanceof GitbeakerRequestError) {
+      throw new BadRequestError({
+        message: `Failed to fetch GitLab projects: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}`
+      });
     }
 
     if (error instanceof InternalServerError) {
@@ -411,22 +286,25 @@ export const listGitLabProjects = async ({
 export const listGitLabGroups = async ({
   appConnection,
   appConnectionDAL,
-  kmsService,
-  includeSubgroups = true,
-  owned = false
+  kmsService
 }: {
   appConnection: TGitLabConnection;
   appConnectionDAL: Pick<TAppConnectionDALFactory, "updateById">;
   kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
-  includeSubgroups?: boolean;
-  owned?: boolean;
 }): Promise<TGitLabGroup[]> => {
   let { accessToken } = appConnection.credentials;
 
+  if (
+    appConnection.method === GitLabConnectionMethod.AccessToken &&
+    appConnection.credentials.accessTokenType === GitLabAccessTokenType.Project
+  ) {
+    return [];
+  }
+
   if (
     appConnection.method === GitLabConnectionMethod.OAuth &&
     appConnection.credentials.refreshToken &&
-    appConnection.credentials.expiresAt < new Date()
+    new Date(appConnection.credentials.expiresAt) < new Date()
   ) {
     accessToken = await refreshGitLabToken(
       appConnection.credentials.refreshToken,
@@ -438,69 +316,28 @@ export const listGitLabGroups = async ({
     );
   }
 
-  const url = await getGitLabInstanceUrl(appConnection.credentials.instanceUrl);
-  const gitLabApiUrl = `${url}/api/v4`;
-
-  const groups: TGitLabGroup[] = [];
-  let page = 1;
-  const perPage = 100;
-  let hasMorePages = true;
-
   try {
-    while (hasMorePages) {
-      const { data } = await request.get<TGitLabGroup[]>(`${gitLabApiUrl}/groups`, {
-        headers: {
-          Authorization: `Bearer ${accessToken}`,
-          Accept: "application/json"
-        },
-        params: {
-          page: page.toString(),
-          per_page: perPage.toString(),
-          order_by: "name",
-          sort: "asc",
-          all_available: (!owned).toString(),
-          owned: owned.toString(),
-          min_access_level: "10",
-          ...(includeSubgroups && { with_custom_attributes: "true" })
-        }
-      });
+    const client = await getGitLabClient(
+      accessToken,
+      appConnection.credentials.instanceUrl,
+      appConnection.method === GitLabConnectionMethod.OAuth
+    );
 
-      if (!data) {
-        throw new InternalServerError({
-          message: "Failed to get groups: Response was empty"
-        });
-      }
+    const groups = await client.Groups.all({
+      orderBy: "name",
+      sort: "asc",
+      minAccessLevel: 50
+    });
 
-      data.forEach((group) => {
-        groups.push({
-          id: group.id.toString(),
-          name: group.name
-        });
-      });
-
-      hasMorePages = data.length === perPage;
-      page += 1;
-    }
-
-    return groups;
+    return groups.map((group) => ({
+      id: group.id.toString(),
+      name: group.name
+    }));
   } catch (error: unknown) {
-    if (error instanceof AxiosError) {
-      const status = error.response?.status;
-      const { message } = error;
-
-      if (status === 401) {
-        throw new BadRequestError({
-          message: `GitLab authentication failed: ${message}`
-        });
-      } else if (status === 403) {
-        throw new BadRequestError({
-          message: `GitLab access forbidden: ${message}`
-        });
-      } else {
-        throw new BadRequestError({
-          message: `Failed to fetch GitLab groups: ${message}`
-        });
-      }
+    if (error instanceof GitbeakerRequestError) {
+      throw new BadRequestError({
+        message: `Failed to fetch GitLab groups: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}`
+      });
     }
 
     if (error instanceof InternalServerError) {

backend/src/services/app-connection/gitlab/gitlab-connection-service.ts

@@ -18,10 +18,10 @@ export const gitlabConnectionService = (
   appConnectionDAL: Pick<TAppConnectionDALFactory, "updateById">,
   kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">
 ) => {
-  const listProjects = async (connectionId: string, actor: OrgServiceActor, teamId?: string) => {
+  const listProjects = async (connectionId: string, actor: OrgServiceActor) => {
     try {
       const appConnection = await getAppConnection(AppConnection.GitLab, connectionId, actor);
-      const projects = await listGitLabProjects({ appConnection, appConnectionDAL, kmsService, teamId });
+      const projects = await listGitLabProjects({ appConnection, appConnectionDAL, kmsService });
       return projects;
     } catch (error) {
       logger.error(error, `Failed to establish connection with GitLab for app ${connectionId}`);

backend/src/services/secret-sync/gitlab/gitlab-sync-enums.ts

@@ -1,4 +1,4 @@
 export enum GitLabSyncScope {
-  Individual = "individual",
+  Project = "project",
   Group = "group"
 }

backend/src/services/secret-sync/gitlab/gitlab-sync-fns.ts

@@ -1,9 +1,13 @@
 /* eslint-disable no-await-in-loop */
-import { request } from "@app/lib/config/request";
-import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
+import { GitbeakerRequestError } from "@gitbeaker/rest";
+
 import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";
-import { GitLabConnectionMethod, refreshGitLabToken, TGitLabConnection } from "@app/services/app-connection/gitlab";
-import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
+import {
+  getGitLabClient,
+  GitLabConnectionMethod,
+  refreshGitLabToken,
+  TGitLabConnection
+} from "@app/services/app-connection/gitlab";
 import { TKmsServiceFactory } from "@app/services/kms/kms-service";
 import { TGitLabSyncWithCredentials, TGitLabVariable } from "@app/services/secret-sync/gitlab/gitlab-sync-types";
 import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
@@ -11,6 +15,7 @@ import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns";
 import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
 
 import { SECRET_SYNC_NAME_MAP } from "../secret-sync-maps";
+import { GitLabSyncScope } from "./gitlab-sync-enums";
 
 interface TGitLabVariablePayload {
   key?: string;
@@ -42,81 +47,64 @@ const getValidAccessToken = async (
   if (
     connection.method === GitLabConnectionMethod.OAuth &&
     connection.credentials.refreshToken &&
-    connection.credentials.expiresAt < new Date()
+    new Date(connection.credentials.expiresAt) < new Date()
   ) {
     const accessToken = await refreshGitLabToken(
       connection.credentials.refreshToken,
       connection.id,
       connection.orgId,
       appConnectionDAL,
-      kmsService
+      kmsService,
+      connection.credentials.instanceUrl
     );
     return accessToken;
   }
   return connection.credentials.accessToken;
 };
 
-const getGitLabApiUrl = async (connection: TGitLabConnection): Promise<string> => {
-  const baseUrl = connection.credentials.instanceUrl || IntegrationUrls.GITLAB_API_URL;
-  await blockLocalAndPrivateIpAddresses(baseUrl);
-  return baseUrl.includes("/api") ? baseUrl : `${baseUrl}/api`;
-};
-
-const buildVariablesEndpoint = (apiUrl: string, projectId: string): string => {
-  return `${apiUrl}/v4/projects/${encodeURIComponent(projectId)}/variables`;
-};
-
 const getGitLabVariables = async ({
   accessToken,
   connection,
-  projectId,
+  scope,
+  resourceId,
   targetEnvironment
 }: {
   accessToken: string;
   connection: TGitLabConnection;
-  projectId: string;
+  scope: GitLabSyncScope;
+  resourceId: string;
   targetEnvironment?: string;
 }): Promise<TGitLabVariable[]> => {
   try {
-    const apiUrl = await getGitLabApiUrl(connection);
-    const baseEndpoint = buildVariablesEndpoint(apiUrl, projectId);
+    const client = await getGitLabClient(
+      accessToken,
+      connection.credentials.instanceUrl,
+      connection.method === GitLabConnectionMethod.OAuth
+    );
 
-    const headers = {
-      Authorization: `Bearer ${accessToken}`,
-      "Accept-Encoding": "application/json",
-      "Content-Type": "application/json"
-    };
+    let variables: TGitLabVariable[] = [];
 
-    let allVariables: TGitLabVariable[] = [];
-    let url: string | null = `${baseEndpoint}?per_page=100`;
-
-    if (targetEnvironment) {
-      url += `&filter[environment_scope]=${encodeURIComponent(targetEnvironment)}`;
-    }
-
-    while (url) {
-      const response = await request.get<TGitLabVariable[]>(url, { headers });
-      allVariables = [...allVariables, ...(response.data || [])];
-
-      const linkHeader = response.headers.link as string;
-      const nextLink = linkHeader?.split(",").find((part: string) => part.includes('rel="next"'));
-
-      if (nextLink) {
-        url = nextLink.trim().split(";")[0].slice(1, -1);
-      } else {
-        url = null;
-      }
+    if (scope === GitLabSyncScope.Project) {
+      variables = await client.ProjectVariables.all(resourceId);
+    } else {
+      variables = await client.GroupVariables.all(resourceId);
     }
 
     if (targetEnvironment) {
-      return allVariables.filter((variable) => variable.environment_scope === targetEnvironment);
+      variables = variables.filter((v) => v.environmentScope === targetEnvironment);
     }
 
-    return allVariables;
+    return variables;
   } catch (error) {
+    if (error instanceof GitbeakerRequestError) {
+      throw new SecretSyncError({
+        error: new Error(
+          `Failed to fetch variables: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}`
+        )
+      });
+    }
     throw new SecretSyncError({
-      error,
-      secretKey: "list_variables"
+      error
     });
   }
 };
@@ -124,38 +112,61 @@ const getGitLabVariables = async ({
 const createGitLabVariable = async ({
   accessToken,
   connection,
-  projectId,
+  scope,
+  resourceId,
   variable
 }: {
   accessToken: string;
   connection: TGitLabConnection;
-  projectId: string;
+  scope: GitLabSyncScope;
+  resourceId: string;
   variable: TGitLabVariableCreate;
 }): Promise<void> => {
   try {
-    const apiUrl = await getGitLabApiUrl(connection);
-    const endpoint = buildVariablesEndpoint(apiUrl, projectId);
+    const client = await getGitLabClient(
+      accessToken,
+      connection.credentials.instanceUrl,
+      connection.method === GitLabConnectionMethod.OAuth
+    );
 
     const payload = {
       key: variable.key,
       value: variable.value,
-      variable_type: variable.variable_type || "env_var",
-      environment_scope: variable.environment_scope || "*",
+      variableType: "env_var",
+      environmentScope: variable.environment_scope || "*",
       protected: variable.protected || false,
       masked: variable.masked || false,
       masked_and_hidden: variable.masked_and_hidden || false,
-      raw: false,
-      ...(variable.description && { description: variable.description })
+      raw: false
     };
 
-    await request.post(endpoint, payload, {
-      headers: {
-        Authorization: `Bearer ${accessToken}`,
-        "Accept-Encoding": "application/json",
-        "Content-Type": "application/json"
-      }
-    });
+    if (scope === GitLabSyncScope.Project) {
+      await client.ProjectVariables.create(resourceId, payload.key, payload.value, {
+        variableType: "env_var",
+        environmentScope: payload.environmentScope,
+        protected: payload.protected,
+        masked: payload.masked,
+        masked_and_hidden: payload.masked_and_hidden,
+        raw: false
+      });
+    } else {
+      await client.GroupVariables.create(resourceId, payload.key, payload.value, {
+        variableType: "env_var",
+        environmentScope: payload.environmentScope,
+        protected: payload.protected,
+        masked: payload.masked,
+        raw: false
+      });
+    }
   } catch (error) {
+    if (error instanceof GitbeakerRequestError) {
+      throw new SecretSyncError({
+        error: new Error(
+          `Failed to create variable: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}`
+        ),
+        secretKey: variable.key
+      });
+    }
     throw new SecretSyncError({
       error,
       secretKey: variable.key
@@ -166,45 +177,57 @@ const createGitLabVariable = async ({
 const updateGitLabVariable = async ({
   accessToken,
   connection,
-  projectId,
+  scope,
+  resourceId,
   key,
   variable,
   targetEnvironment
 }: {
   accessToken: string;
   connection: TGitLabConnection;
-  projectId: string;
+  scope: GitLabSyncScope;
+  resourceId: string;
   key: string;
   variable: TGitLabVariableUpdate;
   targetEnvironment?: string;
 }): Promise<void> => {
   try {
-    const apiUrl = await getGitLabApiUrl(connection);
-    const baseEndpoint = buildVariablesEndpoint(apiUrl, projectId);
-    let url = `${baseEndpoint}/${encodeURIComponent(key)}`;
+    const client = await getGitLabClient(
+      accessToken,
+      connection.credentials.instanceUrl,
+      connection.method === GitLabConnectionMethod.OAuth
+    );
 
-    if (targetEnvironment) {
-      url += `?filter[environment_scope]=${encodeURIComponent(targetEnvironment)}`;
-    }
-
-    const payload = {
-      value: variable.value,
-      ...(variable.variable_type && { variable_type: variable.variable_type }),
-      ...(variable.environment_scope && { environment_scope: variable.environment_scope }),
+    const options = {
+      ...(variable.environment_scope && { environmentScope: variable.environment_scope }),
       ...(variable.protected !== undefined && { protected: variable.protected }),
-      ...(variable.masked !== undefined && { masked: variable.masked }),
-      ...(variable.masked_and_hidden !== undefined && { masked_and_hidden: variable.masked_and_hidden }),
-      ...(variable.description !== undefined && { description: variable.description || "" })
+      ...(variable.masked !== undefined && { masked: variable.masked })
     };
 
-    await request.put(url, payload, {
-      headers: {
-        Authorization: `Bearer ${accessToken}`,
-        "Accept-Encoding": "application/json",
-        "Content-Type": "application/json"
-      }
-    });
+    if (targetEnvironment) {
+      options.environmentScope = targetEnvironment;
+    }
+
+    if (scope === GitLabSyncScope.Project) {
+      await client.ProjectVariables.edit(resourceId, key, variable.value, {
+        ...options,
+        filter: { environment_scope: targetEnvironment || "*" }
+      });
+    } else {
+      await client.GroupVariables.edit(resourceId, key, variable.value, {
+        ...options,
+        filter: { environment_scope: targetEnvironment || "*" }
+      });
+    }
   } catch (error) {
+    if (error instanceof GitbeakerRequestError) {
+      throw new SecretSyncError({
+        error: new Error(
+          `Failed to update variable: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}`
+        ),
+        secretKey: key
+      });
+    }
     throw new SecretSyncError({
       error,
       secretKey: key
@@ -215,35 +238,52 @@ const updateGitLabVariable = async ({
 const deleteGitLabVariable = async ({
   accessToken,
   connection,
-  projectId,
+  scope,
+  resourceId,
   key,
-  targetEnvironment
+  targetEnvironment,
+  allVariables
 }: {
   accessToken: string;
   connection: TGitLabConnection;
-  projectId: string;
+  scope: GitLabSyncScope;
+  resourceId: string;
   key: string;
   targetEnvironment?: string;
+  allVariables?: TGitLabVariable[];
 }): Promise<void> => {
+  if (allVariables && !allVariables.find((v) => v.key === key)) {
+    return;
+  }
   try {
-    const apiUrl = await getGitLabApiUrl(connection);
-    const baseEndpoint = buildVariablesEndpoint(apiUrl, projectId);
-    let url = `${baseEndpoint}/${encodeURIComponent(key)}`;
+    const client = await getGitLabClient(
+      accessToken,
+      connection.credentials.instanceUrl,
+      connection.method === GitLabConnectionMethod.OAuth
+    );
 
+    const options: { filter?: { environment_scope: string } } = {};
     if (targetEnvironment) {
-      url += `?filter[environment_scope]=${encodeURIComponent(targetEnvironment)}`;
+      options.filter = { environment_scope: targetEnvironment || "*" };
     }
 
-    await request.delete(url, {
-      headers: {
-        Authorization: `Bearer ${accessToken}`,
-        "Accept-Encoding": "application/json",
-        "Content-Type": "application/json"
-      }
-    });
+    if (scope === GitLabSyncScope.Project) {
+      await client.ProjectVariables.remove(resourceId, key, options);
+    } else {
+      await client.GroupVariables.remove(resourceId, key);
+    }
   } catch (error: unknown) {
+    if (error instanceof GitbeakerRequestError) {
+      throw new SecretSyncError({
+        error: new Error(
+          `Failed to delete variable: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}`
+        ),
+        secretKey: key
+      });
+    }
     throw new SecretSyncError({
-      error
+      error,
+      secretKey: key
     });
   }
 };
@@ -255,8 +295,9 @@ export const GitLabSyncFns = {
     { appConnectionDAL, kmsService }: TGitLabSyncFactoryDeps
   ): Promise<void> => {
     const { connection, environment, destinationConfig } = secretSync;
+    const { scope, targetEnvironment } = destinationConfig;
 
-    const { projectId, targetEnvironment } = destinationConfig;
+    const resourceId = scope === GitLabSyncScope.Project ? destinationConfig.projectId : destinationConfig.groupId;
 
     const accessToken = await getValidAccessToken(connection, appConnectionDAL, kmsService);
 
@@ -264,13 +305,20 @@ export const GitLabSyncFns = {
       const currentVariables = await getGitLabVariables({
         accessToken,
         connection,
-        projectId,
+        scope,
+        resourceId,
         targetEnvironment
       });
 
       const currentVariableMap = new Map(currentVariables.map((v) => [v.key, v]));
 
       for (const [key, { value }] of Object.entries(secretMap)) {
+        if (value?.length < 8 && destinationConfig.shouldMaskSecrets) {
+          throw new SecretSyncError({
+            message: `Secret ${key} is too short to be masked. GitLab requires a minimum of 8 characters for masked secrets.`,
+            secretKey: key
+          });
+        }
         try {
           const existingVariable = currentVariableMap.get(key);
 
@@ -279,17 +327,14 @@ export const GitLabSyncFns = {
               await updateGitLabVariable({
                 accessToken,
                 connection,
-                projectId,
+                scope,
+                resourceId,
                 key,
                 variable: {
                   value,
-                  variable_type: existingVariable.variable_type,
-                  environment_scope: targetEnvironment || existingVariable.environment_scope,
+                  environment_scope: targetEnvironment,
                   protected: destinationConfig.shouldProtectSecrets ?? existingVariable.protected,
-                  ...(!existingVariable.masked && destinationConfig.shouldMaskSecrets && { masked: value?.length > 8 }),
-                  ...(!existingVariable.hidden &&
-                    destinationConfig.shouldHideSecrets && { masked_and_hidden: value?.length > 8 }),
-                  description: existingVariable.description ?? undefined
+                  masked: existingVariable.masked || destinationConfig.shouldMaskSecrets
                 },
                 targetEnvironment
               });
@@ -298,15 +343,16 @@ export const GitLabSyncFns = {
             await createGitLabVariable({
               accessToken,
               connection,
-              projectId,
+              scope,
+              resourceId,
               variable: {
                 key,
                 value,
                 variable_type: "env_var",
                 environment_scope: targetEnvironment || "*",
                 protected: destinationConfig.shouldProtectSecrets || false,
-                masked: value?.length > 8 ? destinationConfig.shouldMaskSecrets || false : false,
-                masked_and_hidden: value?.length > 8 ? destinationConfig.shouldHideSecrets || false : false
+                masked: destinationConfig.shouldMaskSecrets || false,
+                masked_and_hidden: destinationConfig.shouldHideSecrets || false
               }
             });
           }
@@ -323,13 +369,14 @@ export const GitLabSyncFns = {
           try {
             const shouldDelete =
               matchesSchema(variable.key, environment?.slug || "", secretSync.syncOptions.keySchema) &&
-              !(variable.key in secretMap);
+              variable.key in secretMap;
 
             if (shouldDelete) {
               await deleteGitLabVariable({
                 accessToken,
                 connection,
-                projectId,
+                scope,
+                resourceId,
                 key: variable.key,
                 targetEnvironment
               });
@@ -359,19 +406,30 @@ export const GitLabSyncFns = {
     { appConnectionDAL, kmsService }: TGitLabSyncFactoryDeps
   ): Promise<void> => {
     const { connection, destinationConfig } = secretSync;
+    const { scope, targetEnvironment } = destinationConfig;
 
-    const { projectId, targetEnvironment } = destinationConfig;
+    const resourceId = scope === GitLabSyncScope.Project ? destinationConfig.projectId : destinationConfig.groupId;
 
     const accessToken = await getValidAccessToken(connection, appConnectionDAL, kmsService);
 
+    const allVariables = await getGitLabVariables({
+      accessToken,
+      connection,
+      scope,
+      resourceId,
+      targetEnvironment
+    });
+
     for (const key of Object.keys(secretMap)) {
       try {
         await deleteGitLabVariable({
           accessToken,
           connection,
-          projectId,
+          scope,
+          resourceId,
           key,
-          targetEnvironment
+          targetEnvironment,
+          allVariables
         });
       } catch (error) {
         throw new SecretSyncError({

backend/src/services/secret-sync/gitlab/gitlab-sync-schemas.ts

@@ -14,7 +14,7 @@ import { GitLabSyncScope } from "./gitlab-sync-enums";
 
 const GitLabSyncDestinationConfigSchema = z.discriminatedUnion("scope", [
   z.object({
-    scope: z.literal(GitLabSyncScope.Individual).describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.scope),
+    scope: z.literal(GitLabSyncScope.Project).describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.scope),
     projectId: z.string().min(1, "Project ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.projectId),
     projectName: z
       .string()
@@ -44,11 +44,7 @@ const GitLabSyncDestinationConfigSchema = z.discriminatedUnion("scope", [
   z.object({
     scope: z.literal(GitLabSyncScope.Group).describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.scope),
     groupId: z.string().min(1, "Group ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.groupId),
-    projectId: z.string().min(1, "Project ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.projectId),
-    projectName: z
-      .string()
-      .min(1, "Project name is required")
-      .describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.projectName),
+    groupName: z.string().min(1, "Group name is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.groupName),
     targetEnvironment: z
       .string()
       .optional()

backend/src/services/secret-sync/gitlab/gitlab-sync-types.ts

@@ -15,13 +15,9 @@ export type TGitLabSyncWithCredentials = TGitLabSync & {
 export type TGitLabVariable = {
   key: string;
   value: string;
-  variable_type: "env_var" | "file";
   protected: boolean;
   masked: boolean;
-  hidden: boolean;
-  raw: boolean;
-  environment_scope: string;
-  description: string | null;
+  environmentScope?: string;
 };
 
 export type TGitLabVariableCreate = {