feat(secret-sync): rework GitLab secret-sync to add group variables

backend/src/services/secret-sync/gitlab/gitlab-sync-enums.ts

@@ -1,4 +1,4 @@
 export enum GitLabSyncScope {
-  Individual = "individual",
+  Project = "project",
   Group = "group"
 }

backend/src/services/secret-sync/gitlab/gitlab-sync-fns.ts

@@ -1,9 +1,13 @@
 /* eslint-disable no-await-in-loop */
-import { request } from "@app/lib/config/request";
-import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
+import { GitbeakerRequestError } from "@gitbeaker/rest";
+
 import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";
-import { GitLabConnectionMethod, refreshGitLabToken, TGitLabConnection } from "@app/services/app-connection/gitlab";
-import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
+import {
+  getGitLabClient,
+  GitLabConnectionMethod,
+  refreshGitLabToken,
+  TGitLabConnection
+} from "@app/services/app-connection/gitlab";
 import { TKmsServiceFactory } from "@app/services/kms/kms-service";
 import { TGitLabSyncWithCredentials, TGitLabVariable } from "@app/services/secret-sync/gitlab/gitlab-sync-types";
 import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
@@ -11,6 +15,7 @@ import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns";
 import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
 
 import { SECRET_SYNC_NAME_MAP } from "../secret-sync-maps";
+import { GitLabSyncScope } from "./gitlab-sync-enums";
 
 interface TGitLabVariablePayload {
   key?: string;
@@ -42,81 +47,64 @@ const getValidAccessToken = async (
   if (
     connection.method === GitLabConnectionMethod.OAuth &&
     connection.credentials.refreshToken &&
-    connection.credentials.expiresAt < new Date()
+    new Date(connection.credentials.expiresAt) < new Date()
   ) {
     const accessToken = await refreshGitLabToken(
       connection.credentials.refreshToken,
       connection.id,
       connection.orgId,
       appConnectionDAL,
-      kmsService
+      kmsService,
+      connection.credentials.instanceUrl
     );
     return accessToken;
   }
   return connection.credentials.accessToken;
 };
 
-const getGitLabApiUrl = async (connection: TGitLabConnection): Promise<string> => {
-  const baseUrl = connection.credentials.instanceUrl || IntegrationUrls.GITLAB_API_URL;
-  await blockLocalAndPrivateIpAddresses(baseUrl);
-  return baseUrl.includes("/api") ? baseUrl : `${baseUrl}/api`;
-};
-
-const buildVariablesEndpoint = (apiUrl: string, projectId: string): string => {
-  return `${apiUrl}/v4/projects/${encodeURIComponent(projectId)}/variables`;
-};
-
 const getGitLabVariables = async ({
   accessToken,
   connection,
-  projectId,
+  scope,
+  resourceId,
   targetEnvironment
 }: {
   accessToken: string;
   connection: TGitLabConnection;
-  projectId: string;
+  scope: GitLabSyncScope;
+  resourceId: string;
   targetEnvironment?: string;
 }): Promise<TGitLabVariable[]> => {
   try {
-    const apiUrl = await getGitLabApiUrl(connection);
-    const baseEndpoint = buildVariablesEndpoint(apiUrl, projectId);
+    const client = await getGitLabClient(
+      accessToken,
+      connection.credentials.instanceUrl,
+      connection.method === GitLabConnectionMethod.OAuth
+    );
 
-    const headers = {
-      Authorization: `Bearer ${accessToken}`,
-      "Accept-Encoding": "application/json",
-      "Content-Type": "application/json"
-    };
+    let variables: TGitLabVariable[] = [];
 
-    let allVariables: TGitLabVariable[] = [];
-    let url: string | null = `${baseEndpoint}?per_page=100`;
-
-    if (targetEnvironment) {
-      url += `&filter[environment_scope]=${encodeURIComponent(targetEnvironment)}`;
-    }
-
-    while (url) {
-      const response = await request.get<TGitLabVariable[]>(url, { headers });
-      allVariables = [...allVariables, ...(response.data || [])];
-
-      const linkHeader = response.headers.link as string;
-      const nextLink = linkHeader?.split(",").find((part: string) => part.includes('rel="next"'));
-
-      if (nextLink) {
-        url = nextLink.trim().split(";")[0].slice(1, -1);
-      } else {
-        url = null;
-      }
+    if (scope === GitLabSyncScope.Project) {
+      variables = await client.ProjectVariables.all(resourceId);
+    } else {
+      variables = await client.GroupVariables.all(resourceId);
     }
 
     if (targetEnvironment) {
-      return allVariables.filter((variable) => variable.environment_scope === targetEnvironment);
+      variables = variables.filter((v) => v.environmentScope === targetEnvironment);
     }
 
-    return allVariables;
+    return variables;
   } catch (error) {
+    if (error instanceof GitbeakerRequestError) {
+      throw new SecretSyncError({
+        error: new Error(
+          `Failed to fetch variables: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}`
+        )
+      });
+    }
     throw new SecretSyncError({
-      error,
-      secretKey: "list_variables"
+      error
     });
   }
 };
@@ -124,38 +112,61 @@ const getGitLabVariables = async ({
 const createGitLabVariable = async ({
   accessToken,
   connection,
-  projectId,
+  scope,
+  resourceId,
   variable
 }: {
   accessToken: string;
   connection: TGitLabConnection;
-  projectId: string;
+  scope: GitLabSyncScope;
+  resourceId: string;
   variable: TGitLabVariableCreate;
 }): Promise<void> => {
   try {
-    const apiUrl = await getGitLabApiUrl(connection);
-    const endpoint = buildVariablesEndpoint(apiUrl, projectId);
+    const client = await getGitLabClient(
+      accessToken,
+      connection.credentials.instanceUrl,
+      connection.method === GitLabConnectionMethod.OAuth
+    );
 
     const payload = {
       key: variable.key,
       value: variable.value,
-      variable_type: variable.variable_type || "env_var",
-      environment_scope: variable.environment_scope || "*",
+      variableType: "env_var",
+      environmentScope: variable.environment_scope || "*",
       protected: variable.protected || false,
       masked: variable.masked || false,
       masked_and_hidden: variable.masked_and_hidden || false,
-      raw: false,
-      ...(variable.description && { description: variable.description })
+      raw: false
     };
 
-    await request.post(endpoint, payload, {
-      headers: {
-        Authorization: `Bearer ${accessToken}`,
-        "Accept-Encoding": "application/json",
-        "Content-Type": "application/json"
-      }
-    });
+    if (scope === GitLabSyncScope.Project) {
+      await client.ProjectVariables.create(resourceId, payload.key, payload.value, {
+        variableType: "env_var",
+        environmentScope: payload.environmentScope,
+        protected: payload.protected,
+        masked: payload.masked,
+        masked_and_hidden: payload.masked_and_hidden,
+        raw: false
+      });
+    } else {
+      await client.GroupVariables.create(resourceId, payload.key, payload.value, {
+        variableType: "env_var",
+        environmentScope: payload.environmentScope,
+        protected: payload.protected,
+        masked: payload.masked,
+        raw: false
+      });
+    }
   } catch (error) {
+    if (error instanceof GitbeakerRequestError) {
+      throw new SecretSyncError({
+        error: new Error(
+          `Failed to create variable: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}`
+        ),
+        secretKey: variable.key
+      });
+    }
     throw new SecretSyncError({
       error,
       secretKey: variable.key
@@ -166,45 +177,57 @@ const createGitLabVariable = async ({
 const updateGitLabVariable = async ({
   accessToken,
   connection,
-  projectId,
+  scope,
+  resourceId,
   key,
   variable,
   targetEnvironment
 }: {
   accessToken: string;
   connection: TGitLabConnection;
-  projectId: string;
+  scope: GitLabSyncScope;
+  resourceId: string;
   key: string;
   variable: TGitLabVariableUpdate;
   targetEnvironment?: string;
 }): Promise<void> => {
   try {
-    const apiUrl = await getGitLabApiUrl(connection);
-    const baseEndpoint = buildVariablesEndpoint(apiUrl, projectId);
-    let url = `${baseEndpoint}/${encodeURIComponent(key)}`;
+    const client = await getGitLabClient(
+      accessToken,
+      connection.credentials.instanceUrl,
+      connection.method === GitLabConnectionMethod.OAuth
+    );
 
-    if (targetEnvironment) {
-      url += `?filter[environment_scope]=${encodeURIComponent(targetEnvironment)}`;
-    }
-
-    const payload = {
-      value: variable.value,
-      ...(variable.variable_type && { variable_type: variable.variable_type }),
-      ...(variable.environment_scope && { environment_scope: variable.environment_scope }),
+    const options = {
+      ...(variable.environment_scope && { environmentScope: variable.environment_scope }),
       ...(variable.protected !== undefined && { protected: variable.protected }),
-      ...(variable.masked !== undefined && { masked: variable.masked }),
-      ...(variable.masked_and_hidden !== undefined && { masked_and_hidden: variable.masked_and_hidden }),
-      ...(variable.description !== undefined && { description: variable.description || "" })
+      ...(variable.masked !== undefined && { masked: variable.masked })
     };
 
-    await request.put(url, payload, {
-      headers: {
-        Authorization: `Bearer ${accessToken}`,
-        "Accept-Encoding": "application/json",
-        "Content-Type": "application/json"
-      }
-    });
+    if (targetEnvironment) {
+      options.environmentScope = targetEnvironment;
+    }
+
+    if (scope === GitLabSyncScope.Project) {
+      await client.ProjectVariables.edit(resourceId, key, variable.value, {
+        ...options,
+        filter: { environment_scope: targetEnvironment || "*" }
+      });
+    } else {
+      await client.GroupVariables.edit(resourceId, key, variable.value, {
+        ...options,
+        filter: { environment_scope: targetEnvironment || "*" }
+      });
+    }
   } catch (error) {
+    if (error instanceof GitbeakerRequestError) {
+      throw new SecretSyncError({
+        error: new Error(
+          `Failed to update variable: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}`
+        ),
+        secretKey: key
+      });
+    }
     throw new SecretSyncError({
       error,
       secretKey: key
@@ -215,35 +238,52 @@ const updateGitLabVariable = async ({
 const deleteGitLabVariable = async ({
   accessToken,
   connection,
-  projectId,
+  scope,
+  resourceId,
   key,
-  targetEnvironment
+  targetEnvironment,
+  allVariables
 }: {
   accessToken: string;
   connection: TGitLabConnection;
-  projectId: string;
+  scope: GitLabSyncScope;
+  resourceId: string;
   key: string;
   targetEnvironment?: string;
+  allVariables?: TGitLabVariable[];
 }): Promise<void> => {
+  if (allVariables && !allVariables.find((v) => v.key === key)) {
+    return;
+  }
   try {
-    const apiUrl = await getGitLabApiUrl(connection);
-    const baseEndpoint = buildVariablesEndpoint(apiUrl, projectId);
-    let url = `${baseEndpoint}/${encodeURIComponent(key)}`;
+    const client = await getGitLabClient(
+      accessToken,
+      connection.credentials.instanceUrl,
+      connection.method === GitLabConnectionMethod.OAuth
+    );
 
+    const options: { filter?: { environment_scope: string } } = {};
     if (targetEnvironment) {
-      url += `?filter[environment_scope]=${encodeURIComponent(targetEnvironment)}`;
+      options.filter = { environment_scope: targetEnvironment || "*" };
     }
 
-    await request.delete(url, {
-      headers: {
-        Authorization: `Bearer ${accessToken}`,
-        "Accept-Encoding": "application/json",
-        "Content-Type": "application/json"
-      }
-    });
+    if (scope === GitLabSyncScope.Project) {
+      await client.ProjectVariables.remove(resourceId, key, options);
+    } else {
+      await client.GroupVariables.remove(resourceId, key);
+    }
   } catch (error: unknown) {
+    if (error instanceof GitbeakerRequestError) {
+      throw new SecretSyncError({
+        error: new Error(
+          `Failed to delete variable: ${error.message ?? "Unknown error"}${error.cause?.description && error.message !== "Unauthorized" ? `. Cause: ${error.cause.description}` : ""}`
+        ),
+        secretKey: key
+      });
+    }
     throw new SecretSyncError({
-      error
+      error,
+      secretKey: key
     });
   }
 };
@@ -255,8 +295,9 @@ export const GitLabSyncFns = {
     { appConnectionDAL, kmsService }: TGitLabSyncFactoryDeps
   ): Promise<void> => {
     const { connection, environment, destinationConfig } = secretSync;
+    const { scope, targetEnvironment } = destinationConfig;
 
-    const { projectId, targetEnvironment } = destinationConfig;
+    const resourceId = scope === GitLabSyncScope.Project ? destinationConfig.projectId : destinationConfig.groupId;
 
     const accessToken = await getValidAccessToken(connection, appConnectionDAL, kmsService);
 
@@ -264,13 +305,20 @@ export const GitLabSyncFns = {
       const currentVariables = await getGitLabVariables({
         accessToken,
         connection,
-        projectId,
+        scope,
+        resourceId,
         targetEnvironment
       });
 
       const currentVariableMap = new Map(currentVariables.map((v) => [v.key, v]));
 
       for (const [key, { value }] of Object.entries(secretMap)) {
+        if (value?.length < 8 && destinationConfig.shouldMaskSecrets) {
+          throw new SecretSyncError({
+            message: `Secret ${key} is too short to be masked. GitLab requires a minimum of 8 characters for masked secrets.`,
+            secretKey: key
+          });
+        }
         try {
           const existingVariable = currentVariableMap.get(key);
 
@@ -279,17 +327,14 @@ export const GitLabSyncFns = {
               await updateGitLabVariable({
                 accessToken,
                 connection,
-                projectId,
+                scope,
+                resourceId,
                 key,
                 variable: {
                   value,
-                  variable_type: existingVariable.variable_type,
-                  environment_scope: targetEnvironment || existingVariable.environment_scope,
+                  environment_scope: targetEnvironment,
                   protected: destinationConfig.shouldProtectSecrets ?? existingVariable.protected,
-                  ...(!existingVariable.masked && destinationConfig.shouldMaskSecrets && { masked: value?.length > 8 }),
-                  ...(!existingVariable.hidden &&
-                    destinationConfig.shouldHideSecrets && { masked_and_hidden: value?.length > 8 }),
-                  description: existingVariable.description ?? undefined
+                  masked: existingVariable.masked || destinationConfig.shouldMaskSecrets
                 },
                 targetEnvironment
               });
@@ -298,15 +343,16 @@ export const GitLabSyncFns = {
             await createGitLabVariable({
               accessToken,
               connection,
-              projectId,
+              scope,
+              resourceId,
               variable: {
                 key,
                 value,
                 variable_type: "env_var",
                 environment_scope: targetEnvironment || "*",
                 protected: destinationConfig.shouldProtectSecrets || false,
-                masked: value?.length > 8 ? destinationConfig.shouldMaskSecrets || false : false,
-                masked_and_hidden: value?.length > 8 ? destinationConfig.shouldHideSecrets || false : false
+                masked: destinationConfig.shouldMaskSecrets || false,
+                masked_and_hidden: destinationConfig.shouldHideSecrets || false
               }
             });
           }
@@ -323,13 +369,14 @@ export const GitLabSyncFns = {
           try {
             const shouldDelete =
               matchesSchema(variable.key, environment?.slug || "", secretSync.syncOptions.keySchema) &&
-              !(variable.key in secretMap);
+              variable.key in secretMap;
 
             if (shouldDelete) {
               await deleteGitLabVariable({
                 accessToken,
                 connection,
-                projectId,
+                scope,
+                resourceId,
                 key: variable.key,
                 targetEnvironment
               });
@@ -359,19 +406,30 @@ export const GitLabSyncFns = {
     { appConnectionDAL, kmsService }: TGitLabSyncFactoryDeps
   ): Promise<void> => {
     const { connection, destinationConfig } = secretSync;
+    const { scope, targetEnvironment } = destinationConfig;
 
-    const { projectId, targetEnvironment } = destinationConfig;
+    const resourceId = scope === GitLabSyncScope.Project ? destinationConfig.projectId : destinationConfig.groupId;
 
     const accessToken = await getValidAccessToken(connection, appConnectionDAL, kmsService);
 
+    const allVariables = await getGitLabVariables({
+      accessToken,
+      connection,
+      scope,
+      resourceId,
+      targetEnvironment
+    });
+
     for (const key of Object.keys(secretMap)) {
       try {
         await deleteGitLabVariable({
           accessToken,
           connection,
-          projectId,
+          scope,
+          resourceId,
           key,
-          targetEnvironment
+          targetEnvironment,
+          allVariables
         });
       } catch (error) {
         throw new SecretSyncError({

backend/src/services/secret-sync/gitlab/gitlab-sync-schemas.ts

@@ -14,7 +14,7 @@ import { GitLabSyncScope } from "./gitlab-sync-enums";
 
 const GitLabSyncDestinationConfigSchema = z.discriminatedUnion("scope", [
   z.object({
-    scope: z.literal(GitLabSyncScope.Individual).describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.scope),
+    scope: z.literal(GitLabSyncScope.Project).describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.scope),
     projectId: z.string().min(1, "Project ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.projectId),
     projectName: z
       .string()
@@ -44,11 +44,7 @@ const GitLabSyncDestinationConfigSchema = z.discriminatedUnion("scope", [
   z.object({
     scope: z.literal(GitLabSyncScope.Group).describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.scope),
     groupId: z.string().min(1, "Group ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.groupId),
-    projectId: z.string().min(1, "Project ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.projectId),
-    projectName: z
-      .string()
-      .min(1, "Project name is required")
-      .describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.projectName),
+    groupName: z.string().min(1, "Group name is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.groupName),
     targetEnvironment: z
       .string()
       .optional()

backend/src/services/secret-sync/gitlab/gitlab-sync-types.ts

@@ -15,13 +15,9 @@ export type TGitLabSyncWithCredentials = TGitLabSync & {
 export type TGitLabVariable = {
   key: string;
   value: string;
-  variable_type: "env_var" | "file";
   protected: boolean;
   masked: boolean;
-  hidden: boolean;
-  raw: boolean;
-  environment_scope: string;
-  description: string | null;
+  environmentScope?: string;
 };
 
 export type TGitLabVariableCreate = {