Merge remote-tracking branch 'origin/main' into feat/gateway-v2

backend/src/services/secret-sync/checkly/checkly-sync-fns.ts

@@ -23,56 +23,120 @@ export const ChecklySyncFns = {
 
     const config = secretSync.destinationConfig;
 
-    const variables = await ChecklyPublicAPI.getVariables(secretSync.connection, config.accountId);
+    if (config.groupId) {
+      // Handle group environment variables
+      const groupVars = await ChecklyPublicAPI.getCheckGroupEnvironmentVariables(
+        secretSync.connection,
+        config.accountId,
+        config.groupId
+      );
 
-    const checklySecrets = Object.fromEntries(variables!.map((variable) => [variable.key, variable]));
+      const checklyGroupSecrets = Object.fromEntries(groupVars.map((variable) => [variable.key, variable]));
 
-    for await (const key of Object.keys(secretMap)) {
-      try {
+      // Prepare all variables to update at once
+      const updatedVariables = { ...checklyGroupSecrets };
+
+      for (const key of Object.keys(secretMap)) {
         const entry = secretMap[key];
 
-        // If value is empty, we skip the upsert - checkly does not allow empty values
+        // If value is empty, we skip adding it - checkly does not allow empty values
         if (entry.value.trim() === "") {
-          // Delete the secret from Checkly if its empty
+          // Delete the secret from the group if it's empty
           if (!disableSecretDeletion) {
-            await ChecklyPublicAPI.deleteVariable(secretSync.connection, config.accountId, {
-              key
-            });
+            delete updatedVariables[key];
           }
           continue; // Skip empty values
         }
 
-        await ChecklyPublicAPI.upsertVariable(secretSync.connection, config.accountId, {
+        // Add or update the variable
+        updatedVariables[key] = {
           key,
           value: entry.value,
-          secret: true,
           locked: true
-        });
+        };
+      }
+
+      // Remove secrets that are not in the secretMap if deletion is enabled
+      if (!disableSecretDeletion) {
+        for (const key of Object.keys(checklyGroupSecrets)) {
+          // eslint-disable-next-line no-continue
+          if (!matchesSchema(key, environment?.slug || "", keySchema)) continue;
+
+          if (!secretMap[key]) {
+            delete updatedVariables[key];
+          }
+        }
+      }
+
+      // Update all group environment variables at once
+      try {
+        await ChecklyPublicAPI.updateCheckGroupEnvironmentVariables(
+          secretSync.connection,
+          config.accountId,
+          config.groupId,
+          Object.values(updatedVariables)
+        );
       } catch (error) {
+        if (error instanceof SecretSyncError) throw error;
+
         throw new SecretSyncError({
           error,
-          secretKey: key
+          secretKey: "group_update"
         });
       }
-    }
+    } else {
+      // Handle global variables (existing logic)
+      const variables = await ChecklyPublicAPI.getVariables(secretSync.connection, config.accountId);
 
-    if (disableSecretDeletion) return;
+      const checklySecrets = Object.fromEntries(variables!.map((variable) => [variable.key, variable]));
 
-    for await (const key of Object.keys(checklySecrets)) {
-      try {
-        // eslint-disable-next-line no-continue
-        if (!matchesSchema(key, environment?.slug || "", keySchema)) continue;
+      for await (const key of Object.keys(secretMap)) {
+        try {
+          const entry = secretMap[key];
 
-        if (!secretMap[key]) {
-          await ChecklyPublicAPI.deleteVariable(secretSync.connection, config.accountId, {
-            key
+          // If value is empty, we skip the upsert - checkly does not allow empty values
+          if (entry.value.trim() === "") {
+            // Delete the secret from Checkly if its empty
+            if (!disableSecretDeletion) {
+              await ChecklyPublicAPI.deleteVariable(secretSync.connection, config.accountId, {
+                key
+              });
+            }
+            continue; // Skip empty values
+          }
+
+          await ChecklyPublicAPI.upsertVariable(secretSync.connection, config.accountId, {
+            key,
+            value: entry.value,
+            secret: true,
+            locked: true
+          });
+        } catch (error) {
+          throw new SecretSyncError({
+            error,
+            secretKey: key
+          });
+        }
+      }
+
+      if (disableSecretDeletion) return;
+
+      for await (const key of Object.keys(checklySecrets)) {
+        try {
+          // eslint-disable-next-line no-continue
+          if (!matchesSchema(key, environment?.slug || "", keySchema)) continue;
+
+          if (!secretMap[key]) {
+            await ChecklyPublicAPI.deleteVariable(secretSync.connection, config.accountId, {
+              key
+            });
+          }
+        } catch (error) {
+          throw new SecretSyncError({
+            error,
+            secretKey: key
           });
         }
-      } catch (error) {
-        throw new SecretSyncError({
-          error,
-          secretKey: key
-        });
       }
     }
   },
@@ -80,23 +144,54 @@ export const ChecklySyncFns = {
   async removeSecrets(secretSync: TChecklySyncWithCredentials, secretMap: TSecretMap) {
     const config = secretSync.destinationConfig;
 
-    const variables = await ChecklyPublicAPI.getVariables(secretSync.connection, config.accountId);
+    if (config.groupId) {
+      // Handle group environment variables
+      const groupVars = await ChecklyPublicAPI.getCheckGroupEnvironmentVariables(
+        secretSync.connection,
+        config.accountId,
+        config.groupId
+      );
 
-    const checklySecrets = Object.fromEntries(variables!.map((variable) => [variable.key, variable]));
+      const checklyGroupSecrets = Object.fromEntries(groupVars.map((variable) => [variable.key, variable]));
+
+      // Filter out the secrets to remove
+      const remainingVariables = Object.keys(checklyGroupSecrets)
+        .filter((key) => !(key in secretMap))
+        .map((key) => checklyGroupSecrets[key]);
 
-    for await (const secret of Object.keys(checklySecrets)) {
       try {
-        if (secret in secretMap) {
-          await ChecklyPublicAPI.deleteVariable(secretSync.connection, config.accountId, {
-            key: secret
-          });
-        }
+        await ChecklyPublicAPI.updateCheckGroupEnvironmentVariables(
+          secretSync.connection,
+          config.accountId,
+          config.groupId,
+          remainingVariables
+        );
       } catch (error) {
         throw new SecretSyncError({
           error,
-          secretKey: secret
+          secretKey: "group_remove"
         });
       }
+    } else {
+      // Handle global variables (existing logic)
+      const variables = await ChecklyPublicAPI.getVariables(secretSync.connection, config.accountId);
+
+      const checklySecrets = Object.fromEntries(variables!.map((variable) => [variable.key, variable]));
+
+      for await (const secret of Object.keys(checklySecrets)) {
+        try {
+          if (secret in secretMap) {
+            await ChecklyPublicAPI.deleteVariable(secretSync.connection, config.accountId, {
+              key: secret
+            });
+          }
+        } catch (error) {
+          throw new SecretSyncError({
+            error,
+            secretKey: secret
+          });
+        }
+      }
     }
   }
 };

backend/src/services/secret-sync/checkly/checkly-sync-schemas.ts

@@ -11,7 +11,17 @@ import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types"
 
 const ChecklySyncDestinationConfigSchema = z.object({
   accountId: z.string().min(1, "Account ID is required").max(255, "Account ID must be less than 255 characters"),
-  accountName: z.string().min(1, "Account Name is required").max(255, "Account ID must be less than 255 characters")
+  accountName: z
+    .string()
+    .min(1, "Account Name is required")
+    .max(255, "Account ID must be less than 255 characters")
+    .optional(),
+  groupId: z.string().min(1, "Group ID is required").max(255, "Group ID must be less than 255 characters").optional(),
+  groupName: z
+    .string()
+    .min(1, "Group Name is required")
+    .max(255, "Group Name must be less than 255 characters")
+    .optional()
 });
 
 const ChecklySyncOptionsConfig: TSyncOptionsConfig = { canImportSecrets: false };

backend/src/services/secret-sync/github/github-sync-fns.ts

@@ -214,7 +214,7 @@ export const GithubSyncFns = {
     const token =
       connection.method === GitHubConnectionMethod.OAuth
         ? connection.credentials.accessToken
-        : await getGitHubAppAuthToken(connection);
+        : await getGitHubAppAuthToken(connection, gatewayService);
 
     const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService, gatewayV2Service);
     const publicKey = await getPublicKey(secretSync, gatewayService, gatewayV2Service, token);
@@ -272,7 +272,7 @@ export const GithubSyncFns = {
     const token =
       connection.method === GitHubConnectionMethod.OAuth
         ? connection.credentials.accessToken
-        : await getGitHubAppAuthToken(connection);
+        : await getGitHubAppAuthToken(connection, gatewayService);
 
     const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService, gatewayV2Service);
 

backend/src/services/secret-sync/hc-vault/hc-vault-sync-fns.ts

@@ -1,9 +1,13 @@
 import { isAxiosError } from "axios";
 
-import { request } from "@app/lib/config/request";
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { removeTrailingSlash } from "@app/lib/fn";
-import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
-import { getHCVaultAccessToken, getHCVaultInstanceUrl } from "@app/services/app-connection/hc-vault";
+import {
+  getHCVaultAccessToken,
+  getHCVaultInstanceUrl,
+  requestWithHCVaultGateway,
+  THCVaultConnection
+} from "@app/services/app-connection/hc-vault";
 import {
   THCVaultListVariables,
   THCVaultListVariablesResponse,
@@ -14,19 +18,20 @@ import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
 import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns";
 import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
 
-const listHCVaultVariables = async ({ instanceUrl, namespace, mount, accessToken, path }: THCVaultListVariables) => {
-  await blockLocalAndPrivateIpAddresses(instanceUrl);
-
+const listHCVaultVariables = async (
+  { instanceUrl, namespace, mount, accessToken, path }: THCVaultListVariables,
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
   try {
-    const { data } = await request.get<THCVaultListVariablesResponse>(
-      `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
-      {
-        headers: {
-          "X-Vault-Token": accessToken,
-          ...(namespace ? { "X-Vault-Namespace": namespace } : {})
-        }
+    const { data } = await requestWithHCVaultGateway<THCVaultListVariablesResponse>(connection, gatewayService, {
+      url: `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
+      method: "GET",
+      headers: {
+        "X-Vault-Token": accessToken,
+        ...(namespace ? { "X-Vault-Namespace": namespace } : {})
       }
-    );
+    });
 
     return data.data.data;
   } catch (error: unknown) {
@@ -39,33 +44,29 @@ const listHCVaultVariables = async ({ instanceUrl, namespace, mount, accessToken
 };
 
 // Hashicorp Vault updates all variables in one batch. This is to respect their versioning
-const updateHCVaultVariables = async ({
-  path,
-  instanceUrl,
-  namespace,
-  accessToken,
-  mount,
-  data
-}: TPostHCVaultVariable) => {
-  await blockLocalAndPrivateIpAddresses(instanceUrl);
-
-  return request.post(
-    `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
-    {
-      data
+const updateHCVaultVariables = async (
+  { path, instanceUrl, namespace, accessToken, mount, data }: TPostHCVaultVariable,
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
+  return requestWithHCVaultGateway(connection, gatewayService, {
+    url: `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
+    method: "POST",
+    headers: {
+      "X-Vault-Token": accessToken,
+      ...(namespace ? { "X-Vault-Namespace": namespace } : {}),
+      "Content-Type": "application/json"
     },
-    {
-      headers: {
-        "X-Vault-Token": accessToken,
-        ...(namespace ? { "X-Vault-Namespace": namespace } : {}),
-        "Content-Type": "application/json"
-      }
-    }
-  );
+    data: { data }
+  });
 };
 
 export const HCVaultSyncFns = {
-  syncSecrets: async (secretSync: THCVaultSyncWithCredentials, secretMap: TSecretMap) => {
+  syncSecrets: async (
+    secretSync: THCVaultSyncWithCredentials,
+    secretMap: TSecretMap,
+    gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+  ) => {
     const {
       connection,
       environment,
@@ -74,16 +75,20 @@ export const HCVaultSyncFns = {
     } = secretSync;
 
     const { namespace } = connection.credentials;
-    const accessToken = await getHCVaultAccessToken(connection);
+    const accessToken = await getHCVaultAccessToken(connection, gatewayService);
     const instanceUrl = await getHCVaultInstanceUrl(connection);
 
-    const variables = await listHCVaultVariables({
-      instanceUrl,
-      accessToken,
-      namespace,
-      mount,
-      path
-    });
+    const variables = await listHCVaultVariables(
+      {
+        instanceUrl,
+        accessToken,
+        namespace,
+        mount,
+        path
+      },
+      connection,
+      gatewayService
+    );
     let tainted = false;
 
     for (const entry of Object.entries(secretMap)) {
@@ -110,24 +115,36 @@ export const HCVaultSyncFns = {
     if (!tainted) return;
 
     try {
-      await updateHCVaultVariables({ accessToken, instanceUrl, namespace, mount, path, data: variables });
+      await updateHCVaultVariables(
+        { accessToken, instanceUrl, namespace, mount, path, data: variables },
+        connection,
+        gatewayService
+      );
     } catch (error) {
       throw new SecretSyncError({
         error
       });
     }
   },
-  removeSecrets: async (secretSync: THCVaultSyncWithCredentials, secretMap: TSecretMap) => {
+  removeSecrets: async (
+    secretSync: THCVaultSyncWithCredentials,
+    secretMap: TSecretMap,
+    gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+  ) => {
     const {
       connection,
       destinationConfig: { mount, path }
     } = secretSync;
 
     const { namespace } = connection.credentials;
-    const accessToken = await getHCVaultAccessToken(connection);
+    const accessToken = await getHCVaultAccessToken(connection, gatewayService);
     const instanceUrl = await getHCVaultInstanceUrl(connection);
 
-    const variables = await listHCVaultVariables({ instanceUrl, namespace, accessToken, mount, path });
+    const variables = await listHCVaultVariables(
+      { instanceUrl, namespace, accessToken, mount, path },
+      connection,
+      gatewayService
+    );
 
     for await (const [key] of Object.entries(variables)) {
       if (key in secretMap) {
@@ -136,30 +153,41 @@ export const HCVaultSyncFns = {
     }
 
     try {
-      await updateHCVaultVariables({ accessToken, instanceUrl, namespace, mount, path, data: variables });
+      await updateHCVaultVariables(
+        { accessToken, instanceUrl, namespace, mount, path, data: variables },
+        connection,
+        gatewayService
+      );
     } catch (error) {
       throw new SecretSyncError({
         error
       });
     }
   },
-  getSecrets: async (secretSync: THCVaultSyncWithCredentials) => {
+  getSecrets: async (
+    secretSync: THCVaultSyncWithCredentials,
+    gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+  ) => {
     const {
       connection,
       destinationConfig: { mount, path }
     } = secretSync;
 
     const { namespace } = connection.credentials;
-    const accessToken = await getHCVaultAccessToken(connection);
+    const accessToken = await getHCVaultAccessToken(connection, gatewayService);
     const instanceUrl = await getHCVaultInstanceUrl(connection);
 
-    const variables = await listHCVaultVariables({
-      instanceUrl,
-      namespace,
-      accessToken,
-      mount,
-      path
-    });
+    const variables = await listHCVaultVariables(
+      {
+        instanceUrl,
+        namespace,
+        accessToken,
+        mount,
+        path
+      },
+      connection,
+      gatewayService
+    );
 
     return Object.fromEntries(Object.entries(variables).map(([key, value]) => [key, { value }]));
   }

backend/src/services/secret-sync/secret-sync-fns.ts

@@ -246,7 +246,7 @@ export const SecretSyncFns = {
       case SecretSync.Windmill:
         return WindmillSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.HCVault:
-        return HCVaultSyncFns.syncSecrets(secretSync, schemaSecretMap);
+        return HCVaultSyncFns.syncSecrets(secretSync, schemaSecretMap, gatewayService);
       case SecretSync.TeamCity:
         return TeamCitySyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.OCIVault:
@@ -285,7 +285,7 @@ export const SecretSyncFns = {
   },
   getSecrets: async (
     secretSync: TSecretSyncWithCredentials,
-    { kmsService, appConnectionDAL }: TSyncSecretDeps
+    { kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps
   ): Promise<TSecretMap> => {
     let secretMap: TSecretMap;
     switch (secretSync.destination) {
@@ -343,7 +343,7 @@ export const SecretSyncFns = {
         secretMap = await WindmillSyncFns.getSecrets(secretSync);
         break;
       case SecretSync.HCVault:
-        secretMap = await HCVaultSyncFns.getSecrets(secretSync);
+        secretMap = await HCVaultSyncFns.getSecrets(secretSync, gatewayService);
         break;
       case SecretSync.TeamCity:
         secretMap = await TeamCitySyncFns.getSecrets(secretSync);
@@ -453,7 +453,7 @@ export const SecretSyncFns = {
       case SecretSync.Windmill:
         return WindmillSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.HCVault:
-        return HCVaultSyncFns.removeSecrets(secretSync, schemaSecretMap);
+        return HCVaultSyncFns.removeSecrets(secretSync, schemaSecretMap, gatewayService);
       case SecretSync.TeamCity:
         return TeamCitySyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.OCIVault: