Merge pull request #4591 from Infisical/ENG-3791

Add warning on same destination secret sync

backend/src/lib/fn/object.ts

@@ -53,3 +53,53 @@ export const titleCaseToCamelCase = (obj: unknown): unknown => {
 
   return result;
 };
+
+export const deepEqual = (obj1: unknown, obj2: unknown): boolean => {
+  if (obj1 === obj2) return true;
+
+  if (obj1 === null || obj2 === null || obj1 === undefined || obj2 === undefined) {
+    return obj1 === obj2;
+  }
+
+  if (typeof obj1 !== typeof obj2) return false;
+
+  if (typeof obj1 !== "object") return obj1 === obj2;
+
+  if (Array.isArray(obj1) !== Array.isArray(obj2)) return false;
+
+  if (Array.isArray(obj1)) {
+    const arr1 = obj1 as unknown[];
+    const arr2 = obj2 as unknown[];
+    if (arr1.length !== arr2.length) return false;
+    return arr1.every((val, idx) => deepEqual(val, arr2[idx]));
+  }
+
+  const keys1 = Object.keys(obj1 as Record<string, unknown>).sort();
+  const keys2 = Object.keys(obj2 as Record<string, unknown>).sort();
+
+  if (keys1.length !== keys2.length) return false;
+  if (keys1.some((key, idx) => key !== keys2[idx])) return false;
+
+  return keys1.every((key) =>
+    deepEqual((obj1 as Record<string, unknown>)[key], (obj2 as Record<string, unknown>)[key])
+  );
+};
+
+export const deepEqualSkipFields = (obj1: unknown, obj2: unknown, skipFields: string[] = []): boolean => {
+  if (skipFields.length === 0) {
+    return deepEqual(obj1, obj2);
+  }
+
+  if (typeof obj1 !== "object" || typeof obj2 !== "object" || obj1 === null || obj2 === null) {
+    return deepEqual(obj1, obj2);
+  }
+
+  const filtered1 = Object.fromEntries(
+    Object.entries(obj1 as Record<string, unknown>).filter(([key]) => !skipFields.includes(key))
+  );
+  const filtered2 = Object.fromEntries(
+    Object.entries(obj2 as Record<string, unknown>).filter(([key]) => !skipFields.includes(key))
+  );
+
+  return deepEqual(filtered1, filtered2);
+};

backend/src/server/routes/v1/secret-sync-routers/secret-sync-endpoints.ts

@@ -425,4 +425,42 @@ export const registerSyncSecretsEndpoints = <T extends TSecretSync, I extends TS
       return { secretSync };
     }
   });
+
+  server.route({
+    method: "POST",
+    url: "/check-destination",
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      tags: [ApiDocsTags.SecretSyncs],
+      body: z.object({
+        destinationConfig: z.unknown(),
+        excludeSyncId: z.string().uuid().optional(),
+        projectId: z.string().uuid()
+      }),
+      response: {
+        200: z.object({
+          hasDuplicate: z.boolean(),
+          duplicateProjectId: z.string().uuid().optional()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
+    handler: async (req) => {
+      const { destinationConfig, excludeSyncId, projectId } = req.body;
+
+      const result = await server.services.secretSync.checkDuplicateDestination(
+        {
+          destinationConfig: destinationConfig as Record<string, unknown>,
+          destination,
+          excludeSyncId,
+          projectId
+        },
+        req.permission
+      );
+
+      return result;
+    }
+  });
 };

backend/src/services/secret-sync/secret-sync-dal.ts

@@ -204,5 +204,19 @@ export const secretSyncDALFactory = (
     }
   };
 
-  return { ...secretSyncOrm, findById, findOne, find, create, updateById };
+  const findByDestinationAndOrgId = async (destination: string, orgId: string, tx?: Knex) => {
+    try {
+      const response = await (tx || db.replicaNode())(TableName.SecretSync)
+        .join(TableName.Project, `${TableName.SecretSync}.projectId`, `${TableName.Project}.id`)
+        .where(`${TableName.SecretSync}.destination`, destination)
+        .where(`${TableName.Project}.orgId`, orgId)
+        .select(selectAllTableCols(TableName.SecretSync));
+
+      return response;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find By Destination And Org ID - Secret Sync" });
+    }
+  };
+
+  return { ...secretSyncOrm, findById, findOne, find, create, updateById, findByDestinationAndOrgId };
 };

backend/src/services/secret-sync/secret-sync-maps.ts

@@ -1,5 +1,6 @@
 import { AppConnection } from "@app/services/app-connection/app-connection-enums";
 import { SecretSync, SecretSyncPlanType } from "@app/services/secret-sync/secret-sync-enums";
+import { DestinationDuplicateCheckFn } from "@app/services/secret-sync/secret-sync-types";
 
 export const SECRET_SYNC_NAME_MAP: Record<SecretSync, string> = {
   [SecretSync.AWSParameterStore]: "AWS Parameter Store",
@@ -99,3 +100,104 @@ export const SECRET_SYNC_PLAN_MAP: Record<SecretSync, SecretSyncPlanType> = {
   [SecretSync.Netlify]: SecretSyncPlanType.Regular,
   [SecretSync.Bitbucket]: SecretSyncPlanType.Regular
 };
+
+export const SECRET_SYNC_SKIP_FIELDS_MAP: Record<SecretSync, string[]> = {
+  [SecretSync.AWSParameterStore]: [],
+  [SecretSync.AWSSecretsManager]: ["mappingBehavior", "secretName"],
+  [SecretSync.GitHub]: [],
+  [SecretSync.GCPSecretManager]: [],
+  [SecretSync.AzureKeyVault]: [],
+  [SecretSync.AzureAppConfiguration]: ["label"],
+  [SecretSync.AzureDevOps]: ["devopsProjectName"],
+  [SecretSync.Databricks]: [],
+  [SecretSync.Humanitec]: [],
+  [SecretSync.TerraformCloud]: ["variableSetName", "workspaceName"],
+  [SecretSync.Camunda]: [],
+  [SecretSync.Vercel]: ["appName"],
+  [SecretSync.Windmill]: [],
+  [SecretSync.HCVault]: [],
+  [SecretSync.TeamCity]: [],
+  [SecretSync.OCIVault]: [],
+  [SecretSync.OnePass]: ["valueLabel"],
+  [SecretSync.Heroku]: ["appName"],
+  [SecretSync.Render]: [],
+  [SecretSync.Flyio]: [],
+  [SecretSync.GitLab]: [
+    "projectName",
+    "shouldProtectSecrets",
+    "shouldMaskSecrets",
+    "shouldHideSecrets",
+    "targetEnvironment",
+    "groupName",
+    "groupId",
+    "projectId"
+  ],
+  [SecretSync.CloudflarePages]: [],
+  [SecretSync.CloudflareWorkers]: [],
+  [SecretSync.Supabase]: ["projectName"],
+  [SecretSync.Zabbix]: ["hostName", "macroType"],
+  [SecretSync.Railway]: ["projectName", "environmentName", "serviceName"],
+  [SecretSync.Checkly]: ["groupName", "accountName"],
+  [SecretSync.DigitalOceanAppPlatform]: ["appName"],
+  [SecretSync.Netlify]: ["accountName", "siteName"],
+  [SecretSync.Bitbucket]: []
+};
+
+const defaultDuplicateCheck: DestinationDuplicateCheckFn = () => true;
+
+export const DESTINATION_DUPLICATE_CHECK_MAP: Record<SecretSync, DestinationDuplicateCheckFn> = {
+  [SecretSync.AWSParameterStore]: defaultDuplicateCheck,
+  [SecretSync.AWSSecretsManager]: defaultDuplicateCheck,
+  [SecretSync.GitHub]: defaultDuplicateCheck,
+  [SecretSync.GCPSecretManager]: defaultDuplicateCheck,
+  [SecretSync.AzureKeyVault]: defaultDuplicateCheck,
+  [SecretSync.AzureAppConfiguration]: defaultDuplicateCheck,
+  [SecretSync.AzureDevOps]: defaultDuplicateCheck,
+  [SecretSync.Databricks]: defaultDuplicateCheck,
+  [SecretSync.Humanitec]: defaultDuplicateCheck,
+  [SecretSync.TerraformCloud]: defaultDuplicateCheck,
+  [SecretSync.Camunda]: defaultDuplicateCheck,
+  [SecretSync.Vercel]: defaultDuplicateCheck,
+  [SecretSync.Windmill]: defaultDuplicateCheck,
+  [SecretSync.HCVault]: defaultDuplicateCheck,
+  [SecretSync.TeamCity]: defaultDuplicateCheck,
+  [SecretSync.OCIVault]: defaultDuplicateCheck,
+  [SecretSync.OnePass]: defaultDuplicateCheck,
+  [SecretSync.Heroku]: defaultDuplicateCheck,
+  [SecretSync.Render]: defaultDuplicateCheck,
+  [SecretSync.Flyio]: defaultDuplicateCheck,
+  [SecretSync.GitLab]: (existingConfig, newConfig) => {
+    const existingTargetEnv = existingConfig.targetEnvironment as string | undefined;
+    const newTargetEnv = newConfig.targetEnvironment as string | undefined;
+
+    const wildcardValues = ["*", ""];
+
+    if (
+      (newConfig.scope as string) === "group"
+        ? existingConfig.groupId !== newConfig.groupId
+        : existingConfig.projectId !== newConfig.projectId
+    )
+      return false;
+
+    // If either has wildcard, it conflicts with any targetEnvironment
+    if (
+      !existingTargetEnv ||
+      !newTargetEnv ||
+      wildcardValues.includes(existingTargetEnv) ||
+      wildcardValues.includes(newTargetEnv)
+    ) {
+      return true;
+    }
+
+    return existingTargetEnv === newTargetEnv;
+  },
+  [SecretSync.CloudflarePages]: defaultDuplicateCheck,
+  [SecretSync.CloudflareWorkers]: defaultDuplicateCheck,
+  [SecretSync.Supabase]: defaultDuplicateCheck,
+  [SecretSync.Zabbix]: defaultDuplicateCheck,
+  [SecretSync.Railway]: defaultDuplicateCheck,
+  [SecretSync.Checkly]: defaultDuplicateCheck,
+  [SecretSync.DigitalOceanAppPlatform]: defaultDuplicateCheck,
+  [SecretSync.Netlify]: defaultDuplicateCheck,
+  [SecretSync.Bitbucket]: defaultDuplicateCheck
+};

backend/src/services/secret-sync/secret-sync-service.ts

@@ -12,6 +12,7 @@ import {
 import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
 import { DatabaseErrorCode } from "@app/lib/error-codes";
 import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
+import { deepEqualSkipFields } from "@app/lib/fn/object";
 import { OrgServiceActor } from "@app/lib/types";
 import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service";
 import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
@@ -20,6 +21,7 @@ import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
 import { enterpriseSyncCheck, listSecretSyncOptions } from "@app/services/secret-sync/secret-sync-fns";
 import {
   SecretSyncStatus,
+  TCheckDuplicateDestinationDTO,
   TCreateSecretSyncDTO,
   TDeleteSecretSyncDTO,
   TFindSecretSyncByIdDTO,
@@ -35,7 +37,12 @@ import {
 
 import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
 import { TSecretSyncDALFactory } from "./secret-sync-dal";
-import { SECRET_SYNC_CONNECTION_MAP, SECRET_SYNC_NAME_MAP } from "./secret-sync-maps";
+import {
+  DESTINATION_DUPLICATE_CHECK_MAP,
+  SECRET_SYNC_CONNECTION_MAP,
+  SECRET_SYNC_NAME_MAP,
+  SECRET_SYNC_SKIP_FIELDS_MAP
+} from "./secret-sync-maps";
 import { TSecretSyncQueueFactory } from "./secret-sync-queue";
 
 type TSecretSyncServiceFactoryDep = {
@@ -696,6 +703,61 @@ export const secretSyncServiceFactory = ({
     return updatedSecretSync as TSecretSync;
   };
 
+  const checkDuplicateDestination = async (
+    { destination, destinationConfig, excludeSyncId, projectId }: TCheckDuplicateDestinationDTO,
+    actor: OrgServiceActor
+  ) => {
+    const skipFields = SECRET_SYNC_SKIP_FIELDS_MAP[destination];
+    const { permission } = await permissionService.getProjectPermission({
+      actor: actor.type,
+      actorId: actor.id,
+      actorAuthMethod: actor.authMethod,
+      actorOrgId: actor.orgId,
+      actionProjectType: ActionProjectType.SecretManager,
+      projectId
+    });
+
+    ForbiddenError.from(permission).throwUnlessCan(
+      ProjectPermissionSecretSyncActions.Read,
+      ProjectPermissionSub.SecretSyncs
+    );
+
+    if (!destinationConfig || Object.keys(destinationConfig).length === 0) {
+      return { hasDuplicate: false, duplicateProjectId: undefined };
+    }
+
+    try {
+      const existingSyncs = await secretSyncDAL.findByDestinationAndOrgId(destination, actor.orgId);
+
+      const duplicates = existingSyncs.filter((sync) => {
+        if (sync.id === excludeSyncId) {
+          return false;
+        }
+
+        try {
+          const baseFieldsMatch = deepEqualSkipFields(sync.destinationConfig, destinationConfig, skipFields);
+          if (baseFieldsMatch) {
+            return DESTINATION_DUPLICATE_CHECK_MAP[destination](
+              sync.destinationConfig as Record<string, unknown>,
+              destinationConfig
+            );
+          }
+          return false;
+        } catch {
+          return false;
+        }
+      });
+
+      const hasDuplicate = duplicates.length > 0;
+      return {
+        hasDuplicate,
+        duplicateProjectId: hasDuplicate ? duplicates[0].projectId : undefined
+      };
+    } catch (error) {
+      return { hasDuplicate: false, duplicateProjectId: undefined };
+    }
+  };
+
   return {
     listSecretSyncOptions,
     listSecretSyncsByProjectId,
@@ -707,6 +769,7 @@ export const secretSyncServiceFactory = ({
     deleteSecretSync,
     triggerSecretSyncSyncSecretsById,
     triggerSecretSyncImportSecretsById,
-    triggerSecretSyncRemoveSecretsById
+    triggerSecretSyncRemoveSecretsById,
+    checkDuplicateDestination
   };
 };

backend/src/services/secret-sync/secret-sync-types.ts

@@ -324,6 +324,13 @@ export type TDeleteSecretSyncDTO = {
   removeSecrets: boolean;
 };
 
+export type TCheckDuplicateDestinationDTO = {
+  destination: SecretSync;
+  destinationConfig: Record<string, unknown>;
+  excludeSyncId?: string;
+  projectId: string;
+};
+
 export enum SecretSyncStatus {
   Pending = "pending",
   Running = "running",
@@ -408,3 +415,8 @@ export type TSecretMap = Record<
     secretMetadata?: ResourceMetadataDTO;
   }
 >;
+
+export type DestinationDuplicateCheckFn = (
+  existingConfig: Record<string, unknown>,
+  newConfig: Record<string, unknown>
+) => boolean;