with stripSchema and filterForSchema

backend/src/services/secret-sync/secret-sync-fns.ts

@@ -61,45 +61,63 @@ type TSyncSecretDeps = {
   kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
 };
 
-// const addAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => {
-//   let secretMap = { ...unprocessedSecretMap };
-//
-//   const { appendSuffix, prependPrefix } = secretSync.syncOptions;
-//
-//   if (appendSuffix || prependPrefix) {
-//     secretMap = {};
-//     Object.entries(unprocessedSecretMap).forEach(([key, value]) => {
-//       secretMap[`${prependPrefix || ""}${key}${appendSuffix || ""}`] = value;
-//     });
-//   }
-//
-//   return secretMap;
-// };
-//
-// const stripAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => {
-//   let secretMap = { ...unprocessedSecretMap };
-//
-//   const { appendSuffix, prependPrefix } = secretSync.syncOptions;
-//
-//   if (appendSuffix || prependPrefix) {
-//     secretMap = {};
-//     Object.entries(unprocessedSecretMap).forEach(([key, value]) => {
-//       let processedKey = key;
-//
-//       if (prependPrefix && processedKey.startsWith(prependPrefix)) {
-//         processedKey = processedKey.slice(prependPrefix.length);
-//       }
-//
-//       if (appendSuffix && processedKey.endsWith(appendSuffix)) {
-//         processedKey = processedKey.slice(0, -appendSuffix.length);
-//       }
-//
-//       secretMap[processedKey] = value;
-//     });
-//   }
-//
-//   return secretMap;
-// };
+interface TSyncSecretConfig {
+  filterForSchema?: boolean;
+  stripSchema?: boolean;
+}
+
+// Add schema to secret keys
+const addSchema = (unprocessedSecretMap: TSecretMap, schema?: string): TSecretMap => {
+  if (!schema) return unprocessedSecretMap;
+
+  const processedSecretMap: TSecretMap = {};
+
+  for (const [key, value] of Object.entries(unprocessedSecretMap)) {
+    const newKey = schema.replace("{{secretKey}}", key);
+    processedSecretMap[newKey] = value;
+  }
+
+  return processedSecretMap;
+};
+
+// Strip schema from secret keys
+const stripSchema = (unprocessedSecretMap: TSecretMap, schema?: string): TSecretMap => {
+  if (!schema) return unprocessedSecretMap;
+
+  const [prefix, suffix] = schema.split("{{secretKey}}");
+
+  const strippedMap: TSecretMap = {};
+
+  for (const [key, value] of Object.entries(unprocessedSecretMap)) {
+    if (!key.startsWith(prefix) || !key.endsWith(suffix)) {
+      // eslint-disable-next-line no-continue
+      continue;
+    }
+
+    const strippedKey = key.slice(prefix.length, key.length - suffix.length);
+    strippedMap[strippedKey] = value;
+  }
+
+  return strippedMap;
+};
+
+// Filter only for secrets with keys that match the schema
+const filterForSchema = (secretMap: TSecretMap, schema?: string): TSecretMap => {
+  if (!schema) return secretMap;
+
+  const [prefix, suffix] = schema.split("{{secretKey}}");
+  if (prefix === undefined || suffix === undefined) return secretMap;
+
+  const filteredMap: TSecretMap = {};
+
+  for (const [key, value] of Object.entries(secretMap)) {
+    if (key.startsWith(prefix) && key.endsWith(suffix)) {
+      filteredMap[key] = value;
+    }
+  }
+
+  return filteredMap;
+};
 
 export const SecretSyncFns = {
   syncSecrets: (
@@ -107,51 +125,51 @@ export const SecretSyncFns = {
     secretMap: TSecretMap,
     { kmsService, appConnectionDAL }: TSyncSecretDeps
   ): Promise<void> => {
-    // const affixedSecretMap = addAffixes(secretSync, secretMap);
+    const schemaSecretMap = addSchema(secretMap, secretSync.syncOptions.keySchema);
 
     switch (secretSync.destination) {
       case SecretSync.AWSParameterStore:
-        return AwsParameterStoreSyncFns.syncSecrets(secretSync, secretMap);
+        return AwsParameterStoreSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.AWSSecretsManager:
-        return AwsSecretsManagerSyncFns.syncSecrets(secretSync, secretMap);
+        return AwsSecretsManagerSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.GitHub:
-        return GithubSyncFns.syncSecrets(secretSync, secretMap);
+        return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.GCPSecretManager:
-        return GcpSyncFns.syncSecrets(secretSync, secretMap);
+        return GcpSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.AzureKeyVault:
         return azureKeyVaultSyncFactory({
           appConnectionDAL,
           kmsService
-        }).syncSecrets(secretSync, secretMap);
+        }).syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.AzureAppConfiguration:
         return azureAppConfigurationSyncFactory({
           appConnectionDAL,
           kmsService
-        }).syncSecrets(secretSync, secretMap);
+        }).syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.Databricks:
         return databricksSyncFactory({
           appConnectionDAL,
           kmsService
-        }).syncSecrets(secretSync, secretMap);
+        }).syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.Humanitec:
-        return HumanitecSyncFns.syncSecrets(secretSync, secretMap);
+        return HumanitecSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.TerraformCloud:
-        return TerraformCloudSyncFns.syncSecrets(secretSync, secretMap);
+        return TerraformCloudSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.Camunda:
         return camundaSyncFactory({
           appConnectionDAL,
           kmsService
-        }).syncSecrets(secretSync, secretMap);
+        }).syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.Vercel:
-        return VercelSyncFns.syncSecrets(secretSync, secretMap);
+        return VercelSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.Windmill:
-        return WindmillSyncFns.syncSecrets(secretSync, secretMap);
+        return WindmillSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.HCVault:
-        return HCVaultSyncFns.syncSecrets(secretSync, secretMap);
+        return HCVaultSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.TeamCity:
-        return TeamCitySyncFns.syncSecrets(secretSync, secretMap);
+        return TeamCitySyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.OCIVault:
-        return OCIVaultSyncFns.syncSecrets(secretSync, secretMap);
+        return OCIVaultSyncFns.syncSecrets(secretSync, schemaSecretMap);
       default:
         throw new Error(
           `Unhandled sync destination for sync secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`
@@ -160,7 +178,8 @@ export const SecretSyncFns = {
   },
   getSecrets: async (
     secretSync: TSecretSyncWithCredentials,
-    { kmsService, appConnectionDAL }: TSyncSecretDeps
+    { kmsService, appConnectionDAL }: TSyncSecretDeps,
+    config?: TSyncSecretConfig
   ): Promise<TSecretMap> => {
     let secretMap: TSecretMap;
     switch (secretSync.destination) {
@@ -226,59 +245,68 @@ export const SecretSyncFns = {
         );
     }
 
-    return secretMap;
-    // return stripAffixes(secretSync, secretMap);
+    let processedSecretMap = secretMap;
+
+    if (config?.filterForSchema) {
+      processedSecretMap = filterForSchema(processedSecretMap);
+    }
+
+    if (config?.stripSchema) {
+      return stripSchema(processedSecretMap, secretSync.syncOptions.keySchema);
+    }
+
+    return processedSecretMap;
   },
   removeSecrets: (
     secretSync: TSecretSyncWithCredentials,
     secretMap: TSecretMap,
     { kmsService, appConnectionDAL }: TSyncSecretDeps
   ): Promise<void> => {
-    // const affixedSecretMap = addAffixes(secretSync, secretMap);
+    const schemaSecretMap = addSchema(secretMap, secretSync.syncOptions.keySchema);
 
     switch (secretSync.destination) {
       case SecretSync.AWSParameterStore:
-        return AwsParameterStoreSyncFns.removeSecrets(secretSync, secretMap);
+        return AwsParameterStoreSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.AWSSecretsManager:
-        return AwsSecretsManagerSyncFns.removeSecrets(secretSync, secretMap);
+        return AwsSecretsManagerSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.GitHub:
-        return GithubSyncFns.removeSecrets(secretSync, secretMap);
+        return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.GCPSecretManager:
-        return GcpSyncFns.removeSecrets(secretSync, secretMap);
+        return GcpSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.AzureKeyVault:
         return azureKeyVaultSyncFactory({
           appConnectionDAL,
           kmsService
-        }).removeSecrets(secretSync, secretMap);
+        }).removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.AzureAppConfiguration:
         return azureAppConfigurationSyncFactory({
           appConnectionDAL,
           kmsService
-        }).removeSecrets(secretSync, secretMap);
+        }).removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.Databricks:
         return databricksSyncFactory({
           appConnectionDAL,
           kmsService
-        }).removeSecrets(secretSync, secretMap);
+        }).removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.Humanitec:
-        return HumanitecSyncFns.removeSecrets(secretSync, secretMap);
+        return HumanitecSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.TerraformCloud:
-        return TerraformCloudSyncFns.removeSecrets(secretSync, secretMap);
+        return TerraformCloudSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.Camunda:
         return camundaSyncFactory({
           appConnectionDAL,
           kmsService
-        }).removeSecrets(secretSync, secretMap);
+        }).removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.Vercel:
-        return VercelSyncFns.removeSecrets(secretSync, secretMap);
+        return VercelSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.Windmill:
-        return WindmillSyncFns.removeSecrets(secretSync, secretMap);
+        return WindmillSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.HCVault:
-        return HCVaultSyncFns.removeSecrets(secretSync, secretMap);
+        return HCVaultSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.TeamCity:
-        return TeamCitySyncFns.removeSecrets(secretSync, secretMap);
+        return TeamCitySyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.OCIVault:
-        return OCIVaultSyncFns.removeSecrets(secretSync, secretMap);
+        return OCIVaultSyncFns.removeSecrets(secretSync, schemaSecretMap);
       default:
         throw new Error(
           `Unhandled sync destination for remove secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`

backend/src/services/secret-sync/secret-sync-queue.ts

@@ -319,9 +319,12 @@ export const secretSyncQueueFactory = ({
     );
   };
 
+  // TODO(andrey): Possibly add a "stripSchema" parameter for imports?
   const $importSecrets = async (
     secretSync: TSecretSyncWithCredentials,
-    importBehavior: SecretSyncImportBehavior
+    importBehavior: SecretSyncImportBehavior,
+    filterForSchema: boolean,
+    stripSchema: boolean
   ): Promise<TSecretMap> => {
     const { projectId, environment, folder } = secretSync;
 
@@ -330,10 +333,17 @@ export const secretSyncQueueFactory = ({
         "Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path."
       );
 
-    const importedSecrets = await SecretSyncFns.getSecrets(secretSync, {
-      appConnectionDAL,
-      kmsService
-    });
+    const importedSecrets = await SecretSyncFns.getSecrets(
+      secretSync,
+      {
+        appConnectionDAL,
+        kmsService
+      },
+      {
+        filterForSchema,
+        stripSchema
+      }
+    );
 
     if (!Object.keys(importedSecrets).length) return {};
 
@@ -439,11 +449,14 @@ export const secretSyncQueueFactory = ({
       const secretMap = await $getInfisicalSecrets(secretSync);
 
       if (!lastSyncedAt && initialSyncBehavior !== SecretSyncInitialSyncBehavior.OverwriteDestination) {
+        // TODO(andrey): Possibly add a way to filter / strip schemas on initial sync?
         const importedSecretMap = await $importSecrets(
           secretSyncWithCredentials,
           initialSyncBehavior === SecretSyncInitialSyncBehavior.ImportPrioritizeSource
             ? SecretSyncImportBehavior.PrioritizeSource
-            : SecretSyncImportBehavior.PrioritizeDestination
+            : SecretSyncImportBehavior.PrioritizeDestination,
+          false,
+          false
         );
 
         Object.entries(importedSecretMap).forEach(([key, secretData]) => {
@@ -535,7 +548,7 @@ export const secretSyncQueueFactory = ({
 
   const $handleImportSecretsJob = async (job: TSecretSyncImportSecretsDTO) => {
     const {
-      data: { syncId, auditLogInfo, importBehavior }
+      data: { syncId, auditLogInfo, importBehavior, filterForSchema, stripSchema }
     } = job;
 
     const secretSync = await secretSyncDAL.findById(syncId);
@@ -573,7 +586,9 @@ export const secretSyncQueueFactory = ({
             credentials
           }
         } as TSecretSyncWithCredentials,
-        importBehavior
+        importBehavior,
+        filterForSchema,
+        stripSchema
       );
 
       isSuccess = true;

backend/src/services/secret-sync/secret-sync-schemas.ts

@@ -1,3 +1,4 @@
+import RE2 from "re2";
 import { AnyZodObject, z } from "zod";
 
 import { SecretSyncsSchema } from "@app/db/schemas/secret-syncs";
@@ -24,6 +25,14 @@ const BaseSyncOptionsSchema = <T extends AnyZodObject | undefined = undefined>({
       ? z.nativeEnum(SecretSyncInitialSyncBehavior)
       : z.literal(SecretSyncInitialSyncBehavior.OverwriteDestination)
     ).describe(SecretSyncs.SYNC_OPTIONS(destination).initialSyncBehavior),
+    keySchema: z
+      .string()
+      .optional()
+      .refine((val) => !val || new RE2(/^(?:[a-zA-Z0-9\-/]*)(?:\{\{secretKey\}\})(?:[a-zA-Z0-9\-/]*)$/).test(val), {
+        message:
+          "Key schema must include {{secretKey}} and only contain letters, numbers, dashes, slashes, and the {{secretKey}} placeholder."
+      })
+      .describe(SecretSyncs.SYNC_OPTIONS(destination).keySchema),
     disableSecretDeletion: z.boolean().optional().describe(SecretSyncs.SYNC_OPTIONS(destination).disableSecretDeletion)
   });
 

backend/src/services/secret-sync/secret-sync-types.ts

@@ -231,6 +231,8 @@ export type TQueueSecretSyncImportSecretsByIdDTO = {
   syncId: string;
   importBehavior: SecretSyncImportBehavior;
   auditLogInfo?: AuditLogInfo;
+  filterForSchema: boolean;
+  stripSchema: boolean;
 };
 
 export type TTriggerSecretSyncImportSecretsByIdDTO = {