From bae62421ae61a3ab6a39de54a4c544fa3bf056d2 Mon Sep 17 00:00:00 2001 From: x032205 Date: Tue, 13 May 2025 23:08:54 -0400 Subject: [PATCH 1/8] with stripSchema and filterForSchema --- backend/src/lib/api-docs/constants.ts | 1 + .../secret-sync-endpoints.ts | 18 +- .../services/secret-sync/secret-sync-fns.ts | 176 ++++++++++-------- .../services/secret-sync/secret-sync-queue.ts | 31 ++- .../secret-sync/secret-sync-schemas.ts | 9 + .../services/secret-sync/secret-sync-types.ts | 2 + .../SecretSyncImportSecretsModal.tsx | 90 ++++++++- .../SecretSyncOptionsFields.tsx | 46 ++--- .../SecretSyncReviewFields.tsx | 9 +- .../forms/schemas/base-secret-sync-schema.ts | 23 ++- .../src/hooks/api/secretSyncs/mutations.tsx | 6 +- .../src/hooks/api/secretSyncs/types/index.ts | 2 + .../hooks/api/secretSyncs/types/root-sync.ts | 3 +- .../SecretSyncOptionsSection.tsx | 10 +- 14 files changed, 275 insertions(+), 151 deletions(-) diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index a4556a75d..5075233e9 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -2109,6 +2109,7 @@ export const SecretSyncs = { const destinationName = SECRET_SYNC_NAME_MAP[destination]; return { initialSyncBehavior: `Specify how Infisical should resolve the initial sync to the ${destinationName} destination.`, + keySchema: `Specify the format to use for structuring secret keys in the ${destinationName} destination.`, disableSecretDeletion: `Enable this flag to prevent removal of secrets from the ${destinationName} destination when syncing.` }; }, diff --git a/backend/src/server/routes/v1/secret-sync-routers/secret-sync-endpoints.ts b/backend/src/server/routes/v1/secret-sync-routers/secret-sync-endpoints.ts index 6ab9b5939..e4de0fa51 100644 --- a/backend/src/server/routes/v1/secret-sync-routers/secret-sync-endpoints.ts +++ b/backend/src/server/routes/v1/secret-sync-routers/secret-sync-endpoints.ts @@ -367,7 +367,17 @@ export const registerSyncSecretsEndpoints = v === "true"), + stripSchema: z + .enum(["true", "false"]) + .optional() + .default("false") + .transform((v) => v === "true") }), response: { 200: z.object({ secretSync: responseSchema }) @@ -376,13 +386,15 @@ export const registerSyncSecretsEndpoints = { const { syncId } = req.params; - const { importBehavior } = req.query; + const { importBehavior, filterForSchema, stripSchema } = req.query; const secretSync = (await server.services.secretSync.triggerSecretSyncImportSecretsById( { syncId, destination, - importBehavior + importBehavior, + filterForSchema, + stripSchema }, req.permission )) as T; diff --git a/backend/src/services/secret-sync/secret-sync-fns.ts b/backend/src/services/secret-sync/secret-sync-fns.ts index d029a000a..87f187124 100644 --- a/backend/src/services/secret-sync/secret-sync-fns.ts +++ b/backend/src/services/secret-sync/secret-sync-fns.ts @@ -61,45 +61,63 @@ type TSyncSecretDeps = { kmsService: Pick; }; -// const addAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => { -// let secretMap = { ...unprocessedSecretMap }; -// -// const { appendSuffix, prependPrefix } = secretSync.syncOptions; -// -// if (appendSuffix || prependPrefix) { -// secretMap = {}; -// Object.entries(unprocessedSecretMap).forEach(([key, value]) => { -// secretMap[`${prependPrefix || ""}${key}${appendSuffix || ""}`] = value; -// }); -// } -// -// return secretMap; -// }; -// -// const stripAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => { -// let secretMap = { ...unprocessedSecretMap }; -// -// const { appendSuffix, prependPrefix } = secretSync.syncOptions; -// -// if (appendSuffix || prependPrefix) { -// secretMap = {}; -// Object.entries(unprocessedSecretMap).forEach(([key, value]) => { -// let processedKey = key; -// -// if (prependPrefix && processedKey.startsWith(prependPrefix)) { -// processedKey = processedKey.slice(prependPrefix.length); -// } -// -// if (appendSuffix && processedKey.endsWith(appendSuffix)) { -// processedKey = processedKey.slice(0, -appendSuffix.length); -// } -// -// secretMap[processedKey] = value; -// }); -// } -// -// return secretMap; -// }; +interface TSyncSecretConfig { + filterForSchema?: boolean; + stripSchema?: boolean; +} + +// Add schema to secret keys +const addSchema = (unprocessedSecretMap: TSecretMap, schema?: string): TSecretMap => { + if (!schema) return unprocessedSecretMap; + + const processedSecretMap: TSecretMap = {}; + + for (const [key, value] of Object.entries(unprocessedSecretMap)) { + const newKey = schema.replace("{{secretKey}}", key); + processedSecretMap[newKey] = value; + } + + return processedSecretMap; +}; + +// Strip schema from secret keys +const stripSchema = (unprocessedSecretMap: TSecretMap, schema?: string): TSecretMap => { + if (!schema) return unprocessedSecretMap; + + const [prefix, suffix] = schema.split("{{secretKey}}"); + + const strippedMap: TSecretMap = {}; + + for (const [key, value] of Object.entries(unprocessedSecretMap)) { + if (!key.startsWith(prefix) || !key.endsWith(suffix)) { + // eslint-disable-next-line no-continue + continue; + } + + const strippedKey = key.slice(prefix.length, key.length - suffix.length); + strippedMap[strippedKey] = value; + } + + return strippedMap; +}; + +// Filter only for secrets with keys that match the schema +const filterForSchema = (secretMap: TSecretMap, schema?: string): TSecretMap => { + if (!schema) return secretMap; + + const [prefix, suffix] = schema.split("{{secretKey}}"); + if (prefix === undefined || suffix === undefined) return secretMap; + + const filteredMap: TSecretMap = {}; + + for (const [key, value] of Object.entries(secretMap)) { + if (key.startsWith(prefix) && key.endsWith(suffix)) { + filteredMap[key] = value; + } + } + + return filteredMap; +}; export const SecretSyncFns = { syncSecrets: ( @@ -107,51 +125,51 @@ export const SecretSyncFns = { secretMap: TSecretMap, { kmsService, appConnectionDAL }: TSyncSecretDeps ): Promise => { - // const affixedSecretMap = addAffixes(secretSync, secretMap); + const schemaSecretMap = addSchema(secretMap, secretSync.syncOptions.keySchema); switch (secretSync.destination) { case SecretSync.AWSParameterStore: - return AwsParameterStoreSyncFns.syncSecrets(secretSync, secretMap); + return AwsParameterStoreSyncFns.syncSecrets(secretSync, schemaSecretMap); case SecretSync.AWSSecretsManager: - return AwsSecretsManagerSyncFns.syncSecrets(secretSync, secretMap); + return AwsSecretsManagerSyncFns.syncSecrets(secretSync, schemaSecretMap); case SecretSync.GitHub: - return GithubSyncFns.syncSecrets(secretSync, secretMap); + return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap); case SecretSync.GCPSecretManager: - return GcpSyncFns.syncSecrets(secretSync, secretMap); + return GcpSyncFns.syncSecrets(secretSync, schemaSecretMap); case SecretSync.AzureKeyVault: return azureKeyVaultSyncFactory({ appConnectionDAL, kmsService - }).syncSecrets(secretSync, secretMap); + }).syncSecrets(secretSync, schemaSecretMap); case SecretSync.AzureAppConfiguration: return azureAppConfigurationSyncFactory({ appConnectionDAL, kmsService - }).syncSecrets(secretSync, secretMap); + }).syncSecrets(secretSync, schemaSecretMap); case SecretSync.Databricks: return databricksSyncFactory({ appConnectionDAL, kmsService - }).syncSecrets(secretSync, secretMap); + }).syncSecrets(secretSync, schemaSecretMap); case SecretSync.Humanitec: - return HumanitecSyncFns.syncSecrets(secretSync, secretMap); + return HumanitecSyncFns.syncSecrets(secretSync, schemaSecretMap); case SecretSync.TerraformCloud: - return TerraformCloudSyncFns.syncSecrets(secretSync, secretMap); + return TerraformCloudSyncFns.syncSecrets(secretSync, schemaSecretMap); case SecretSync.Camunda: return camundaSyncFactory({ appConnectionDAL, kmsService - }).syncSecrets(secretSync, secretMap); + }).syncSecrets(secretSync, schemaSecretMap); case SecretSync.Vercel: - return VercelSyncFns.syncSecrets(secretSync, secretMap); + return VercelSyncFns.syncSecrets(secretSync, schemaSecretMap); case SecretSync.Windmill: - return WindmillSyncFns.syncSecrets(secretSync, secretMap); + return WindmillSyncFns.syncSecrets(secretSync, schemaSecretMap); case SecretSync.HCVault: - return HCVaultSyncFns.syncSecrets(secretSync, secretMap); + return HCVaultSyncFns.syncSecrets(secretSync, schemaSecretMap); case SecretSync.TeamCity: - return TeamCitySyncFns.syncSecrets(secretSync, secretMap); + return TeamCitySyncFns.syncSecrets(secretSync, schemaSecretMap); case SecretSync.OCIVault: - return OCIVaultSyncFns.syncSecrets(secretSync, secretMap); + return OCIVaultSyncFns.syncSecrets(secretSync, schemaSecretMap); default: throw new Error( `Unhandled sync destination for sync secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}` @@ -160,7 +178,8 @@ export const SecretSyncFns = { }, getSecrets: async ( secretSync: TSecretSyncWithCredentials, - { kmsService, appConnectionDAL }: TSyncSecretDeps + { kmsService, appConnectionDAL }: TSyncSecretDeps, + config?: TSyncSecretConfig ): Promise => { let secretMap: TSecretMap; switch (secretSync.destination) { @@ -226,59 +245,68 @@ export const SecretSyncFns = { ); } - return secretMap; - // return stripAffixes(secretSync, secretMap); + let processedSecretMap = secretMap; + + if (config?.filterForSchema) { + processedSecretMap = filterForSchema(processedSecretMap); + } + + if (config?.stripSchema) { + return stripSchema(processedSecretMap, secretSync.syncOptions.keySchema); + } + + return processedSecretMap; }, removeSecrets: ( secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap, { kmsService, appConnectionDAL }: TSyncSecretDeps ): Promise => { - // const affixedSecretMap = addAffixes(secretSync, secretMap); + const schemaSecretMap = addSchema(secretMap, secretSync.syncOptions.keySchema); switch (secretSync.destination) { case SecretSync.AWSParameterStore: - return AwsParameterStoreSyncFns.removeSecrets(secretSync, secretMap); + return AwsParameterStoreSyncFns.removeSecrets(secretSync, schemaSecretMap); case SecretSync.AWSSecretsManager: - return AwsSecretsManagerSyncFns.removeSecrets(secretSync, secretMap); + return AwsSecretsManagerSyncFns.removeSecrets(secretSync, schemaSecretMap); case SecretSync.GitHub: - return GithubSyncFns.removeSecrets(secretSync, secretMap); + return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap); case SecretSync.GCPSecretManager: - return GcpSyncFns.removeSecrets(secretSync, secretMap); + return GcpSyncFns.removeSecrets(secretSync, schemaSecretMap); case SecretSync.AzureKeyVault: return azureKeyVaultSyncFactory({ appConnectionDAL, kmsService - }).removeSecrets(secretSync, secretMap); + }).removeSecrets(secretSync, schemaSecretMap); case SecretSync.AzureAppConfiguration: return azureAppConfigurationSyncFactory({ appConnectionDAL, kmsService - }).removeSecrets(secretSync, secretMap); + }).removeSecrets(secretSync, schemaSecretMap); case SecretSync.Databricks: return databricksSyncFactory({ appConnectionDAL, kmsService - }).removeSecrets(secretSync, secretMap); + }).removeSecrets(secretSync, schemaSecretMap); case SecretSync.Humanitec: - return HumanitecSyncFns.removeSecrets(secretSync, secretMap); + return HumanitecSyncFns.removeSecrets(secretSync, schemaSecretMap); case SecretSync.TerraformCloud: - return TerraformCloudSyncFns.removeSecrets(secretSync, secretMap); + return TerraformCloudSyncFns.removeSecrets(secretSync, schemaSecretMap); case SecretSync.Camunda: return camundaSyncFactory({ appConnectionDAL, kmsService - }).removeSecrets(secretSync, secretMap); + }).removeSecrets(secretSync, schemaSecretMap); case SecretSync.Vercel: - return VercelSyncFns.removeSecrets(secretSync, secretMap); + return VercelSyncFns.removeSecrets(secretSync, schemaSecretMap); case SecretSync.Windmill: - return WindmillSyncFns.removeSecrets(secretSync, secretMap); + return WindmillSyncFns.removeSecrets(secretSync, schemaSecretMap); case SecretSync.HCVault: - return HCVaultSyncFns.removeSecrets(secretSync, secretMap); + return HCVaultSyncFns.removeSecrets(secretSync, schemaSecretMap); case SecretSync.TeamCity: - return TeamCitySyncFns.removeSecrets(secretSync, secretMap); + return TeamCitySyncFns.removeSecrets(secretSync, schemaSecretMap); case SecretSync.OCIVault: - return OCIVaultSyncFns.removeSecrets(secretSync, secretMap); + return OCIVaultSyncFns.removeSecrets(secretSync, schemaSecretMap); default: throw new Error( `Unhandled sync destination for remove secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}` diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts index 62b4ba3cc..895afd45f 100644 --- a/backend/src/services/secret-sync/secret-sync-queue.ts +++ b/backend/src/services/secret-sync/secret-sync-queue.ts @@ -319,9 +319,12 @@ export const secretSyncQueueFactory = ({ ); }; + // TODO(andrey): Possibly add a "stripSchema" parameter for imports? const $importSecrets = async ( secretSync: TSecretSyncWithCredentials, - importBehavior: SecretSyncImportBehavior + importBehavior: SecretSyncImportBehavior, + filterForSchema: boolean, + stripSchema: boolean ): Promise => { const { projectId, environment, folder } = secretSync; @@ -330,10 +333,17 @@ export const secretSyncQueueFactory = ({ "Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path." ); - const importedSecrets = await SecretSyncFns.getSecrets(secretSync, { - appConnectionDAL, - kmsService - }); + const importedSecrets = await SecretSyncFns.getSecrets( + secretSync, + { + appConnectionDAL, + kmsService + }, + { + filterForSchema, + stripSchema + } + ); if (!Object.keys(importedSecrets).length) return {}; @@ -439,11 +449,14 @@ export const secretSyncQueueFactory = ({ const secretMap = await $getInfisicalSecrets(secretSync); if (!lastSyncedAt && initialSyncBehavior !== SecretSyncInitialSyncBehavior.OverwriteDestination) { + // TODO(andrey): Possibly add a way to filter / strip schemas on initial sync? const importedSecretMap = await $importSecrets( secretSyncWithCredentials, initialSyncBehavior === SecretSyncInitialSyncBehavior.ImportPrioritizeSource ? SecretSyncImportBehavior.PrioritizeSource - : SecretSyncImportBehavior.PrioritizeDestination + : SecretSyncImportBehavior.PrioritizeDestination, + false, + false ); Object.entries(importedSecretMap).forEach(([key, secretData]) => { @@ -535,7 +548,7 @@ export const secretSyncQueueFactory = ({ const $handleImportSecretsJob = async (job: TSecretSyncImportSecretsDTO) => { const { - data: { syncId, auditLogInfo, importBehavior } + data: { syncId, auditLogInfo, importBehavior, filterForSchema, stripSchema } } = job; const secretSync = await secretSyncDAL.findById(syncId); @@ -573,7 +586,9 @@ export const secretSyncQueueFactory = ({ credentials } } as TSecretSyncWithCredentials, - importBehavior + importBehavior, + filterForSchema, + stripSchema ); isSuccess = true; diff --git a/backend/src/services/secret-sync/secret-sync-schemas.ts b/backend/src/services/secret-sync/secret-sync-schemas.ts index 50ff3f307..114bd2322 100644 --- a/backend/src/services/secret-sync/secret-sync-schemas.ts +++ b/backend/src/services/secret-sync/secret-sync-schemas.ts @@ -1,3 +1,4 @@ +import RE2 from "re2"; import { AnyZodObject, z } from "zod"; import { SecretSyncsSchema } from "@app/db/schemas/secret-syncs"; @@ -24,6 +25,14 @@ const BaseSyncOptionsSchema = ({ ? z.nativeEnum(SecretSyncInitialSyncBehavior) : z.literal(SecretSyncInitialSyncBehavior.OverwriteDestination) ).describe(SecretSyncs.SYNC_OPTIONS(destination).initialSyncBehavior), + keySchema: z + .string() + .optional() + .refine((val) => !val || new RE2(/^(?:[a-zA-Z0-9\-/]*)(?:\{\{secretKey\}\})(?:[a-zA-Z0-9\-/]*)$/).test(val), { + message: + "Key schema must include {{secretKey}} and only contain letters, numbers, dashes, slashes, and the {{secretKey}} placeholder." + }) + .describe(SecretSyncs.SYNC_OPTIONS(destination).keySchema), disableSecretDeletion: z.boolean().optional().describe(SecretSyncs.SYNC_OPTIONS(destination).disableSecretDeletion) }); diff --git a/backend/src/services/secret-sync/secret-sync-types.ts b/backend/src/services/secret-sync/secret-sync-types.ts index 64d027e18..a25dfa3e0 100644 --- a/backend/src/services/secret-sync/secret-sync-types.ts +++ b/backend/src/services/secret-sync/secret-sync-types.ts @@ -231,6 +231,8 @@ export type TQueueSecretSyncImportSecretsByIdDTO = { syncId: string; importBehavior: SecretSyncImportBehavior; auditLogInfo?: AuditLogInfo; + filterForSchema: boolean; + stripSchema: boolean; }; export type TTriggerSecretSyncImportSecretsByIdDTO = { diff --git a/frontend/src/components/secret-syncs/SecretSyncImportSecretsModal.tsx b/frontend/src/components/secret-syncs/SecretSyncImportSecretsModal.tsx index 8b587c62f..87742c0f1 100644 --- a/frontend/src/components/secret-syncs/SecretSyncImportSecretsModal.tsx +++ b/frontend/src/components/secret-syncs/SecretSyncImportSecretsModal.tsx @@ -10,7 +10,8 @@ import { ModalClose, ModalContent, Select, - SelectItem + SelectItem, + Switch } from "@app/components/v2"; import { SECRET_SYNC_IMPORT_BEHAVIOR_MAP, SECRET_SYNC_MAP } from "@app/helpers/secretSyncs"; import { @@ -31,30 +32,49 @@ type ContentProps = { }; const FormSchema = z.object({ - importBehavior: z.nativeEnum(SecretSyncImportBehavior) + importBehavior: z.nativeEnum(SecretSyncImportBehavior), + filterForSchema: z.boolean(), + stripSchema: z.boolean() }); type TFormData = z.infer; const Content = ({ secretSync, onComplete }: ContentProps) => { - const { id: syncId, destination, projectId } = secretSync; + const { + id: syncId, + destination, + projectId, + syncOptions: { keySchema } + } = secretSync; const destinationName = SECRET_SYNC_MAP[destination].name; const { handleSubmit, control, formState: { isSubmitting, isDirty } - } = useForm({ resolver: zodResolver(FormSchema) }); + } = useForm({ + resolver: zodResolver(FormSchema), + defaultValues: { + filterForSchema: false, + stripSchema: false + } + }); const triggerImportSecrets = useTriggerSecretSyncImportSecrets(); - const handleTriggerImportSecrets = async ({ importBehavior }: TFormData) => { + const handleTriggerImportSecrets = async ({ + importBehavior, + filterForSchema, + stripSchema + }: TFormData) => { try { await triggerImportSecrets.mutateAsync({ syncId, destination, importBehavior, - projectId + projectId, + filterForSchema, + stripSchema }); createNotification({ @@ -131,6 +151,64 @@ const Content = ({ secretSync, onComplete }: ContentProps) => { )} /> + {keySchema && ( + <> + ( + +

+ If enabled, Infisical will only import destination secrets that match your key + schema: +

+ {keySchema} + + } + label="Filter Keys for Schema" + isError={Boolean(error)} + errorText={error?.message} + > + + Only import destination secrets that match schema + + + )} + /> + ( + + + Strip schema from imported secret keys + + + )} + /> + + )}
- } - label="Filter Keys for Schema" - isError={Boolean(error)} - errorText={error?.message} - > - - Only import destination secrets that match schema - - - )} - /> - ( - - - Strip schema from imported secret keys - - - )} - /> - - )}