From 48e5f550e90d873d4d61b28042cdfe0689684bab Mon Sep 17 00:00:00 2001 From: Scott Wilson Date: Thu, 14 Aug 2025 15:46:41 -0700 Subject: [PATCH 1/3] fix: handle keyschema with path segments for aws parameter store --- .../aws-parameter-store-sync-fns.ts | 69 +++++++++++++++---- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts index d0e97afa1..83bf28ae7 100644 --- a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts +++ b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts @@ -34,18 +34,41 @@ const sleep = async () => setTimeout(resolve, 1000); }); -const getParametersByPath = async (ssm: AWS.SSM, path: string): Promise => { +const getFullPath = ({ path, keySchema }: { path: string; keySchema?: string }) => { + if (!keySchema || !keySchema.includes("/")) return path; + + const keySchemaSegments = keySchema.split("/"); + + const pathSegments = keySchemaSegments.slice(0, keySchemaSegments.length - 1); + + if (pathSegments.some((segment) => segment.includes("{{"))) { + throw new SecretSyncError({ + message: "Key schema cannot contain '/' after keys: ie {{secretKey}} or {{environment}}", + shouldRetry: false + }); + } + + return `${path}${pathSegments.join("/")}/`; +}; + +const getParametersByPath = async ( + ssm: AWS.SSM, + path: string, + keySchema: string | undefined +): Promise => { const awsParameterStoreSecretsRecord: TAWSParameterStoreRecord = {}; let hasNext = true; let nextToken: string | undefined; let attempt = 0; + const fullPath = getFullPath({ path, keySchema }); + while (hasNext) { try { // eslint-disable-next-line no-await-in-loop const parameters = await ssm .getParametersByPath({ - Path: path, + Path: fullPath, Recursive: false, WithDecryption: true, MaxResults: BATCH_SIZE, @@ -59,7 +82,7 @@ const getParametersByPath = async (ssm: AWS.SSM, path: string): Promise { if (parameter.Name) { // no leading slash if path is '/' - const secKey = path.length > 1 ? parameter.Name.substring(path.length) : parameter.Name; + const secKey = fullPath.length > 1 ? parameter.Name.substring(path.length) : parameter.Name; awsParameterStoreSecretsRecord[secKey] = parameter; } }); @@ -83,12 +106,18 @@ const getParametersByPath = async (ssm: AWS.SSM, path: string): Promise => { +const getParameterMetadataByPath = async ( + ssm: AWS.SSM, + path: string, + keySchema: string | undefined +): Promise => { const awsParameterStoreMetadataRecord: TAWSParameterStoreMetadataRecord = {}; let hasNext = true; let nextToken: string | undefined; let attempt = 0; + const fullPath = getFullPath({ path, keySchema }); + while (hasNext) { try { // eslint-disable-next-line no-await-in-loop @@ -100,7 +129,7 @@ const getParameterMetadataByPath = async (ssm: AWS.SSM, path: string): Promise { if (parameter.Name) { // no leading slash if path is '/' - const secKey = path.length > 1 ? parameter.Name.substring(path.length) : parameter.Name; + const secKey = fullPath.length > 1 ? parameter.Name.substring(path.length) : parameter.Name; awsParameterStoreMetadataRecord[secKey] = parameter; } }); @@ -298,9 +327,17 @@ export const AwsParameterStoreSyncFns = { const ssm = await getSSM(secretSync); - const awsParameterStoreSecretsRecord = await getParametersByPath(ssm, destinationConfig.path); + const awsParameterStoreSecretsRecord = await getParametersByPath( + ssm, + destinationConfig.path, + syncOptions.keySchema + ); - const awsParameterStoreMetadataRecord = await getParameterMetadataByPath(ssm, destinationConfig.path); + const awsParameterStoreMetadataRecord = await getParameterMetadataByPath( + ssm, + destinationConfig.path, + syncOptions.keySchema + ); const { shouldManageTags, awsParameterStoreTagsRecord } = await getParameterStoreTagsRecord( ssm, @@ -400,22 +437,30 @@ export const AwsParameterStoreSyncFns = { await deleteParametersBatch(ssm, parametersToDelete); }, getSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials): Promise => { - const { destinationConfig } = secretSync; + const { destinationConfig, syncOptions } = secretSync; const ssm = await getSSM(secretSync); - const awsParameterStoreSecretsRecord = await getParametersByPath(ssm, destinationConfig.path); + const awsParameterStoreSecretsRecord = await getParametersByPath( + ssm, + destinationConfig.path, + syncOptions.keySchema + ); return Object.fromEntries( Object.entries(awsParameterStoreSecretsRecord).map(([key, value]) => [key, { value: value.Value ?? "" }]) ); }, removeSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials, secretMap: TSecretMap) => { - const { destinationConfig } = secretSync; + const { destinationConfig, syncOptions } = secretSync; const ssm = await getSSM(secretSync); - const awsParameterStoreSecretsRecord = await getParametersByPath(ssm, destinationConfig.path); + const awsParameterStoreSecretsRecord = await getParametersByPath( + ssm, + destinationConfig.path, + syncOptions.keySchema + ); const parametersToDelete: AWS.SSM.Parameter[] = []; From a7847f177c831a3d6c82d6b9b27a3c3a188d5d2e Mon Sep 17 00:00:00 2001 From: Scott Wilson Date: Thu, 14 Aug 2025 16:25:00 -0700 Subject: [PATCH 2/3] improvements: address feedback --- .../aws-parameter-store-sync-fns.ts | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts index 83bf28ae7..d51340408 100644 --- a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts +++ b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts @@ -1,4 +1,5 @@ import AWS, { AWSError } from "aws-sdk"; +import handlebars from "handlebars"; import { getAwsConnectionConfig } from "@app/services/app-connection/aws/aws-connection-fns"; import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors"; @@ -34,16 +35,21 @@ const sleep = async () => setTimeout(resolve, 1000); }); -const getFullPath = ({ path, keySchema }: { path: string; keySchema?: string }) => { +const getFullPath = ({ path, keySchema, environment }: { path: string; keySchema?: string; environment: string }) => { if (!keySchema || !keySchema.includes("/")) return path; - const keySchemaSegments = keySchema.split("/"); + const keySchemaSegments = handlebars + .compile(keySchema)({ + environment, + secretKey: "{{secretKey}}" + }) + .split("/"); const pathSegments = keySchemaSegments.slice(0, keySchemaSegments.length - 1); - if (pathSegments.some((segment) => segment.includes("{{"))) { + if (pathSegments.some((segment) => segment.includes("{{secretKey}}"))) { throw new SecretSyncError({ - message: "Key schema cannot contain '/' after keys: ie {{secretKey}} or {{environment}}", + message: "Key schema cannot contain '/' after {{secretKey}}", shouldRetry: false }); } @@ -54,14 +60,15 @@ const getFullPath = ({ path, keySchema }: { path: string; keySchema?: string }) const getParametersByPath = async ( ssm: AWS.SSM, path: string, - keySchema: string | undefined + keySchema: string | undefined, + environment: string ): Promise => { const awsParameterStoreSecretsRecord: TAWSParameterStoreRecord = {}; let hasNext = true; let nextToken: string | undefined; let attempt = 0; - const fullPath = getFullPath({ path, keySchema }); + const fullPath = getFullPath({ path, keySchema, environment }); while (hasNext) { try { @@ -109,14 +116,15 @@ const getParametersByPath = async ( const getParameterMetadataByPath = async ( ssm: AWS.SSM, path: string, - keySchema: string | undefined + keySchema: string | undefined, + environment: string ): Promise => { const awsParameterStoreMetadataRecord: TAWSParameterStoreMetadataRecord = {}; let hasNext = true; let nextToken: string | undefined; let attempt = 0; - const fullPath = getFullPath({ path, keySchema }); + const fullPath = getFullPath({ path, keySchema, environment }); while (hasNext) { try { @@ -330,13 +338,15 @@ export const AwsParameterStoreSyncFns = { const awsParameterStoreSecretsRecord = await getParametersByPath( ssm, destinationConfig.path, - syncOptions.keySchema + syncOptions.keySchema, + environment!.slug ); const awsParameterStoreMetadataRecord = await getParameterMetadataByPath( ssm, destinationConfig.path, - syncOptions.keySchema + syncOptions.keySchema, + environment!.slug ); const { shouldManageTags, awsParameterStoreTagsRecord } = await getParameterStoreTagsRecord( @@ -437,14 +447,15 @@ export const AwsParameterStoreSyncFns = { await deleteParametersBatch(ssm, parametersToDelete); }, getSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials): Promise => { - const { destinationConfig, syncOptions } = secretSync; + const { destinationConfig, syncOptions, environment } = secretSync; const ssm = await getSSM(secretSync); const awsParameterStoreSecretsRecord = await getParametersByPath( ssm, destinationConfig.path, - syncOptions.keySchema + syncOptions.keySchema, + environment!.slug ); return Object.fromEntries( @@ -452,14 +463,15 @@ export const AwsParameterStoreSyncFns = { ); }, removeSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials, secretMap: TSecretMap) => { - const { destinationConfig, syncOptions } = secretSync; + const { destinationConfig, syncOptions, environment } = secretSync; const ssm = await getSSM(secretSync); const awsParameterStoreSecretsRecord = await getParametersByPath( ssm, destinationConfig.path, - syncOptions.keySchema + syncOptions.keySchema, + environment!.slug ); const parametersToDelete: AWS.SSM.Parameter[] = []; From d24f5a57a8d03d99ef80b0828cd3eaad2fdbc883 Mon Sep 17 00:00:00 2001 From: Scott Wilson Date: Thu, 14 Aug 2025 16:59:26 -0700 Subject: [PATCH 3/3] improvement: throw on keyschema leading slash --- .../aws-parameter-store/aws-parameter-store-sync-fns.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts index d51340408..680dd4256 100644 --- a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts +++ b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts @@ -38,6 +38,10 @@ const sleep = async () => const getFullPath = ({ path, keySchema, environment }: { path: string; keySchema?: string; environment: string }) => { if (!keySchema || !keySchema.includes("/")) return path; + if (keySchema.startsWith("/")) { + throw new SecretSyncError({ message: `Key schema cannot contain leading '/'`, shouldRetry: false }); + } + const keySchemaSegments = handlebars .compile(keySchema)({ environment,