Merge pull request #3086 from Infisical/aws-secrets-manager-sync

Feature: AWS Secrets Manager Sync

backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts

@@ -69,6 +69,8 @@ const getParametersByPath = async (ssm: AWS.SSM, path: string): Promise<TAWSPara
         attempt += 1;
         // eslint-disable-next-line no-await-in-loop
         await sleep();
+        // eslint-disable-next-line no-continue
+        continue;
       }
 
       throw e;

backend/src/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-constants.ts

@@ -0,0 +1,10 @@
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
+import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
+
+export const AWS_SECRETS_MANAGER_SYNC_LIST_OPTION: TSecretSyncListItem = {
+  name: "AWS Secrets Manager",
+  destination: SecretSync.AWSSecretsManager,
+  connection: AppConnection.AWS,
+  canImportSecrets: true
+};

backend/src/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-enums.ts

@@ -0,0 +1,4 @@
+export enum AwsSecretsManagerSyncMappingBehavior {
+  OneToOne = "one-to-one",
+  ManyToOne = "many-to-one"
+}

backend/src/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-fns.ts

@@ -0,0 +1,352 @@
+import {
+  BatchGetSecretValueCommand,
+  CreateSecretCommand,
+  CreateSecretCommandInput,
+  DeleteSecretCommand,
+  DeleteSecretResponse,
+  ListSecretsCommand,
+  SecretsManagerClient,
+  UpdateSecretCommand,
+  UpdateSecretCommandInput
+} from "@aws-sdk/client-secrets-manager";
+import { AWSError } from "aws-sdk";
+import { CreateSecretResponse, SecretListEntry, SecretValueEntry } from "aws-sdk/clients/secretsmanager";
+
+import { getAwsConnectionConfig } from "@app/services/app-connection/aws/aws-connection-fns";
+import { AwsSecretsManagerSyncMappingBehavior } from "@app/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-enums";
+import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
+import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
+
+import { TAwsSecretsManagerSyncWithCredentials } from "./aws-secrets-manager-sync-types";
+
+type TAwsSecretsRecord = Record<string, SecretListEntry>;
+type TAwsSecretValuesRecord = Record<string, SecretValueEntry>;
+
+const MAX_RETRIES = 5;
+const BATCH_SIZE = 20;
+
+const getSecretsManagerClient = async (secretSync: TAwsSecretsManagerSyncWithCredentials) => {
+  const { destinationConfig, connection } = secretSync;
+
+  const config = await getAwsConnectionConfig(connection, destinationConfig.region);
+
+  const secretsManagerClient = new SecretsManagerClient({
+    region: config.region,
+    credentials: config.credentials!
+  });
+
+  return secretsManagerClient;
+};
+
+const sleep = async () =>
+  new Promise((resolve) => {
+    setTimeout(resolve, 1000);
+  });
+
+const getSecretsRecord = async (client: SecretsManagerClient): Promise<TAwsSecretsRecord> => {
+  const awsSecretsRecord: TAwsSecretsRecord = {};
+  let hasNext = true;
+  let nextToken: string | undefined;
+  let attempt = 0;
+
+  while (hasNext) {
+    try {
+      // eslint-disable-next-line no-await-in-loop
+      const output = await client.send(new ListSecretsCommand({ NextToken: nextToken }));
+
+      attempt = 0;
+
+      if (output.SecretList) {
+        output.SecretList.forEach((secretEntry) => {
+          if (secretEntry.Name) {
+            awsSecretsRecord[secretEntry.Name] = secretEntry;
+          }
+        });
+      }
+
+      hasNext = Boolean(output.NextToken);
+      nextToken = output.NextToken;
+    } catch (e) {
+      if ((e as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
+        attempt += 1;
+        // eslint-disable-next-line no-await-in-loop
+        await sleep();
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+
+      throw e;
+    }
+  }
+
+  return awsSecretsRecord;
+};
+
+const getSecretValuesRecord = async (
+  client: SecretsManagerClient,
+  awsSecretsRecord: TAwsSecretsRecord
+): Promise<TAwsSecretValuesRecord> => {
+  const awsSecretValuesRecord: TAwsSecretValuesRecord = {};
+  let attempt = 0;
+
+  const secretIdList = Object.keys(awsSecretsRecord);
+
+  for (let i = 0; i < secretIdList.length; i += BATCH_SIZE) {
+    const batchSecretIds = secretIdList.slice(i, i + BATCH_SIZE);
+    let hasNext = true;
+    let nextToken: string | undefined;
+
+    while (hasNext) {
+      try {
+        // eslint-disable-next-line no-await-in-loop
+        const output = await client.send(
+          new BatchGetSecretValueCommand({
+            SecretIdList: batchSecretIds,
+            NextToken: nextToken
+          })
+        );
+
+        attempt = 0;
+
+        if (output.SecretValues) {
+          output.SecretValues.forEach((secretValueEntry) => {
+            if (secretValueEntry.Name) {
+              awsSecretValuesRecord[secretValueEntry.Name] = secretValueEntry;
+            }
+          });
+        }
+
+        hasNext = Boolean(output.NextToken);
+        nextToken = output.NextToken;
+      } catch (e) {
+        if ((e as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
+          attempt += 1;
+          // eslint-disable-next-line no-await-in-loop
+          await sleep();
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        throw e;
+      }
+    }
+  }
+
+  return awsSecretValuesRecord;
+};
+
+const createSecret = async (
+  client: SecretsManagerClient,
+  input: CreateSecretCommandInput,
+  attempt = 0
+): Promise<CreateSecretResponse> => {
+  try {
+    return await client.send(new CreateSecretCommand(input));
+  } catch (error) {
+    if ((error as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
+      await sleep();
+
+      // retry
+      return createSecret(client, input, attempt + 1);
+    }
+    throw error;
+  }
+};
+
+const updateSecret = async (
+  client: SecretsManagerClient,
+  input: UpdateSecretCommandInput,
+  attempt = 0
+): Promise<CreateSecretResponse> => {
+  try {
+    return await client.send(new UpdateSecretCommand(input));
+  } catch (error) {
+    if ((error as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
+      await sleep();
+
+      // retry
+      return updateSecret(client, input, attempt + 1);
+    }
+    throw error;
+  }
+};
+
+const deleteSecret = async (
+  client: SecretsManagerClient,
+  secretKey: string,
+  attempt = 0
+): Promise<DeleteSecretResponse> => {
+  try {
+    return await client.send(new DeleteSecretCommand({ SecretId: secretKey, ForceDeleteWithoutRecovery: true }));
+  } catch (error) {
+    if ((error as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
+      await sleep();
+
+      // retry
+      return deleteSecret(client, secretKey, attempt + 1);
+    }
+    throw error;
+  }
+};
+
+export const AwsSecretsManagerSyncFns = {
+  syncSecrets: async (secretSync: TAwsSecretsManagerSyncWithCredentials, secretMap: TSecretMap) => {
+    const { destinationConfig } = secretSync;
+
+    const client = await getSecretsManagerClient(secretSync);
+
+    const awsSecretsRecord = await getSecretsRecord(client);
+
+    const awsValuesRecord = await getSecretValuesRecord(client, awsSecretsRecord);
+
+    if (destinationConfig.mappingBehavior === AwsSecretsManagerSyncMappingBehavior.OneToOne) {
+      for await (const entry of Object.entries(secretMap)) {
+        const [key, { value }] = entry;
+
+        // skip secrets that don't have a value set
+        if (!value) {
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        if (awsSecretsRecord[key]) {
+          // skip secrets that haven't changed
+          if (awsValuesRecord[key]?.SecretString === value) {
+            // eslint-disable-next-line no-continue
+            continue;
+          }
+
+          try {
+            await updateSecret(client, {
+              SecretId: key,
+              SecretString: value
+            });
+          } catch (error) {
+            throw new SecretSyncError({
+              error,
+              secretKey: key
+            });
+          }
+        } else {
+          try {
+            await createSecret(client, {
+              Name: key,
+              SecretString: value
+            });
+          } catch (error) {
+            throw new SecretSyncError({
+              error,
+              secretKey: key
+            });
+          }
+        }
+      }
+
+      for await (const secretKey of Object.keys(awsSecretsRecord)) {
+        if (!(secretKey in secretMap) || !secretMap[secretKey].value) {
+          try {
+            await deleteSecret(client, secretKey);
+          } catch (error) {
+            throw new SecretSyncError({
+              error,
+              secretKey
+            });
+          }
+        }
+      }
+    } else {
+      // Many-To-One Mapping
+
+      const secretValue = JSON.stringify(
+        Object.fromEntries(Object.entries(secretMap).map(([key, secretData]) => [key, secretData.value]))
+      );
+
+      if (awsValuesRecord[destinationConfig.secretName]) {
+        await updateSecret(client, {
+          SecretId: destinationConfig.secretName,
+          SecretString: secretValue
+        });
+      } else {
+        await createSecret(client, {
+          Name: destinationConfig.secretName,
+          SecretString: secretValue
+        });
+      }
+
+      for await (const secretKey of Object.keys(awsSecretsRecord)) {
+        if (secretKey === destinationConfig.secretName) {
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        try {
+          await deleteSecret(client, secretKey);
+        } catch (error) {
+          throw new SecretSyncError({
+            error,
+            secretKey
+          });
+        }
+      }
+    }
+  },
+  getSecrets: async (secretSync: TAwsSecretsManagerSyncWithCredentials): Promise<TSecretMap> => {
+    const client = await getSecretsManagerClient(secretSync);
+
+    const awsSecretsRecord = await getSecretsRecord(client);
+    const awsValuesRecord = await getSecretValuesRecord(client, awsSecretsRecord);
+
+    const { destinationConfig } = secretSync;
+
+    if (destinationConfig.mappingBehavior === AwsSecretsManagerSyncMappingBehavior.OneToOne) {
+      return Object.fromEntries(
+        Object.keys(awsSecretsRecord).map((key) => [key, { value: awsValuesRecord[key].SecretString ?? "" }])
+      );
+    }
+
+    // Many-To-One Mapping
+
+    const secretValueEntry = awsValuesRecord[destinationConfig.secretName];
+
+    if (!secretValueEntry) return {};
+
+    try {
+      const parsedValue = (secretValueEntry.SecretString ? JSON.parse(secretValueEntry.SecretString) : {}) as Record<
+        string,
+        string
+      >;
+
+      return Object.fromEntries(Object.entries(parsedValue).map(([key, value]) => [key, { value }]));
+    } catch {
+      throw new SecretSyncError({
+        message:
+          "Failed to import secrets. Invalid format for Many-To-One mapping behavior: requires key/value configuration.",
+        shouldRetry: false
+      });
+    }
+  },
+  removeSecrets: async (secretSync: TAwsSecretsManagerSyncWithCredentials, secretMap: TSecretMap) => {
+    const { destinationConfig } = secretSync;
+
+    const client = await getSecretsManagerClient(secretSync);
+
+    const awsSecretsRecord = await getSecretsRecord(client);
+
+    if (destinationConfig.mappingBehavior === AwsSecretsManagerSyncMappingBehavior.OneToOne) {
+      for await (const secretKey of Object.keys(awsSecretsRecord)) {
+        if (secretKey in secretMap) {
+          try {
+            await deleteSecret(client, secretKey);
+          } catch (error) {
+            throw new SecretSyncError({
+              error,
+              secretKey
+            });
+          }
+        }
+      }
+    } else {
+      await deleteSecret(client, destinationConfig.secretName);
+    }
+  }
+};

backend/src/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-schemas.ts

@@ -0,0 +1,63 @@
+import { z } from "zod";
+
+import { SecretSyncs } from "@app/lib/api-docs";
+import { AppConnection, AWSRegion } from "@app/services/app-connection/app-connection-enums";
+import { AwsSecretsManagerSyncMappingBehavior } from "@app/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-enums";
+import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
+import {
+  BaseSecretSyncSchema,
+  GenericCreateSecretSyncFieldsSchema,
+  GenericUpdateSecretSyncFieldsSchema
+} from "@app/services/secret-sync/secret-sync-schemas";
+
+const AwsSecretsManagerSyncDestinationConfigSchema = z
+  .discriminatedUnion("mappingBehavior", [
+    z.object({
+      mappingBehavior: z
+        .literal(AwsSecretsManagerSyncMappingBehavior.OneToOne)
+        .describe(SecretSyncs.DESTINATION_CONFIG.AWS_SECRETS_MANAGER.MAPPING_BEHAVIOR)
+    }),
+    z.object({
+      mappingBehavior: z
+        .literal(AwsSecretsManagerSyncMappingBehavior.ManyToOne)
+        .describe(SecretSyncs.DESTINATION_CONFIG.AWS_SECRETS_MANAGER.MAPPING_BEHAVIOR),
+      secretName: z
+        .string()
+        .regex(
+          /^[a-zA-Z0-9/_+=.@-]+$/,
+          "Secret name must contain only alphanumeric characters and the characters /_+=.@-"
+        )
+        .min(1, "Secret name is required")
+        .max(256, "Secret name cannot exceed 256 characters")
+        .describe(SecretSyncs.DESTINATION_CONFIG.AWS_SECRETS_MANAGER.SECRET_NAME)
+    })
+  ])
+  .and(
+    z.object({
+      region: z.nativeEnum(AWSRegion).describe(SecretSyncs.DESTINATION_CONFIG.AWS_SECRETS_MANAGER.REGION)
+    })
+  );
+
+export const AwsSecretsManagerSyncSchema = BaseSecretSyncSchema(SecretSync.AWSSecretsManager).extend({
+  destination: z.literal(SecretSync.AWSSecretsManager),
+  destinationConfig: AwsSecretsManagerSyncDestinationConfigSchema
+});
+
+export const CreateAwsSecretsManagerSyncSchema = GenericCreateSecretSyncFieldsSchema(
+  SecretSync.AWSSecretsManager
+).extend({
+  destinationConfig: AwsSecretsManagerSyncDestinationConfigSchema
+});
+
+export const UpdateAwsSecretsManagerSyncSchema = GenericUpdateSecretSyncFieldsSchema(
+  SecretSync.AWSSecretsManager
+).extend({
+  destinationConfig: AwsSecretsManagerSyncDestinationConfigSchema.optional()
+});
+
+export const AwsSecretsManagerSyncListItemSchema = z.object({
+  name: z.literal("AWS Secrets Manager"),
+  connection: z.literal(AppConnection.AWS),
+  destination: z.literal(SecretSync.AWSSecretsManager),
+  canImportSecrets: z.literal(true)
+});

backend/src/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-types.ts

@@ -0,0 +1,19 @@
+import { z } from "zod";
+
+import { TAwsConnection } from "@app/services/app-connection/aws";
+
+import {
+  AwsSecretsManagerSyncListItemSchema,
+  AwsSecretsManagerSyncSchema,
+  CreateAwsSecretsManagerSyncSchema
+} from "./aws-secrets-manager-sync-schemas";
+
+export type TAwsSecretsManagerSync = z.infer<typeof AwsSecretsManagerSyncSchema>;
+
+export type TAwsSecretsManagerSyncInput = z.infer<typeof CreateAwsSecretsManagerSyncSchema>;
+
+export type TAwsSecretsManagerSyncListItem = z.infer<typeof AwsSecretsManagerSyncListItemSchema>;
+
+export type TAwsSecretsManagerSyncWithCredentials = TAwsSecretsManagerSync & {
+  connection: TAwsConnection;
+};

backend/src/services/secret-sync/aws-secrets-manager/index.ts

@@ -0,0 +1,4 @@
+export * from "./aws-secrets-manager-sync-constants";
+export * from "./aws-secrets-manager-sync-fns";
+export * from "./aws-secrets-manager-sync-schemas";
+export * from "./aws-secrets-manager-sync-types";

backend/src/services/secret-sync/secret-sync-enums.ts

@@ -1,5 +1,6 @@
 export enum SecretSync {
   AWSParameterStore = "aws-parameter-store",
+  AWSSecretsManager = "aws-secrets-manager",
   GitHub = "github",
   GCPSecretManager = "gcp-secret-manager"
 }

backend/src/services/secret-sync/secret-sync-fns.ts

@@ -4,6 +4,10 @@ import {
   AWS_PARAMETER_STORE_SYNC_LIST_OPTION,
   AwsParameterStoreSyncFns
 } from "@app/services/secret-sync/aws-parameter-store";
+import {
+  AWS_SECRETS_MANAGER_SYNC_LIST_OPTION,
+  AwsSecretsManagerSyncFns
+} from "@app/services/secret-sync/aws-secrets-manager";
 import { GITHUB_SYNC_LIST_OPTION, GithubSyncFns } from "@app/services/secret-sync/github";
 import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
 import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
@@ -18,6 +22,7 @@ import { GcpSyncFns } from "./gcp/gcp-sync-fns";
 
 const SECRET_SYNC_LIST_OPTIONS: Record<SecretSync, TSecretSyncListItem> = {
   [SecretSync.AWSParameterStore]: AWS_PARAMETER_STORE_SYNC_LIST_OPTION,
+  [SecretSync.AWSSecretsManager]: AWS_SECRETS_MANAGER_SYNC_LIST_OPTION,
   [SecretSync.GitHub]: GITHUB_SYNC_LIST_OPTION,
   [SecretSync.GCPSecretManager]: GCP_SYNC_LIST_OPTION
 };
@@ -73,6 +78,8 @@ export const SecretSyncFns = {
     switch (secretSync.destination) {
       case SecretSync.AWSParameterStore:
         return AwsParameterStoreSyncFns.syncSecrets(secretSync, secretMap);
+      case SecretSync.AWSSecretsManager:
+        return AwsSecretsManagerSyncFns.syncSecrets(secretSync, secretMap);
       case SecretSync.GitHub:
         return GithubSyncFns.syncSecrets(secretSync, secretMap);
       case SecretSync.GCPSecretManager:
@@ -89,6 +96,9 @@ export const SecretSyncFns = {
       case SecretSync.AWSParameterStore:
         secretMap = await AwsParameterStoreSyncFns.getSecrets(secretSync);
         break;
+      case SecretSync.AWSSecretsManager:
+        secretMap = await AwsSecretsManagerSyncFns.getSecrets(secretSync);
+        break;
       case SecretSync.GitHub:
         secretMap = await GithubSyncFns.getSecrets(secretSync);
         break;
@@ -110,6 +120,8 @@ export const SecretSyncFns = {
     switch (secretSync.destination) {
       case SecretSync.AWSParameterStore:
         return AwsParameterStoreSyncFns.removeSecrets(secretSync, secretMap);
+      case SecretSync.AWSSecretsManager:
+        return AwsSecretsManagerSyncFns.removeSecrets(secretSync, secretMap);
       case SecretSync.GitHub:
         return GithubSyncFns.removeSecrets(secretSync, secretMap);
       case SecretSync.GCPSecretManager:

backend/src/services/secret-sync/secret-sync-maps.ts

@@ -3,12 +3,14 @@ import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
 
 export const SECRET_SYNC_NAME_MAP: Record<SecretSync, string> = {
   [SecretSync.AWSParameterStore]: "AWS Parameter Store",
+  [SecretSync.AWSSecretsManager]: "AWS Secrets Manager",
   [SecretSync.GitHub]: "GitHub",
   [SecretSync.GCPSecretManager]: "GCP Secret Manager"
 };
 
 export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
   [SecretSync.AWSParameterStore]: AppConnection.AWS,
+  [SecretSync.AWSSecretsManager]: AppConnection.AWS,
   [SecretSync.GitHub]: AppConnection.GitHub,
   [SecretSync.GCPSecretManager]: AppConnection.GCP
 };

backend/src/services/secret-sync/secret-sync-types.ts

@@ -2,6 +2,12 @@ import { Job } from "bullmq";
 
 import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
 import { QueueJobs } from "@app/queue";
+import {
+  TAwsSecretsManagerSync,
+  TAwsSecretsManagerSyncInput,
+  TAwsSecretsManagerSyncListItem,
+  TAwsSecretsManagerSyncWithCredentials
+} from "@app/services/secret-sync/aws-secrets-manager";
 import {
   TGitHubSync,
   TGitHubSyncInput,
@@ -19,16 +25,25 @@ import {
 } from "./aws-parameter-store";
 import { TGcpSync, TGcpSyncInput, TGcpSyncListItem, TGcpSyncWithCredentials } from "./gcp";
 
-export type TSecretSync = TAwsParameterStoreSync | TGitHubSync | TGcpSync;
+export type TSecretSync = TAwsParameterStoreSync | TAwsSecretsManagerSync | TGitHubSync | TGcpSync;
 
 export type TSecretSyncWithCredentials =
   | TAwsParameterStoreSyncWithCredentials
+  | TAwsSecretsManagerSyncWithCredentials
   | TGitHubSyncWithCredentials
   | TGcpSyncWithCredentials;
 
-export type TSecretSyncInput = TAwsParameterStoreSyncInput | TGitHubSyncInput | TGcpSyncInput;
+export type TSecretSyncInput =
+  | TAwsParameterStoreSyncInput
+  | TAwsSecretsManagerSyncInput
+  | TGitHubSyncInput
+  | TGcpSyncInput;
 
-export type TSecretSyncListItem = TAwsParameterStoreSyncListItem | TGitHubSyncListItem | TGcpSyncListItem;
+export type TSecretSyncListItem =
+  | TAwsParameterStoreSyncListItem
+  | TAwsSecretsManagerSyncListItem
+  | TGitHubSyncListItem
+  | TGcpSyncListItem;
 
 export type TSyncOptionsConfig = {
   canImportSecrets: boolean;