feat: aws secret manager metadata sync

backend/src/lib/api-docs/constants.ts

@@ -1136,7 +1136,8 @@ export const INTEGRATION = {
       shouldMaskSecrets: "Specifies if the secrets synced from Infisical to Gitlab should be marked as 'Masked'.",
       shouldProtectSecrets: "Specifies if the secrets synced from Infisical to Gitlab should be marked as 'Protected'.",
       shouldEnableDelete: "The flag to enable deletion of secrets.",
-      octopusDeployScopeValues: "Specifies the scope values to set on synced secrets to Octopus Deploy."
+      octopusDeployScopeValues: "Specifies the scope values to set on synced secrets to Octopus Deploy.",
+      metadataSyncMode: "The mode for syncing metadata to external system"
     }
   },
   UPDATE: {

backend/src/services/integration-auth/integration-list.ts

@@ -427,3 +427,8 @@ export const getIntegrationOptions = async () => {
 
   return INTEGRATION_OPTIONS;
 };
+
+export enum IntegrationMetadataSyncMode {
+  CUSTOM = "custom",
+  SECRET_METADATA = "secret-metadata"
+}

backend/src/services/integration-auth/integration-sync-secret.ts

@@ -38,6 +38,7 @@ import { TCreateManySecretsRawFn, TUpdateManySecretsRawFn } from "@app/services/
 
 import { TIntegrationDALFactory } from "../integration/integration-dal";
 import { IntegrationMetadataSchema } from "../integration/integration-schema";
+import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema";
 import { IntegrationAuthMetadataSchema } from "./integration-auth-schema";
 import {
   CircleCiScope,
@@ -48,6 +49,7 @@ import {
 import {
   IntegrationInitialSyncBehavior,
   IntegrationMappingBehavior,
+  IntegrationMetadataSyncMode,
   Integrations,
   IntegrationUrls
 } from "./integration-list";
@@ -1074,14 +1076,14 @@ const syncSecretsAWSSecretManager = async ({
   projectId
 }: {
   integration: TIntegrations;
-  secrets: Record<string, { value: string; comment?: string }>;
+  secrets: Record<string, { value: string; comment?: string; secretMetadata?: ResourceMetadataDTO }>;
   accessId: string | null;
   accessToken: string;
   awsAssumeRoleArn: string | null;
   projectId?: string;
 }) => {
   const appCfg = getConfig();
-  const metadata = z.record(z.any()).parse(integration.metadata || {});
+  const metadata = IntegrationMetadataSchema.parse(integration.metadata || {});
 
   if (!accessId && !awsAssumeRoleArn) {
     throw new Error("AWS access ID/AWS Assume Role is required");
@@ -1129,8 +1131,25 @@ const syncSecretsAWSSecretManager = async ({
 
   const processAwsSecret = async (
     secretId: string,
-    secretValue: Record<string, string | null | undefined> | string
+    secretValue: Record<string, string | null | undefined> | string,
+    secretMetadata?: ResourceMetadataDTO
   ) => {
+    const secretAWSTag = metadata.secretAWSTag as { key: string; value: string }[] | undefined;
+    const shouldTag =
+      (secretAWSTag && secretAWSTag.length) ||
+      (metadata.metadataSyncMode === IntegrationMetadataSyncMode.SECRET_METADATA &&
+        metadata.mappingBehavior === IntegrationMappingBehavior.ONE_TO_ONE);
+    const tagArray =
+      (metadata.metadataSyncMode === IntegrationMetadataSyncMode.SECRET_METADATA ? secretMetadata : secretAWSTag) ?? [];
+
+    const integrationTagObj = tagArray.reduce(
+      (acc, item) => {
+        acc[item.key] = item.value;
+        return acc;
+      },
+      {} as Record<string, string>
+    );
+
     try {
       const awsSecretManagerSecret = await secretsManager.send(
         new GetSecretValueCommand({
@@ -1165,9 +1184,7 @@ const syncSecretsAWSSecretManager = async ({
         }
       }
 
-      const secretAWSTag = metadata.secretAWSTag as { key: string; value: string }[] | undefined;
+      if (shouldTag) {
-
-      if (secretAWSTag && secretAWSTag.length) {
         const describedSecret = await secretsManager.send(
           // requires secretsmanager:DescribeSecret policy
           new DescribeSecretCommand({
@@ -1177,14 +1194,6 @@ const syncSecretsAWSSecretManager = async ({
 
         if (!describedSecret.Tags) return;
 
-        const integrationTagObj = secretAWSTag.reduce(
-          (acc, item) => {
-            acc[item.key] = item.value;
-            return acc;
-          },
-          {} as Record<string, string>
-        );
-
         const awsTagObj = (describedSecret.Tags || []).reduce(
           (acc, item) => {
             if (item.Key && item.Value) {
@@ -1216,7 +1225,7 @@ const syncSecretsAWSSecretManager = async ({
           }
         });
 
-        secretAWSTag?.forEach((tag) => {
+        tagArray.forEach((tag) => {
           if (!(tag.key in awsTagObj)) {
             // create tag in AWS secret manager
             tagsToUpdate.push({
@@ -1253,8 +1262,8 @@ const syncSecretsAWSSecretManager = async ({
               Name: secretId,
               SecretString: typeof secretValue === "string" ? secretValue : JSON.stringify(secretValue),
               ...(metadata.kmsKeyId && { KmsKeyId: metadata.kmsKeyId }),
-              Tags: metadata.secretAWSTag
+              Tags: shouldTag
-                ? metadata.secretAWSTag.map((tag: { key: string; value: string }) => ({
+                ? tagArray.map((tag: { key: string; value: string }) => ({
                     Key: tag.key,
                     Value: tag.value
                   }))
@@ -1271,7 +1280,7 @@ const syncSecretsAWSSecretManager = async ({
 
   if (metadata.mappingBehavior === IntegrationMappingBehavior.ONE_TO_ONE) {
     for await (const [key, value] of Object.entries(secrets)) {
-      await processAwsSecret(key, value.value);
+      await processAwsSecret(key, value.value, value.secretMetadata);
     }
   } else {
     await processAwsSecret(integration.app as string, getSecretKeyValuePair(secrets));
@@ -4392,7 +4401,7 @@ export const syncIntegrationSecrets = async ({
     secretPath: string;
   };
   integrationAuth: TIntegrationAuths;
-  secrets: Record<string, { value: string; comment?: string }>;
+  secrets: Record<string, { value: string; comment?: string; secretMetadata?: ResourceMetadataDTO }>;
   accessId: string | null;
   awsAssumeRoleArn: string | null;
   accessToken: string;

backend/src/services/integration/integration-schema.ts

@@ -2,7 +2,7 @@ import { z } from "zod";
 
 import { INTEGRATION } from "@app/lib/api-docs";
 
-import { IntegrationMappingBehavior } from "../integration-auth/integration-list";
+import { IntegrationMappingBehavior, IntegrationMetadataSyncMode } from "../integration-auth/integration-list";
 
 export const IntegrationMetadataSchema = z.object({
   initialSyncBehavior: z.string().optional().describe(INTEGRATION.CREATE.metadata.initialSyncBehavoir),
@@ -50,6 +50,11 @@ export const IntegrationMetadataSchema = z.object({
   shouldMaskSecrets: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldMaskSecrets),
   shouldProtectSecrets: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldProtectSecrets),
 
+  metadataSyncMode: z
+    .nativeEnum(IntegrationMetadataSyncMode)
+    .optional()
+    .describe(INTEGRATION.CREATE.metadata.metadataSyncMode),
+
   octopusDeployScopeValues: z
     .object({
       // in Octopus Deploy Scope Value Format

backend/src/services/secret/secret-queue.ts

@@ -47,6 +47,7 @@ import { TProjectKeyDALFactory } from "../project-key/project-key-dal";
 import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
 import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal";
 import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal";
+import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
 import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns";
@@ -121,7 +122,12 @@ export const uniqueSecretQueueKey = (environment: string, secretPath: string) =>
 
 type TIntegrationSecret = Record<
   string,
-  { value: string; comment?: string; skipMultilineEncoding?: boolean | null | undefined }
+  {
+    value: string;
+    comment?: string;
+    skipMultilineEncoding?: boolean | null | undefined;
+    secretMetadata?: ResourceMetadataDTO;
+  }
 >;
 
 // TODO(akhilmhdh): split this into multiple queue
@@ -370,6 +376,7 @@ export const secretQueueFactory = ({
         }
 
         content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
+        content[secretKey].secretMetadata = secret.secretMetadata;
       })
     );
 
@@ -395,7 +402,8 @@ export const secretQueueFactory = ({
           content[importedSecret.key] = {
             skipMultilineEncoding: importedSecret.skipMultilineEncoding,
             comment: importedSecret.secretComment,
-            value: importedSecret.secretValue || ""
+            value: importedSecret.secretValue || "",
+            secretMetadata: importedSecret.secretMetadata
           };
         }
       }

docs/integrations/cloud/aws-secret-manager.mdx

@@ -17,6 +17,7 @@ Prerequisites:
 If your instance is deployed on AWS, the aws-sdk will automatically retrieve the credentials. Ensure that you assign the provided permission policy to your deployed instance, such as ECS or EC2.
 
 The following steps are for instances not deployed on AWS
+
 <Steps>
     <Step title="Create an IAM User">
       Navigate to [Create IAM User](https://console.aws.amazon.com/iamv2/home#/users/create) in your AWS Console.
@@ -40,9 +41,10 @@ The following steps are for instances not deployed on AWS
 <Step title="Obtain the IAM User Credentials">
   Obtain the AWS access key ID and secret access key for your IAM User by navigating to IAM > Users > [Your User] > Security credentials > Access keys.
 
-  ![Access Key Step 1](../../images/integrations/aws/integrations-aws-access-key-1.png)
+![Access Key Step 1](../../images/integrations/aws/integrations-aws-access-key-1.png)
-  ![Access Key Step 2](../../images/integrations/aws/integrations-aws-access-key-2.png)
+![Access Key Step 2](../../images/integrations/aws/integrations-aws-access-key-2.png)
-  ![Access Key Step 3](../../images/integrations/aws/integrations-aws-access-key-3.png)
+![Access Key Step 3](../../images/integrations/aws/integrations-aws-access-key-3.png)
+
 </Step>
 <Step title="Set Up Integration Keys">
   1. Set the access key as **CLIENT_ID_AWS_INTEGRATION**.
@@ -59,6 +61,7 @@ The following steps are for instances not deployed on AWS
     2. Select **AWS Account** as the **Trusted Entity Type**.
     3. Choose **Another AWS Account** and enter **381492033652** (Infisical AWS Account ID). This restricts the role to be assumed only by Infisical. If self-hosting, provide your AWS account number instead.
     4. Optionally, enable **Require external ID** and enter your **project ID** to further enhance security.
+
   </Step>
 
   <Step title="Add Required Permissions for the IAM Role">
@@ -89,11 +92,13 @@ The following steps are for instances not deployed on AWS
       ]
     }
     ```
+
   </Step>
 
-  <Step title="Copy the AWS IAM Role ARN">
+<Step title="Copy the AWS IAM Role ARN">
-    ![Copy IAM Role ARN](../../images/integrations/aws/integration-aws-iam-assume-arn.png)
+  ![Copy IAM Role
-  </Step>
+  ARN](../../images/integrations/aws/integration-aws-iam-assume-arn.png)
+</Step>
 
   <Step title="Authorize Infisical for AWS Secrets Manager">
     1. Navigate to your project's integrations tab in Infisical.
@@ -104,6 +109,7 @@ The following steps are for instances not deployed on AWS
        ![Select Assume Role](../../images/integrations/aws/integration-aws-iam-assume-select.png)
 
     4. Provide the **AWS IAM Role ARN** obtained from the previous step.
+
   </Step>  <Step title="Start integration">
     Select how you want to integration to work by specifying a number of parameters:
 
@@ -127,6 +133,12 @@ The following steps are for instances not deployed on AWS
 
     Optionally, you can add tags or specify the encryption key of all the secrets created via this integration:
 
+    <ParamField path="Tag Sync Mode" type="string" optional>
+      The sync mode for AWS tags. The supported options are `Secret Metadata` and `Custom`. If `Secret Metadata` is selected,
+      the metadata of the Infisical secrets are used as tags in AWS. If custom is selected, then the key/value of the **Secret Tag** field is used. `Secret Metadata` mode
+      is only supported for one-to-one integrations.
+     </ParamField>
+
     <ParamField path="Secret Tag" type="string" optional>
       The Key/Value of a tag that will be added to secrets in AWS. Please note that it is possible to add multiple tags via API.
     </ParamField>

frontend/src/hooks/api/integrations/queries.tsx

@@ -4,7 +4,12 @@ import { createNotification } from "@app/components/notifications";
 import { apiRequest } from "@app/config/request";
 
 import { workspaceKeys } from "../workspace";
-import { TCloudIntegration, TIntegrationWithEnv, TOctopusDeployScopeValues } from "./types";
+import {
+  IntegrationMetadataSyncMode,
+  TCloudIntegration,
+  TIntegrationWithEnv,
+  TOctopusDeployScopeValues
+} from "./types";
 
 export const integrationQueryKeys = {
   getIntegrations: () => ["integrations"] as const,
@@ -89,6 +94,7 @@ export const useCreateIntegration = () => {
         shouldProtectSecrets?: boolean;
         shouldEnableDelete?: boolean;
         octopusDeployScopeValues?: TOctopusDeployScopeValues;
+        metadataSyncMode?: IntegrationMetadataSyncMode;
       };
     }) => {
       const {

frontend/src/hooks/api/integrations/types.ts

@@ -62,6 +62,7 @@ export type TIntegration = {
     octopusDeployScopeValues?: TOctopusDeployScopeValues;
     awsIamRole?: string;
     region?: string;
+    metadataSyncMode?: IntegrationMetadataSyncMode;
   };
 };
 
@@ -92,3 +93,8 @@ export enum IntegrationMappingBehavior {
   ONE_TO_ONE = "one-to-one",
   MANY_TO_ONE = "many-to-one"
 }
+
+export enum IntegrationMetadataSyncMode {
+  CUSTOM = "custom",
+  SECRET_METADATA = "secret-metadata"
+}

frontend/src/pages/integrations/aws-secret-manager/create.tsx

@@ -19,7 +19,10 @@ import z from "zod";
 import { SecretPathInput } from "@app/components/v2/SecretPathInput";
 import { useCreateIntegration } from "@app/hooks/api";
 import { useGetIntegrationAuthAwsKmsKeys } from "@app/hooks/api/integrationAuth/queries";
-import { IntegrationMappingBehavior } from "@app/hooks/api/integrations/types";
+import {
+  IntegrationMappingBehavior,
+  IntegrationMetadataSyncMode
+} from "@app/hooks/api/integrations/types";
 
 import {
   Badge,
@@ -97,6 +100,7 @@ const schema = z
     mappingBehavior: z.nativeEnum(IntegrationMappingBehavior),
     kmsKeyId: z.string().optional(),
     shouldTag: z.boolean().optional(),
+    metadataSyncMode: z.nativeEnum(IntegrationMetadataSyncMode).optional(),
     tags: z
       .object({
         key: z.string(),
@@ -139,6 +143,7 @@ export default function AWSSecretManagerCreateIntegrationPage() {
   });
 
   const shouldTagState = watch("shouldTag");
+  const selectedMetadataSyncMode = watch("metadataSyncMode");
   const selectedSourceEnvironment = watch("sourceEnvironment");
   const selectedAWSRegion = watch("awsRegion");
   const selectedMappingBehavior = watch("mappingBehavior");
@@ -171,7 +176,8 @@ export default function AWSSecretManagerCreateIntegrationPage() {
     tags,
     secretPrefix,
     kmsKeyId,
-    mappingBehavior
+    mappingBehavior,
+    metadataSyncMode
   }: TFormSchema) => {
     try {
       if (!integrationAuth?.id) return;
@@ -186,7 +192,8 @@ export default function AWSSecretManagerCreateIntegrationPage() {
         metadata: {
           ...(shouldTag
             ? {
-                secretAWSTag: tags
+                secretAWSTag: tags,
+                metadataSyncMode
               }
             : {}),
           ...(secretPrefix && { secretPrefix }),
@@ -339,7 +346,12 @@ export default function AWSSecretManagerCreateIntegrationPage() {
                     >
                       <Select
                         defaultValue={field.value}
-                        onValueChange={(e) => onChange(e)}
+                        onValueChange={(e) => {
+                          if (e === IntegrationMappingBehavior.MANY_TO_ONE) {
+                            setValue("metadataSyncMode", IntegrationMetadataSyncMode.CUSTOM);
+                          }
+                          onChange(e);
+                        }}
                         className="w-full border border-mineshaft-500"
                         dropdownContainerClassName="max-w-full"
                       >
@@ -386,7 +398,7 @@ export default function AWSSecretManagerCreateIntegrationPage() {
                 animate={{ opacity: 1, translateX: 0 }}
                 exit={{ opacity: 0, translateX: 30 }}
               >
-                <div className="mt-2 ml-1">
+                <div className="mt-2 ml-1 mb-3">
                   <Controller
                     control={control}
                     name="shouldTag"
@@ -401,37 +413,76 @@ export default function AWSSecretManagerCreateIntegrationPage() {
                     )}
                   />
                 </div>
-                {shouldTagState && (
+                {shouldTagState &&
-                  <div className="mt-4 flex justify-between">
+                  selectedMappingBehavior === IntegrationMappingBehavior.ONE_TO_ONE && (
                     <Controller
                       control={control}
-                      name="tags.0.key"
+                      name="metadataSyncMode"
-                      render={({ field, fieldState: { error } }) => (
+                      render={({ field: { onChange, ...field }, fieldState: { error } }) => (
                         <FormControl
-                          label="Tag Key"
+                          label="Tag Sync Mode"
                           errorText={error?.message}
                           isError={Boolean(error)}
                         >
-                          <Input placeholder="managed-by" {...field} />
+                          <Select
+                            defaultValue={field.value}
+                            onValueChange={(e) => {
+                              setValue("tags", []);
+                              onChange(e);
+                            }}
+                            className="w-full border border-mineshaft-500"
+                            dropdownContainerClassName="max-w-full"
+                          >
+                            <SelectItem
+                              value={IntegrationMetadataSyncMode.SECRET_METADATA}
+                              className="text-left"
+                              key={`sync-mode-${IntegrationMetadataSyncMode.SECRET_METADATA}`}
+                            >
+                              Secret Metadata
+                            </SelectItem>
+                            <SelectItem
+                              value={IntegrationMetadataSyncMode.CUSTOM}
+                              className="text-left"
+                              key={`sync-mode-${IntegrationMetadataSyncMode.CUSTOM}`}
+                            >
+                              Custom
+                            </SelectItem>
+                          </Select>
                         </FormControl>
                       )}
                     />
-                    <Controller
+                  )}
-                      control={control}
+                {shouldTagState &&
-                      name="tags.0.value"
+                  selectedMetadataSyncMode === IntegrationMetadataSyncMode.CUSTOM && (
-                      render={({ field, fieldState: { error } }) => (
+                    <div className="mt-4 flex justify-between">
-                        <FormControl
+                      <Controller
-                          label="Tag Value"
+                        control={control}
-                          errorText={error?.message}
+                        name="tags.0.key"
-                          isError={Boolean(error)}
+                        render={({ field, fieldState: { error } }) => (
-                        >
+                          <FormControl
-                          <Input placeholder="infisical" {...field} />
+                            label="Tag Key"
-                        </FormControl>
+                            errorText={error?.message}
-                      )}
+                            isError={Boolean(error)}
-                    />
+                          >
-                  </div>
+                            <Input placeholder="managed-by" {...field} />
-                )}
+                          </FormControl>
-
+                        )}
+                      />
+                      <Controller
+                        control={control}
+                        name="tags.0.value"
+                        render={({ field, fieldState: { error } }) => (
+                          <FormControl
+                            label="Tag Value"
+                            errorText={error?.message}
+                            isError={Boolean(error)}
+                          >
+                            <Input placeholder="infisical" {...field} />
+                          </FormControl>
+                        )}
+                      />
+                    </div>
+                  )}
                 <Controller
                   control={control}
                   name="secretPrefix"

frontend/src/views/IntegrationsPage/IntegrationDetailsPage/components/IntegrationSettingsSection.tsx

@@ -31,7 +31,8 @@ const metadataMappings: Record<keyof NonNullable<TIntegrationWithEnv["metadata"]
   shouldEnableDelete: "GitHub Secret Deletion Enabled",
   octopusDeployScopeValues: "Octopus Deploy Scope Values",
   awsIamRole: "AWS IAM Role",
-  region: "Region"
+  region: "Region",
+  metadataSyncMode: "Metadata Sync Mode"
 } as const;
 
 export const IntegrationSettingsSection = ({ integration }: Props) => {